feat(mdtest run/auto): support specifying multiple test scripts in the
test spec file and add --coverage option for mdtest to report code
coverage
mdtest now supports the execution of multiple test scripts. This
feature can be used if running multiple test scripts only requires an
application to be started once before the first execution. Subsequent
tests will not cause the application restart.
Code coverage support is also added in mdtest for both 'run' and 'auto'
commands. mdtest is able to read code coverage hitmap from the
application after test execution and create a coverage info file in lcov
format so that coverage report tools like lcov/genhtml can be used to
create coverage report for the application.
Change-Id: I85cdb4df5112f7edb62397f552342b245c37d0c2
diff --git a/mdtest/lib/src/base/common.dart b/mdtest/lib/src/base/common.dart
index 84dfc97..de1d4d3 100644
--- a/mdtest/lib/src/base/common.dart
+++ b/mdtest/lib/src/base/common.dart
@@ -3,3 +3,4 @@
// license that can be found in the LICENSE file.
const String defaultTempSpecsName = 'tmp.spec';
+const String defaultCodeCoverageDirectoryPath = 'coverage/code_coverage';
diff --git a/mdtest/lib/src/commands/auto.dart b/mdtest/lib/src/commands/auto.dart
index a176229..0b01dcf 100644
--- a/mdtest/lib/src/commands/auto.dart
+++ b/mdtest/lib/src/commands/auto.dart
@@ -13,6 +13,7 @@
import '../algorithms/matching.dart';
import '../globals.dart';
import '../runner/mdtest_command.dart';
+import '../test/coverage_collector.dart';
class AutoCommand extends MDTestCommand {
@override
@@ -61,6 +62,9 @@
= findMinimumMappings(cov2match, clusterInfo);
printMatches(chosenMappings);
+ Map<String, CoverageCollector> collectorPool
+ = <String, CoverageCollector>{};
+
List<int> errRounds = [];
int roundNum = 1;
for (Map<DeviceSpec, Device> deviceMapping in chosenMappings) {
@@ -75,13 +79,19 @@
await storeMatches(deviceMapping);
- if (await runner.runTest(_specs['test-path']) != 0) {
- printError('Test execution exit with error.');
+ if (await runner.runAllTests(_specs['test-paths']) != 0) {
+ printError('Tests execution exit with error.');
await uninstallTestedApps(deviceMapping);
errRounds.add(roundNum++);
continue;
}
+ if (argResults['coverage']) {
+ print('Collecting code coverage hitmap ...');
+ buildCoverageCollectionTasks(deviceMapping, collectorPool);
+ await runCoverageCollectionTasks(collectorPool);
+ }
+
await uninstallTestedApps(deviceMapping);
}
@@ -90,10 +100,17 @@
return 1;
}
+ if (argResults['coverage']) {
+ print('Computing code coverage for each application ...');
+ if (await computeAppsCoverage(collectorPool, name) != 0)
+ return 1;
+ }
+
return 0;
}
AutoCommand() {
usesSpecsOption();
+ usesCoverageFlag();
}
}
diff --git a/mdtest/lib/src/commands/helper.dart b/mdtest/lib/src/commands/helper.dart
index ee5bd5d..fd7dc55 100644
--- a/mdtest/lib/src/commands/helper.dart
+++ b/mdtest/lib/src/commands/helper.dart
@@ -6,10 +6,13 @@
import 'dart:convert';
import 'dart:io';
+import '../base/common.dart';
import '../mobile/device.dart';
import '../mobile/device_spec.dart';
import '../mobile/android.dart';
+import '../test/coverage_collector.dart';
import '../globals.dart';
+import '../util.dart';
class MDTestRunner {
List<Process> appProcesses;
@@ -71,6 +74,15 @@
return 0;
}
+ /// Run all tests
+ Future<int> runAllTests(Iterable<String> testPaths) async {
+ int result = 0;
+ for (String testPath in testPaths) {
+ result += await runTest(testPath);
+ }
+ return result == 0 ? 0 : 1;
+ }
+
/// Create a process and invoke 'dart testPath' to run the test script. After
/// test result is returned (either pass or fail), kill all app processes and
/// return the current process exit code
@@ -85,7 +97,6 @@
if (testStopPattern.hasMatch(line.toString()))
break;
}
- killAppProcesses();
process.stderr.drain();
return await process.exitCode;
}
@@ -97,3 +108,64 @@
}
}
}
+
+/// Create a coverage collector for each application and assign a coverage
+/// collection task for the coverage collector
+void buildCoverageCollectionTasks(
+ Map<DeviceSpec, Device> deviceMapping,
+ Map<String, CoverageCollector> collectorPool
+) {
+ assert(collectorPool != null);
+ // Build app path to coverage collector mapping and add collection tasks
+ deviceMapping.keys.forEach((DeviceSpec spec) {
+ collectorPool.putIfAbsent(
+ spec.appRootPath,
+ () => new CoverageCollector()
+ ).collectCoverage(spec.observatoryUrl);
+ });
+}
+
+/// Run coverage collection tasks for each application
+Future<Null> runCoverageCollectionTasks(
+ Map<String, CoverageCollector> collectorPool
+) async {
+ assert(collectorPool.isNotEmpty);
+ // Collect coverage for every application
+ for (CoverageCollector collector in collectorPool.values) {
+ await collector.finishPendingJobs();
+ }
+}
+
+/// Compute application code coverage and write coverage info in lcov format
+Future<int> computeAppsCoverage(
+ Map<String, CoverageCollector> collectorPool,
+ String commandName
+) async {
+ if (collectorPool.isEmpty)
+ return 1;
+ // Write coverage info to coverage/code_coverage folder under each
+ // application folder
+ for (String appRootPath in collectorPool.keys) {
+ CoverageCollector collector = collectorPool[appRootPath];
+ String coverageData = await collector.finalizeCoverage(appRootPath);
+ if (coverageData == null)
+ return 1;
+
+ String coveragePath = normalizePath(
+ appRootPath,
+ '$defaultCodeCoverageDirectoryPath',
+ 'cov_${commandName}_${generateTimeStamp()}.info'
+ );
+ try {
+ // Write coverage info to code_coverage folder
+ new File(coveragePath)
+ ..createSync(recursive: true)
+ ..writeAsStringSync(coverageData, flush: true);
+ print('Writing code coverage to $coveragePath');
+ } on FileSystemException {
+ printError('Cannot write code coverage info to $coveragePath');
+ return 1;
+ }
+ }
+ return 0;
+}
diff --git a/mdtest/lib/src/commands/run.dart b/mdtest/lib/src/commands/run.dart
index edbdf7a..45ca296 100644
--- a/mdtest/lib/src/commands/run.dart
+++ b/mdtest/lib/src/commands/run.dart
@@ -11,6 +11,7 @@
import '../algorithms/matching.dart';
import '../globals.dart';
import '../runner/mdtest_command.dart';
+import '../test/coverage_collector.dart';
class RunCommand extends MDTestCommand {
@@ -58,12 +59,23 @@
await storeMatches(deviceMapping);
- if (await runner.runTest(_specs['test-path']) != 0) {
- printError('Test execution exit with error.');
+ if (await runner.runAllTests(_specs['test-paths']) != 0) {
+ printError('Tests execution exit with error.');
await uninstallTestedApps(deviceMapping);
return 1;
}
+ if (argResults['coverage']) {
+ Map<String, CoverageCollector> collectorPool
+ = <String, CoverageCollector>{};
+ buildCoverageCollectionTasks(deviceMapping, collectorPool);
+ print('Collecting code coverage hitmap ...');
+ await runCoverageCollectionTasks(collectorPool);
+ print('Computing code coverage for each application ...');
+ if (await computeAppsCoverage(collectorPool, name) != 0)
+ return 1;
+ }
+
await uninstallTestedApps(deviceMapping);
return 0;
@@ -71,5 +83,6 @@
RunCommand() {
usesSpecsOption();
+ usesCoverageFlag();
}
}
diff --git a/mdtest/lib/src/commands/runner.dart b/mdtest/lib/src/commands/runner.dart
deleted file mode 100644
index f57876a..0000000
--- a/mdtest/lib/src/commands/runner.dart
+++ /dev/null
@@ -1,94 +0,0 @@
-// Copyright 2016 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-import 'dart:async';
-import 'dart:convert';
-import 'dart:io';
-
-import '../mobile/device.dart';
-import '../mobile/device_spec.dart';
-import '../globals.dart';
-
-class MDTestRunner {
- List<Process> appProcesses;
-
- MDTestRunner() {
- appProcesses = <Process>[];
- }
-
- /// Invoke runApp function for each device spec to device mapping in parallel
- Future<int> runAllApps(Map<DeviceSpec, Device> deviceMapping) async {
- List<Future<int>> runAppList = <Future<int>>[];
- for (DeviceSpec deviceSpec in deviceMapping.keys) {
- Device device = deviceMapping[deviceSpec];
- runAppList.add(runApp(deviceSpec, device));
- }
- int res = 0;
- List<int> results = await Future.wait(runAppList);
- for (int result in results)
- res += result;
- return res == 0 ? 0 : 1;
- }
-
- /// Create a process that runs 'flutter run ...' command which installs and
- /// starts the app on the device. The function finds a observatory port
- /// through the process output. If no observatory port is found, then report
- /// error.
- Future<int> runApp(DeviceSpec deviceSpec, Device device) async {
- Process process = await Process.start(
- 'flutter',
- ['run', '-d', device.id, '--target=${deviceSpec.appPath}'],
- workingDirectory: deviceSpec.appRootPath
- );
- appProcesses.add(process);
- Stream lineStream = process.stdout
- .transform(new Utf8Decoder())
- .transform(new LineSplitter());
- RegExp portPattern = new RegExp(r'Observatory listening on (http.*)');
- await for (var line in lineStream) {
- print(line.toString().trim());
- Match portMatch = portPattern.firstMatch(line.toString());
- if (portMatch != null) {
- deviceSpec.observatoryUrl = portMatch.group(1);
- break;
- }
- }
-
- process.stderr.drain();
-
- if (deviceSpec.observatoryUrl == null) {
- printError('No observatory url is found.');
- return 1;
- }
-
- return 0;
- }
-
- /// Create a process and invoke 'dart testPath' to run the test script. After
- /// test result is returned (either pass or fail), kill all app processes and
- /// return the current process exit code
- Future<int> runTest(String testPath) async {
- Process process = await Process.start('dart', ['$testPath']);
- RegExp testStopPattern = new RegExp(r'All tests passed|Some tests failed');
- Stream stdoutStream = process.stdout
- .transform(new Utf8Decoder())
- .transform(new LineSplitter());
- await for (var line in stdoutStream) {
- print(line.toString().trim());
- if (testStopPattern.hasMatch(line.toString())) {
- process.stderr.drain();
- killAppProcesses();
- break;
- }
- }
- return await process.exitCode;
- }
-
- /// Kill all app processes
- Future<Null> killAppProcesses() async {
- for (Process process in appProcesses) {
- process.kill();
- }
- }
-}
diff --git a/mdtest/lib/src/mobile/device_spec.dart b/mdtest/lib/src/mobile/device_spec.dart
index 042b055..697daa9 100644
--- a/mdtest/lib/src/mobile/device_spec.dart
+++ b/mdtest/lib/src/mobile/device_spec.dart
@@ -69,7 +69,11 @@
// Get the parent directory of the specs file
String rootPath = new File(specsPath).parent.absolute.path;
// Normalize the 'test-path' in the specs file
- newSpecs['test-path'] = normalizePath(rootPath, newSpecs['test-path']);
+ // newSpecs['test-path'] = normalizePath(rootPath, newSpecs['test-path']);
+ newSpecs['test-paths']
+ = newSpecs['test-paths'].map(
+ (String testPath) => normalizePath(rootPath, testPath)
+ );
// Normalize the 'app-path' in the specs file
newSpecs['devices'].forEach((String name, Map<String, String> map) {
map['app-path'] = normalizePath(rootPath, map['app-path']);
diff --git a/mdtest/lib/src/runner/mdtest_command.dart b/mdtest/lib/src/runner/mdtest_command.dart
index 7610572..c20c5e4 100644
--- a/mdtest/lib/src/runner/mdtest_command.dart
+++ b/mdtest/lib/src/runner/mdtest_command.dart
@@ -35,6 +35,14 @@
_usesSpecsOption = true;
}
+ void usesCoverageFlag() {
+ argParser.addFlag('coverage',
+ defaultsTo: false,
+ negatable: false,
+ help: 'Whether to collect coverage information.'
+ );
+ }
+
@override
Future<int> run() {
Stopwatch stopwatch = new Stopwatch()..start();
diff --git a/mdtest/lib/src/test/coverage_collector.dart b/mdtest/lib/src/test/coverage_collector.dart
new file mode 100644
index 0000000..4060711
--- /dev/null
+++ b/mdtest/lib/src/test/coverage_collector.dart
@@ -0,0 +1,59 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:coverage/coverage.dart';
+import 'package:path/path.dart' as path;
+
+class CoverageCollector {
+ List<Future<Null>> _jobs = <Future<Null>>[];
+ Map<String, dynamic> _globalHitmap;
+
+ void collectCoverage(String observatoryUrl) {
+ RegExp urlPattern = new RegExp(r'http://(.*):(\d+)');
+ Match urlMatcher = urlPattern.firstMatch(observatoryUrl);
+ if (urlMatcher == null) {
+ print('Cannot parse host name and port '
+ 'from observatory url $observatoryUrl');
+ return;
+ }
+ String host = urlMatcher.group(1);
+ int port = int.parse(urlMatcher.group(2));
+ _jobs.add(_startJob(
+ host: host,
+ port: port
+ ));
+ }
+
+ Future<Null> _startJob({
+ String host,
+ int port
+ }) async {
+ Map<String, dynamic> data = await collect(host, port, false, false);
+ Map<String, dynamic> hitmap = createHitmap(data['coverage']);
+ if (_globalHitmap == null)
+ _globalHitmap = hitmap;
+ else
+ mergeHitmaps(hitmap, _globalHitmap);
+ }
+
+ Future<Null> finishPendingJobs() async {
+ await Future.wait(_jobs.toList(), eagerError: true);
+ }
+
+ Future<String> finalizeCoverage(String appRootPath) async {
+ if (_globalHitmap == null)
+ return null;
+ Resolver resolver
+ = new Resolver(packagesPath: path.join(appRootPath, '.packages'));
+ LcovFormatter formatter = new LcovFormatter(resolver);
+ List<String> reportOn = <String>[path.join(appRootPath, 'lib')];
+ return await formatter.format(
+ _globalHitmap,
+ reportOn: reportOn,
+ basePath: appRootPath
+ );
+ }
+}
diff --git a/mdtest/lib/src/util.dart b/mdtest/lib/src/util.dart
index 7f85405..bf0359c 100644
--- a/mdtest/lib/src/util.dart
+++ b/mdtest/lib/src/util.dart
@@ -2,11 +2,13 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-import 'dart:io' show Platform;
+import 'dart:io';
import 'dart:math';
import 'package:path/path.dart' as path;
+import 'globals.dart';
+
int minLength(List<String> elements) {
if (elements == null || elements.isEmpty) return -1;
return elements.map((String e) => e.length).reduce(min);
@@ -35,6 +37,27 @@
return minL;
}
-String normalizePath(String rootPath, String relativePath) {
- return path.normalize(path.join(rootPath, relativePath));
+String normalizePath(
+ String rootPath,
+ [String relativePath1, relativePath2]
+) {
+ return path.normalize(
+ path.join(rootPath, relativePath1, relativePath2)
+ );
+}
+
+String generateTimeStamp() {
+ return new DateTime.now().toIso8601String();
+}
+
+bool deleteDirectories(Iterable<String> dirPaths) {
+ for (String dirPath in dirPaths) {
+ try {
+ new Directory(dirPath).deleteSync(recursive: true);
+ } on FileSystemException {
+ printError('Cannot delete directory $dirPath');
+ return false;
+ }
+ }
+ return true;
}
diff --git a/mdtest/pubspec.yaml b/mdtest/pubspec.yaml
index 163108c..ac93572 100644
--- a/mdtest/pubspec.yaml
+++ b/mdtest/pubspec.yaml
@@ -4,6 +4,7 @@
dependencies:
args: ^0.13.4
stack_trace: ^1.4.0
+ coverage: ^0.7.9
dlog:
path: ../../../../third_party/dart/dlog
flutter_driver:
