feat(mdtest auto): introduce app-device coverage concept
Introduce app-device coverage concept. We first cluster device specs
based on the app path, all specs with the same app path is clustered
together. We also cluster devices based on model name, devices with
same model name is clustered together. The clustering rule can be
easily changed. In the current version, only the above clustering rule
is supported. Some app-device coverage definition:
* Coverage Definition:
If an app in an app cluster A is installed and run on a device in a
device cluster D, then we say app cluster A covers device cluster D
and the app-device path A-D is covered.
* Spec-Device Mapping Suite Definition:
A spec-device mapping suite is defined as the union of all spec to
device mappings for the same test script. For example, a single
application can be run on different types of devices. In such case,
the test script and spec never change but the spec-device mapping is
changed so an application can be run on different devices.
* Full Coverage Definition:
If a test script is repeatedly executed according to a spec-device
mapping suite S, in all such runs, if every app cluster covers
every device cluster, then we say the spec-device mapping suite S
gives full app-device coverage.
* Coverage Score Definition:
Suppose there are N app clusters, M device clusters, and a
spec-device mapping suite S covers X app-device paths, then the
coverage score is computed by C(S) = X / (N * M)
An algorithm is implemented to find the minimum set of spec-device
matches that gives the maximum app-device coverage. A new command
"mdtest auto ..." is implemented to invoke the algorithm.
Change-Id: Idcc7e33daba4754d26e409d8701f8e3b857935de
diff --git a/mdtest/lib/src/driver/driver_util.dart b/mdtest/lib/driver_util.dart
similarity index 95%
rename from mdtest/lib/src/driver/driver_util.dart
rename to mdtest/lib/driver_util.dart
index 1b84816..ae8ea2d 100644
--- a/mdtest/lib/src/driver/driver_util.dart
+++ b/mdtest/lib/driver_util.dart
@@ -8,8 +8,8 @@
import 'package:flutter_driver/flutter_driver.dart';
-import '../base/common.dart';
-import '../globals.dart';
+import 'src/base/common.dart';
+import 'src/globals.dart';
class DriverUtil {
static Future<FlutterDriver> connectByName(String deviceNickname) async {
diff --git a/mdtest/lib/executable.dart b/mdtest/lib/executable.dart
index 47ef009..46fbe09 100644
--- a/mdtest/lib/executable.dart
+++ b/mdtest/lib/executable.dart
@@ -8,11 +8,13 @@
import 'package:stack_trace/stack_trace.dart';
import 'src/commands/run.dart';
+import 'src/commands/auto.dart';
import 'src/runner/mdtest_command_runner.dart';
Future<Null> main(List<String> args) async {
MDTestCommandRunner runner = new MDTestCommandRunner()
- ..addCommand(new RunCommand());
+ ..addCommand(new RunCommand())
+ ..addCommand(new AutoCommand());
return Chain.capture(() async {
dynamic result = await runner.run(args);
diff --git a/mdtest/lib/src/algorithms/coverage.dart b/mdtest/lib/src/algorithms/coverage.dart
new file mode 100644
index 0000000..6791af3
--- /dev/null
+++ b/mdtest/lib/src/algorithms/coverage.dart
@@ -0,0 +1,138 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'package:dlog/dlog.dart' show Table;
+
+import '../mobile/device.dart' show Device;
+import '../mobile/device_spec.dart' show DeviceSpec;
+import '../util.dart';
+
+class ClusterInfo {
+ Map<String, List<Device>> _deviceClusters;
+ Map<String, List<DeviceSpec>> _deviceSpecClusters;
+ List<String> _deviceClustersOrder;
+ List<String> _deviceSpecClustersOrder;
+
+ ClusterInfo(
+ Map<String, List<Device>> deviceClusters,
+ Map<String, List<DeviceSpec>> deviceSpecClusters
+ ) {
+ _deviceClusters = deviceClusters;
+ _deviceSpecClusters = deviceSpecClusters;
+ _deviceClustersOrder = new List.from(_deviceClusters.keys);
+ _deviceSpecClustersOrder = new List.from(_deviceSpecClusters.keys);
+ }
+
+ Map<String, List<Device>> get deviceClusters => _deviceClusters;
+ Map<String, List<DeviceSpec>> get deviceSpecClusters => _deviceSpecClusters;
+ List<String> get deviceClustersOrder => _deviceClustersOrder;
+ List<String> get deviceSpecClustersOrder => _deviceSpecClustersOrder;
+}
+
+class CoverageMatrix {
+
+ CoverageMatrix(this.clusterInfo) {
+ this.matrix = new List<List<int>>(clusterInfo.deviceSpecClusters.length);
+ for (int i = 0; i < matrix.length; i++) {
+ matrix[i] = new List<int>.filled(clusterInfo.deviceClusters.length, 0);
+ }
+ }
+
+ ClusterInfo clusterInfo;
+ // Coverage matrix, where a row indicats an app cluster and a column
+ // indicates a device cluster
+ List<List<int>> matrix;
+
+ void fill(Map<DeviceSpec, Device> match) {
+ match.forEach((DeviceSpec spec, Device device) {
+ int rowNum = clusterInfo.deviceSpecClustersOrder
+ .indexOf(spec.clusterKey());
+ int colNum = clusterInfo.deviceClustersOrder
+ .indexOf(device.clusterKey());
+ matrix[rowNum][colNum] = 1;
+ });
+ }
+
+ void union(CoverageMatrix newCoverage) {
+ for (int i = 0; i < matrix.length; i++) {
+ List<int> row = matrix[i];
+ for (int j = 0; j < row.length; j++) {
+ matrix[i][j] |= newCoverage.matrix[i][j];
+ }
+ }
+ }
+
+ void printMatrix() {
+ Table prettyMatrix = new Table(1);
+ prettyMatrix.columns.add('app key \\ device key');
+ prettyMatrix.columns.addAll(clusterInfo.deviceClustersOrder);
+ int startIndx = beginOfDiff(clusterInfo.deviceSpecClustersOrder);
+ for (int i = 0; i < matrix.length; i++) {
+ prettyMatrix.data.add(clusterInfo.deviceSpecClustersOrder[i].substring(startIndx));
+ prettyMatrix.data.addAll(matrix[i]);
+ }
+ print(prettyMatrix);
+ }
+}
+
+Map<CoverageMatrix, Map<DeviceSpec, Device>> buildCoverage2MatchMapping(
+ List<Map<DeviceSpec, Device>> allMatches,
+ ClusterInfo clusterInfo
+) {
+ Map<CoverageMatrix, Map<DeviceSpec, Device>> cov2match
+ = <CoverageMatrix, Map<DeviceSpec, Device>>{};
+ for (Map<DeviceSpec, Device> match in allMatches) {
+ CoverageMatrix cov = new CoverageMatrix(clusterInfo);
+ cov.fill(match);
+ cov2match[cov] = match;
+ }
+ return cov2match;
+}
+
+/// Find a small number of mappings which cover the maximum app-device coverage
+/// feasible in given the available devices and specs. The problem can be
+/// treated as a set cover problem which is NP-complete and the implementation
+/// follow the spirit of greedy algorithm which is O(log(n)).
+/// [ref link]: https://en.wikipedia.org/wiki/Set_cover_problem
+Set<Map<DeviceSpec, Device>> findMinimumMappings(
+ Map<CoverageMatrix, Map<DeviceSpec, Device>> cov2match,
+ ClusterInfo clusterInfo
+) {
+ Set<CoverageMatrix> minSet = new Set<CoverageMatrix>();
+ CoverageMatrix base = new CoverageMatrix(clusterInfo);
+ while (true) {
+ CoverageMatrix currentBestCoverage = null;
+ int maxReward = 0;
+ for (CoverageMatrix coverage in cov2match.keys) {
+ if (minSet.contains(coverage)) continue;
+ int reward = computeReward(base, coverage);
+ if (maxReward < reward) {
+ maxReward = reward;
+ currentBestCoverage = coverage;
+ }
+ }
+ if (currentBestCoverage == null) break;
+ minSet.add(currentBestCoverage);
+ base.union(currentBestCoverage);
+ }
+ print('Best coverage matrix:');
+ base.printMatrix();
+ Set<Map<DeviceSpec, Device>> bestMatches = new Set<Map<DeviceSpec, Device>>();
+ for (CoverageMatrix coverage in minSet) {
+ bestMatches.add(cov2match[coverage]);
+ }
+ return bestMatches;
+}
+
+int computeReward(CoverageMatrix base, CoverageMatrix newCoverage) {
+ int reward = 0;
+ for (int i = 0; i < base.matrix.length; i++) {
+ List<int> row = base.matrix[i];
+ for (int j = 0; j < row.length; j++) {
+ if (base.matrix[i][j] == 0 && newCoverage.matrix[i][j] == 1)
+ reward++;
+ }
+ }
+ return reward;
+}
diff --git a/mdtest/lib/src/algorithms/matching.dart b/mdtest/lib/src/algorithms/matching.dart
new file mode 100644
index 0000000..e75db48
--- /dev/null
+++ b/mdtest/lib/src/algorithms/matching.dart
@@ -0,0 +1,153 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import '../base/common.dart';
+import '../mobile/device.dart';
+import '../mobile/device_spec.dart';
+
+/// Find all matched devices for each device spec
+Map<DeviceSpec, Set<Device>> findIndividualMatches(
+ List<DeviceSpec> deviceSpecs,
+ List<Device> devices) {
+ Map<DeviceSpec, Set<Device>> individualMatches
+ = new Map<DeviceSpec, Set<Device>>();
+ for(DeviceSpec deviceSpecs in deviceSpecs) {
+ Set<Device> matchedDevices = new Set<Device>();
+ for(Device device in devices) {
+ if(deviceSpecs.matches(device))
+ matchedDevices.add(device);
+ }
+ individualMatches[deviceSpecs] = matchedDevices;
+ }
+ return individualMatches;
+}
+
+/// Return the first device spec to device matching, null if no such matching
+Map<DeviceSpec, Device> findMatchingDeviceMapping(
+ List<DeviceSpec> deviceSpecs,
+ Map<DeviceSpec, Set<Device>> individualMatches) {
+ Map<DeviceSpec, Device> deviceMapping = <DeviceSpec, Device>{};
+ Set<Device> visited = new Set<Device>();
+ if (!_findMatchingDeviceMapping(0, deviceSpecs, individualMatches,
+ visited, deviceMapping)) {
+ return null;
+ }
+ return deviceMapping;
+}
+
+/// Find a mapping that matches every device spec to a device. If such
+/// mapping is not found, return false, otherwise return true.
+bool _findMatchingDeviceMapping(
+ int order,
+ List<DeviceSpec> deviceSpecs,
+ Map<DeviceSpec, Set<Device>> individualMatches,
+ Set<Device> visited,
+ Map<DeviceSpec, Device> deviceMapping
+) {
+ if(order == deviceSpecs.length) return true;
+ DeviceSpec deviceSpec = deviceSpecs[order];
+ Set<Device> matchedDevices = individualMatches[deviceSpec];
+ for(Device candidate in matchedDevices) {
+ if(visited.add(candidate)) {
+ deviceMapping[deviceSpec] = candidate;
+ if(_findMatchingDeviceMapping(order + 1, deviceSpecs, individualMatches,
+ visited, deviceMapping))
+ return true;
+ else {
+ visited.remove(candidate);
+ deviceMapping.remove(deviceSpec);
+ }
+ }
+ }
+ return false;
+}
+
+/// Store the specs to device mapping as a system temporary file. The file
+/// stores device nickname as well as device id and observatory url for
+/// each device
+Future<Null> storeMatches(Map<DeviceSpec, Device> deviceMapping) async {
+ Map<String, dynamic> matchesData = new Map<String, dynamic>();
+ deviceMapping.forEach((DeviceSpec specs, Device device) {
+ matchesData[specs.nickName] =
+ {
+ 'device-id': device.id,
+ 'observatory-url': specs.observatoryUrl
+ };
+ });
+ Directory systemTempDir = Directory.systemTemp;
+ File tempFile = new File('${systemTempDir.path}/$defaultTempSpecsName');
+ if(await tempFile.exists())
+ await tempFile.delete();
+ File file = await tempFile.create();
+ await file.writeAsString(JSON.encode(matchesData));
+}
+
+/// Return all spec to device mappings, return empty list if no such mapping
+/// exists
+List<Map<DeviceSpec, Device>> findAllMatchingDeviceMappings(
+ List<DeviceSpec> deviceSpecs,
+ Map<DeviceSpec, Set<Device>> individualMatches) {
+ Map<DeviceSpec, Device> deviceMapping = <DeviceSpec, Device>{};
+ Set<Device> visited = new Set<Device>();
+ List<Map<DeviceSpec, Device>> allMatches = <Map<DeviceSpec, Device>>[];
+ _findAllMatchingDeviceMappings(
+ 0, deviceSpecs, individualMatches,
+ visited, deviceMapping, allMatches
+ );
+ return allMatches;
+}
+
+/// Recursively find every spec to device mapping and collect such mappings
+/// as [allMatches]. Return true if a mapping is found, false otherwise.
+/// Invoking this method will always return false, the return value is used
+/// during the recursive invocation. The real return value is stored as
+/// [foundAllMatches].
+bool _findAllMatchingDeviceMappings(
+ int order,
+ List<DeviceSpec> deviceSpecs,
+ Map<DeviceSpec, Set<Device>> individualMatches,
+ Set<Device> visited,
+ Map<DeviceSpec, Device> deviceMapping,
+ List<Map<DeviceSpec, Device>> allMatches
+) {
+ if(order == deviceSpecs.length)
+ return true;
+ DeviceSpec deviceSpec = deviceSpecs[order];
+ Set<Device> matchedDevices = individualMatches[deviceSpec];
+ for(Device candidate in matchedDevices) {
+ if(visited.add(candidate)) {
+ deviceMapping[deviceSpec] = candidate;
+ if(_findAllMatchingDeviceMappings(
+ order + 1, deviceSpecs, individualMatches,
+ visited, deviceMapping, allMatches)) {
+ allMatches.add(new Map.from(deviceMapping));
+ }
+ visited.remove(candidate);
+ deviceMapping.remove(deviceSpec);
+ }
+ }
+ return false;
+}
+
+/// Print a collection of matches which is iterable.
+void printMatches(Iterable<Map<DeviceSpec, Device>> matches) {
+ StringBuffer sb = new StringBuffer();
+ int roundNum = 1;
+ sb.writeln('**********');
+ for (Map<DeviceSpec, Device> match in matches) {
+ sb.writeln('Round $roundNum:');
+ match.forEach((DeviceSpec spec, Device device) {
+ sb.writeln('[Spec Cluster Key: ${spec.clusterKey()}]'
+ ' -> '
+ '[Device Cluster Key: ${device.clusterKey()}]');
+ });
+ roundNum++;
+ }
+ sb.write('**********');
+ print(sb.toString());
+}
diff --git a/mdtest/lib/src/commands/auto.dart b/mdtest/lib/src/commands/auto.dart
new file mode 100644
index 0000000..16adf00
--- /dev/null
+++ b/mdtest/lib/src/commands/auto.dart
@@ -0,0 +1,94 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:async';
+
+import 'runner.dart';
+import '../mobile/device.dart';
+import '../mobile/device_spec.dart';
+import '../mobile/key_provider.dart';
+import '../algorithms/coverage.dart';
+import '../algorithms/matching.dart';
+import '../globals.dart';
+import '../runner/mdtest_command.dart';
+
+class AutoCommand extends MDTestCommand {
+ @override
+ final String name = 'auto';
+
+ @override
+ final String description
+ = 'Automatically run applications based on a subset of spec to device '
+ 'settings that maximize the device coverage';
+
+ dynamic _specs;
+
+ List<Device> _devices;
+
+ @override
+ Future<int> runCore() async {
+ print('Running "mdtest auto command" ...');
+
+ this._specs = await loadSpecs(argResults['specs']);
+
+ this._devices = await getDevices();
+ if (_devices.isEmpty) {
+ printError('No device found.');
+ return 1;
+ }
+
+ List<DeviceSpec> allDeviceSpecs
+ = await constructAllDeviceSpecs(_specs['devices']);
+ Map<DeviceSpec, Set<Device>> individualMatches
+ = findIndividualMatches(allDeviceSpecs, _devices);
+ List<Map<DeviceSpec, Device>> allDeviceMappings
+ = findAllMatchingDeviceMappings(allDeviceSpecs, individualMatches);
+ if(allDeviceMappings.isEmpty) {
+ printError('No device specs to devices mapping is found.');
+ return 1;
+ }
+
+ Map<String, List<Device>> deviceClusters = buildCluster(_devices);
+ Map<String, List<DeviceSpec>> deviceSpecClusters
+ = buildCluster(allDeviceSpecs);
+
+ ClusterInfo clusterInfo = new ClusterInfo(deviceClusters, deviceSpecClusters);
+ Map<CoverageMatrix, Map<DeviceSpec, Device>> cov2match
+ = buildCoverage2MatchMapping(allDeviceMappings, clusterInfo);
+ Set<Map<DeviceSpec, Device>> chosenMappings
+ = findMinimumMappings(cov2match, clusterInfo);
+ printMatches(chosenMappings);
+
+ List<int> errRounds = [];
+ int roundNum = 1;
+ for (Map<DeviceSpec, Device> deviceMapping in chosenMappings) {
+ MDTestRunner runner = new MDTestRunner();
+
+ if (await runner.runAllApps(deviceMapping) != 0) {
+ printError('Error when running applications');
+ errRounds.add(roundNum);
+ continue;
+ }
+
+ await storeMatches(deviceMapping);
+
+ if (await runner.runTest(_specs['test-path']) != 0) {
+ printError('Test execution exit with error.');
+ errRounds.add(roundNum);
+ continue;
+ }
+ }
+
+ if (errRounds.isNotEmpty) {
+ printError('Error in Round #${errRounds.join(", #")}');
+ return 1;
+ }
+
+ return 0;
+ }
+
+ AutoCommand() {
+ usesSpecsOption();
+ }
+}
diff --git a/mdtest/lib/src/commands/run.dart b/mdtest/lib/src/commands/run.dart
index 119a3e8..6f698e4 100644
--- a/mdtest/lib/src/commands/run.dart
+++ b/mdtest/lib/src/commands/run.dart
@@ -3,15 +3,11 @@
// license that can be found in the LICENSE file.
import 'dart:async';
-import 'dart:convert';
-import 'dart:io';
-import 'package:path/path.dart' as path;
-
-import '../base/common.dart';
+import 'runner.dart';
import '../mobile/device.dart';
import '../mobile/device_spec.dart';
-import '../mobile/device_util.dart';
+import '../algorithms/matching.dart';
import '../globals.dart';
import '../runner/mdtest_command.dart';
@@ -51,14 +47,16 @@
return 1;
}
- if (await runAllApps(deviceMapping) != 0) {
+ MDTestRunner runner = new MDTestRunner();
+
+ if (await runner.runAllApps(deviceMapping) != 0) {
printError('Error when running applications');
return 1;
}
await storeMatches(deviceMapping);
- if (await runTest(_specs['test-path']) != 0) {
+ if (await runner.runTest(_specs['test-path']) != 0) {
printError('Test execution exit with error.');
return 1;
}
@@ -70,204 +68,3 @@
usesSpecsOption();
}
}
-
-Future<dynamic> loadSpecs(String specsPath) async {
- try {
- // Read specs file into json format
- dynamic newSpecs = JSON.decode(await new File(specsPath).readAsString());
- // Get the parent directory of the specs file
- String rootPath = new File(specsPath).parent.absolute.path;
- // Normalize the 'test-path' in the specs file
- newSpecs['test-path'] = normalizePath(rootPath, newSpecs['test-path']);
- // Normalize the 'app-path' in the specs file
- newSpecs['devices'].forEach((String name, Map<String, String> map) {
- map['app-path'] = normalizePath(rootPath, map['app-path']);
- map['app-root'] = normalizePath(rootPath, map['app-root']);
- });
- return newSpecs;
- } on FileSystemException {
- printError('File $specsPath does not exist.');
- exit(1);
- } on FormatException {
- printError('File $specsPath is not in JSON format.');
- exit(1);
- } catch (e) {
- print('Unknown Exception details:\n $e');
- exit(1);
- }
-}
-
-String normalizePath(String rootPath, String relativePath) {
- return path.normalize(path.join(rootPath, relativePath));
-}
-
-/// Build a list of device specs from mappings loaded from JSON .spec file
-Future<List<DeviceSpec>> constructAllDeviceSpecs(dynamic allSpecs) async {
- List<DeviceSpec> deviceSpecs = <DeviceSpec>[];
- for(String name in allSpecs.keys) {
- Map<String, String> specs = allSpecs[name];
- deviceSpecs.add(
- new DeviceSpec(
- nickName: name,
- deviceID: specs['device-id'],
- deviceModelName: specs['model-name'],
- appRootPath: specs['app-root'],
- appPath: specs['app-path']
- )
- );
- }
- return deviceSpecs;
-}
-
-/// Find all matched devices for each device spec
-Map<DeviceSpec, Set<Device>> findIndividualMatches(
- List<DeviceSpec> deviceSpecs,
- List<Device> devices) {
- Map<DeviceSpec, Set<Device>> individualMatches
- = new Map<DeviceSpec, Set<Device>>();
- for(DeviceSpec deviceSpecs in deviceSpecs) {
- Set<Device> matchedDevices = new Set<Device>();
- for(Device device in devices) {
- if(deviceSpecs.matches(device))
- matchedDevices.add(device);
- }
- individualMatches[deviceSpecs] = matchedDevices;
- }
- return individualMatches;
-}
-
-/// Return the first device spec to device matching, null if no such matching
-Map<DeviceSpec, Device> findMatchingDeviceMapping(
- List<DeviceSpec> deviceSpecs,
- Map<DeviceSpec, Set<Device>> individualMatches) {
- Map<DeviceSpec, Device> deviceMapping = <DeviceSpec, Device>{};
- Set<Device> visited = new Set<Device>();
- if (!_findMatchingDeviceMapping(0, deviceSpecs, individualMatches,
- visited, deviceMapping)) {
- return null;
- }
- return deviceMapping;
-}
-
-/// Find a mapping that matches every device spec to a device. If such
-/// mapping is not found, return false, otherwise return true.
-bool _findMatchingDeviceMapping(
- int order,
- List<DeviceSpec> deviceSpecs,
- Map<DeviceSpec, Set<Device>> individualMatches,
- Set<Device> visited,
- Map<DeviceSpec, Device> deviceMapping
-) {
- if(order == deviceSpecs.length) return true;
- DeviceSpec deviceSpec = deviceSpecs[order];
- Set<Device> matchedDevices = individualMatches[deviceSpec];
- for(Device candidate in matchedDevices) {
- if(visited.add(candidate)) {
- deviceMapping[deviceSpec] = candidate;
- if(_findMatchingDeviceMapping(order + 1, deviceSpecs, individualMatches,
- visited, deviceMapping))
- return true;
- else {
- visited.remove(candidate);
- deviceMapping.remove(deviceSpec);
- }
- }
- }
- return false;
-}
-
-List<Process> appProcesses = <Process>[];
-
-Future<int> runAllApps(Map<DeviceSpec, Device> deviceMapping) async {
- List<Future<int>> runAppList = <Future<int>>[];
- for (DeviceSpec deviceSpec in deviceMapping.keys) {
- Device device = deviceMapping[deviceSpec];
- runAppList.add(runApp(deviceSpec, device));
- }
- int res = 0;
- List<int> results = await Future.wait(runAppList);
- for (int result in results)
- res += result;
- return res == 0 ? 0 : 1;
-}
-
-/// Create a process that runs 'flutter run ...' command which installs and
-/// starts the app on the device. The function finds a observatory port
-/// through the process output. If no observatory port is found, then report
-/// error.
-Future<int> runApp(DeviceSpec deviceSpec, Device device) async {
- Process process = await Process.start(
- 'flutter',
- ['run', '-d', device.id, '--target=${deviceSpec.appPath}'],
- workingDirectory: deviceSpec.appRootPath
- );
- appProcesses.add(process);
- Stream lineStream = process.stdout
- .transform(new Utf8Decoder())
- .transform(new LineSplitter());
- RegExp portPattern = new RegExp(r'Observatory listening on (http.*)');
- await for (var line in lineStream) {
- print(line.toString().trim());
- Match portMatch = portPattern.firstMatch(line.toString());
- if (portMatch != null) {
- deviceSpec.observatoryUrl = portMatch.group(1);
- break;
- }
- }
-
- process.stderr.drain();
-
- if (deviceSpec.observatoryUrl == null) {
- printError('No observatory url is found.');
- return 1;
- }
-
- return 0;
-}
-
-/// Store the specs to device mapping as a system temporary file. The file
-/// stores device nickname as well as device id and observatory port for
-/// each device
-Future<Null> storeMatches(Map<DeviceSpec, Device> deviceMapping) async {
- Map<String, dynamic> matchesData = new Map<String, dynamic>();
- deviceMapping.forEach((DeviceSpec specs, Device device) {
- matchesData[specs.nickName] =
- {
- 'device-id': device.id,
- 'observatory-url': specs.observatoryUrl
- };
- });
- Directory systemTempDir = Directory.systemTemp;
- File tempFile = new File('${systemTempDir.path}/$defaultTempSpecsName');
- if(await tempFile.exists())
- await tempFile.delete();
- File file = await tempFile.create();
- await file.writeAsString(JSON.encode(matchesData));
-}
-
-/// Create a process and invoke 'dart testPath' to run the test script. After
-/// test result is returned (either pass or fail), kill all app processes and
-/// return the current process exit code
-Future<int> runTest(String testPath) async {
- Process process = await Process.start('dart', ['$testPath']);
- RegExp testStopPattern = new RegExp(r'All tests passed|Some tests failed');
- Stream stdoutStream = process.stdout
- .transform(new Utf8Decoder())
- .transform(new LineSplitter());
- await for (var line in stdoutStream) {
- print(line.toString().trim());
- if (testStopPattern.hasMatch(line.toString())) {
- process.stderr.drain();
- killAllProcesses(appProcesses);
- break;
- }
- }
- return await process.exitCode;
-}
-
-/// Kill all given processes
-Future<Null> killAllProcesses(List<Process> processes) async {
- for (Process process in processes) {
- process.kill();
- }
-}
diff --git a/mdtest/lib/src/commands/runner.dart b/mdtest/lib/src/commands/runner.dart
new file mode 100644
index 0000000..f57876a
--- /dev/null
+++ b/mdtest/lib/src/commands/runner.dart
@@ -0,0 +1,94 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import '../mobile/device.dart';
+import '../mobile/device_spec.dart';
+import '../globals.dart';
+
+class MDTestRunner {
+ List<Process> appProcesses;
+
+ MDTestRunner() {
+ appProcesses = <Process>[];
+ }
+
+ /// Invoke runApp function for each device spec to device mapping in parallel
+ Future<int> runAllApps(Map<DeviceSpec, Device> deviceMapping) async {
+ List<Future<int>> runAppList = <Future<int>>[];
+ for (DeviceSpec deviceSpec in deviceMapping.keys) {
+ Device device = deviceMapping[deviceSpec];
+ runAppList.add(runApp(deviceSpec, device));
+ }
+ int res = 0;
+ List<int> results = await Future.wait(runAppList);
+ for (int result in results)
+ res += result;
+ return res == 0 ? 0 : 1;
+ }
+
+ /// Create a process that runs 'flutter run ...' command which installs and
+ /// starts the app on the device. The function finds a observatory port
+ /// through the process output. If no observatory port is found, then report
+ /// error.
+ Future<int> runApp(DeviceSpec deviceSpec, Device device) async {
+ Process process = await Process.start(
+ 'flutter',
+ ['run', '-d', device.id, '--target=${deviceSpec.appPath}'],
+ workingDirectory: deviceSpec.appRootPath
+ );
+ appProcesses.add(process);
+ Stream lineStream = process.stdout
+ .transform(new Utf8Decoder())
+ .transform(new LineSplitter());
+ RegExp portPattern = new RegExp(r'Observatory listening on (http.*)');
+ await for (var line in lineStream) {
+ print(line.toString().trim());
+ Match portMatch = portPattern.firstMatch(line.toString());
+ if (portMatch != null) {
+ deviceSpec.observatoryUrl = portMatch.group(1);
+ break;
+ }
+ }
+
+ process.stderr.drain();
+
+ if (deviceSpec.observatoryUrl == null) {
+ printError('No observatory url is found.');
+ return 1;
+ }
+
+ return 0;
+ }
+
+ /// Create a process and invoke 'dart testPath' to run the test script. After
+ /// test result is returned (either pass or fail), kill all app processes and
+ /// return the current process exit code
+ Future<int> runTest(String testPath) async {
+ Process process = await Process.start('dart', ['$testPath']);
+ RegExp testStopPattern = new RegExp(r'All tests passed|Some tests failed');
+ Stream stdoutStream = process.stdout
+ .transform(new Utf8Decoder())
+ .transform(new LineSplitter());
+ await for (var line in stdoutStream) {
+ print(line.toString().trim());
+ if (testStopPattern.hasMatch(line.toString())) {
+ process.stderr.drain();
+ killAppProcesses();
+ break;
+ }
+ }
+ return await process.exitCode;
+ }
+
+ /// Kill all app processes
+ Future<Null> killAppProcesses() async {
+ for (Process process in appProcesses) {
+ process.kill();
+ }
+ }
+}
diff --git a/mdtest/lib/src/mobile/device.dart b/mdtest/lib/src/mobile/device.dart
index 1286580..ca8b4ad 100644
--- a/mdtest/lib/src/mobile/device.dart
+++ b/mdtest/lib/src/mobile/device.dart
@@ -1,8 +1,13 @@
// Copyright 2016 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
-class Device {
+import 'key_provider.dart';
+
+class Device implements ClusterKeyProvider {
Device({
this.id,
this.modelName
@@ -12,5 +17,68 @@
final String modelName;
@override
+ String clusterKey() {
+ return id;
+ }
+
+ @override
String toString() => '<id: $id, model-name: $modelName>';
}
+
+Future<List<Device>> getDevices() async {
+ List<Device> devices = <Device>[];
+ await _getDeviceIDs().then((List<String> ids) async {
+ for(String id in ids) {
+ devices.add(await _collectDeviceProps(id));
+ }
+ });
+ return devices;
+}
+
+/// Run "flutter devices -v" to collect device ids from the output
+Future<List<String>> _getDeviceIDs() async {
+ List<String> deviceIDs = <String>[];
+ Process process = await Process.start('flutter', ['devices', '-v']);
+ Stream lineStream = process.stdout
+ .transform(new Utf8Decoder())
+ .transform(new LineSplitter());
+ bool startReading = false;
+ RegExp startPattern = new RegExp(r'List of devices attached');
+ RegExp deviceIDPattern = new RegExp(r'\s+(\w+)\s+.*');
+ RegExp stopPatternWithDevices = new RegExp(r'\d+ connected devices?');
+ RegExp stopPatternWithoutDevices = new RegExp(r'No devices detected');
+ await for (var line in lineStream) {
+ if (!startReading && startPattern.hasMatch(line.toString())) {
+ startReading = true;
+ continue;
+ }
+
+ if (stopPatternWithDevices.hasMatch(line.toString())
+ || stopPatternWithoutDevices.hasMatch(line.toString()))
+ break;
+
+ if (startReading) {
+ Match idMatch = deviceIDPattern.firstMatch(line.toString());
+ if (idMatch != null) {
+ String deviceID = idMatch.group(1);
+ deviceIDs.add(deviceID);
+ }
+ }
+ }
+
+ process.stderr.drain();
+
+ return deviceIDs;
+}
+
+Future<Device> _collectDeviceProps(String deviceID) async {
+ return new Device(
+ id: deviceID,
+ modelName: await _getProperty(deviceID, 'ro.product.model')
+ );
+}
+
+Future<String> _getProperty(String deviceID, String propName) async {
+ ProcessResult results = await Process.run('adb', ['-s', deviceID, 'shell', 'getprop', propName]);
+ return results.stdout.toString().trim();
+}
diff --git a/mdtest/lib/src/mobile/device_spec.dart b/mdtest/lib/src/mobile/device_spec.dart
index ef1f825..44543a8 100644
--- a/mdtest/lib/src/mobile/device_spec.dart
+++ b/mdtest/lib/src/mobile/device_spec.dart
@@ -2,9 +2,17 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-import 'device.dart';
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
-class DeviceSpec {
+import 'package:path/path.dart' as path;
+
+import 'device.dart';
+import 'key_provider.dart';
+import '../globals.dart';
+
+class DeviceSpec implements ClusterKeyProvider {
DeviceSpec(
{
this.nickName,
@@ -37,6 +45,60 @@
}
@override
- String toString() => '<nickname: $nickName, iD: $deviceID, '
- 'model name: $deviceModelName, port: $observatoryUrl>';
+ String clusterKey() {
+ return appPath;
+ }
+
+ @override
+ String toString() => '<nickname: $nickName, id: $deviceID, '
+ 'model name: $deviceModelName, port: $observatoryUrl, '
+ 'app path: $appPath>';
+}
+
+Future<dynamic> loadSpecs(String specsPath) async {
+ try {
+ // Read specs file into json format
+ dynamic newSpecs = JSON.decode(await new File(specsPath).readAsString());
+ // Get the parent directory of the specs file
+ String rootPath = new File(specsPath).parent.absolute.path;
+ // Normalize the 'test-path' in the specs file
+ newSpecs['test-path'] = normalizePath(rootPath, newSpecs['test-path']);
+ // Normalize the 'app-path' in the specs file
+ newSpecs['devices'].forEach((String name, Map<String, String> map) {
+ map['app-path'] = normalizePath(rootPath, map['app-path']);
+ map['app-root'] = normalizePath(rootPath, map['app-root']);
+ });
+ return newSpecs;
+ } on FileSystemException {
+ printError('File $specsPath does not exist.');
+ exit(1);
+ } on FormatException {
+ printError('File $specsPath is not in JSON format.');
+ exit(1);
+ } catch (e) {
+ print('Unknown Exception details:\n $e');
+ exit(1);
+ }
+}
+
+String normalizePath(String rootPath, String relativePath) {
+ return path.normalize(path.join(rootPath, relativePath));
+}
+
+/// Build a list of device specs from mappings loaded from JSON .spec file
+Future<List<DeviceSpec>> constructAllDeviceSpecs(dynamic allSpecs) async {
+ List<DeviceSpec> deviceSpecs = <DeviceSpec>[];
+ for(String name in allSpecs.keys) {
+ Map<String, String> specs = allSpecs[name];
+ deviceSpecs.add(
+ new DeviceSpec(
+ nickName: name,
+ deviceID: specs['device-id'],
+ deviceModelName: specs['model-name'],
+ appRootPath: specs['app-root'],
+ appPath: specs['app-path']
+ )
+ );
+ }
+ return deviceSpecs;
}
diff --git a/mdtest/lib/src/mobile/device_util.dart b/mdtest/lib/src/mobile/device_util.dart
deleted file mode 100644
index 783e47c..0000000
--- a/mdtest/lib/src/mobile/device_util.dart
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2016 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-import 'dart:async';
-import 'dart:convert';
-import 'dart:io';
-
-import 'device.dart';
-
-Future<List<Device>> getDevices() async {
- List<Device> devices = <Device>[];
- await _getDeviceIDs().then((List<String> ids) async {
- for(String id in ids) {
- devices.add(await _collectDeviceProps(id));
- }
- });
- return devices;
-}
-
-/// Run "flutter devices -v" to collect device ids from the output
-Future<List<String>> _getDeviceIDs() async {
- List<String> deviceIDs = <String>[];
- Process process = await Process.start('flutter', ['devices', '-v']);
- Stream lineStream = process.stdout
- .transform(new Utf8Decoder())
- .transform(new LineSplitter());
- bool startReading = false;
- RegExp startPattern = new RegExp(r'List of devices attached');
- RegExp deviceIDPattern = new RegExp(r'\s+(\w+)\s+.*');
- RegExp stopPatternWithDevices = new RegExp(r'\d+ connected devices?');
- RegExp stopPatternWithoutDevices = new RegExp(r'No devices detected');
- await for (var line in lineStream) {
- if (!startReading && startPattern.hasMatch(line.toString())) {
- startReading = true;
- continue;
- }
-
- if (stopPatternWithDevices.hasMatch(line.toString())
- || stopPatternWithoutDevices.hasMatch(line.toString()))
- break;
-
- if (startReading) {
- Match idMatch = deviceIDPattern.firstMatch(line.toString());
- if (idMatch != null) {
- String deviceID = idMatch.group(1);
- deviceIDs.add(deviceID);
- }
- }
- }
-
- process.stderr.drain();
-
- return deviceIDs;
-}
-
-Future<Device> _collectDeviceProps(String deviceID) async {
- return new Device(
- id: deviceID,
- modelName: await _getProperty(deviceID, 'ro.product.model')
- );
-}
-
-Future<String> _getProperty(String deviceID, String propName) async {
- ProcessResult results = await Process.run('adb', ['-s', deviceID, 'shell', 'getprop', propName]);
- return results.stdout.toString().trim();
-}
diff --git a/mdtest/lib/src/mobile/key_provider.dart b/mdtest/lib/src/mobile/key_provider.dart
new file mode 100644
index 0000000..3974539
--- /dev/null
+++ b/mdtest/lib/src/mobile/key_provider.dart
@@ -0,0 +1,16 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+abstract class ClusterKeyProvider {
+ String clusterKey();
+}
+
+Map<String, List<dynamic>> buildCluster(List<dynamic> elements) {
+ Map<String, List<dynamic>> clusters = <String, List<dynamic>>{};
+ elements.forEach((dynamic element) {
+ clusters.putIfAbsent(element.clusterKey(), () => <dynamic>[])
+ .add(element);
+ });
+ return clusters;
+}
diff --git a/mdtest/lib/src/util.dart b/mdtest/lib/src/util.dart
new file mode 100644
index 0000000..8c53afe
--- /dev/null
+++ b/mdtest/lib/src/util.dart
@@ -0,0 +1,34 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:io' show Platform;
+import 'dart:math';
+
+int minLength(List<String> elements) {
+ if (elements == null || elements.isEmpty) return -1;
+ return elements.map((String e) => e.length).reduce(min);
+}
+
+bool isSystemSeparator(String letter) {
+ return letter == Platform.pathSeparator;
+}
+
+int beginOfDiff(List<String> elements) {
+ if (elements.length == 1)
+ return elements[0].lastIndexOf(Platform.pathSeparator) + 1;
+ int minL = minLength(elements);
+ int lastSlash = 0;
+ for (int i = 0; i < minL; i++) {
+ String letter = elements[0][i];
+ if (isSystemSeparator(letter)) {
+ lastSlash = i;
+ }
+ for (String element in elements) {
+ if (letter != element[i]) {
+ return lastSlash + 1;
+ }
+ }
+ }
+ return minL;
+}
diff --git a/mdtest/pubspec.yaml b/mdtest/pubspec.yaml
index c06b8c2..163108c 100644
--- a/mdtest/pubspec.yaml
+++ b/mdtest/pubspec.yaml
@@ -4,5 +4,7 @@
dependencies:
args: ^0.13.4
stack_trace: ^1.4.0
+ dlog:
+ path: ../../../../third_party/dart/dlog
flutter_driver:
path: ../deps/flutter/packages/flutter_driver
