feat(mdtest run): support device matching and test script automation
Automatically match each device spec with an available device, then
launch applications on devices based on the matching. After all
applications are correctly installed and launched, run test script
which automates the UI. mdtest requires the user to use its
driver/driver_util.dart to create flutter driver.
Change-Id: Ib3b895dd162ca5584b2a208dedaaed6535ed91c2
diff --git a/mdtest/lib/src/base/common.dart b/mdtest/lib/src/base/common.dart
new file mode 100644
index 0000000..84dfc97
--- /dev/null
+++ b/mdtest/lib/src/base/common.dart
@@ -0,0 +1,5 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+const String defaultTempSpecsName = 'tmp.spec';
diff --git a/mdtest/lib/src/base/logger.dart b/mdtest/lib/src/base/logger.dart
index c3baacf..71bd53b 100644
--- a/mdtest/lib/src/base/logger.dart
+++ b/mdtest/lib/src/base/logger.dart
@@ -12,7 +12,7 @@
class StdoutLogger extends Logger {
@override
void info(String message) {
- print('[info] $message');
+ stderr.writeln('[info ] $message');
}
@override
diff --git a/mdtest/lib/src/commands/run.dart b/mdtest/lib/src/commands/run.dart
index 60da707..119a3e8 100644
--- a/mdtest/lib/src/commands/run.dart
+++ b/mdtest/lib/src/commands/run.dart
@@ -8,7 +8,9 @@
import 'package:path/path.dart' as path;
+import '../base/common.dart';
import '../mobile/device.dart';
+import '../mobile/device_spec.dart';
import '../mobile/device_util.dart';
import '../globals.dart';
import '../runner/mdtest_command.dart';
@@ -28,13 +30,39 @@
@override
Future<int> runCore() async {
print('Running "mdtest run command" ...');
+
this._specs = await loadSpecs(argResults['specs']);
print(_specs);
+
this._devices = await getDevices();
if (_devices.isEmpty) {
printError('No device found.');
return 1;
}
+
+ List<DeviceSpec> allDeviceSpecs
+ = await constructAllDeviceSpecs(_specs['devices']);
+ Map<DeviceSpec, Set<Device>> individualMatches
+ = findIndividualMatches(allDeviceSpecs, _devices);
+ Map<DeviceSpec, Device> deviceMapping
+ = findMatchingDeviceMapping(allDeviceSpecs, individualMatches);
+ if(deviceMapping == null) {
+ printError('No device specs to devices mapping is found.');
+ return 1;
+ }
+
+ if (await runAllApps(deviceMapping) != 0) {
+ printError('Error when running applications');
+ return 1;
+ }
+
+ await storeMatches(deviceMapping);
+
+ if (await runTest(_specs['test-path']) != 0) {
+ printError('Test execution exit with error.');
+ return 1;
+ }
+
return 0;
}
@@ -72,3 +100,174 @@
String normalizePath(String rootPath, String relativePath) {
return path.normalize(path.join(rootPath, relativePath));
}
+
+/// Build a list of device specs from mappings loaded from JSON .spec file
+Future<List<DeviceSpec>> constructAllDeviceSpecs(dynamic allSpecs) async {
+ List<DeviceSpec> deviceSpecs = <DeviceSpec>[];
+ for(String name in allSpecs.keys) {
+ Map<String, String> specs = allSpecs[name];
+ deviceSpecs.add(
+ new DeviceSpec(
+ nickName: name,
+ deviceID: specs['device-id'],
+ deviceModelName: specs['model-name'],
+ appRootPath: specs['app-root'],
+ appPath: specs['app-path']
+ )
+ );
+ }
+ return deviceSpecs;
+}
+
+/// Find all matched devices for each device spec
+Map<DeviceSpec, Set<Device>> findIndividualMatches(
+ List<DeviceSpec> deviceSpecs,
+ List<Device> devices) {
+ Map<DeviceSpec, Set<Device>> individualMatches
+ = new Map<DeviceSpec, Set<Device>>();
+ for(DeviceSpec deviceSpecs in deviceSpecs) {
+ Set<Device> matchedDevices = new Set<Device>();
+ for(Device device in devices) {
+ if(deviceSpecs.matches(device))
+ matchedDevices.add(device);
+ }
+ individualMatches[deviceSpecs] = matchedDevices;
+ }
+ return individualMatches;
+}
+
+/// Return the first device spec to device matching, null if no such matching
+Map<DeviceSpec, Device> findMatchingDeviceMapping(
+ List<DeviceSpec> deviceSpecs,
+ Map<DeviceSpec, Set<Device>> individualMatches) {
+ Map<DeviceSpec, Device> deviceMapping = <DeviceSpec, Device>{};
+ Set<Device> visited = new Set<Device>();
+ if (!_findMatchingDeviceMapping(0, deviceSpecs, individualMatches,
+ visited, deviceMapping)) {
+ return null;
+ }
+ return deviceMapping;
+}
+
+/// Find a mapping that matches every device spec to a device. If such
+/// mapping is not found, return false, otherwise return true.
+bool _findMatchingDeviceMapping(
+ int order,
+ List<DeviceSpec> deviceSpecs,
+ Map<DeviceSpec, Set<Device>> individualMatches,
+ Set<Device> visited,
+ Map<DeviceSpec, Device> deviceMapping
+) {
+ if(order == deviceSpecs.length) return true;
+ DeviceSpec deviceSpec = deviceSpecs[order];
+ Set<Device> matchedDevices = individualMatches[deviceSpec];
+ for(Device candidate in matchedDevices) {
+ if(visited.add(candidate)) {
+ deviceMapping[deviceSpec] = candidate;
+ if(_findMatchingDeviceMapping(order + 1, deviceSpecs, individualMatches,
+ visited, deviceMapping))
+ return true;
+ else {
+ visited.remove(candidate);
+ deviceMapping.remove(deviceSpec);
+ }
+ }
+ }
+ return false;
+}
+
+List<Process> appProcesses = <Process>[];
+
+Future<int> runAllApps(Map<DeviceSpec, Device> deviceMapping) async {
+ List<Future<int>> runAppList = <Future<int>>[];
+ for (DeviceSpec deviceSpec in deviceMapping.keys) {
+ Device device = deviceMapping[deviceSpec];
+ runAppList.add(runApp(deviceSpec, device));
+ }
+ int res = 0;
+ List<int> results = await Future.wait(runAppList);
+ for (int result in results)
+ res += result;
+ return res == 0 ? 0 : 1;
+}
+
+/// Create a process that runs 'flutter run ...' command which installs and
+/// starts the app on the device. The function finds a observatory port
+/// through the process output. If no observatory port is found, then report
+/// error.
+Future<int> runApp(DeviceSpec deviceSpec, Device device) async {
+ Process process = await Process.start(
+ 'flutter',
+ ['run', '-d', device.id, '--target=${deviceSpec.appPath}'],
+ workingDirectory: deviceSpec.appRootPath
+ );
+ appProcesses.add(process);
+ Stream lineStream = process.stdout
+ .transform(new Utf8Decoder())
+ .transform(new LineSplitter());
+ RegExp portPattern = new RegExp(r'Observatory listening on (http.*)');
+ await for (var line in lineStream) {
+ print(line.toString().trim());
+ Match portMatch = portPattern.firstMatch(line.toString());
+ if (portMatch != null) {
+ deviceSpec.observatoryUrl = portMatch.group(1);
+ break;
+ }
+ }
+
+ process.stderr.drain();
+
+ if (deviceSpec.observatoryUrl == null) {
+ printError('No observatory url is found.');
+ return 1;
+ }
+
+ return 0;
+}
+
+/// Store the specs to device mapping as a system temporary file. The file
+/// stores device nickname as well as device id and observatory port for
+/// each device
+Future<Null> storeMatches(Map<DeviceSpec, Device> deviceMapping) async {
+ Map<String, dynamic> matchesData = new Map<String, dynamic>();
+ deviceMapping.forEach((DeviceSpec specs, Device device) {
+ matchesData[specs.nickName] =
+ {
+ 'device-id': device.id,
+ 'observatory-url': specs.observatoryUrl
+ };
+ });
+ Directory systemTempDir = Directory.systemTemp;
+ File tempFile = new File('${systemTempDir.path}/$defaultTempSpecsName');
+ if(await tempFile.exists())
+ await tempFile.delete();
+ File file = await tempFile.create();
+ await file.writeAsString(JSON.encode(matchesData));
+}
+
+/// Create a process and invoke 'dart testPath' to run the test script. After
+/// test result is returned (either pass or fail), kill all app processes and
+/// return the current process exit code
+Future<int> runTest(String testPath) async {
+ Process process = await Process.start('dart', ['$testPath']);
+ RegExp testStopPattern = new RegExp(r'All tests passed|Some tests failed');
+ Stream stdoutStream = process.stdout
+ .transform(new Utf8Decoder())
+ .transform(new LineSplitter());
+ await for (var line in stdoutStream) {
+ print(line.toString().trim());
+ if (testStopPattern.hasMatch(line.toString())) {
+ process.stderr.drain();
+ killAllProcesses(appProcesses);
+ break;
+ }
+ }
+ return await process.exitCode;
+}
+
+/// Kill all given processes
+Future<Null> killAllProcesses(List<Process> processes) async {
+ for (Process process in processes) {
+ process.kill();
+ }
+}
diff --git a/mdtest/lib/src/driver/driver_util.dart b/mdtest/lib/src/driver/driver_util.dart
new file mode 100644
index 0000000..1b84816
--- /dev/null
+++ b/mdtest/lib/src/driver/driver_util.dart
@@ -0,0 +1,39 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+import 'dart:convert' show JSON;
+
+import 'package:flutter_driver/flutter_driver.dart';
+
+import '../base/common.dart';
+import '../globals.dart';
+
+class DriverUtil {
+ static Future<FlutterDriver> connectByName(String deviceNickname) async {
+ // read the temp spec file to find the device nickname -> observatory port
+ // mapping
+ Directory systemTempDir = Directory.systemTemp;
+ File tempFile = new File('${systemTempDir.path}/$defaultTempSpecsName');
+ // if temp spec file is not found, report error and exit
+ if(!await tempFile.exists()) {
+ printError('Multi-Drive temporary specs file not found.');
+ exit(1);
+ }
+ // decode specs
+ dynamic configs = JSON.decode(await tempFile.readAsString());
+ // report error if nickname is not found
+ if(!configs.containsKey(deviceNickname)) {
+ printError('Device nickname $deviceNickname not found.');
+ exit(1);
+ }
+ // read device id and observatory port
+ String deviceID = configs[deviceNickname]['device-id'];
+ String observatoryUrl = configs[deviceNickname]['observatory-url'];
+ printInfo('$deviceNickname refers to device $deviceID running on url $observatoryUrl');
+ // delegate to flutter driver connect method
+ return await FlutterDriver.connect(dartVmServiceUrl: '$observatoryUrl');
+ }
+}
diff --git a/mdtest/lib/src/mobile/device_spec.dart b/mdtest/lib/src/mobile/device_spec.dart
new file mode 100644
index 0000000..ef1f825
--- /dev/null
+++ b/mdtest/lib/src/mobile/device_spec.dart
@@ -0,0 +1,42 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'device.dart';
+
+class DeviceSpec {
+ DeviceSpec(
+ {
+ this.nickName,
+ this.deviceID,
+ this.deviceModelName,
+ this.appRootPath,
+ this.appPath,
+ this.observatoryUrl
+ }
+ );
+
+ final String nickName;
+ final String deviceID;
+ final String deviceModelName;
+ final String appRootPath;
+ final String appPath;
+ String observatoryUrl;
+
+ // TODO(kaiyuanw): rewrite matches function later if necessary
+ bool matches(Device device) {
+ if(deviceID == device.id) {
+ return deviceModelName == null ?
+ true : deviceModelName == device.modelName;
+ } else {
+ return deviceID == null ?
+ (deviceModelName == null ?
+ true : deviceModelName == device.modelName)
+ : false;
+ }
+ }
+
+ @override
+ String toString() => '<nickname: $nickName, iD: $deviceID, '
+ 'model name: $deviceModelName, port: $observatoryUrl>';
+}
diff --git a/mdtest/pubspec.yaml b/mdtest/pubspec.yaml
index 3099215..c06b8c2 100644
--- a/mdtest/pubspec.yaml
+++ b/mdtest/pubspec.yaml
@@ -4,3 +4,5 @@
dependencies:
args: ^0.13.4
stack_trace: ^1.4.0
+ flutter_driver:
+ path: ../deps/flutter/packages/flutter_driver
