feat(mdtest run/auto): add test spec validation checker
Check if test spec meets the requirements. If user does not specify any
valid test paths neither from the test spec nor from the command line,
report error. If 'devices' property is not specified, report error. If
no device spec is specified, report error. If screen size property is
not one of 'small', 'normal', 'large' and 'xlarge', report error. If
app-root is not specified or is not a directory, report error. If
apppath is not specified or is not a file, report error.
Change-Id: I37a6f82aa926ab09d417f77c0eb8948c60907b72
diff --git a/mdtest/lib/src/commands/auto.dart b/mdtest/lib/src/commands/auto.dart
index 88323fe..088d8c2 100644
--- a/mdtest/lib/src/commands/auto.dart
+++ b/mdtest/lib/src/commands/auto.dart
@@ -33,6 +33,10 @@
printInfo('Running "mdtest auto command" ...');
this._specs = await loadSpecs(argResults);
+ if (sanityCheckSpecs(_specs, argResults['spec']) != 0) {
+ printError('Test spec does not meet requirements.');
+ return 1;
+ }
this._devices = await getDevices();
if (_devices.isEmpty) {
diff --git a/mdtest/lib/src/commands/run.dart b/mdtest/lib/src/commands/run.dart
index 07d332e..e6e9890 100644
--- a/mdtest/lib/src/commands/run.dart
+++ b/mdtest/lib/src/commands/run.dart
@@ -31,6 +31,10 @@
this._specs = await loadSpecs(argResults);
printTrace(_specs.toString());
+ if (sanityCheckSpecs(_specs, argResults['specs']) != 0) {
+ printError('Test spec does not meet requirements.');
+ return 1;
+ }
this._devices = await getDevices();
if (_devices.isEmpty) {
diff --git a/mdtest/lib/src/mobile/device.dart b/mdtest/lib/src/mobile/device.dart
index 574114f..6629eec 100644
--- a/mdtest/lib/src/mobile/device.dart
+++ b/mdtest/lib/src/mobile/device.dart
@@ -47,9 +47,9 @@
.transform(new Utf8Decoder())
.transform(new LineSplitter());
bool startReading = false;
- RegExp startPattern = new RegExp(r'List of devices attached');
- RegExp deviceIDPattern = new RegExp(r'\s+(\w+)\s+.*');
- RegExp stopPattern = new RegExp(r'\d+ connected devices?|No devices detected');
+ RegExp startPattern = new RegExp(r'\d+ connected device|No devices detected');
+ RegExp deviceIDPattern = new RegExp(r'\d+ ms •.*•\s+(\S+)\s+•.*');
+ RegExp stopPattern = new RegExp(r"'flutter devices' took \d+ms; exiting with code");
await for (var line in lineStream) {
if (!startReading && startPattern.hasMatch(line.toString())) {
startReading = true;
diff --git a/mdtest/lib/src/mobile/device_spec.dart b/mdtest/lib/src/mobile/device_spec.dart
index 8522159..2f7e924 100644
--- a/mdtest/lib/src/mobile/device_spec.dart
+++ b/mdtest/lib/src/mobile/device_spec.dart
@@ -75,16 +75,16 @@
// from the command line argument
List<String> testPathsFromSpec
= listFilePathsFromGlobPatterns(rootPath, newSpecs['test-paths']);
- print('Test paths from spec: $testPathsFromSpec');
+ printTrace('Test paths from spec: $testPathsFromSpec');
List<String> testPathsFromCommandLine
= listFilePathsFromGlobPatterns(Directory.current.path, argResults.rest);
- print('Test paths from command line: $testPathsFromCommandLine');
+ printTrace('Test paths from command line: $testPathsFromCommandLine');
newSpecs['test-paths'] = mergeWithoutDuplicate(
testPathsFromSpec,
testPathsFromCommandLine
);
// Normalize the 'app-path' in the specs file
- newSpecs['devices'].forEach((String name, Map<String, String> map) {
+ newSpecs['devices']?.forEach((String name, Map<String, String> map) {
map['app-path'] = normalizePath(rootPath, map['app-path']);
map['app-root'] = normalizePath(rootPath, map['app-root']);
});
@@ -101,6 +101,66 @@
}
}
+/// Check if test spec meets the requirements. If user does not specify any
+/// valid test paths neither from the test spec nor from the command line,
+/// report error. If 'devices' property is not specified, report error. If
+/// no device spec is specified, report error. If screen size property is not
+/// one of 'small', 'normal', 'large' and 'xlarge', report error. If app-root
+/// is not specified or is not a directory, report error. If appPath is not
+/// specified or is not a file, report error.
+///
+/// Note: If a test path does not exist, it will be ignored and thus does not
+/// count as a valid test path. If device nickname is not unique, json decoder
+/// will overwrite the previous device spec associated with the same nickname,
+/// thus only the last nickname to device spec pair is used.
+int sanityCheckSpecs(dynamic spec, String specsPath) {
+ if (spec['test-paths'].isEmpty) {
+ printError(
+ 'No test paths found. '
+ 'You must specify at least one test path.'
+ );
+ return 1;
+ }
+ dynamic deviceSpecs = spec['devices'];
+ if (deviceSpecs == null) {
+ printError('"devices" property is not specified in $specsPath');
+ return 1;
+ }
+ if (deviceSpecs.isEmpty) {
+ printError('No device spec is found in $specsPath');
+ return 1;
+ }
+ for (String nickname in deviceSpecs.keys) {
+ dynamic individualDeviceSpec = deviceSpecs[nickname];
+ List<String> screenSizes = <String>['small', 'normal', 'large', 'xlarge'];
+ if (individualDeviceSpec['screen-size'] != null
+ &&
+ !screenSizes.contains(individualDeviceSpec['screen-size'])) {
+ printError('Screen size must be one of $screenSizes');
+ return 1;
+ }
+ String appRootPath = individualDeviceSpec['app-root'];
+ if (appRootPath == null) {
+ printError('Application root path is not specified.');
+ return 1;
+ }
+ if (!FileSystemEntity.isDirectorySync(appRootPath)) {
+ printError('Application root path is not a directory.');
+ return 1;
+ }
+ String appPath = individualDeviceSpec['app-path'];
+ if (appPath == null) {
+ printError('Application path is not specified.');
+ return 1;
+ }
+ if (!FileSystemEntity.isFileSync(appPath)) {
+ printError('Application path is not a file.');
+ return 1;
+ }
+ }
+ return 0;
+}
+
/// Build a list of device specs from mappings loaded from JSON .spec file
Future<List<DeviceSpec>> constructAllDeviceSpecs(dynamic allSpecs) async {
List<DeviceSpec> deviceSpecs = <DeviceSpec>[];
diff --git a/mdtest/lib/src/util.dart b/mdtest/lib/src/util.dart
index 2840353..f9f6402 100644
--- a/mdtest/lib/src/util.dart
+++ b/mdtest/lib/src/util.dart
@@ -39,6 +39,9 @@
}
String normalizePath(String rootPath, String relativePath) {
+ if (rootPath == null || relativePath == null) {
+ return null;
+ }
return path.normalize(
path.join(rootPath, relativePath)
);
