feat(mdtest create): mdtest helps users to create a template test spec
and provide relevant instructions on how to create valid spec from
the template

mdtest now support "mdtest create" command which is used to create
either a spec or a test template for the user.  --spec-template option
is added to specify the path to the template spec;  --test-template
option is added to specify the path to the template test.

mdtest will print out information regarding to each attribute in the
test spec and provide instructions to make the template a real test
spec.  mdtest will also print out some guidance for creating test
scripts and the test template is detailed with more instructions.

Now mdtest return 1 if some test fails.

Change-Id: Ie88635c2274bd6acf6c25afa89b281d8cdeb4652
diff --git a/mdtest/lib/executable.dart b/mdtest/lib/executable.dart
index 7c6f3ba..8399afe 100644
--- a/mdtest/lib/executable.dart
+++ b/mdtest/lib/executable.dart
@@ -8,6 +8,7 @@
 import 'package:args/command_runner.dart';
 import 'package:stack_trace/stack_trace.dart';
 
+import 'src/commands/create.dart';
 import 'src/commands/run.dart';
 import 'src/commands/auto.dart';
 import 'src/runner/mdtest_command_runner.dart';
@@ -15,6 +16,7 @@
 
 Future<Null> main(List<String> args) async {
   MDTestCommandRunner runner = new MDTestCommandRunner()
+    ..addCommand(new CreateCommand())
     ..addCommand(new RunCommand())
     ..addCommand(new AutoCommand());
 
diff --git a/mdtest/lib/src/algorithms/matching.dart b/mdtest/lib/src/algorithms/matching.dart
index 25054e4..1951b68 100644
--- a/mdtest/lib/src/algorithms/matching.dart
+++ b/mdtest/lib/src/algorithms/matching.dart
@@ -2,7 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
@@ -72,7 +71,7 @@
 /// Store the specs to device mapping as a system temporary file.  The file
 /// stores device nickname as well as device id and observatory url for
 /// each device
-Future<Null> storeMatches(Map<DeviceSpec, Device> deviceMapping) async {
+void storeMatches(Map<DeviceSpec, Device> deviceMapping) {
   Map<String, dynamic> matchesData = new Map<String, dynamic>();
   deviceMapping.forEach((DeviceSpec specs, Device device) {
     matchesData[specs.nickName] =
@@ -82,11 +81,8 @@
     };
   });
   Directory systemTempDir = Directory.systemTemp;
-  File tempFile = new File('${systemTempDir.path}/$defaultTempSpecsName');
-  if(await tempFile.exists())
-    await tempFile.delete();
-  File file = await tempFile.create();
-  await file.writeAsString(JSON.encode(matchesData));
+  File file = createNewFile('${systemTempDir.path}/$defaultTempSpecsName');
+  file.writeAsStringSync(JSON.encode(matchesData));
 }
 
 /// Return all spec to device mappings, return empty list if no such mapping
@@ -143,7 +139,8 @@
   }
   StringBuffer sb = new StringBuffer();
   int roundNum = 1;
-  sb.writeln('=' * 10);
+  sb.writeln(doubleLineSeparator);
+  String intermediateSeparator = '';
   for (Map<DeviceSpec, Device> match in matches) {
     int startIndx = beginOfDiff(
       new List.from(
@@ -154,14 +151,16 @@
         )
       )
     );
+    sb.write(intermediateSeparator);
     sb.writeln('Round $roundNum:');
     match.forEach((DeviceSpec spec, Device device) {
       sb.writeln('<Spec Group Key: ${spec.groupKey().substring(startIndx)}>'
                  ' -> '
                  '<Device Group Key: ${device.groupKey()}>');
     });
+    intermediateSeparator = '$singleLineSeparator\n';
     roundNum++;
   }
-  sb.write('=' * 10);
+  sb.write(doubleLineSeparator);
   print(sb.toString());
 }
diff --git a/mdtest/lib/src/commands/auto.dart b/mdtest/lib/src/commands/auto.dart
index 84663aa..1c1c0f7 100644
--- a/mdtest/lib/src/commands/auto.dart
+++ b/mdtest/lib/src/commands/auto.dart
@@ -33,7 +33,7 @@
     printInfo('Running "mdtest auto command" ...');
 
     this._specs = await loadSpecs(argResults);
-    if (sanityCheckSpecs(_specs, argResults['specs']) != 0) {
+    if (sanityCheckSpecs(_specs, argResults['spec']) != 0) {
       printError('Test spec does not meet requirements.');
       return 1;
     }
@@ -137,7 +137,7 @@
         return 1;
     }
 
-    return 0;
+    return failRounds.isNotEmpty ? 1 : 0;
   }
 
   AutoCommand() {
diff --git a/mdtest/lib/src/commands/create.dart b/mdtest/lib/src/commands/create.dart
new file mode 100644
index 0000000..005a8c0
--- /dev/null
+++ b/mdtest/lib/src/commands/create.dart
@@ -0,0 +1,197 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+
+import '../runner/mdtest_command.dart';
+import '../globals.dart';
+import '../util.dart';
+
+const String specTemplate =
+'''
+{
+  "devices": {
+    "{nickname}": {
+      "device-id": "{optional}",
+      "model-name": "{optional}",
+      "os-version": "{optional}",
+      "api-level": "{optional}",
+      "screen-size": "{optional}",
+      "app-root": "{required}",
+      "app-path": "{required}"
+    },
+    "{nickname}": {
+      "app-root": "{required}",
+      "app-path": "{required}"
+    }
+  }
+}
+''';
+
+const String specGuide =
+'Everything in the curly braces can be replaced with your own value.\n'
+'"device-id", "model-name", "os-version", "api-level" and "screem-size" '
+'are optional.\n'
+'"app-root" and "app-path" are required.\n'
+'An example spec would be\n'
+'''
+{
+  "devices": {
+    "Alice": {
+      "device-id": "HT4CWJT03204",
+      "model-name": "Nexus 9",
+      "os-version": "6.0",
+      "api-level": "23",
+      "screen-size": "xlarge",
+      "app-root": "/path/to/flutter-app",
+      "app-path": "/path/to/main.dart"
+    },
+    "Bob": {
+      ...
+    }
+    ...
+  }
+}
+'''
+'"nickname" will be used as the identifier of the device that matches '
+'the corresponding properties in the test spec.  You will use nicknames '
+'to establish connections between flutter drivers and devices in your '
+'test scripts.\n'
+'"device-id" is the unique id of your device.\n'
+'"model-name" is the device model name.\n'
+'"os-version" is the operating system version of your device.\n'
+'"api-level" is Android specific and refers to the API level of your device.\n'
+'"screen-size" is the screen diagonal size measured in inches.  The candidate '
+'values are "small"(<3.5"), "normal"(>=3.5" && <5"), "large"(>=5" && <8") '
+'and "xlarge"(>=8").\n'
+'"app-root" is the path of your flutter application directory.\n'
+'"app-path" is the path of the instrumented version of your app main function.\n'
+;
+
+const String testTemplate =
+'''
+import 'dart:async';
+
+// add flutter_driver, test, mdtest to your pubspec/yaml
+import 'package:flutter_driver/flutter_driver.dart';
+import 'package:test/test.dart';
+import 'package:mdtest/driver_util.dart';
+
+void main() {
+  group('Group 1', () {
+    // Create a flutter driver map that maps each nickname with a flutter driver
+    DriverMap driverMap;
+
+    setUpAll(() async {
+      driverMap = new DriverMap();
+      // Other setup functions
+      // ...
+    });
+
+    tearDownAll(() async {
+      if (driverMap != null) {
+        driverMap.closeAll();
+      }
+      // Other tear down functions
+      // ...
+    });
+
+    test('test 1', () async {
+      // Send tap request to each connected driver
+      List<FlutterDriver> drivers = await Future.wait(driverMap.values);
+      await Future.forEach(drivers, (FlutterDriver driver) async {
+        await driver.tap(find.byValueKey(...));
+      });
+
+      // Get text from each connected driver and compare the result
+      await Future.forEach(drivers, (FlutterDriver driver) async {
+        String result = await driver.getText(find.byValueKey(textKey));
+        expect(result, equals('...'));
+      });
+
+      // An alternative to send tap request to each connected driver
+      await Future.wait(
+        drivers.map(
+          (FlutterDriver driver) => driver.tap(find.byValueKey(buttonKey))
+        )
+      );
+    });
+
+    test('test 2', () async {
+      FlutterDriver driver1 = await driverMap['nickname1'];
+      FlutterDriver driver2 = await driverMap['nickname2'];
+      await driver1.tap(find.byValueKey(...));
+      await driver2.tap(find.byValueKey(...));
+      String result1 = await driver1.getText(find.byValueKey(...));
+      expect(result1, equals('...'));
+      String result2 = await driver2.getText(find.byValueKey(...));
+      expect(result2, equals('...'));
+    });
+
+    // More tests go here
+    // ...
+  });
+
+  // More groups go here
+  // ...
+}
+''';
+
+const String testGuide =
+'mdtest provide a DriverMap class which maps each nickname in the test spec '
+'to a flutter driver.  Users can get a flutter driver instance by saying `'
+'FlutterDriver driver = await driverMap[nickname];`  '
+'DriverMap will lazy initialize a flutter driver instance the first time '
+'you invoke the [] operator.\n'
+'Once you get access to a flutter driver, you can use it to automate your '
+'flutter app.  For more detailed usage, please refer to the generated test '
+'template.'
+;
+
+class CreateCommand extends MDTestCommand {
+
+  @override
+  final String name = 'create';
+
+  @override
+  final String description = 'create a test spec/script template for the user to fill in';
+
+  @override
+  Future<int> runCore() async {
+    printInfo('Running "mdtest create command" ...');
+    String specTemplatePath = argResults['spec-template'];
+    String testTemplatePath = argResults['test-template'];
+    if (specTemplatePath == null && testTemplatePath == null) {
+      printError('You must provide a path for either spec or test template.');
+      return 1;
+    }
+
+    if (specTemplatePath != null) {
+      File file = createNewFile('$specTemplatePath');
+      file.writeAsStringSync(specTemplate);
+      String absolutePath = normalizePath(Directory.current.path, specTemplatePath);
+      printInfo('Template test spec written to $absolutePath');
+      printGuide(specGuide);
+    }
+
+    if (testTemplatePath != null) {
+      File file = createNewFile('$testTemplatePath');
+      file.writeAsStringSync(testTemplate);
+      String absolutePath = normalizePath(Directory.current.path, testTemplatePath);
+      printInfo('Template test written to $absolutePath');
+      printGuide(testGuide);
+    }
+    return 0;
+  }
+
+  void printGuide(String guide) {
+    guide.split('\n').forEach((String line) => printInfo(line));
+  }
+
+  CreateCommand() {
+    usesSpecTemplateOption();
+    usesTestTemplateOption();
+  }
+}
diff --git a/mdtest/lib/src/commands/helper.dart b/mdtest/lib/src/commands/helper.dart
index 9e4d73f..015fac8 100644
--- a/mdtest/lib/src/commands/helper.dart
+++ b/mdtest/lib/src/commands/helper.dart
@@ -116,7 +116,8 @@
         break;
     }
     process.stderr.drain();
-    return await process.exitCode;
+    process.kill();
+    return 0;
   }
 
   /// Create a process and invoke 'pub run test --reporter json [testPath]' to
@@ -128,13 +129,23 @@
       'pub',
       ['run', 'test', '--reporter', 'json', '$testPath']
     );
-    await reporter.report(
+    bool hasTestOutput = await reporter.report(
       process.stdout
              .transform(new Utf8Decoder())
              .transform(new LineSplitter())
     );
-    process.stderr.drain();
-    return await process.exitCode;
+    if (hasTestOutput) {
+      process.stderr.drain();
+      return await process.exitCode;
+    }
+
+    Stream stderrStream = process.stderr
+                                 .transform(new Utf8Decoder())
+                                 .transform(new LineSplitter());
+    await for (var line in stderrStream) {
+      print(line.toString().trim());
+    }
+    return 1;
   }
 
   /// Kill all app processes
diff --git a/mdtest/lib/src/commands/run.dart b/mdtest/lib/src/commands/run.dart
index 773aa75..2ddb855 100644
--- a/mdtest/lib/src/commands/run.dart
+++ b/mdtest/lib/src/commands/run.dart
@@ -31,7 +31,7 @@
 
     this._specs = await loadSpecs(argResults);
     printTrace(_specs.toString());
-    if (sanityCheckSpecs(_specs, argResults['specs']) != 0) {
+    if (sanityCheckSpecs(_specs, argResults['spec']) != 0) {
       printError('Test spec does not meet requirements.');
       return 1;
     }
@@ -92,7 +92,7 @@
 
     await uninstallTestedApps(deviceMapping);
 
-    return 0;
+    return testsFailed ? 1 : 0;
   }
 
   RunCommand() {
diff --git a/mdtest/lib/src/mobile/device_spec.dart b/mdtest/lib/src/mobile/device_spec.dart
index 700281c..44da67a 100644
--- a/mdtest/lib/src/mobile/device_spec.dart
+++ b/mdtest/lib/src/mobile/device_spec.dart
@@ -23,6 +23,8 @@
   String get nickName => specProperties['nickname'];
   String get deviceID => specProperties['device-id'];
   String get deviceModelName => specProperties['model-name'];
+  String get deviceOSVersion => specProperties['os-version'];
+  String get deviceAPILevel => specProperties['api-level'];
   String get deviceScreenSize => specProperties['screen-size'];
   String get appRootPath => specProperties['app-root'];
   String get appPath => specProperties['app-path'];
@@ -61,13 +63,15 @@
   String toString() => '<nickname: $nickName, '
                        'id: $deviceID, '
                        'model name: $deviceModelName, '
+                       'os version: $deviceOSVersion, '
+                       'api level: $deviceAPILevel, '
                        'screen size: $deviceScreenSize, '
                        'port: $observatoryUrl, '
                        'app path: $appPath>';
 }
 
 Future<dynamic> loadSpecs(ArgResults argResults) async {
-  String specsPath = argResults['specs'];
+  String specsPath = argResults['spec'];
   try {
     // Read specs file into json format
     dynamic newSpecs = JSON.decode(await new File(specsPath).readAsString());
@@ -80,8 +84,8 @@
     newSpecs['test-paths'] = testPathsFromCommandLine;
     // Normalize the 'app-path' in the specs file
     newSpecs['devices']?.forEach((String name, Map<String, String> map) {
-      map['app-path'] = normalizePath(rootPath, map['app-path']);
       map['app-root'] = normalizePath(rootPath, map['app-root']);
+      map['app-path'] = normalizePath(map['app-root'], map['app-path']);
     });
     return newSpecs;
   } on FileSystemException {
diff --git a/mdtest/lib/src/runner/mdtest_command.dart b/mdtest/lib/src/runner/mdtest_command.dart
index 36b310b..78a1619 100644
--- a/mdtest/lib/src/runner/mdtest_command.dart
+++ b/mdtest/lib/src/runner/mdtest_command.dart
@@ -22,10 +22,12 @@
   MDTestCommandRunner get runner => super.runner;
 
   bool _usesSpecsOption = false;
+  bool _usesSpecTemplateOption = false;
+  bool _usesTestTemplateOption = false;
 
   void usesSpecsOption() {
     argParser.addOption(
-      'specs',
+      'spec',
       defaultsTo: null,
       help:
         'Path to the config file that specifies the devices, '
@@ -52,6 +54,26 @@
     );
   }
 
+  void usesSpecTemplateOption() {
+    argParser.addOption(
+      'spec-template',
+      defaultsTo: null,
+      help:
+        'Path to create the test spec template.'
+    );
+    _usesSpecTemplateOption = true;
+  }
+
+  void usesTestTemplateOption() {
+    argParser.addOption(
+      'test-template',
+      defaultsTo: null,
+      help:
+        'Path to create the test script template.'
+    );
+    _usesTestTemplateOption = true;
+  }
+
   @override
   Future<int> run() {
     Stopwatch stopwatch = new Stopwatch()..start();
@@ -74,16 +96,51 @@
 
   bool _commandValidator() {
     if (_usesSpecsOption) {
-      String specsPath = argResults['specs'];
+      String specsPath = argResults['spec'];
       if (specsPath == null) {
-        printError('Specs file is not set.');
+        printError('Spec file is not set.');
         return false;
       }
+      if (!specsPath.endsWith('.spec')) {
+        printError('Spec file must have .spec suffix');
+      }
       if (!FileSystemEntity.isFileSync(specsPath)) {
-        printError('Specs file "$specsPath" not found.');
+        printError('Spec file "$specsPath" not found.');
         return false;
       }
     }
+
+    if (_usesSpecTemplateOption) {
+      String specTemplatePath = argResults['spec-template'];
+      if (specTemplatePath != null) {
+        if (!specTemplatePath.endsWith('.spec')) {
+          printError(
+            'Spec template path must have .spec suffix (found "$specTemplatePath").'
+          );
+          return false;
+        }
+        if (FileSystemEntity.isDirectorySync(specTemplatePath)) {
+          printError('Spec template file "$specTemplatePath" is a directory.');
+          return false;
+        }
+      }
+    }
+
+    if (_usesTestTemplateOption) {
+      String testTemplatePath = argResults['test-template'];
+      if (testTemplatePath != null) {
+        if (!testTemplatePath.endsWith('.dart')) {
+          printError(
+            'Test template path must have .dart suffix (found "$testTemplatePath").'
+          );
+          return false;
+        }
+        if (FileSystemEntity.isDirectorySync(testTemplatePath)) {
+          printError('Test template file "$testTemplatePath" is a directory.');
+          return false;
+        }
+      }
+    }
     return true;
   }
 }
diff --git a/mdtest/lib/src/test/reporter.dart b/mdtest/lib/src/test/reporter.dart
index d7f8a43..e946845 100644
--- a/mdtest/lib/src/test/reporter.dart
+++ b/mdtest/lib/src/test/reporter.dart
@@ -25,11 +25,14 @@
     );
   }
 
-  Future<Null> report(Stream jsonOutput) async {
+  Future<bool> report(Stream jsonOutput) async {
+    bool hasTestOutput = false;
     await for (var line in jsonOutput) {
       convertToTAPFormat(line.toString().trim());
+      hasTestOutput = true;
     }
     testEventMapping.clear();
+    return hasTestOutput;
   }
 
   void printSummary() {
diff --git a/mdtest/lib/src/util.dart b/mdtest/lib/src/util.dart
index 1482d20..3b38263 100644
--- a/mdtest/lib/src/util.dart
+++ b/mdtest/lib/src/util.dart
@@ -71,7 +71,6 @@
 /// Get a file with unique name under the given directory.
 File getUniqueFile(Directory dir, String baseName, String ext) {
   int i = 1;
-
   while (true) {
     String name = '${baseName}_${i.toString().padLeft(2, '0')}.$ext';
     File file = new File(path.join(dir.path, name));
@@ -81,6 +80,15 @@
   }
 }
 
+/// Create a file if it does not exist.  If the path points to a file, delete
+/// it and create a new file.  Otherwise, report
+File createNewFile(String path) {
+  File file = new File('$path');;
+  if(file.existsSync())
+    file.deleteSync();
+  return file..createSync(recursive: true);
+}
+
 /// Return the absolute paths of a list of files based on a list of glob
 /// patterns.  The order of the result follow the order of the given glob
 /// patterns, but the order of file paths corresponding to the same glob