feat(mdtest run/auto generate): mdtest is able to encode and store
test output as json format for test report generation. mdtest also
implements a test report generating algorithm as well as a code
coverage report generating algorithm.
mdtest allows users to store the test output into JSON format by
providing --save-report-data option for both `run` and `auto` command.
Users can use --save-report-data option to specify a path to store test
output data. However, the --save-report-data option can only be used with
--format=tap because of the implementation limitation.
mdtest provide another command `mdtest generate` command which can
generate test report and code covereage report. The generate command
support --report-type option which could be either set to `test` or
`coverage` for test report or coverage report, respectively.
--load-report option is provided and should points to either the test
report in json format, or a coverage report in lcov format. --output
option points to the directory to generate test/coverage report. --lib
is points to the source code library related to the coverage report and
can only be specified if --report-type is set to `coverage`.
Change-Id: I5f108888340a83ad537ef2c5fd42991c2e5d6396
diff --git a/mdtest/lib/assets/emerald.png b/mdtest/lib/assets/emerald.png
new file mode 100644
index 0000000..38ad4f4
--- /dev/null
+++ b/mdtest/lib/assets/emerald.png
Binary files differ
diff --git a/mdtest/lib/assets/locator.dart b/mdtest/lib/assets/locator.dart
new file mode 100644
index 0000000..5692bd8
--- /dev/null
+++ b/mdtest/lib/assets/locator.dart
@@ -0,0 +1,19 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import '../src/util.dart';
+
+String mdtestScriptPath = Platform.script.toFilePath();
+int binStart = mdtestScriptPath.lastIndexOf('bin');
+String mdtestRootPath = mdtestScriptPath.substring(0, binStart);
+String libPath = normalizePath(mdtestRootPath, 'lib');
+String assetsPath = normalizePath(libPath, 'assets');
+
+List<String> get relatedPaths => <String>[
+ normalizePath(assetsPath, 'emerald.png'),
+ normalizePath(assetsPath, 'ruby.png'),
+ normalizePath(assetsPath, 'report.css')
+];
diff --git a/mdtest/lib/assets/report.css b/mdtest/lib/assets/report.css
new file mode 100644
index 0000000..30c5a8b
--- /dev/null
+++ b/mdtest/lib/assets/report.css
@@ -0,0 +1,47 @@
+/* Copyright 2015 The Vanadium Authors. All rights reserved. */
+/* Use of this source code is governed by a BSD-style */
+/* license that can be found in the LICENSE file. */
+.just-padding {
+ padding: 15px;
+}
+.list-group.list-group-root {
+ padding: 0;
+ overflow: hidden;
+}
+.list-group.list-group-root .list-group {
+ margin-bottom: 0;
+}
+.list-group.list-group-root .list-group-item {
+ border-radius: 0;
+ border-width: 1px 0 0 0;
+}
+.list-group.list-group-root > .list-group-item:first-child {
+ border-top-width: 0;
+}
+.list-group.list-group-root > .list-group > .list-group-item {
+ padding-left: 30px;
+}
+.list-group.list-group-root > .list-group > .list-group > .list-group-item {
+ padding-left: 60px;
+}
+.list-group.list-group-root > .list-group > .list-group > .list-group > .list-group-item {
+ padding-left: 90px;
+}
+.list-group-item .glyphicon {
+ margin-right: 5px;
+}
+td.title {
+ text-align: center;
+ padding-bottom: 10px;
+ font-family: sans-serif;
+ font-size: 20pt;
+ font-style: italic;
+ font-weight: bold;
+}
+td.ruler {
+ background-color: #6688D4;
+}
+.tooltip-inner {
+ white-space:pre;
+ max-width: none;
+}
diff --git a/mdtest/lib/assets/ruby.png b/mdtest/lib/assets/ruby.png
new file mode 100644
index 0000000..991b6d4
--- /dev/null
+++ b/mdtest/lib/assets/ruby.png
Binary files differ
diff --git a/mdtest/lib/executable.dart b/mdtest/lib/executable.dart
index 8399afe..d390341 100644
--- a/mdtest/lib/executable.dart
+++ b/mdtest/lib/executable.dart
@@ -11,6 +11,7 @@
import 'src/commands/create.dart';
import 'src/commands/run.dart';
import 'src/commands/auto.dart';
+import 'src/commands/generate.dart';
import 'src/runner/mdtest_command_runner.dart';
import 'src/util.dart';
@@ -18,7 +19,8 @@
MDTestCommandRunner runner = new MDTestCommandRunner()
..addCommand(new CreateCommand())
..addCommand(new RunCommand())
- ..addCommand(new AutoCommand());
+ ..addCommand(new AutoCommand())
+ ..addCommand(new GenerateCommand());
return Chain.capture(() async {
dynamic result = await runner.run(args);
diff --git a/mdtest/lib/src/algorithms/coverage.dart b/mdtest/lib/src/algorithms/coverage.dart
index 70836a0..38e2ca6 100644
--- a/mdtest/lib/src/algorithms/coverage.dart
+++ b/mdtest/lib/src/algorithms/coverage.dart
@@ -93,6 +93,49 @@
}
}
}
+
+ /// Convert coverage matrix into JSON format. Return a dictionary that
+ /// stores the title, data and legend of the table.
+ dynamic toJson(String title, f(int e)) {
+ List<List<String>> data = <List<String>>[];
+ List<String> firstRow = <String>[];
+ firstRow.add('app key \\ device key');
+ firstRow.addAll(groupInfo.deviceClustersOrder);
+ data.add(firstRow);
+ int startIndx = beginOfDiff(groupInfo.deviceSpecClustersOrder);
+ for (int i = 0; i < matrix.length; i++) {
+ List<String> row = <String>[];
+ row.add(
+ groupInfo.deviceSpecClustersOrder[i].substring(startIndx)
+ );
+ row.addAll(matrix[i].map(f));
+ data.add(row);
+ }
+ CoverageScore coverageScore = computeCoverageScore(this);
+ return {
+ 'title': title,
+ 'data': data,
+ 'legend': legend,
+ 'reachable-score': coverageScore.reachablePathsPercentage,
+ 'covered-score': coverageScore.coverageScore
+ };
+ }
+}
+
+class CoverageScore {
+ NumberFormat _scoreFormat;
+ final double _reachablePathsPercentage;
+ final double _coverageScore;
+
+ String get reachablePathsPercentage
+ => _scoreFormat.format(_reachablePathsPercentage);
+
+ String get coverageScore
+ => _scoreFormat.format(_coverageScore);
+
+ CoverageScore(this._reachablePathsPercentage, this._coverageScore) {
+ this._scoreFormat = new NumberFormat('%##.0#', 'en_US');
+ }
}
int _countNumberInCoverageMatrix(List<List<int>> matrix, bool test(int e)) {
@@ -103,13 +146,13 @@
return result;
}
-/// Compute and print the app-device coverage.
-void computeAndReportCoverage(CoverageMatrix coverageMatrix) {
+/// Compute the percentage of reachable app-device paths and app-device
+/// coverage score. Return null if coverage matrix is null.
+CoverageScore computeCoverageScore(CoverageMatrix coverageMatrix) {
if (coverageMatrix == null) {
printError('Coverage matrix is null');
- return;
+ return null;
}
-
List<List<int>> matrix = coverageMatrix.matrix;
int rowNum = matrix.length;
int colNum = matrix[0].length;
@@ -118,39 +161,39 @@
= _countNumberInCoverageMatrix(matrix, (int e) => e != cannotBeCovered);
int coveredCombinationNum
= _countNumberInCoverageMatrix(matrix, (int e) => e > isNotCovered);
- StringBuffer scoreReport = new StringBuffer();
- NumberFormat percentFormat = new NumberFormat('%##.0#', 'en_US');
- scoreReport.writeln('App-Device Path Coverage (ADPC) score:');
double reachableCoverageScore = reachableCombinationNum / totalPathNum;
- scoreReport.writeln(
- 'Reachable ADPC score: ${percentFormat.format(reachableCoverageScore)}, '
- 'defined by #reachable / #total.'
- );
double coveredCoverageScore
= coveredCombinationNum / reachableCombinationNum;
- scoreReport.writeln(
- 'Covered ADPC score: ${percentFormat.format(coveredCoverageScore)}, '
- 'defined by #covered / #reachable.'
- );
- print(scoreReport.toString());
+ return new CoverageScore(reachableCoverageScore, coveredCoverageScore);
}
+/// Print the coverage score
+void printCoverageScore(CoverageScore coverageScore) {
+ if (coverageScore == null) {
+ printError('Coverage score is null');
+ return;
+ }
+ print(
+ 'App-Device Path Coverage (ADPC) score:\n'
+ 'Reachable ADPC score: ${coverageScore.reachablePathsPercentage}, '
+ 'defined by #reachable / #total.\n'
+ 'Covered ADPC score: ${coverageScore.coverageScore}, '
+ 'defined by #covered / #reachable.\n'
+ );
+}
+
+const String legend =
+'Meaning of the number in the coverage matrix:\n'
+'$cannotBeCovered: an app-device path is not reachable '
+'given the connected devices.\n'
+' $isNotCovered: an app-device path is reachable but '
+'not covered by any test run.\n'
+'>$isNotCovered: the number of times an app-device path '
+'is covered by some test runs.\n'
+;
+
void printLegend() {
- StringBuffer legendInfo = new StringBuffer();
- legendInfo.writeln('Meaning of the number in the coverage matrix:');
- legendInfo.writeln(
- '$cannotBeCovered: an app-device path is not reachable '
- 'given the connected devices.'
- );
- legendInfo.writeln(
- ' $isNotCovered: an app-device path is reachable but '
- 'not covered by any test run.'
- );
- legendInfo.writeln(
- '>$isNotCovered: the number of times an app-device path '
- 'is covered by some test runs.'
- );
- print(legendInfo.toString());
+ print(legend);
}
void printCoverageMatrix(String title, CoverageMatrix coverageMatrix) {
@@ -179,7 +222,9 @@
}
);
printLegend();
- computeAndReportCoverage(coverageMatrix);
+ printCoverageScore(
+ computeCoverageScore(coverageMatrix)
+ );
}
void printMatrix(String title, CoverageMatrix coverageMatrix, f(int e)) {
diff --git a/mdtest/lib/src/commands/auto.dart b/mdtest/lib/src/commands/auto.dart
index 26278fa..8101b2c 100644
--- a/mdtest/lib/src/commands/auto.dart
+++ b/mdtest/lib/src/commands/auto.dart
@@ -3,6 +3,7 @@
// license that can be found in the LICENSE file.
import 'dart:async';
+import 'dart:io';
import 'helper.dart';
import '../mobile/device.dart';
@@ -11,8 +12,10 @@
import '../algorithms/coverage.dart';
import '../algorithms/matching.dart';
import '../globals.dart';
+import '../util.dart';
import '../runner/mdtest_command.dart';
import '../test/coverage_collector.dart';
+import '../test/reporter.dart';
class AutoCommand extends MDTestCommand {
@override
@@ -31,7 +34,7 @@
Future<int> runCore() async {
printInfo('Running "mdtest auto command" ...');
- this._specs = await loadSpecs(argResults);
+ this._specs = loadSpecs(argResults);
if (sanityCheckSpecs(_specs, argResults['spec']) != 0) {
printError('Test spec does not meet requirements.');
return 1;
@@ -69,6 +72,8 @@
Map<String, CoverageCollector> collectorPool
= <String, CoverageCollector>{};
+ List<TAPReporter> allTAPReporters = <dynamic>[];
+
List<int> errRounds = [];
List<int> failRounds = [];
int roundNum = 0;
@@ -89,7 +94,10 @@
bool testsFailed;
if (argResults['format'] == 'tap') {
- testsFailed = await runner.runAllTestsToTAP(_specs['test-paths']) != 0;
+ TAPReporter reporter = new TAPReporter(deviceMapping);
+ testsFailed
+ = await runner.runAllTestsToTAP(_specs['test-paths'], reporter) != 0;
+ allTAPReporters.add(reporter);
} else {
testsFailed = await runner.runAllTests(_specs['test-paths']) != 0;
}
@@ -114,9 +122,10 @@
printInfo('End of Round #$roundNum\n');
}
+ String hitmapTitle = 'App-device coverage hit matrix:';
if (!briefMode) {
printHitmap(
- 'App-device coverage hit matrix:',
+ hitmapTitle,
appDeviceCoverageMatrix
);
}
@@ -132,6 +141,27 @@
printInfo('All tests in all rounds passed');
}
+ String reportDataPath = argResults['save-report-data'];
+ if (reportDataPath != null) {
+ reportDataPath
+ = normalizePath(Directory.current.path, reportDataPath);
+ File file = createNewFile(reportDataPath);
+ printInfo('Writing report data to $reportDataPath');
+ file.writeAsStringSync(
+ dumpToJSONString(
+ {
+ 'hitmap': appDeviceCoverageMatrix.toJson(
+ hitmapTitle,
+ (int e) => '$e'
+ ),
+ 'rounds-info': allTAPReporters.map(
+ (TAPReporter reporter) => reporter.toJson()
+ ).toList()
+ }
+ )
+ );
+ }
+
if (argResults['coverage']) {
printInfo('Computing code coverage for each application ...');
if (await computeAppsCoverage(collectorPool, name) != 0)
@@ -145,6 +175,7 @@
usesSpecsOption();
usesCoverageFlag();
usesTAPReportOption();
+ usesSaveTestReportOption();
argParser.addOption('groupby',
defaultsTo: 'device-id',
allowed: [
@@ -154,8 +185,9 @@
'os-version',
'screen-size'
],
- help: 'Device property used to group devices to'
- 'adjust app-device coverage criterion.'
+ help: 'Device property used to group devices that applications will run '
+ 'on. Each application is guaranteed to be run on at least one '
+ 'device from each of all device groups that satisfy the test spec.'
);
}
}
diff --git a/mdtest/lib/src/commands/create.dart b/mdtest/lib/src/commands/create.dart
index 4727b9a..7d603b4 100644
--- a/mdtest/lib/src/commands/create.dart
+++ b/mdtest/lib/src/commands/create.dart
@@ -156,7 +156,7 @@
final String name = 'create';
@override
- final String description = 'create a test spec/script template for the user to fill in';
+ final String description = 'Create a test spec/script template for the user to fill in';
@override
Future<int> runCore() async {
@@ -169,18 +169,20 @@
}
if (specTemplatePath != null) {
+ specTemplatePath
+ = normalizePath(Directory.current.path, specTemplatePath);
File file = createNewFile('$specTemplatePath');
file.writeAsStringSync(specTemplate);
- String absolutePath = normalizePath(Directory.current.path, specTemplatePath);
- printInfo('Template test spec written to $absolutePath');
+ printInfo('Template test spec written to $specTemplatePath');
printGuide(specGuide);
}
if (testTemplatePath != null) {
+ testTemplatePath
+ = normalizePath(Directory.current.path, testTemplatePath);
File file = createNewFile('$testTemplatePath');
file.writeAsStringSync(testTemplate);
- String absolutePath = normalizePath(Directory.current.path, testTemplatePath);
- printInfo('Template test written to $absolutePath');
+ printInfo('Template test written to $testTemplatePath');
printGuide(testGuide);
}
return 0;
diff --git a/mdtest/lib/src/commands/generate.dart b/mdtest/lib/src/commands/generate.dart
new file mode 100644
index 0000000..910de9b
--- /dev/null
+++ b/mdtest/lib/src/commands/generate.dart
@@ -0,0 +1,80 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:async';
+
+import '../runner/mdtest_command.dart';
+import '../globals.dart';
+import '../report/test_report.dart';
+import '../report/coverage_report.dart';
+
+class GenerateCommand extends MDTestCommand {
+
+ @override
+ final String name = 'generate';
+
+ @override
+ final String description
+ = 'Generate code coverage or test output web report. Examples:\n'
+ 'mdtest generate --report-type coverage '
+ '--load-report-data path/to/coverage.lcov '
+ '--lib path/to/lib --output out\n'
+ 'mdtest generate --report-type test '
+ '--load-report-data path/to/report_data.json --output out';
+
+ @override
+ Future<int> runCore() async {
+ printInfo('Running "mdtest generate command" ...');
+ String reportDataPath = argResults['load-report-data'];
+ String outputPath = argResults['output'];
+ String reportType = argResults['report-type'];
+ if (reportType == 'test') {
+ printInfo('Generating test report to $outputPath.');
+ TestReport testReport = new TestReport(reportDataPath, outputPath);
+ testReport.writeReport();
+ }
+ if (reportType == 'coverage') {
+ printInfo('Generating code coverage report to $outputPath.');
+ String libPath = argResults['lib'];
+ CoverageReport coverageReport
+ = new CoverageReport(reportDataPath, libPath, outputPath);
+ coverageReport.writeReport();
+ }
+ return 0;
+ }
+
+ void printGuide(String guide) {
+ guide.split('\n').forEach((String line) => printInfo(line));
+ }
+
+ GenerateCommand() {
+ usesReportTypeOption();
+ argParser.addOption(
+ 'load-report-data',
+ defaultsTo: null,
+ help:
+ 'Path to load the report data. '
+ 'The report data could be either lcov format for code coverage, '
+ 'or JSON format for test output.'
+ );
+ argParser.addOption(
+ 'lib',
+ defaultsTo: null,
+ help:
+ 'Path to the flutter lib folder that contains all source code of your '
+ 'flutter application. The source code should be what the code coverage '
+ 'report data refers to. This option is only used when --report-type '
+ 'is set to `coverage`.'
+ );
+ argParser.addOption(
+ 'output',
+ abbr: 'o',
+ defaultsTo: null,
+ help:
+ 'Path to generate a web report. The path should either not exist or '
+ 'point to a directory. If the path does not exist, a new directory '
+ 'will be created using that path.'
+ );
+ }
+}
diff --git a/mdtest/lib/src/commands/helper.dart b/mdtest/lib/src/commands/helper.dart
index aa33421..72a7ef0 100644
--- a/mdtest/lib/src/commands/helper.dart
+++ b/mdtest/lib/src/commands/helper.dart
@@ -88,12 +88,15 @@
}
/// Run all tests with test output in TAP format
- Future<int> runAllTestsToTAP(Iterable<String> testPaths) async {
+ Future<int> runAllTestsToTAP(
+ Iterable<String> testPaths,
+ TAPReporter reporter
+ ) async {
+ int diffFrom = beginOfDiff(testPaths);
int result = 0;
- TAPReporter reporter = new TAPReporter();
reporter.printHeader();
for (String testPath in testPaths) {
- result += await runTestToTAP(testPath, reporter);
+ result += await runTestToTAP(testPath, diffFrom, reporter);
}
reporter.printSummary();
return result == 0 ? 0 : 1;
@@ -125,12 +128,17 @@
/// run the test script and convert json output into tap output on the fly.
/// After test result is returned (either pass or fail), return the current
/// process exit code (0 if success, otherwise failure)
- Future<int> runTestToTAP(String testPath, TAPReporter reporter) async {
+ Future<int> runTestToTAP(
+ String testPath,
+ int diffFrom,
+ TAPReporter reporter
+ ) async {
Process process = await Process.start(
'pub',
['run', 'test', '--reporter', 'json', '$testPath']
);
bool hasTestOutput = await reporter.report(
+ testPath.substring(diffFrom),
process.stdout
.transform(new Utf8Decoder())
.transform(new LineSplitter())
@@ -206,7 +214,7 @@
File codeCoverageReport = getUniqueFile(
new Directory(codeCoverageDirPath),
'cov_$commandName',
- 'info'
+ 'lcov'
);
try {
// Write coverage info to code_coverage folder
diff --git a/mdtest/lib/src/commands/run.dart b/mdtest/lib/src/commands/run.dart
index bf5fea5..852259e 100644
--- a/mdtest/lib/src/commands/run.dart
+++ b/mdtest/lib/src/commands/run.dart
@@ -3,14 +3,17 @@
// license that can be found in the LICENSE file.
import 'dart:async';
+import 'dart:io';
import 'helper.dart';
import '../mobile/device.dart';
import '../mobile/device_spec.dart';
import '../algorithms/matching.dart';
import '../globals.dart';
+import '../util.dart';
import '../runner/mdtest_command.dart';
import '../test/coverage_collector.dart';
+import '../test/reporter.dart';
class RunCommand extends MDTestCommand {
@@ -28,7 +31,7 @@
Future<int> runCore() async {
printInfo('Running "mdtest run command" ...');
- this._specs = await loadSpecs(argResults);
+ this._specs = loadSpecs(argResults);
if (sanityCheckSpecs(_specs, argResults['spec']) != 0) {
printError('Test spec does not meet requirements.');
return 1;
@@ -61,9 +64,11 @@
await storeMatches(deviceMapping);
+ TAPReporter reporter = new TAPReporter(deviceMapping);
bool testsFailed;
if (argResults['format'] == 'tap') {
- testsFailed = await runner.runAllTestsToTAP(_specs['test-paths']) != 0;
+ testsFailed
+ = await runner.runAllTestsToTAP(_specs['test-paths'], reporter) != 0;
} else {
testsFailed = await runner.runAllTests(_specs['test-paths']) != 0;
}
@@ -75,6 +80,19 @@
printInfo('All tests passed');
}
+ String reportDataPath = argResults['save-report-data'];
+ if (reportDataPath != null) {
+ reportDataPath
+ = normalizePath(Directory.current.path, reportDataPath);
+ File file = createNewFile(reportDataPath);
+ printInfo('Writing report data to $reportDataPath');
+ file.writeAsStringSync(
+ dumpToJSONString(
+ {'rounds-info': [reporter.toJson()]}
+ )
+ );
+ }
+
if (argResults['coverage']) {
Map<String, CoverageCollector> collectorPool
= <String, CoverageCollector>{};
@@ -97,5 +115,6 @@
usesSpecsOption();
usesCoverageFlag();
usesTAPReportOption();
+ usesSaveTestReportOption();
}
}
diff --git a/mdtest/lib/src/mobile/device_spec.dart b/mdtest/lib/src/mobile/device_spec.dart
index a70c6ba..52a4d7a 100644
--- a/mdtest/lib/src/mobile/device_spec.dart
+++ b/mdtest/lib/src/mobile/device_spec.dart
@@ -84,13 +84,14 @@
'app path: $appPath>';
}
-Future<dynamic> loadSpecs(ArgResults argResults) async {
+dynamic loadSpecs(ArgResults argResults) {
String specsPath = argResults['spec'];
try {
- // Read specs file into json format
- dynamic newSpecs = JSON.decode(await new File(specsPath).readAsString());
+ File specsFile = new File(specsPath);
+ // Read specs file into json object
+ dynamic newSpecs = JSON.decode(specsFile.readAsStringSync());
// Get the parent directory of the specs file
- String rootPath = new File(specsPath).parent.absolute.path;
+ String rootPath = specsFile.parent.absolute.path;
// Normalize the 'test-path' specified from the command line argument
List<String> testPathsFromCommandLine
= listFilePathsFromGlobPatterns(Directory.current.path, argResults.rest);
diff --git a/mdtest/lib/src/report/coverage_report.dart b/mdtest/lib/src/report/coverage_report.dart
new file mode 100644
index 0000000..e16fb8e
--- /dev/null
+++ b/mdtest/lib/src/report/coverage_report.dart
@@ -0,0 +1,33 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'report.dart';
+import '../globals.dart';
+
+class CoverageReport extends Report {
+ Directory libDirectory;
+
+ CoverageReport(String reportDataPath, String libPath, String outputPath)
+ : super(reportDataPath, outputPath) {
+ this.libDirectory = new Directory(libPath);
+ }
+
+ @override
+ void writeReport() {
+ // Move the lib folder into the output directory
+ Process.runSync('cp', ['-r', libDirectory.path, outputDirectory.path]);
+ ProcessResult result = Process.runSync(
+ 'genhtml',
+ ['-o', outputDirectory.path, reportDataFile.path]
+ );
+ result.stdout.toString().trim().split('\n').forEach(printInfo);
+ if (result.stderr.isNotEmpty) {
+ result.stderr.trim().split('\n').forEach(
+ (String line) => printError(line)
+ );
+ }
+ }
+}
diff --git a/mdtest/lib/src/report/report.dart b/mdtest/lib/src/report/report.dart
new file mode 100644
index 0000000..ac30d5d
--- /dev/null
+++ b/mdtest/lib/src/report/report.dart
@@ -0,0 +1,19 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import '../util.dart';
+
+abstract class Report {
+ File reportDataFile;
+ Directory outputDirectory;
+
+ Report(String reportDataPath, String outputPath) {
+ reportDataFile = new File(reportDataPath);
+ outputDirectory = createNewDirectory(outputPath);
+ }
+
+ void writeReport();
+}
diff --git a/mdtest/lib/src/report/test_report.dart b/mdtest/lib/src/report/test_report.dart
new file mode 100644
index 0000000..9934a4c
--- /dev/null
+++ b/mdtest/lib/src/report/test_report.dart
@@ -0,0 +1,461 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'report.dart';
+import '../globals.dart';
+import '../util.dart';
+import '../../assets/locator.dart';
+
+class TestReport extends Report {
+ HitmapInfo hitmapInfo;
+ List<RoundInfo> roundsInfo;
+
+ TestReport(String reportDataPath, String outputPath)
+ : super(reportDataPath, outputPath) {
+ this.roundsInfo = <RoundInfo>[];
+ _decodeData();
+ }
+
+ void _decodeData() {
+ try {
+ // Read report data file into json object
+ dynamic reportData = JSON.decode(reportDataFile.readAsStringSync());
+ dynamic hitmap = reportData['hitmap'];
+ if (hitmap != null) {
+ hitmapInfo = new HitmapInfo(
+ hitmap['title'],
+ hitmap['data'],
+ hitmap['legend'],
+ hitmap['reachable-score'],
+ hitmap['covered-score']
+ );
+ }
+ int roundNum = 1;
+ for (dynamic roundInfo in reportData['rounds-info']) {
+ roundsInfo.add(new RoundInfo(roundNum++, roundInfo));
+ }
+ } on FormatException {
+ printError('File ${reportDataFile.absolute.path} is not in JSON format.');
+ exit(1);
+ } catch (exception, stackTrace) {
+ print(exception);
+ print(stackTrace);
+ exit(1);
+ }
+ }
+
+ @override
+ void writeReport() {
+ File indexHTML = createNewFile(
+ normalizePath(outputDirectory.path, 'index.html')
+ );
+ indexHTML.writeAsStringSync(toHTML());
+ relatedPaths.forEach(
+ (String imagePath) => copyPathToDirectory(imagePath, outputDirectory.path)
+ );
+ }
+
+ /// Generate the entire HTML report.
+ /// TODO(kaiyuanw): Could use local css and js files so that this works
+ /// without network connections
+ String toHTML() {
+ StringBuffer html = new StringBuffer();
+ html.writeln(
+ '''
+ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+ <html lang="en">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+ <title>MDTest - ${fileBaseName(reportDataFile.path)}</title>
+ <link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css">
+ <link rel="stylesheet" type="text/css" href="report.css">
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
+ <script src="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js"></script>
+ <script>
+ \$(function() {
+ \$(\'.list-group-item\').on(\'click\', function() {
+ \$(\'.glyphicon\', this)
+ .toggleClass(\'glyphicon-chevron-right\')
+ .toggleClass(\'glyphicon-chevron-down\');
+ });
+ });
+ </script>
+ </head>
+ <body>
+ <table width="100%" border=0 cellspacing=0 cellpadding=0>
+ <tr><td class="title">MDTest - test report</td></tr>
+ <tr><td class="ruler"><img width=3 height=3 alt=""></td></tr>
+ </table>
+ '''
+ );
+ if (hitmapInfo != null) {
+ html.writeln(
+ '''
+ <div class="container">
+ <div class="just-padding">
+ ${hitmapInfo.toHTML()}
+ </div>
+ </div>
+ <table width="100%" border=0 cellspacing=0 cellpadding=0>
+ <tr><td class="ruler"><img width=3 height=3 alt=""></td></tr>
+ </table>
+ '''
+ );
+ }
+ html.writeln(
+ '''
+ <div class="container">
+ <div class="just-padding">
+ ${roundsInfo.map((RoundInfo round) {
+ return
+ '''
+ <h3>${round.name}<h3>
+ <h4>${
+ round.highlight.trim().split('\n').map(
+ (String line) => HTML_ESCAPE.convert(line)
+ ).join('</h4>\n<h4>')
+ }</h4>
+ ''';
+ }).join('\n')}
+ </div>
+ </div>
+
+ <table width="100%" border=0 cellspacing=0 cellpadding=0>
+ <tr><td class="ruler"><img width=3 height=3 alt=""></td></tr>
+ </table>
+
+ <div class="container">
+ <div class="just-padding">
+ <div class="list-group list-group-root well">
+ ${roundsInfo.map((RoundInfo round) => round.toHTML()).join('\n')}
+ </div>
+ </div>
+ </div>
+ <script>
+ \$(document).ready(function(){
+ \$('[data-toggle="tooltip"]').tooltip({
+ html: true,
+ container: 'body'
+ });
+ });
+ </script>
+ </body>
+ </html>
+ '''
+ );
+ return html.toString();
+ }
+}
+
+class HitmapInfo {
+ String title;
+ List<List<String>> data;
+ String legend;
+ String reachableScore;
+ String coveredScore;
+
+ HitmapInfo(
+ this.title,
+ dynamic hitmapData,
+ this.legend,
+ this.reachableScore,
+ this.coveredScore
+ ) {
+ this.data = <List<String>>[];
+ for (Iterable<String> iterString in hitmapData) {
+ data.add(iterString.toList());
+ }
+ }
+
+ String toHTML() {
+ if (data == null || data.isEmpty || data.isNotEmpty && data[0].isEmpty) {
+ printError('No hitmap data is found.');
+ return '<h3>No hitmap data is found. '
+ 'Please rerun `mdtest auto` to collect app-device hitmap.</h3>';
+ }
+ int rowNum = data.length;
+ int colNum = data[0].length ?? 0;
+ StringBuffer html = new StringBuffer();
+ html.writeln('<h3>$title</h3>');
+ html.writeln('<table class="table table-striped">');
+ html.writeln('<thead>');
+ html.writeln('<tr>${'<td></td>' * colNum}</tr>');
+ html.writeln('<tr><th>${data[0].join('</th>\n<th>')}</th></tr>');
+ html.writeln('</thead>');
+ html.writeln('<tbody>');
+ for (int i = 1; i < rowNum; i++) {
+ html.writeln('<tr>');
+ html.writeln('<th>${data[i][0]}</th>');
+ for (int j = 1; j < colNum; j++) {
+ String value = data[i][j];
+ html.writeln('<td class=\"${tdColorClass(value)}\">$value</td>');
+ }
+ html.writeln('</tr>');
+ }
+ html.writeln('</tbody>');
+ html.writeln('</table>');
+ List<String> legendLines = legend.trim().split('\n');
+ html.writeln('<h4>${legendLines.join('</h4>\n<h4>')}</h4>');
+ html.writeln(
+ '<br>\n'
+ '<h4>App-Device Path Coverage (ADPC) score:</h4>\n'
+ '<h4>Reachable ADPC score: $reachableScore, '
+ 'defined by #reachable / #total.</h4>\n'
+ '<h4>Covered ADPC score: $coveredScore, '
+ 'defined by #covered / #reachable.</h4>\n'
+ );
+ return html.toString();
+ }
+
+ String tdColorClass(String value) {
+ int val = int.parse(value);
+ if (val == -1) {
+ return 'warning';
+ }
+ if (val == 0) {
+ return 'danger';
+ }
+ if (val > 0) {
+ return 'success';
+ }
+ return 'unknown';
+ }
+}
+
+abstract class Info {
+ String id;
+ String name;
+ String status;
+
+ String toHTML();
+}
+
+class RoundInfo extends Info {
+ String highlight;
+ int skipNum;
+ int passNum;
+ int failNum;
+ List<TestSuiteInfo> testSuitesInfo;
+
+ RoundInfo(int roundNum, dynamic roundInfo) {
+ this.id = 'round-$roundNum';
+ this.name = 'Round #$roundNum';
+ this.highlight = roundInfo['highlight'];
+ this.skipNum = roundInfo['skip-num'];
+ this.passNum = roundInfo['pass-num'];
+ this.failNum = roundInfo['fail-num'];
+ this.status = roundInfo['status'];
+ this.testSuitesInfo = <TestSuiteInfo>[];
+ int suiteNum = 1;
+ for (dynamic suiteInfo in roundInfo['suites-info']) {
+ testSuitesInfo.add(
+ new TestSuiteInfo('$id-suite-${suiteNum++}', suiteInfo)
+ );
+ }
+ }
+
+ @override
+ String toHTML() {
+ StringBuffer html = new StringBuffer();
+ String imgUrl = status == 'fail' ? 'ruby.png' : 'emerald.png';
+ html.writeln(
+ '''
+ <a href="#$id" class="list-group-item" data-toggle="collapse">
+ <div class="row">
+ <div class="col-sm-3">
+ <i class="glyphicon glyphicon-chevron-right"></i>$name
+ </div>
+ <div class="col-sm-2">Status: $status</div>
+ <div class="col-sm-2">#Passed: $passNum</div>
+ <div class="col-sm-2">#Failed: $failNum</div>
+ <div class="col-sm-2">#Skipped: $skipNum</div>
+ <div class="col-sm-1"><img src="$imgUrl" height=20></div>
+ </div>
+ </a>
+ <div class="list-group collapse" id="$id">
+ ${
+ testSuitesInfo.map(
+ (TestSuiteInfo suite) => suite.toHTML()
+ ).join('\n')
+ }
+ </div>
+ '''
+ );
+ return html.toString();
+ }
+}
+
+class TestSuiteInfo extends Info {
+ int skipNum;
+ int passNum;
+ int failNum;
+ List<Info> testSuiteChildrenInfo;
+
+ TestSuiteInfo(String id, dynamic suiteInfo) {
+ this.id = id;
+ this.name = suiteInfo['name'];
+ this.skipNum = suiteInfo['skip-num'];
+ this.passNum = suiteInfo['pass-num'];
+ this.failNum = suiteInfo['fail-num'];
+ this.status = suiteInfo['status'];
+ this.testSuiteChildrenInfo = <Info>[];
+ int childNum = 1;
+ for (dynamic childInfo in suiteInfo['children-info']) {
+ String type = childInfo['type'];
+ if (type == 'test-group') {
+ testSuiteChildrenInfo.add(
+ new TestGroupInfo('$id-child-${childNum++}', childInfo)
+ );
+ } else if (type == 'test-method') {
+ testSuiteChildrenInfo.add(
+ new TestMethodInfo('$id-child-${childNum++}', childInfo)
+ );
+ }
+ }
+ }
+
+ @override
+ String toHTML() {
+ StringBuffer html = new StringBuffer();
+ String imgUrl = status == 'fail' ? 'ruby.png' : 'emerald.png';
+ html.writeln(
+ '''
+ <a href="#$id" class="list-group-item" data-toggle="collapse">
+ <div class="row">
+ <div class="col-sm-3">
+ <i class="glyphicon glyphicon-chevron-right"></i>$name
+ </div>
+ <div class="col-sm-2">Status: $status</div>
+ <div class="col-sm-2">#Passed: $passNum</div>
+ <div class="col-sm-2">#Failed: $failNum</div>
+ <div class="col-sm-2">#Skipped: $skipNum</div>
+ <div class="col-sm-1"><img src="$imgUrl" height=20></div>
+ </div>
+ </a>
+ <div class="list-group collapse" id="$id">
+ ${testSuiteChildrenInfo.map((Info child) => child.toHTML()).join('\n')}
+ </div>
+ '''
+ );
+ return html.toString();
+ }
+}
+
+class TestGroupInfo extends Info {
+ int skipNum;
+ int passNum;
+ int failNum;
+ // Only for skip reason
+ String reason;
+ List<TestMethodInfo> testMethodsInfo;
+
+ TestGroupInfo(String id, dynamic groupInfo) {
+ this.id = id;
+ this.name = groupInfo['name'];
+ this.skipNum = groupInfo['skip-num'];
+ this.passNum = groupInfo['pass-num'];
+ this.failNum = groupInfo['fail-num'];
+ this.status = groupInfo['status'];
+ this.reason = groupInfo['reason'];
+ if (reason != null) {
+ reason = reason.replaceAll(new RegExp(r'\n'), '<br>');
+ }
+ this.testMethodsInfo = <TestMethodInfo>[];
+ int methodNum = 1;
+ for (dynamic testMethodInfo in groupInfo['methods-info']) {
+ String type = testMethodInfo['type'];
+ // Ignore nested group. Nested test groups are not supported yet
+ if (type == 'test-method') {
+ testMethodsInfo.add(
+ new TestMethodInfo('$id-method-${methodNum++}', testMethodInfo)
+ );
+ } else if (type == 'test-group') {
+ throw new UnsupportedError('Nested test groups are not supported yet.');
+ }
+ }
+ }
+
+ @override
+ String toHTML() {
+ StringBuffer html = new StringBuffer();
+ String imgUrl = status == 'fail' ? 'ruby.png' : 'emerald.png';
+ html.writeln(
+ '<a href="#$id" class="list-group-item" data-toggle="collapse">'
+ );
+ if (reason != null) {
+ html.writeln(
+ '<span data-toggle="tooltip" data-placement="right" title="$reason"/>'
+ );
+ }
+ html.writeln(
+ '''
+ <div class="row">
+ <div class="col-sm-3">
+ <i class="glyphicon glyphicon-chevron-right"></i>$name
+ </div>
+ <div class="col-sm-2">Status: $status</div>
+ <div class="col-sm-2">#Passed: $passNum</div>
+ <div class="col-sm-2">#Failed: $failNum</div>
+ <div class="col-sm-2">#Skipped: $skipNum</div>
+ <div class="col-sm-1"><img src="$imgUrl" height=20></div>
+ </div>
+ </a>
+ <div class="list-group collapse" id="$id">
+ ${
+ testMethodsInfo.map(
+ (TestMethodInfo method) => method.toHTML()
+ ).join('\n')
+ }
+ </div>
+ '''
+ );
+ return html.toString();
+ }
+}
+
+class TestMethodInfo extends Info {
+ String reason;
+ TestMethodInfo(String id, dynamic testMethodInfo) {
+ this.id = id;
+ this.name = testMethodInfo['name'];
+ this.status = testMethodInfo['status'];
+ this.reason = testMethodInfo['reason'];
+ if (reason != null) {
+ reason = reason.replaceAll(new RegExp(r'\n'), '<br>');
+ }
+ }
+
+ @override
+ String toHTML() {
+ StringBuffer html = new StringBuffer();
+ String imgUrl = status == 'fail' ? 'ruby.png' : 'emerald.png';
+ if (reason == null) {
+ html.writeln('<a class="list-group-item">');
+ } else {
+ html.writeln(
+ '<a class="list-group-item" data-toggle="tooltip" '
+ 'data-placement="right" title="$reason">'
+ );
+ }
+ html.writeln(
+ '''
+ <div class="row">
+ <div class="col-sm-3">
+ <i class="glyphicon glyphicon-chevron-right"></i>$name
+ </div>
+ <div class="col-sm-2">Status: $status</div>
+ <div class="col-sm-2"></div>
+ <div class="col-sm-2"></div>
+ <div class="col-sm-2"></div>
+ <div class="col-sm-1"><img src="$imgUrl" height=20></div>
+ </div>
+ </a>
+ '''
+ );
+ return html.toString();
+ }
+}
diff --git a/mdtest/lib/src/runner/mdtest_command.dart b/mdtest/lib/src/runner/mdtest_command.dart
index 78a1619..7a5b3f0 100644
--- a/mdtest/lib/src/runner/mdtest_command.dart
+++ b/mdtest/lib/src/runner/mdtest_command.dart
@@ -24,14 +24,16 @@
bool _usesSpecsOption = false;
bool _usesSpecTemplateOption = false;
bool _usesTestTemplateOption = false;
+ bool _usesSaveTestReportOption = false;
+ bool _usesReportTypeOption = false;
void usesSpecsOption() {
argParser.addOption(
'spec',
defaultsTo: null,
help:
- 'Path to the config file that specifies the devices, '
- 'apps and debug-ports for testing.'
+ 'Path to the test spec file that specifies devices that you '
+ 'want your applications to run on.'
);
_usesSpecsOption = true;
}
@@ -54,6 +56,17 @@
);
}
+ void usesSaveTestReportOption() {
+ argParser.addOption(
+ 'save-report-data',
+ defaultsTo: null,
+ help:
+ 'Path to save the test output report data. '
+ 'The report will be saved in JSON format.'
+ );
+ _usesSaveTestReportOption = true;
+ }
+
void usesSpecTemplateOption() {
argParser.addOption(
'spec-template',
@@ -74,6 +87,18 @@
_usesTestTemplateOption = true;
}
+ void usesReportTypeOption() {
+ argParser.addOption('report-type',
+ defaultsTo: null,
+ allowed: [
+ 'test',
+ 'coverage'
+ ],
+ help: 'Whether to generate a test report or a code coverage report.'
+ );
+ _usesReportTypeOption = true;
+ }
+
@override
Future<int> run() {
Stopwatch stopwatch = new Stopwatch()..start();
@@ -98,7 +123,7 @@
if (_usesSpecsOption) {
String specsPath = argResults['spec'];
if (specsPath == null) {
- printError('Spec file is not set.');
+ printError('Spec file path is not specified.');
return false;
}
if (!specsPath.endsWith('.spec')) {
@@ -120,7 +145,10 @@
return false;
}
if (FileSystemEntity.isDirectorySync(specTemplatePath)) {
- printError('Spec template file "$specTemplatePath" is a directory.');
+ printError(
+ 'Spec template file "$specTemplatePath" is a directory. '
+ 'A file path is expected.'
+ );
return false;
}
}
@@ -136,11 +164,117 @@
return false;
}
if (FileSystemEntity.isDirectorySync(testTemplatePath)) {
- printError('Test template file "$testTemplatePath" is a directory.');
+ printError(
+ 'Test template file "$testTemplatePath" is a directory. '
+ 'A file path is expected.'
+ );
return false;
}
}
}
+
+ if (_usesSaveTestReportOption) {
+ String savedReportPath = argResults['save-report-data'];
+ if (savedReportPath != null) {
+ if (argResults['format'] != 'tap') {
+ printError(
+ 'The --save-report-data option must be used with TAP test output '
+ 'format. Please set --format to tap.'
+ );
+ return false;
+ }
+ if (!savedReportPath.endsWith('.json')) {
+ printError(
+ 'Report data file must have .json suffix (found "$savedReportPath").'
+ );
+ return false;
+ }
+ if (FileSystemEntity.isDirectorySync(savedReportPath)) {
+ printError('Report data file "$savedReportPath" is a directory.');
+ return false;
+ }
+ }
+ }
+
+ if (_usesReportTypeOption) {
+ String reportType = argResults['report-type'];
+ if (reportType == null) {
+ printError(
+ 'You must specify a report-type. '
+ 'Only "test" and "coverage" is allowed.'
+ );
+ return false;
+ }
+ // Report data path cannot be null and must be an existing file
+ String loadReportPath = argResults['load-report-data'];
+ if (loadReportPath == null) {
+ printError('You must specify a path to load the report data.');
+ return false;
+ }
+ if (!FileSystemEntity.isFileSync(loadReportPath)) {
+ printError(
+ 'Report data path $loadReportPath is not a file. '
+ 'An existing file path is expected.'
+ );
+ return false;
+ }
+ // Output path cannot be null and must either point to an empty directory,
+ // or not exist
+ String outputPath = argResults['output'];
+ if (outputPath == null) {
+ printError('You must specify a path to generate the web report.');
+ return false;
+ }
+ if (FileSystemEntity.isFileSync(outputPath)) {
+ printError(
+ 'Output path $outputPath is a file. '
+ 'An empty directory path or non-existing path is expected.'
+ );
+ return false;
+ }
+
+ // Lib path that points to the source code that code coverage report
+ // refers to
+ String libPath = argResults['lib'];
+
+ if (reportType == 'coverage') {
+ if (!loadReportPath.endsWith('.lcov')) {
+ printError(
+ 'Coverage report data path $loadReportPath must have .lcov suffix'
+ );
+ return false;
+ }
+ if (libPath == null) {
+ printError(
+ 'A lib path is expected in code coverage report generating mode.'
+ );
+ return false;
+ }
+ if (!FileSystemEntity.isDirectorySync(libPath)) {
+ printError(
+ 'Lib path $libPath is not a directory. '
+ 'A source code directory path is expected.'
+ );
+ return false;
+ }
+ }
+
+ if (reportType == 'test') {
+ if (!loadReportPath.endsWith('.json')) {
+ printError(
+ 'Test report data path $loadReportPath must have .json suffix'
+ );
+ return false;
+ }
+ if (libPath != null) {
+ printError(
+ 'A lib path is not expected in test report generating mode.'
+ );
+ return false;
+ }
+ }
+
+ }
return true;
}
}
diff --git a/mdtest/lib/src/test/reporter.dart b/mdtest/lib/src/test/reporter.dart
index e946845..43c8c61 100644
--- a/mdtest/lib/src/test/reporter.dart
+++ b/mdtest/lib/src/test/reporter.dart
@@ -5,17 +5,37 @@
import 'dart:async';
import 'dart:convert';
+import '../mobile/device_spec.dart';
+import '../mobile/device.dart';
+import 'test_result.dart';
import '../globals.dart';
+import '../util.dart';
class TAPReporter {
int currentTestNum;
int passingTestsNum;
- Map<int, TestEvent> testEventMapping;
+ Map<int, TestMethodResult> testMethodResultMapping;
+ Map<int, GroupResult> groupResultMapping;
+ List<TestSuiteResult> suites;
+ StringBuffer roundHighlight;
- TAPReporter() {
+ TAPReporter(Map<DeviceSpec, Device> deviceMapping) {
this.currentTestNum = 0;
this.passingTestsNum = 0;
- this.testEventMapping = <int, TestEvent>{};
+ this.testMethodResultMapping = <int, TestMethodResult>{};
+ this.groupResultMapping = <int, GroupResult>{};
+ this.suites = <TestSuiteResult>[];
+ roundHighlight = new StringBuffer();
+ int diffFrom = beginOfDiff(
+ deviceMapping.keys.map((DeviceSpec spec) => spec.groupKey()).toList()
+ );
+ deviceMapping.forEach((DeviceSpec spec, Device device) {
+ roundHighlight.writeln(
+ '<Spec Group Key: ${spec.groupKey().substring(diffFrom)}>'
+ ' -> '
+ '<Device Group Key: ${device.groupKey()}> (${spec.nickName})'
+ );
+ });
}
void printHeader() {
@@ -25,13 +45,15 @@
);
}
- Future<bool> report(Stream jsonOutput) async {
+ Future<bool> report(String testScriptPath, Stream jsonOutput) async {
+ testMethodResultMapping.clear();
+ groupResultMapping.clear();
+ suites.add(new TestSuiteResult(testScriptPath));
bool hasTestOutput = false;
await for (var line in jsonOutput) {
convertToTAPFormat(line.toString().trim());
hasTestOutput = true;
}
- testEventMapping.clear();
return hasTestOutput;
}
@@ -55,32 +77,63 @@
return;
}
+ TestSuiteResult lastSuite = suites.last;
+
if (_isGroupEvent(event) && !_isGroupRootEvent(event)) {
dynamic groupInfo = event['group'];
+ String name = groupInfo['name'];
bool skip = groupInfo['metadata']['skip'];
+ String skipReason = groupInfo['metadata']['skipReason'] ?? '';
if (skip) {
- String skipReason = groupInfo['metadata']['skipReason'] ?? '';
print('# skip ${groupInfo['name']} $skipReason');
+ } else {
+ print('# ${groupInfo['name']}');
}
- print('# ${groupInfo['name']}');
+ int groupID = groupInfo['id'];
+ GroupResult groupEvent = new GroupResult(name, skip, skipReason);
+ groupResultMapping[groupID] = groupEvent;
+ lastSuite.addEvent(groupEvent);
} else if (_isTestStartEvent(event)) {
dynamic testInfo = event['test'];
int testID = testInfo['id'];
String name = testInfo['name'];
+ List<int> groupIDs = testInfo['groupIDs'];
+ int directParentGroupID;
+ // Associate the test event to its parent group event if any
+ if (groupIDs.isNotEmpty) {
+ directParentGroupID = groupIDs.last;
+ // Remove group name prefix if any
+ GroupResult directParentGroup = groupResultMapping[directParentGroupID];
+ String groupName = directParentGroup.name;
+ if (name.startsWith(groupName)) {
+ name = name.substring(groupName.length).trim();
+ }
+ }
bool skip = testInfo['metadata']['skip'];
String skipReason = testInfo['metadata']['skipReason'] ?? '';
- testEventMapping[testID] = new TestEvent(name, skip, skipReason);
+ testMethodResultMapping[testID]
+ = new TestMethodResult(name, directParentGroupID, skip, skipReason);
} else if (_isErrorEvent(event)) {
int testID = event['testID'];
- TestEvent testEvent = testEventMapping[testID];
+ TestMethodResult testEvent = testMethodResultMapping[testID];
String errorReason = event['error'];
testEvent.fillError(errorReason);
} else if (_isTestDoneEvent(event)) {
int testID = event['testID'];
- TestEvent testEvent = testEventMapping[testID];
+ TestMethodResult testEvent = testMethodResultMapping[testID];
testEvent.hidden = event['hidden'];
testEvent.result = event['result'];
printTestResult(testEvent);
+ if (testEvent.hidden) {
+ return;
+ }
+ int directParentGroupID = testEvent.directParentGroupID;
+ if (!groupResultMapping.containsKey(directParentGroupID)) {
+ lastSuite.addEvent(testEvent);
+ } else {
+ GroupResult groupEvent = groupResultMapping[directParentGroupID];
+ groupEvent.addTestEvent(testEvent);
+ }
}
}
@@ -109,7 +162,7 @@
return event['type'] == 'testDone';
}
- void printTestResult(TestEvent event) {
+ void printTestResult(TestMethodResult event) {
if (event.hidden)
return;
if (event.result != 'success') {
@@ -131,29 +184,29 @@
print('ok ${++currentTestNum} - ${event.name}');
passingTestsNum++;
}
-}
-class TestEvent {
- // Known at TestStartEvent
- String name;
- bool skip;
- String skipReason;
- // Known at ErrorEvent
- bool error;
- String errorReason;
- // Known at TestDoneEvents
- String result;
- bool hidden;
-
- TestEvent(String name, bool skip, String skipReason) {
- this.name = name;
- this.skip = skip;
- this.skipReason = skipReason;
- this.error = false;
+ int skipNum() {
+ return sum(suites.map((TestSuiteResult e) => e.skipNum()));
}
- void fillError(String errorReason) {
- this.error = true;
- this.errorReason = errorReason;
+ int failNum() {
+ return sum(suites.map((TestSuiteResult e) => e.failNum()));
+ }
+
+ int passNum() {
+ return sum(suites.map((TestSuiteResult e) => e.passNum()));
+ }
+
+ dynamic toJson() {
+ int failures = failNum();
+ return {
+ 'type': 'test-round',
+ 'highlight': roundHighlight.toString(),
+ 'skip-num': skipNum(),
+ 'fail-num': failures,
+ 'pass-num': passNum(),
+ 'status': failures > 0 ? 'fail' : 'pass',
+ 'suites-info': suites.map((TestSuiteResult suite) => suite.toJson()).toList()
+ };
}
}
diff --git a/mdtest/lib/src/test/test_result.dart b/mdtest/lib/src/test/test_result.dart
new file mode 100644
index 0000000..635317e
--- /dev/null
+++ b/mdtest/lib/src/test/test_result.dart
@@ -0,0 +1,188 @@
+// Copyright 2016 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import 'dart:convert';
+
+import '../util.dart';
+
+
+abstract class Result {
+ String name;
+
+
+ Result(this.name);
+
+ Map toJson();
+
+ int skipNum();
+ int failNum();
+ int passNum();
+
+ @override
+ String toString() {
+ JsonEncoder encoder = const JsonEncoder.withIndent(' ');
+ return encoder.convert(toJson());
+ }
+}
+
+class TestMethodResult extends Result {
+ // // Known at TestStartEvent
+ int directParentGroupID;
+ bool skip;
+ String skipReason;
+ // Known at ErrorEvent
+ bool error;
+ String errorReason;
+ // Known at TestDoneEvents
+ String result;
+ bool hidden;
+
+ TestMethodResult(String name, this.directParentGroupID, this.skip, this.skipReason)
+ : super(name) {
+ this.error = false;
+ }
+
+ void fillError(String errorReason) {
+ this.error = true;
+ this.errorReason = errorReason;
+ }
+
+ @override
+ int skipNum() {
+ return skip ? 1 : 0;
+ }
+
+ @override
+ int failNum() {
+ if (skip) {
+ return 0;
+ }
+ return error ? 1 : 0;
+ }
+
+ @override
+ int passNum() {
+ if (skip) {
+ return 0;
+ }
+ return error ? 0 : 1;
+ }
+
+ @override
+ Map<String, String> toJson() {
+ Map<String, String> map = <String, String>{};
+ map['name'] = name;
+ map['type'] = 'test-method';
+ if (skip) {
+ map['status'] = 'skip';
+ map['reason'] = skipReason;
+ } else {
+ if (error) {
+ map['status'] = 'fail';
+ map['reason'] = errorReason;
+ } else {
+ map['status'] = 'pass';
+ }
+ }
+ map['result'] = result;
+ return map;
+ }
+}
+
+
+class GroupResult extends Result {
+ bool skip;
+ String skipReason;
+ List<TestMethodResult> testsInGroup;
+
+ GroupResult(String name, this.skip, this.skipReason) : super(name) {
+ this.testsInGroup = <TestMethodResult>[];
+ }
+
+ void addTestEvent(TestMethodResult testEvent) {
+ testsInGroup.add(testEvent);
+ }
+
+ @override
+ int skipNum() {
+ return skip ? 0 : sum(testsInGroup.map((TestMethodResult e) => e.skipNum()));
+ }
+
+ @override
+ int failNum() {
+ return skip ? 0 : sum(testsInGroup.map((TestMethodResult e) => e.failNum()));
+ }
+
+ @override
+ int passNum() {
+ return skip ? 0 : sum(testsInGroup.map((TestMethodResult e) => e.passNum()));
+ }
+
+ @override
+ Map<String, dynamic> toJson() {
+ Map<String, dynamic> map = <String, dynamic>{};
+ map['name'] = name;
+ map['type'] = 'test-group';
+ if (skip) {
+ map['status'] = 'skip';
+ map['reason'] = skipReason;
+ return map;
+ }
+ int failures = failNum();
+ if (failures > 0) {
+ map['status'] = 'fail';
+ } else {
+ map['status'] = 'pass';
+ }
+ map['skip-num'] = skipNum();
+ map['fail-num'] = failures;
+ map['pass-num'] = passNum();
+ map['methods-info'] = testsInGroup.map(
+ (TestMethodResult e) => e.toJson()
+ ).toList();
+ return map;
+ }
+}
+
+class TestSuiteResult extends Result {
+ List<Result> events;
+
+ TestSuiteResult(String name) : super(name) {
+ this.events = <Result>[];
+ }
+
+ int skipNum() {
+ return sum(events.map((Result e) => e.skipNum()));
+ }
+
+ int failNum() {
+ return sum(events.map((Result e) => e.failNum()));
+ }
+
+ int passNum() {
+ return sum(events.map((Result e) => e.passNum()));
+ }
+
+ Map<String, dynamic> toJson() {
+ Map<String, dynamic> map = <String, dynamic>{};
+ map['name'] = name;
+ map['type'] = 'test-suite';
+ map['skip-num'] = skipNum();
+ map['fail-num'] = failNum();
+ map['pass-num'] = passNum();
+ map['status'] = map['fail-num'] > 0 ? 'fail' : 'pass';
+ map['children-info'] = events.map((Result e) => e.toJson()).toList();
+ return map;
+ }
+
+ void addEvent(Result event) {
+ events.add(event);
+ }
+
+ @override
+ String toString() {
+ JsonEncoder encoder = const JsonEncoder.withIndent(' ');
+ return encoder.convert(toJson());
+ }
+}
diff --git a/mdtest/lib/src/util.dart b/mdtest/lib/src/util.dart
index af88eb0..ec60738 100644
--- a/mdtest/lib/src/util.dart
+++ b/mdtest/lib/src/util.dart
@@ -4,6 +4,7 @@
import 'dart:io';
import 'dart:math';
+import 'dart:convert';
import 'package:path/path.dart' as path;
import 'package:glob/glob.dart';
@@ -43,6 +44,21 @@
// '-' * 20
const String singleLineSeparator = '--------------------';
+int sum(Iterable<num> nums) {
+ if (nums.isEmpty) {
+ return 0;
+ }
+ return nums.reduce((num x, num y) => x + y);
+}
+
+String directoryName(String filePath) {
+ return path.dirname(filePath);
+}
+
+String fileBaseName(String filePath) {
+ return path.basename(filePath);
+}
+
int minLength(List<String> elements) {
if (elements == null || elements.isEmpty) return -1;
return elements.map((String e) => e.length).reduce(min);
@@ -109,7 +125,7 @@
}
/// Create a file if it does not exist. If the path points to a file, delete
-/// it and create a new file. Otherwise, report
+/// it and create a new file. Otherwise, report error
File createNewFile(String path) {
File file = new File('$path');;
if(file.existsSync())
@@ -117,6 +133,15 @@
return file..createSync(recursive: true);
}
+/// Create a directory if it does not exist. If the path points to a directory,
+/// delete it and create a new directory. Otherwise, report error
+Directory createNewDirectory(String path) {
+ Directory directory = new Directory('$path');;
+ if(directory.existsSync())
+ directory.deleteSync(recursive: true);
+ return directory..createSync(recursive: true);
+}
+
/// Return the absolute paths of a list of files based on a list of glob
/// patterns. The order of the result follow the order of the given glob
/// patterns, but the order of file paths corresponding to the same glob
@@ -154,3 +179,12 @@
result.addAll(second.where((String e) => !first.contains(e)));
return result;
}
+
+String dumpToJSONString(dynamic jsonObject) {
+ JsonEncoder encoder = const JsonEncoder.withIndent(' ');
+ return encoder.convert(jsonObject);
+}
+
+void copyPathToDirectory(String fPath, String dirPath) {
+ Process.runSync('cp', ['-r', fPath, dirPath]);
+}