feat(mdtest driver api): mdtest computes the maximum app-device coverage
score and generate app-device hitmap
mdtest is now able to report a app-device coverage matrix that shows
both reachable and unreachable app-device paths before test execution.
mdtest also reports an app-device hitmap based on the real execution
and spits the maximum app-device coverage score as well as the
percentage of app-device paths that are covered by the execution out of
all reachable app-device paths. The hitmap as well as the app-device
coverage score helps users to discover under which app-device combination
some test executions fail. It also tells the user which app-device
combination is tested and which is not.
mdtest also supports --brief flag which will only report test execution
output if set to true.
mdtest now does not support specifying test paths from test spec. test
paths can only be specified from the command line.
Change-Id: I4b414d0f1efa17948a8b2846586f6e304a417e31
diff --git a/mdtest/lib/src/algorithms/coverage.dart b/mdtest/lib/src/algorithms/coverage.dart
index 32e79bc..70836a0 100644
--- a/mdtest/lib/src/algorithms/coverage.dart
+++ b/mdtest/lib/src/algorithms/coverage.dart
@@ -2,11 +2,13 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+import 'package:intl/intl.dart';
import 'package:dlog/dlog.dart' show Table;
import '../mobile/device.dart' show Device;
import '../mobile/device_spec.dart' show DeviceSpec;
import '../util.dart';
+import '../globals.dart';
class GroupInfo {
Map<String, List<Device>> _deviceClusters;
@@ -30,61 +32,185 @@
List<String> get deviceSpecClustersOrder => _deviceSpecClustersOrder;
}
+// -1 indicates that an app-device path can never be covered given the available
+// devices. 0 indicates that an app-device path can be covered, but not covered
+// yet. (No test script is run under this setting) A positive number indicates
+// the number of times an app-device path is hit by some test runs.
+const int cannotBeCovered = -1;
+const int isNotCovered = 0;
+
class CoverageMatrix {
- CoverageMatrix(this.clusterInfo) {
- this.matrix = new List<List<int>>(clusterInfo.deviceSpecClusters.length);
+ CoverageMatrix(this.groupInfo) {
+ this.matrix = new List<List<int>>(groupInfo.deviceSpecClusters.length);
for (int i = 0; i < matrix.length; i++) {
- matrix[i] = new List<int>.filled(clusterInfo.deviceClusters.length, 0);
+ matrix[i]
+ = new List<int>.filled(groupInfo.deviceClusters.length, cannotBeCovered);
}
}
- GroupInfo clusterInfo;
- // Coverage matrix, where a row indicats an app cluster and a column
- // indicates a device cluster
+ GroupInfo groupInfo;
+ // Coverage matrix, where a row indicats an app group and a column
+ // indicates a device group
List<List<int>> matrix;
- void fill(Map<DeviceSpec, Device> match) {
+ /// Fill the corresponding elements baesd on the given match in the
+ /// matrix with [value].
+ void fill(Map<DeviceSpec, Device> match, int value) {
match.forEach((DeviceSpec spec, Device device) {
- int rowNum = clusterInfo.deviceSpecClustersOrder
+ int rowNum = groupInfo.deviceSpecClustersOrder
.indexOf(spec.groupKey());
- int colNum = clusterInfo.deviceClustersOrder
+ int colNum = groupInfo.deviceClustersOrder
.indexOf(device.groupKey());
- matrix[rowNum][colNum] = 1;
+ matrix[rowNum][colNum] = value;
});
}
- void union(CoverageMatrix newCoverage) {
+ /// Increate the corresponding elements' values by 1 given the match
+ void hit(Map<DeviceSpec, Device> match) {
+ match.forEach((DeviceSpec spec, Device device) {
+ int rowNum = groupInfo.deviceSpecClustersOrder
+ .indexOf(spec.groupKey());
+ int colNum = groupInfo.deviceClustersOrder
+ .indexOf(device.groupKey());
+ matrix[rowNum][colNum]++;
+ });
+ }
+
+ /// Merge the new coverage matrix with this. Each element value in the
+ /// matrix is set to [isNotCovered] if the matching algorithm finds that
+ /// the corresponding app-device path is reachable. The goal is to accumulate
+ /// reachable app-device paths.
+ void merge(CoverageMatrix newCoverage) {
for (int i = 0; i < matrix.length; i++) {
List<int> row = matrix[i];
for (int j = 0; j < row.length; j++) {
- matrix[i][j] |= newCoverage.matrix[i][j];
+ if (matrix[i][j] == cannotBeCovered
+ &&
+ newCoverage.matrix[i][j] == isNotCovered) {
+ matrix[i][j] = isNotCovered;
+ }
}
}
}
+}
- void printMatrix() {
- Table prettyMatrix = new Table(1);
- prettyMatrix.columns.add('app key \\ device key');
- prettyMatrix.columns.addAll(clusterInfo.deviceClustersOrder);
- int startIndx = beginOfDiff(clusterInfo.deviceSpecClustersOrder);
- for (int i = 0; i < matrix.length; i++) {
- prettyMatrix.data.add(clusterInfo.deviceSpecClustersOrder[i].substring(startIndx));
- prettyMatrix.data.addAll(matrix[i]);
- }
- print(prettyMatrix);
+int _countNumberInCoverageMatrix(List<List<int>> matrix, bool test(int e)) {
+ int result = 0;
+ matrix.forEach((List<int> row) {
+ result += row.where((int element) => test(element)).length;
+ });
+ return result;
+}
+
+/// Compute and print the app-device coverage.
+void computeAndReportCoverage(CoverageMatrix coverageMatrix) {
+ if (coverageMatrix == null) {
+ printError('Coverage matrix is null');
+ return;
}
+
+ List<List<int>> matrix = coverageMatrix.matrix;
+ int rowNum = matrix.length;
+ int colNum = matrix[0].length;
+ int totalPathNum = rowNum * colNum;
+ int reachableCombinationNum
+ = _countNumberInCoverageMatrix(matrix, (int e) => e != cannotBeCovered);
+ int coveredCombinationNum
+ = _countNumberInCoverageMatrix(matrix, (int e) => e > isNotCovered);
+ StringBuffer scoreReport = new StringBuffer();
+ NumberFormat percentFormat = new NumberFormat('%##.0#', 'en_US');
+ scoreReport.writeln('App-Device Path Coverage (ADPC) score:');
+ double reachableCoverageScore = reachableCombinationNum / totalPathNum;
+ scoreReport.writeln(
+ 'Reachable ADPC score: ${percentFormat.format(reachableCoverageScore)}, '
+ 'defined by #reachable / #total.'
+ );
+ double coveredCoverageScore
+ = coveredCombinationNum / reachableCombinationNum;
+ scoreReport.writeln(
+ 'Covered ADPC score: ${percentFormat.format(coveredCoverageScore)}, '
+ 'defined by #covered / #reachable.'
+ );
+ print(scoreReport.toString());
+}
+
+void printLegend() {
+ StringBuffer legendInfo = new StringBuffer();
+ legendInfo.writeln('Meaning of the number in the coverage matrix:');
+ legendInfo.writeln(
+ '$cannotBeCovered: an app-device path is not reachable '
+ 'given the connected devices.'
+ );
+ legendInfo.writeln(
+ ' $isNotCovered: an app-device path is reachable but '
+ 'not covered by any test run.'
+ );
+ legendInfo.writeln(
+ '>$isNotCovered: the number of times an app-device path '
+ 'is covered by some test runs.'
+ );
+ print(legendInfo.toString());
+}
+
+void printCoverageMatrix(String title, CoverageMatrix coverageMatrix) {
+ printMatrix(
+ title,
+ coverageMatrix,
+ (int e) {
+ if (e == -1) {
+ return 'unreachable';
+ } else {
+ return 'reachable';
+ }
+ }
+ );
+}
+
+void printHitmap(String title, CoverageMatrix coverageMatrix) {
+ if (briefMode) {
+ return;
+ }
+ printMatrix(
+ title,
+ coverageMatrix,
+ (int e) {
+ return '$e';
+ }
+ );
+ printLegend();
+ computeAndReportCoverage(coverageMatrix);
+}
+
+void printMatrix(String title, CoverageMatrix coverageMatrix, f(int e)) {
+ if (coverageMatrix == null) {
+ return;
+ }
+ GroupInfo groupInfo = coverageMatrix.groupInfo;
+ List<List<int>> matrix = coverageMatrix.matrix;
+ Table prettyMatrix = new Table(1);
+ prettyMatrix.columns.add('app key \\ device key');
+ prettyMatrix.columns.addAll(groupInfo.deviceClustersOrder);
+ int startIndx = beginOfDiff(groupInfo.deviceSpecClustersOrder);
+ for (int i = 0; i < matrix.length; i++) {
+ prettyMatrix.data.add(
+ groupInfo.deviceSpecClustersOrder[i].substring(startIndx)
+ );
+ prettyMatrix.data.addAll(matrix[i].map(f));
+ }
+ print(title);
+ print(prettyMatrix);
}
Map<CoverageMatrix, Map<DeviceSpec, Device>> buildCoverage2MatchMapping(
List<Map<DeviceSpec, Device>> allMatches,
- GroupInfo clusterInfo
+ GroupInfo groupInfo
) {
Map<CoverageMatrix, Map<DeviceSpec, Device>> cov2match
= <CoverageMatrix, Map<DeviceSpec, Device>>{};
for (Map<DeviceSpec, Device> match in allMatches) {
- CoverageMatrix cov = new CoverageMatrix(clusterInfo);
- cov.fill(match);
+ CoverageMatrix cov = new CoverageMatrix(groupInfo);
+ cov.fill(match, isNotCovered);
cov2match[cov] = match;
}
return cov2match;
@@ -97,10 +223,9 @@
/// [ref link]: https://en.wikipedia.org/wiki/Set_cover_problem
Set<Map<DeviceSpec, Device>> findMinimumMappings(
Map<CoverageMatrix, Map<DeviceSpec, Device>> cov2match,
- GroupInfo clusterInfo
+ CoverageMatrix base
) {
Set<CoverageMatrix> minSet = new Set<CoverageMatrix>();
- CoverageMatrix base = new CoverageMatrix(clusterInfo);
while (true) {
CoverageMatrix currentBestCoverage = null;
int maxReward = 0;
@@ -114,10 +239,14 @@
}
if (currentBestCoverage == null) break;
minSet.add(currentBestCoverage);
- base.union(currentBestCoverage);
+ base.merge(currentBestCoverage);
}
- print('Best coverage matrix:');
- base.printMatrix();
+ if (!briefMode) {
+ printCoverageMatrix(
+ 'Best app-device coverage matrix:',
+ base
+ );
+ }
Set<Map<DeviceSpec, Device>> bestMatches = new Set<Map<DeviceSpec, Device>>();
for (CoverageMatrix coverage in minSet) {
bestMatches.add(cov2match[coverage]);
@@ -130,7 +259,9 @@
for (int i = 0; i < base.matrix.length; i++) {
List<int> row = base.matrix[i];
for (int j = 0; j < row.length; j++) {
- if (base.matrix[i][j] == 0 && newCoverage.matrix[i][j] == 1)
+ if (base.matrix[i][j] == cannotBeCovered
+ &&
+ newCoverage.matrix[i][j] == isNotCovered)
reward++;
}
}
diff --git a/mdtest/lib/src/algorithms/matching.dart b/mdtest/lib/src/algorithms/matching.dart
index 447cfea..25054e4 100644
--- a/mdtest/lib/src/algorithms/matching.dart
+++ b/mdtest/lib/src/algorithms/matching.dart
@@ -9,6 +9,8 @@
import '../base/common.dart';
import '../mobile/device.dart';
import '../mobile/device_spec.dart';
+import '../util.dart';
+import '../globals.dart';
/// Find all matched devices for each device spec
Map<DeviceSpec, Set<Device>> findIndividualMatches(
@@ -136,15 +138,27 @@
/// Print a collection of matches which is iterable.
void printMatches(Iterable<Map<DeviceSpec, Device>> matches) {
+ if (briefMode) {
+ return;
+ }
StringBuffer sb = new StringBuffer();
int roundNum = 1;
sb.writeln('=' * 10);
for (Map<DeviceSpec, Device> match in matches) {
+ int startIndx = beginOfDiff(
+ new List.from(
+ match.keys.map(
+ (DeviceSpec spec) {
+ return spec.groupKey();
+ }
+ )
+ )
+ );
sb.writeln('Round $roundNum:');
match.forEach((DeviceSpec spec, Device device) {
- sb.writeln('[Spec Cluster Key: ${spec.groupKey()}]'
+ sb.writeln('<Spec Group Key: ${spec.groupKey().substring(startIndx)}>'
' -> '
- '[Device Cluster Key: ${device.groupKey()}]');
+ '<Device Group Key: ${device.groupKey()}>');
});
roundNum++;
}
diff --git a/mdtest/lib/src/base/logger.dart b/mdtest/lib/src/base/logger.dart
index 2c318e1..5a8e355 100644
--- a/mdtest/lib/src/base/logger.dart
+++ b/mdtest/lib/src/base/logger.dart
@@ -41,3 +41,16 @@
stderr.writeln('[TRACE] $message');
}
}
+
+class DumbLogger extends Logger {
+ @override
+ void info(String message) {}
+
+ @override
+ void error(String message) {
+ stderr.writeln('[ERROR] $message');
+ }
+
+ @override
+ void trace(String message) {}
+}
diff --git a/mdtest/lib/src/commands/auto.dart b/mdtest/lib/src/commands/auto.dart
index 1fe236d..84663aa 100644
--- a/mdtest/lib/src/commands/auto.dart
+++ b/mdtest/lib/src/commands/auto.dart
@@ -55,15 +55,16 @@
return 1;
}
- Map<String, List<Device>> deviceClusters = buildCluster(_devices);
- Map<String, List<DeviceSpec>> deviceSpecClusters
- = buildCluster(allDeviceSpecs);
+ Map<String, List<Device>> deviceGroups = buildGroups(_devices);
+ Map<String, List<DeviceSpec>> deviceSpecGroups
+ = buildGroups(allDeviceSpecs);
- GroupInfo clusterInfo = new GroupInfo(deviceClusters, deviceSpecClusters);
+ GroupInfo groupInfo = new GroupInfo(deviceGroups, deviceSpecGroups);
Map<CoverageMatrix, Map<DeviceSpec, Device>> cov2match
- = buildCoverage2MatchMapping(allDeviceMappings, clusterInfo);
+ = buildCoverage2MatchMapping(allDeviceMappings, groupInfo);
+ CoverageMatrix appDeviceCoverageMatrix = new CoverageMatrix(groupInfo);
Set<Map<DeviceSpec, Device>> chosenMappings
- = findMinimumMappings(cov2match, clusterInfo);
+ = findMinimumMappings(cov2match, appDeviceCoverageMatrix);
printMatches(chosenMappings);
Map<String, CoverageCollector> collectorPool
@@ -73,10 +74,11 @@
List<int> failRounds = [];
int roundNum = 1;
for (Map<DeviceSpec, Device> deviceMapping in chosenMappings) {
+ printInfo('Begining of Round #$roundNum');
MDTestRunner runner = new MDTestRunner();
if (await runner.runAllApps(deviceMapping) != 0) {
- printError('Error when running applications');
+ printError('Error when running applications on #Round $roundNum');
await uninstallTestedApps(deviceMapping);
errRounds.add(roundNum++);
continue;
@@ -99,6 +101,8 @@
printInfo('All tests in Round #${roundNum++} passed');
}
+ appDeviceCoverageMatrix.hit(deviceMapping);
+
if (argResults['coverage']) {
printTrace('Collecting code coverage hitmap (this may take some time)');
buildCoverageCollectionTasks(deviceMapping, collectorPool);
@@ -106,10 +110,18 @@
}
await uninstallTestedApps(deviceMapping);
+ printInfo('End of Round #$roundNum\n');
+ }
+
+ if (!briefMode) {
+ printHitmap(
+ 'App-device coverage hit matrix:',
+ appDeviceCoverageMatrix
+ );
}
if (errRounds.isNotEmpty) {
- printError('Error in Round #${errRounds.join(', #')}');
+ printInfo('Error in Round #${errRounds.join(', #')}');
return 1;
}
@@ -134,7 +146,13 @@
usesTAPReportOption();
argParser.addOption('groupby',
defaultsTo: 'device-id',
- allowed: ['device-id', 'model-name', 'os-version', 'api-level', 'screen-size'],
+ allowed: [
+ 'device-id',
+ 'model-name',
+ 'os-version',
+ 'api-level',
+ 'screen-size'
+ ],
help: 'Device property used to group devices to'
'adjust app-device coverage criterion.'
);
diff --git a/mdtest/lib/src/commands/helper.dart b/mdtest/lib/src/commands/helper.dart
index a954d89..9e4d73f 100644
--- a/mdtest/lib/src/commands/helper.dart
+++ b/mdtest/lib/src/commands/helper.dart
@@ -46,6 +46,7 @@
return 1;
}
+ printInfo('Start application ${deviceSpec.appPath} on device ${device.id}');
Process process = await Process.start(
'flutter',
['run', '-d', '${device.id}', '--target=${deviceSpec.appPath}'],
@@ -57,12 +58,13 @@
.transform(new LineSplitter());
RegExp portPattern = new RegExp(r'Observatory listening on (http.*)');
await for (var line in lineStream) {
- print(line.toString().trim());
Match portMatch = portPattern.firstMatch(line.toString());
if (portMatch != null) {
+ printInfo('${line.toString().trim()} (${deviceSpec.nickName}: ${device.id})');
deviceSpec.observatoryUrl = portMatch.group(1);
break;
}
+ printTrace(line.toString().trim());
}
process.stderr.drain();
diff --git a/mdtest/lib/src/commands/run.dart b/mdtest/lib/src/commands/run.dart
index e6e9890..773aa75 100644
--- a/mdtest/lib/src/commands/run.dart
+++ b/mdtest/lib/src/commands/run.dart
@@ -72,7 +72,7 @@
assert(testsFailed != null);
if (testsFailed) {
- printError('Some tests failed');
+ printInfo('Some tests failed');
} else {
printInfo('All tests passed');
}
diff --git a/mdtest/lib/src/globals.dart b/mdtest/lib/src/globals.dart
index 7ad36ef..11961ef 100644
--- a/mdtest/lib/src/globals.dart
+++ b/mdtest/lib/src/globals.dart
@@ -12,3 +12,5 @@
void printError(String message) => logger.error(message);
void printTrace(String message) => logger.trace(message);
+
+bool briefMode = false;
diff --git a/mdtest/lib/src/mobile/device_spec.dart b/mdtest/lib/src/mobile/device_spec.dart
index 251eba0..700281c 100644
--- a/mdtest/lib/src/mobile/device_spec.dart
+++ b/mdtest/lib/src/mobile/device_spec.dart
@@ -73,18 +73,11 @@
dynamic newSpecs = JSON.decode(await new File(specsPath).readAsString());
// Get the parent directory of the specs file
String rootPath = new File(specsPath).parent.absolute.path;
- // Normalize the 'test-path' in the specs file and add extra test paths
- // from the command line argument
- List<String> testPathsFromSpec
- = listFilePathsFromGlobPatterns(rootPath, newSpecs['test-paths']);
- printTrace('Test paths from spec: $testPathsFromSpec');
+ // Normalize the 'test-path' specified from the command line argument
List<String> testPathsFromCommandLine
= listFilePathsFromGlobPatterns(Directory.current.path, argResults.rest);
printTrace('Test paths from command line: $testPathsFromCommandLine');
- newSpecs['test-paths'] = mergeWithoutDuplicate(
- testPathsFromSpec,
- testPathsFromCommandLine
- );
+ newSpecs['test-paths'] = testPathsFromCommandLine;
// Normalize the 'app-path' in the specs file
newSpecs['devices']?.forEach((String name, Map<String, String> map) {
map['app-path'] = normalizePath(rootPath, map['app-path']);
diff --git a/mdtest/lib/src/mobile/key_provider.dart b/mdtest/lib/src/mobile/key_provider.dart
index d0a9dc1..44218a4 100644
--- a/mdtest/lib/src/mobile/key_provider.dart
+++ b/mdtest/lib/src/mobile/key_provider.dart
@@ -7,11 +7,11 @@
String groupKey();
}
-Map<String, List<dynamic>> buildCluster(List<dynamic> elements) {
- Map<String, List<dynamic>> clusters = <String, List<dynamic>>{};
+Map<String, List<dynamic>> buildGroups(List<dynamic> elements) {
+ Map<String, List<dynamic>> groups = <String, List<dynamic>>{};
elements.forEach((dynamic element) {
- clusters.putIfAbsent(element.groupKey(), () => <dynamic>[])
+ groups.putIfAbsent(element.groupKey(), () => <dynamic>[])
.add(element);
});
- return clusters;
+ return groups;
}
diff --git a/mdtest/lib/src/runner/mdtest_command.dart b/mdtest/lib/src/runner/mdtest_command.dart
index 3c01a34..36b310b 100644
--- a/mdtest/lib/src/runner/mdtest_command.dart
+++ b/mdtest/lib/src/runner/mdtest_command.dart
@@ -35,7 +35,8 @@
}
void usesCoverageFlag() {
- argParser.addFlag('coverage',
+ argParser.addFlag(
+ 'coverage',
defaultsTo: false,
negatable: false,
help: 'Whether to collect coverage information.'
@@ -43,7 +44,8 @@
}
void usesTAPReportOption() {
- argParser.addOption('format',
+ argParser.addOption(
+ 'format',
defaultsTo: 'none',
allowed: ['none', 'tap'],
help: 'Format to be used to display test output result.'
diff --git a/mdtest/lib/src/runner/mdtest_command_runner.dart b/mdtest/lib/src/runner/mdtest_command_runner.dart
index 94c9a21..e5c9981 100644
--- a/mdtest/lib/src/runner/mdtest_command_runner.dart
+++ b/mdtest/lib/src/runner/mdtest_command_runner.dart
@@ -15,10 +15,19 @@
'mdtest',
'Launch mdtest and run tests'
) {
- argParser.addFlag('verbose',
- abbr: 'v',
- negatable: false,
- help: 'Noisy logging, including all shell commands executed.');
+ argParser.addFlag(
+ 'verbose',
+ abbr: 'v',
+ negatable: false,
+ help: 'Noisy logging, including detailed information '
+ 'through the entire execution.'
+ );
+ argParser.addFlag(
+ 'brief',
+ abbr: 'b',
+ negatable: false,
+ help: 'Disable logging, only report test execution output.'
+ );
}
@override
@@ -30,8 +39,24 @@
@override
Future<int> runCommand(ArgResults globalResults) async {
- if (globalResults['verbose'])
+ if (!_commandValidator(globalResults)) {
+ return 1;
+ }
+ if (globalResults['verbose']) {
defaultLogger = new VerboseLogger();
+ }
+ if (globalResults['brief']) {
+ defaultLogger = new DumbLogger();
+ briefMode = true;
+ }
return await super.runCommand(globalResults);
}
+
+ bool _commandValidator(ArgResults globalResults) {
+ if (globalResults['verbose'] && globalResults['brief']) {
+ printError('--verbose flag conflicts with --brief flag');
+ return false;
+ }
+ return true;
+ }
}
diff --git a/mdtest/lib/src/util.dart b/mdtest/lib/src/util.dart
index f9f6402..1482d20 100644
--- a/mdtest/lib/src/util.dart
+++ b/mdtest/lib/src/util.dart
@@ -10,6 +10,11 @@
import 'globals.dart';
+// '=' * 20
+const String doubleLineSeparator = '====================';
+// '-' * 20
+const String singleLineSeparator = '--------------------';
+
int minLength(List<String> elements) {
if (elements == null || elements.isEmpty) return -1;
return elements.map((String e) => e.length).reduce(min);
@@ -63,6 +68,7 @@
return true;
}
+/// Get a file with unique name under the given directory.
File getUniqueFile(Directory dir, String baseName, String ext) {
int i = 1;
diff --git a/mdtest/pubspec.yaml b/mdtest/pubspec.yaml
index 4ad70a0..aad05ef 100644
--- a/mdtest/pubspec.yaml
+++ b/mdtest/pubspec.yaml
@@ -6,6 +6,7 @@
stack_trace: ^1.4.0
coverage: ^0.7.9
glob: ">=1.1.3"
+ intl: ">=0.12.4+2"
dlog:
path: ../../../../third_party/dart/dlog
flutter_driver:
