| // Copyright 2016 The Vanadium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| import 'dart:async'; |
| |
| import 'helper.dart'; |
| import '../mobile/device.dart'; |
| import '../mobile/device_spec.dart'; |
| import '../mobile/key_provider.dart'; |
| import '../mobile/android.dart'; |
| import '../algorithms/coverage.dart'; |
| import '../algorithms/matching.dart'; |
| import '../globals.dart'; |
| import '../runner/mdtest_command.dart'; |
| import '../test/coverage_collector.dart'; |
| |
| class AutoCommand extends MDTestCommand { |
| @override |
| final String name = 'auto'; |
| |
| @override |
| final String description |
| = 'Automatically run applications based on a subset of spec to device ' |
| 'settings that maximize the device coverage'; |
| |
| dynamic _specs; |
| |
| List<Device> _devices; |
| |
| @override |
| Future<int> runCore() async { |
| printInfo('Running "mdtest auto command" ...'); |
| |
| this._specs = await loadSpecs(argResults); |
| if (sanityCheckSpecs(_specs, argResults['spec']) != 0) { |
| printError('Test spec does not meet requirements.'); |
| return 1; |
| } |
| |
| this._devices = await getDevices(groupKey: argResults['groupby']); |
| if (_devices.isEmpty) { |
| printError('No device found.'); |
| return 1; |
| } |
| |
| List<DeviceSpec> allDeviceSpecs |
| = await constructAllDeviceSpecs(_specs['devices']); |
| Map<DeviceSpec, Set<Device>> individualMatches |
| = findIndividualMatches(allDeviceSpecs, _devices); |
| List<Map<DeviceSpec, Device>> allDeviceMappings |
| = findAllMatchingDeviceMappings(allDeviceSpecs, individualMatches); |
| if(allDeviceMappings.isEmpty) { |
| printError('No device specs to devices mapping is found.'); |
| return 1; |
| } |
| |
| Map<String, List<Device>> deviceGroups = buildGroups(_devices); |
| Map<String, List<DeviceSpec>> deviceSpecGroups |
| = buildGroups(allDeviceSpecs); |
| |
| GroupInfo groupInfo = new GroupInfo(deviceGroups, deviceSpecGroups); |
| Map<CoverageMatrix, Map<DeviceSpec, Device>> cov2match |
| = buildCoverage2MatchMapping(allDeviceMappings, groupInfo); |
| CoverageMatrix appDeviceCoverageMatrix = new CoverageMatrix(groupInfo); |
| Set<Map<DeviceSpec, Device>> chosenMappings |
| = findMinimumMappings(cov2match, appDeviceCoverageMatrix); |
| printMatches(chosenMappings); |
| |
| Map<String, CoverageCollector> collectorPool |
| = <String, CoverageCollector>{}; |
| |
| List<int> errRounds = []; |
| List<int> failRounds = []; |
| int roundNum = 1; |
| for (Map<DeviceSpec, Device> deviceMapping in chosenMappings) { |
| printInfo('Begining of Round #$roundNum'); |
| MDTestRunner runner = new MDTestRunner(); |
| |
| if (await runner.runAllApps(deviceMapping) != 0) { |
| printError('Error when running applications on #Round $roundNum'); |
| await uninstallTestedApps(deviceMapping); |
| errRounds.add(roundNum++); |
| continue; |
| } |
| |
| await storeMatches(deviceMapping); |
| |
| bool testsFailed; |
| if (argResults['format'] == 'tap') { |
| testsFailed = await runner.runAllTestsToTAP(_specs['test-paths']) != 0; |
| } else { |
| testsFailed = await runner.runAllTests(_specs['test-paths']) != 0; |
| } |
| |
| assert(testsFailed != null); |
| if (testsFailed) { |
| printInfo('Some tests in Round #$roundNum failed'); |
| failRounds.add(roundNum++); |
| } else { |
| printInfo('All tests in Round #${roundNum++} passed'); |
| } |
| |
| appDeviceCoverageMatrix.hit(deviceMapping); |
| |
| if (argResults['coverage']) { |
| printTrace('Collecting code coverage hitmap (this may take some time)'); |
| buildCoverageCollectionTasks(deviceMapping, collectorPool); |
| await runCoverageCollectionTasks(collectorPool); |
| } |
| |
| await uninstallTestedApps(deviceMapping); |
| printInfo('End of Round #$roundNum\n'); |
| } |
| |
| if (!briefMode) { |
| printHitmap( |
| 'App-device coverage hit matrix:', |
| appDeviceCoverageMatrix |
| ); |
| } |
| |
| if (errRounds.isNotEmpty) { |
| printInfo('Error in Round #${errRounds.join(', #')}'); |
| return 1; |
| } |
| |
| if (failRounds.isNotEmpty) { |
| printInfo('Some tests failed in Round #${failRounds.join(', #')}'); |
| } else { |
| printInfo('All tests in all rounds passed'); |
| } |
| |
| if (argResults['coverage']) { |
| printInfo('Computing code coverage for each application ...'); |
| if (await computeAppsCoverage(collectorPool, name) != 0) |
| return 1; |
| } |
| |
| return failRounds.isNotEmpty ? 1 : 0; |
| } |
| |
| AutoCommand() { |
| usesSpecsOption(); |
| usesCoverageFlag(); |
| usesTAPReportOption(); |
| argParser.addOption('groupby', |
| defaultsTo: 'device-id', |
| allowed: [ |
| 'device-id', |
| 'model-name', |
| 'os-version', |
| 'api-level', |
| 'screen-size' |
| ], |
| help: 'Device property used to group devices to' |
| 'adjust app-device coverage criterion.' |
| ); |
| } |
| } |