blob: 6c5aa7587b03454775626f169fa0aa1b98a462d2 [file] [log] [blame]
// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package test
import (
"encoding/xml"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"regexp"
"runtime"
"strings"
"testing"
"v.io/jiri"
"v.io/jiri/jiritest"
"v.io/jiri/tool"
"v.io/x/devtools/internal/test"
"v.io/x/devtools/internal/xunit"
"v.io/x/devtools/tooldata"
)
// failuresMatch checks whether the given test failures match. The Message
// field must match exactly, whereas fs1's Data field must contain
// fs2's; fs1 is the 'got' and fs2 the 'want'.
func failuresMatch(fs1, fs2 []xunit.Failure) bool {
if len(fs1) != len(fs2) {
return false
}
for i := 0; i < len(fs1); i++ {
if fs1[i].Message != fs2[i].Message {
return false
}
if !strings.Contains(fs1[i].Data, fs2[i].Data) {
return false
}
}
return true
}
// caseMatch checks whether the given test cases match modulo their
// execution time.
func caseMatch(c1, c2 xunit.TestCase) bool {
// Test names can have a CPU count appended to them (e.g. TestFoo-12)
// so we take care to strip that out when comparing with
// expected results.
ncpu := runtime.NumCPU()
re := regexp.MustCompile(fmt.Sprintf("(.*)-%d(.*)", ncpu))
stripNumCPU := func(s string) string {
parts := re.FindStringSubmatch(s)
switch len(parts) {
case 3:
return strings.TrimRight(parts[1]+parts[2], " ")
default:
return s
}
}
if stripNumCPU(c1.Name) != stripNumCPU(c2.Name) {
return false
}
if c1.Classname != c2.Classname {
return false
}
if !reflect.DeepEqual(c1.Errors, c2.Errors) {
return false
}
if !failuresMatch(c1.Failures, c2.Failures) {
return false
}
return true
}
// coverageMatch checks whether the given test coverages match modulo
// their timestamps and sources.
func coverageMatch(c1, c2 testCoverage) bool {
if c1.BranchRate != c2.BranchRate {
return false
}
if c1.LineRate != c2.LineRate {
return false
}
if !reflect.DeepEqual(c1.Packages, c2.Packages) {
return false
}
return true
}
// suiteMatch checks whether the given test suites match modulo their
// execution time.
func suiteMatch(s1, s2 xunit.TestSuite) bool {
if s1.Name != s2.Name {
return false
}
if s1.Errors != s2.Errors {
return false
}
if s1.Failures != s2.Failures {
return false
}
if s1.Skip != s2.Skip {
return false
}
if s1.Tests != s2.Tests {
return false
}
if len(s1.Cases) != len(s2.Cases) {
return false
}
for i := 0; i < len(s1.Cases); i++ {
found := false
for j := 0; j < len(s2.Cases); j++ {
if caseMatch(s1.Cases[i], s2.Cases[j]) {
found = true
break
}
}
if !found {
return false
}
}
return true
}
// suitesMatch checks whether the given test suites match modulo their
// execution time.
func suitesMatch(ss1, ss2 xunit.TestSuites) bool {
if len(ss1.Suites) != len(ss2.Suites) {
return false
}
for i := 0; i < len(ss1.Suites); i++ {
if !suiteMatch(ss1.Suites[i], ss2.Suites[i]) {
return false
}
}
return true
}
var (
wantBuild = xunit.TestSuites{
Suites: []xunit.TestSuite{
xunit.TestSuite{
Name: "v.io/x/devtools/jiri-test/internal/test/testdata/foo2",
Cases: []xunit.TestCase{
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo2",
Name: "Build",
Failures: []xunit.Failure{
xunit.Failure{
Message: "build failure",
Data: "missing return at end of function",
},
},
},
},
Tests: 1,
Failures: 1,
},
},
}
wantTest = xunit.TestSuites{
Suites: []xunit.TestSuite{
xunit.TestSuite{
Name: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Cases: []xunit.TestCase{
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "Test1",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "Test2",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "Test3",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23B",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23Hello",
},
},
Tests: 6,
Skip: 3,
},
},
}
wantV23Test = xunit.TestSuites{
Suites: []xunit.TestSuite{
xunit.TestSuite{
Name: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Cases: []xunit.TestCase{
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23B",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23Hello",
},
},
Tests: 3,
Skip: 0,
},
},
}
wantV23TestWithExcludedTests = xunit.TestSuites{
Suites: []xunit.TestSuite{
xunit.TestSuite{
Name: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Cases: []xunit.TestCase{
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23Hello",
},
},
Tests: 2,
Skip: 0,
},
},
}
wantRegressionTest = xunit.TestSuites{
Suites: []xunit.TestSuite{
xunit.TestSuite{
Name: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Cases: []xunit.TestCase{
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23Hello",
},
},
Tests: 1,
Skip: 0,
},
},
}
wantTestWithSuffix = xunit.TestSuites{
Suites: []xunit.TestSuite{
xunit.TestSuite{
Name: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Cases: []xunit.TestCase{
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "Test1 [Suffix]",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "Test2 [Suffix]",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "Test3 [Suffix]",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23 [Suffix]",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23B [Suffix]",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23Hello [Suffix]",
},
},
Tests: 6,
Skip: 3,
},
},
}
wantTestWithExcludedTests = xunit.TestSuites{
Suites: []xunit.TestSuite{
xunit.TestSuite{
Name: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Cases: []xunit.TestCase{
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "Test1",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23B",
},
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
Name: "TestV23Hello",
},
},
Tests: 4,
Skip: 3,
},
},
}
wantExcludedPackage = xunit.TestSuites{
Suites: []xunit.TestSuite{},
}
wantTestWithTimeout = xunit.TestSuites{
Suites: []xunit.TestSuite{
xunit.TestSuite{
Name: "v.io/x/devtools/jiri-test/internal/test/testdata/foo_timeout",
Cases: []xunit.TestCase{
xunit.TestCase{
Classname: "v.io/x/devtools/jiri-test/internal/test/testdata/foo_timeout",
Name: "TestWithSleep",
Failures: []xunit.Failure{
xunit.Failure{
Message: "error",
Data: "test timed out after 1s",
},
},
},
},
Tests: 1,
Failures: 1,
},
},
}
wantCoverage = testCoverage{
LineRate: 0,
BranchRate: 0,
Packages: []testCoveragePkg{
testCoveragePkg{
Name: "v.io/x/devtools/jiri-test/internal/test/testdata/foo",
LineRate: 0,
BranchRate: 0,
Complexity: 0,
Classes: []testCoverageClass{
testCoverageClass{
Name: "-",
Filename: "v.io/x/devtools/jiri-test/internal/test/testdata/foo/foo.go",
LineRate: 0,
BranchRate: 0,
Complexity: 0,
Methods: []testCoverageMethod{
testCoverageMethod{
Name: "Foo",
LineRate: 0,
BranchRate: 0,
Signature: "",
Lines: []testCoverageLine{
testCoverageLine{Number: 7, Hits: 1},
testCoverageLine{Number: 8, Hits: 1},
testCoverageLine{Number: 9, Hits: 1},
},
},
},
},
},
},
},
}
)
var skipProfiles = jiriGoOpt([]string{"-skip-profiles"})
// TestGoBuild checks the Go build based test logic.
func TestGoBuild(t *testing.T) {
fake, cleanupFake := jiritest.NewFakeJiriRoot(t)
defer cleanupFake()
if err := tooldata.SaveConfig(fake.X, tooldata.NewConfig()); err != nil {
t.Fatal(err)
}
testName := "test-go-build"
cleanupTest, err := initTestImpl(fake.X, false, false, false, testName, nil, "")
if err != nil {
t.Fatalf("%v", err)
}
defer cleanupTest()
// This package will pass.
{
pkgName := "v.io/x/devtools/jiri-test/internal/test/testdata/foo"
result, err := goBuild(fake.X, testName, pkgsOpt([]string{pkgName}), skipProfiles)
if err != nil {
t.Fatalf("%v", err)
}
if got, want := result.Status, test.Passed; got != want {
t.Fatalf("unexpected result: got %s, want %s", got, want)
}
// When test passes, there shouldn't be any xunit report.
xUnitFile := xunit.ReportPath(testName)
if _, err := os.Stat(xUnitFile); err == nil {
t.Fatalf("want no xunit report, but got one %q", xUnitFile)
}
}
// This package will fail.
{
pkgName := "v.io/x/devtools/jiri-test/internal/test/testdata/foo2"
result, err := goBuild(fake.X, testName, pkgsOpt([]string{pkgName}), skipProfiles)
if err != nil {
t.Fatalf("%v", err)
}
if got, want := result.Status, test.Failed; got != want {
t.Fatalf("unexpected result: got %s, want %s", got, want)
}
// Check the xUnit report.
xUnitFile := xunit.ReportPath(testName)
data, err := ioutil.ReadFile(xUnitFile)
if err != nil {
t.Fatalf("ReadFile(%v) failed: %v", xUnitFile, err)
}
defer os.RemoveAll(xUnitFile)
var gotBuild xunit.TestSuites
if err := xml.Unmarshal(data, &gotBuild); err != nil {
t.Fatalf("Unmarshal() failed: %v\n%v", err, string(data))
}
if !suitesMatch(gotBuild, wantBuild) {
t.Fatalf("unexpected result:\ngot\n%#v\nwant\n%#v", gotBuild, wantBuild)
}
}
}
// TestGoCoverage checks the Go test coverage based test logic.
func TestGoCoverage(t *testing.T) {
jirix := newJiriXWithRealRoot(t)
testName, pkgName := "test-go-coverage", "v.io/x/devtools/jiri-test/internal/test/testdata/foo"
cleanupTest, err := initTestImpl(jirix, false, false, false, testName, nil, "")
if err != nil {
t.Fatalf("%v", err)
}
defer cleanupTest()
result, err := goCoverage(jirix, testName, pkgsOpt([]string{pkgName}), skipProfiles)
if err != nil {
t.Fatalf("%v", err)
}
if got, want := result.Status, test.Passed; got != want {
t.Fatalf("unexpected result: got %s, want %s", got, want)
}
// Check the xUnit report.
xUnitFile := xunit.ReportPath(testName)
data, err := ioutil.ReadFile(xUnitFile)
if err != nil {
t.Fatalf("ReadFile(%v) failed: %v", xUnitFile, err)
}
defer os.RemoveAll(xUnitFile)
var gotTest xunit.TestSuites
if err := xml.Unmarshal(data, &gotTest); err != nil {
t.Fatalf("Unmarshal() failed: %v\n%v", err, string(data))
}
if !suitesMatch(gotTest, wantTest) {
t.Fatalf("unexpected result:\ngot\n%v\nwant\n%v", gotTest, wantTest)
}
// Check the cobertura report.
coberturaFile := coberturaReportPath(testName)
data, err = ioutil.ReadFile(coberturaFile)
if err != nil {
t.Fatalf("ReadFile(%v) failed: %v", coberturaFile, err)
}
var gotCoverage testCoverage
if err := xml.Unmarshal(data, &gotCoverage); err != nil {
t.Fatalf("Unmarshal() failed: %v\n%v", err, string(data))
}
if !coverageMatch(gotCoverage, wantCoverage) {
t.Fatalf("unexpected result:\ngot\n%v\nwant\n%v", gotCoverage, wantCoverage)
}
}
// TestGoTest checks the Go test based test logic.
func TestGoTest(t *testing.T) {
runGoTest(t, "", nil, wantTest, test.Passed, "foo")
}
// TestGoTestWithSuffix checks the suffix mode of Go test based test
// logic.
func TestGoTestWithSuffix(t *testing.T) {
runGoTest(t, "[Suffix]", nil, wantTestWithSuffix, test.Passed, "foo")
}
// TestGoTestWithExcludedTests checks the excluded test mode of Go
// test based test logic.
func TestGoTestWithExcludedTests(t *testing.T) {
exclusions := []exclusion{
newExclusion("v.io/x/devtools/jiri-test/internal/test/testdata/foo", "Test1", false),
newExclusion("v.io/x/devtools/jiri-test/internal/test/testdata/foo", "Test2", true),
newExclusion("v.io/x/devtools/jiri-test/internal/test/testdata/foo", "Test3", true),
}
runGoTest(t, "", exclusions, wantTestWithExcludedTests, test.Passed, "foo")
}
func TestGoTestWithExcludedTestsWithWildcards(t *testing.T) {
exclusions := []exclusion{
newExclusion("v.io/x/devtools/jiri-test/internal/test/testdata/foo", "Test[23]$", true),
}
runGoTest(t, "", exclusions, wantTestWithExcludedTests, test.Passed, "foo")
}
func TestGoTestExcludedPackage(t *testing.T) {
exclusions := []exclusion{
newExclusion("v.io/x/devtools/jiri-test/internal/test/testdata/foo", ".*", true),
}
runGoTest(t, "", exclusions, wantExcludedPackage, test.Passed, "foo")
}
func TestGoTestWithTimeout(t *testing.T) {
runGoTest(t, "", nil, wantTestWithTimeout, test.Failed, "foo_timeout", timeoutOpt("1s"))
}
func TestGoTestV23(t *testing.T) {
runGoTest(t, "", nil, wantV23Test, test.Passed, "foo", funcMatcherOpt{&matchV23TestFunc{testNameRE: integrationTestNameRE}}, nonTestArgsOpt([]string{"--v23.tests"}))
}
func TestGoTestV23WithExcludedTests(t *testing.T) {
exclusions := []exclusion{
newExclusion("v.io/x/devtools/jiri-test/internal/test/testdata/foo", "TestV23B", true),
}
runGoTest(t, "", exclusions, wantV23TestWithExcludedTests, test.Passed, "foo", funcMatcherOpt{&matchV23TestFunc{testNameRE: integrationTestNameRE}}, nonTestArgsOpt([]string{"--v23.tests"}))
}
func TestRegressionTest(t *testing.T) {
config := defaultRegressionConfig()
runGoTest(t, "", nil, wantRegressionTest, test.Passed, "foo", funcMatcherOpt{&matchV23TestFunc{testNameRE: regexp.MustCompile(config.Tests)}}, nonTestArgsOpt([]string{"--v23.tests"}))
}
func runGoTest(t *testing.T, suffix string, exclusions []exclusion, expectedTestSuite xunit.TestSuites, expectedStatus test.Status, subPkg string, testOpts ...goTestOpt) {
jirix := newJiriXWithRealRoot(t)
testName, pkgName := "test-go-test", "v.io/x/devtools/jiri-test/internal/test/testdata/"+subPkg
cleanupTest, err := initTestImpl(jirix, false, false, false, testName, nil, "")
if err != nil {
t.Fatalf("%v", err)
}
defer cleanupTest()
opts := []goTestOpt{
pkgsOpt([]string{pkgName}),
suffixOpt(suffix),
exclusionsOpt(exclusions),
suppressTestOutputOpt(true),
skipProfiles,
}
opts = append(opts, testOpts...)
result, err := goTestAndReport(jirix, testName, opts...)
if err != nil {
t.Fatalf("%v", err)
}
if got, want := result.Status, expectedStatus; got != want {
t.Fatalf("unexpected result: got %s, want %s", got, want)
}
// Check the xUnit report.
xUnitFile := xunit.ReportPath(testName)
data, err := ioutil.ReadFile(xUnitFile)
if err != nil {
t.Fatalf("ReadFile(%v) failed: %v", xUnitFile, err)
}
defer os.RemoveAll(xUnitFile)
var gotTest xunit.TestSuites
if err := xml.Unmarshal(data, &gotTest); err != nil {
t.Fatalf("Unmarshal() failed: %v\n%v", err, string(data))
}
if !suitesMatch(gotTest, expectedTestSuite) {
t.Fatalf("unexpected result:\ngot\n%v\nwant\n%v", gotTest, expectedTestSuite)
}
}
func newJiriXWithRealRoot(t *testing.T) *jiri.X {
// Capture JIRI_ROOT using a relative path. We need the real JIRI_ROOT for
// test that build and use tools from third_party.
root, err := filepath.Abs(filepath.Join("..", "..", "..", "..", "..", "..", "..", "..", ".."))
if err != nil {
t.Fatal(err)
}
return &jiri.X{Context: tool.NewDefaultContext(), Root: root}
}