presubmit/test.go

// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"math"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"

	"v.io/jiri"
	"v.io/jiri/collect"
	"v.io/jiri/gerrit"
	"v.io/jiri/gitutil"
	"v.io/jiri/project"
	"v.io/jiri/runutil"
	"v.io/jiri/tool"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/xunit"
	"v.io/x/lib/cmdline"
)

const (
	// gsPrefix identifies the prefix of a Google Storage location where
	// the test results are stored.
	gsPrefix = "gs://vanadium-test-results/v0/"

	// Timeout value for jiri-test command.
	jiriTestTimeout = time.Minute * 55

	// Timeout value for "jiri profile" command.
	jiriProfileTimeout = time.Minute * 5
)

var (
	numWorkersFlag       int
	reviewTargetRefsFlag string
	testFlag             string
	testPartRE           = regexp.MustCompile(`(.*)-part(\d)$`)

	// The variables below are used for testing presubmit only.
	testMode                 = false
	testFilePaths            = ""
	testFileExpectedContents = ""
)

func init() {
	cmdTest.Flags.IntVar(&jenkinsBuildNumberFlag, "build-number", -1, "The number of the Jenkins build.")
	cmdTest.Flags.IntVar(&numWorkersFlag, "num-test-workers", runtime.NumCPU(), "Set the number of test workers to use when running sub-tests.")
	cmdTest.Flags.Lookup("num-test-workers").DefValue = "<runtime.NumCPU()>"
	cmdTest.Flags.StringVar(&projectsFlag, "projects", "", "The base names of the remote projects containing the CLs pointed by the refs, separated by ':'.")
	cmdTest.Flags.StringVar(&reviewTargetRefsFlag, "refs", "", "The review references separated by ':'.")
	cmdTest.Flags.StringVar(&testFlag, "test", "", "The name of a single test to run.")

	tool.InitializeProjectFlags(&cmdTest.Flags)
}

// cmdTest represents the 'test' command of the presubmit tool.
var cmdTest = &cmdline.Command{
	Name:  "test",
	Short: "Run tests for a CL",
	Long: `
This subcommand pulls the open CLs from Gerrit, runs tests specified in a config
file, and posts test results back to the corresponding Gerrit review thread.
`,
	Runner: jiri.RunnerFunc(runTest),
}

const (
	mergeConflictMessageTmpl     = "Possible merge conflict detected in %s.\nPresubmit tests will be executed after a new patchset that resolves the conflicts is submitted."
	toolsBuildFailureMessageTmpl = "Failed to build required tools. This is likely caused by your changes.\n%s"
	nanoToMiliSeconds            = 1000000
	prepareTestBranchAttempts    = 3
)

type cl struct {
	clNumber int
	patchset int
	ref      string
	project  string
}

func (c cl) String() string {
	return fmt.Sprintf("http://go/vcl/%d/%d", c.clNumber, c.patchset)
}

// runTest implements the 'test' subcommand.
func runTest(jirix *jiri.X, args []string) (e error) {
	hostname, err := os.Hostname()
	if err != nil {
		return fmt.Errorf("Hostname() failed: %v", err)
	}
	printf(jirix.Stdout(), "### Running the presubmit binary on %s\n", hostname)

	// Basic sanity checks.
	if err := sanityChecks(jirix); err != nil {
		return err
	}

	// Warn users that presubmit will delete all non-master branches when
	// running on their local machines.
	if !testMode && os.Getenv("USER") != "veyron" {
		fmt.Printf("WARNING: Presubmit will delete all non-master branches.\nContinue? y/N:")
		var response string
		if _, err := fmt.Scanf("%s\n", &response); err != nil || response != "y" {
			return fmt.Errorf("Test aborted by user.")
		}
	}

	// Record the current timestamp so we can get the correct postsubmit build
	// when processing the results.
	curTimestamp := time.Now().UnixNano() / nanoToMiliSeconds

	// Generate cls from the refs and projects flags.
	cls, err := parseCLs()
	if err != nil {
		return err
	}

	projects, tools, err := project.LoadManifest(jirix)
	if err != nil {
		return err
	}

	// tmpBinDir is where developer tools are built after changes are
	// pulled from the target CLs.
	tmpBinDir := filepath.Join(jirix.Root, "tmpBin")

	// Setup cleanup function for cleaning up presubmit test branch.
	cleanupFn := func() error {
		os.RemoveAll(tmpBinDir)
		return cleanupAllPresubmitTestBranches(jirix, projects)
	}
	defer collect.Error(func() error { return cleanupFn() }, &e)

	// Trap SIGTERM and SIGINT signal when the program is aborted
	// on Jenkins.
	go func() {
		sigchan := make(chan os.Signal, 1)
		signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
		<-sigchan
		if err := cleanupFn(); err != nil {
			fmt.Fprintf(os.Stderr, "%v\n", err)
		}
		// Linux convention is to use 128+signal as the exit
		// code. We use 0 here to let Jenkins properly mark a
		// run as "Aborted" instead of "Failed".
		os.Exit(0)
	}()

	// Extract test name without part suffix and part index from the original
	// test name (stored in testFlag).
	testName, partIndex, err := processTestPartSuffix(testFlag)
	if err != nil {
		return err
	}

	// Prepare presubmit test branch.
	for i := 1; i <= prepareTestBranchAttempts; i++ {
		if failedCL, err := preparePresubmitTestBranch(jirix, cls, projects); err != nil {
			if i > 1 {
				fmt.Fprintf(jirix.Stdout(), "Attempt #%d:\n", i)
			}
			if failedCL != nil {
				fmt.Fprintf(jirix.Stderr(), "%s: %v\n", failedCL.String(), err)
			}
			errMsg := err.Error()
			if strings.Contains(errMsg, "unable to access") {
				// Cannot access googlesource.com, try again.
				continue
			}
			if strings.Contains(errMsg, "hung up") {
				// The remote end hung up unexpectedly, try again.
				continue
			}
			if strings.Contains(errMsg, "git pull") {
				// Possible merge conflict.
				message := fmt.Sprintf(mergeConflictMessageTmpl, failedCL.String())
				result := test.Result{
					Status:          test.MergeConflict,
					MergeConflictCL: failedCL.String(),
				}
				if err := recordPresubmitFailure(jirix, "MergeConflict", "Merge conflict detected", message, testName, -1, result); err != nil {
					return err
				}
				return nil
			}
			return err
		}
		break
	}

	// Rebuild developer tools and override PATH to point there.
	env := map[string]string{}
	if !testMode {
		var err error
		env, err = rebuildDeveloperTools(jirix, tools, projects, tmpBinDir)
		if err != nil {
			message := fmt.Sprintf(toolsBuildFailureMessageTmpl, err.Error())
			result := test.Result{
				Status:               test.ToolsBuildFailure,
				ToolsBuildFailureMsg: err.Error(),
			}
			if err := recordPresubmitFailure(jirix, "BuildTools", "Failed to build tools", message, testName, -1, result); err != nil {
				return err
			}
			fmt.Fprintf(jirix.Stderr(), "failed to build tools:\n%s\n", err.Error())
			return nil
		}
	}

	// Run the tests via "jiri test run" and collect the test results.
	printf(jirix.Stdout(), "### Running the presubmit test\n")
	s := jirix.NewSeq()
	outputDir, err := s.TempDir("", "")
	if err != nil {
		return err
	}
	defer collect.Error(func() error { return jirix.NewSeq().RemoveAll(outputDir).Done() }, &e)

	jiriArgs := []string{
		"run",
		"-output-dir", outputDir,
		"-num-test-workers", fmt.Sprintf("%d", numWorkersFlag),
	}
	if testMode {
		jiriArgs = append(jiriArgs,
			"-mock-file-paths", testFilePaths, "-mock-file-contents", testFileExpectedContents)
	}
	if partIndex != -1 {
		jiriArgs = append(jiriArgs, "-part", fmt.Sprintf("%d", partIndex))
	}
	jiriArgs = append(jiriArgs, testName)

	var out bytes.Buffer
	out.Grow(1 << 20)
	stdout := io.MultiWriter(&out, jirix.Stdout())
	stderr := io.MultiWriter(&out, jirix.Stderr())
	if err := s.Env(env).Capture(stdout, stderr).Timeout(jiriTestTimeout).
		Last("jiri-test", jiriArgs...); err != nil {
		// Clean up profiles if any of the CLs modified profile related files.
		profilesModified, pmErr := profileFilesModified(jirix, cls)
		if pmErr != nil {
			fmt.Fprintf(jirix.Stderr(), "%v\n", pmErr)
		}
		if (pmErr == nil && profilesModified) || pmErr != nil {
			if err := cleanupProfiles(jirix, env, cls); err != nil {
				fmt.Fprintf(jirix.Stderr(), "%v\n", err)
			}
		}
		// jiri-test command times out.
		if runutil.IsTimeout(err) {
			result := test.Result{
				Status:       test.TimedOut,
				TimeoutValue: jiriTestTimeout,
			}
			failureMessage := fmt.Sprintf("Test timed out after %v", jiriTestTimeout)
			if err := recordPresubmitFailure(jirix, "Timeout", failureMessage, out.String(), testName, partIndex, result); err != nil {
				return err
			}
		}
		oe := runutil.GetOriginalError(err)
		// Check the error status to differentiate failed test errors.
		exiterr, ok := oe.(*exec.ExitError)
		if !ok {
			return err
		}
		status, ok := exiterr.Sys().(syscall.WaitStatus)
		if !ok {
			return err
		}
		if status.ExitStatus() != test.FailedExitCode {
			return err
		}
	}
	var results map[string]*test.Result
	resultsFile := filepath.Join(outputDir, "results")
	bytes, err := s.ReadFile(resultsFile)
	if err != nil {
		return err
	}

	if err := json.Unmarshal(bytes, &results); err != nil {
		return fmt.Errorf("Unmarshal() failed: %v\n%v", err, string(bytes))
	}
	result, ok := results[testName]
	if !ok {
		return fmt.Errorf("no test result found for %q", testName)
	}

	// Upload the test results to Google Storage.
	path := gsPrefix + fmt.Sprintf("presubmit/%d/%s/%s", jenkinsBuildNumberFlag, os.Getenv("OS"), os.Getenv("ARCH"))
	if err := persistTestData(jirix, outputDir, testName, partIndex, path); err != nil {
		fmt.Fprintf(jirix.Stderr(), "failed to store test results: %v\n", err)
	}

	return writeTestStatusFile(jirix, *result, curTimestamp, testName, partIndex)
}

// profileFilesModified checks any of the given CLs modified files under the
// "jiri-v23-profile/" or "jiri-profile-v23/" directories in the
// "release.go.x.devtools" project.
func profileFilesModified(jirix *jiri.X, cls []cl) (bool, error) {
	gUrl, err := gerritBaseUrl()
	if err != nil {
		return false, err
	}
	for _, curCL := range cls {
		results, err := jirix.Gerrit(gUrl).Query(fmt.Sprintf("change:%d", curCL.clNumber))
		if err != nil {
			return false, err
		}
		for _, result := range results {
			if result.Project != "release.go.x.devtools" {
				continue
			}
			for _, revision := range result.Revisions {
				for filename := range revision.Files {
					if strings.HasPrefix(filename, "jiri-v23-profile/") {
						return true, nil
					}
					if strings.HasPrefix(filename, "jiri-profile-v23/") {
						return true, nil
					}
				}
			}
		}
	}
	return false, nil
}

func cleanupProfiles(jirix *jiri.X, env map[string]string, cls []cl) error {
	fmt.Fprintf(jirix.Stdout(), "### Cleanning up profiles ###")
	return jirix.NewSeq().Env(env).Timeout(jiriProfileTimeout).
		Last("jiri", "profile", "cleanup", "--rm-all")
}

// persistTestData uploads test data to Google Storage.
func persistTestData(jirix *jiri.X, outputDir string, testName string, partIndex int, path string) error {
	// Write out a file that records the host configuration.
	conf := struct {
		Arch string
		OS   string
	}{
		Arch: runtime.GOARCH,
		OS:   runtime.GOOS,
	}
	bytes, err := json.Marshal(conf)
	if err != nil {
		return fmt.Errorf("Marshal(%v) failed: %v", conf, err)
	}
	confFile := filepath.Join(outputDir, "conf")
	s := jirix.NewSeq()
	if err := s.WriteFile(confFile, bytes, os.FileMode(0600)).Done(); err != nil {
		return err
	}
	if partIndex == -1 {
		partIndex = 0
	}
	// Upload test data to Google Storage.
	dstDir := fmt.Sprintf("%s/%s/%d", path, testName, partIndex)
	args := []string{"-q", "-m", "cp", filepath.Join(outputDir, "*"), dstDir}
	if err := s.Last("gsutil", args...); err != nil {
		return err
	}
	xUnitFile := xunit.ReportPath(testName)
	if _, err := s.Stat(xUnitFile); err == nil {
		args := []string{"-q", "cp", xUnitFile, dstDir + "/" + "xunit.xml"}
		if err := s.Last("gsutil", args...); err != nil {
			return err
		}
	} else if !runutil.IsNotExist(err) {
		return err
	}
	return nil
}

// sanityChecks performs basic sanity checks for various flags.
func sanityChecks(jirix *jiri.X) error {
	if projectsFlag == "" {
		return jirix.UsageErrorf("-projects flag is required")
	}
	if reviewTargetRefsFlag == "" {
		return jirix.UsageErrorf("-refs flag is required")
	}
	return nil
}

// parseCLs parses cl info from refs and projects flag, and returns a
// slice of "cl" objects.
func parseCLs() ([]cl, error) {
	refs := strings.Split(reviewTargetRefsFlag, ":")
	projects := strings.Split(projectsFlag, ":")
	if got, want := len(refs), len(projects); got != want {
		return nil, fmt.Errorf("Mismatching lengths of %v and %v: %v vs. %v", refs, projects, len(refs), len(projects))
	}
	cls := []cl{}
	for i, ref := range refs {
		project := projects[i]
		clNumber, patchset, err := gerrit.ParseRefString(ref)
		if err != nil {
			return nil, err
		}
		cls = append(cls, cl{
			clNumber: clNumber,
			patchset: patchset,
			ref:      ref,
			project:  project,
		})
	}
	return cls, nil
}

// presubmitTestBranchName returns the name of the branch where the cl
// content is pulled.
func presubmitTestBranchName(ref string) string {
	return "presubmit_" + ref
}

// preparePresubmitTestBranch creates and checks out the presubmit
// test branch and pulls the CL there.
func preparePresubmitTestBranch(jirix *jiri.X, cls []cl, projects project.Projects) (_ *cl, e error) {
	strCLs := []string{}
	for _, cl := range cls {
		strCLs = append(strCLs, cl.String())
	}
	wd, err := os.Getwd()
	if err != nil {
		return nil, fmt.Errorf("Getwd() failed: %v", err)
	}
	defer collect.Error(func() error { return jirix.NewSeq().Chdir(wd).Done() }, &e)
	if err := cleanupAllPresubmitTestBranches(jirix, projects); err != nil {
		return nil, fmt.Errorf("%v\n", err)
	}
	// Pull changes for each cl.
	printf(jirix.Stdout(), "### Preparing to test %s\n", strings.Join(strCLs, ", "))
	prepareFn := func(curCL cl) error {
		localProject, err := projects.FindUnique(curCL.project)
		if err != nil {
			return fmt.Errorf("error finding project %q: %v", curCL.project, err)
		}
		s := jirix.NewSeq()
		if err := s.Chdir(localProject.Path).Done(); err != nil {
			return fmt.Errorf("Chdir(%v) failed: %v", localProject.Path, err)
		}
		git := gitutil.New(s)
		branchName := presubmitTestBranchName(curCL.ref)
		if err := git.CreateAndCheckoutBranch(branchName); err != nil {
			return err
		}
		if err := git.Pull(localProject.Remote, curCL.ref); err != nil {
			return err
		}
		return nil
	}
	for _, cl := range cls {
		if err := prepareFn(cl); err != nil {
			test.Fail(jirix.Context, "pull changes from %s\n", cl.String())
			return &cl, err
		}
		test.Pass(jirix.Context, "pull changes from %s\n", cl.String())
	}
	return nil, nil
}

// recordPresubmitFailure records failure from presubmit binary itself
// (not from the test it runs) in the test status file and xUnit report.
func recordPresubmitFailure(jirix *jiri.X, testCaseName, failureMessage, failureOutput, testName string, partIndex int, result test.Result) error {
	if err := xunit.CreateFailureReport(jirix, testName, testName, testCaseName, failureMessage, failureOutput); err != nil {
		return nil
	}
	// We use math.MaxInt64 here so that the logic that tries to find the newest
	// build before the given timestamp terminates after the first iteration.
	if err := writeTestStatusFile(jirix, result, math.MaxInt64, testName, partIndex); err != nil {
		return err
	}
	return nil
}

// rebuildDeveloperTools rebuilds developer tools (e.g. jiri, vdl..) in a
// temporary directory, and overrides the PATH to use that directory.
func rebuildDeveloperTools(jirix *jiri.X, tools project.Tools, projects project.Projects, tmpBinDir string) (map[string]string, error) {
	if err := project.BuildTools(jirix, projects, tools, tmpBinDir); err != nil {
		return nil, err
	}
	// Create a new PATH that replaces JIRI_ROOT/devtools/bin and
	// JIRI_ROOT/.jiri_root/bin with the temporary bin directory.
	//
	// TODO(toddw): Remove replacement of devtools/bin when the transition to
	// .jiri_root is done.
	oldBinDir := filepath.Join(jirix.Root, "devtools", "bin")
	path := os.Getenv("PATH")
	path = strings.Replace(path, oldBinDir, tmpBinDir, -1)
	path = strings.Replace(path, jirix.BinDir(), tmpBinDir, -1)
	env := map[string]string{
		"PATH":               path,
		jiri.PreservePathEnv: "true",
	}
	return env, nil
}

// processTestPartSuffix extracts the test name without part suffix as well
// as the part index from the given test name that might have part suffix
// (vanadium-go-race_part0). If the given test name doesn't have part suffix,
// the returned test name will be the same as the given test name, and the
// returned part index will be -1.
func processTestPartSuffix(testName string) (string, int, error) {
	matches := testPartRE.FindStringSubmatch(testName)
	partIndex := -1
	if matches != nil {
		testName = matches[1]
		strPartIndex := matches[2]
		var err error
		if partIndex, err = strconv.Atoi(strPartIndex); err != nil {
			return "", partIndex, err
		}
	}
	return testName, partIndex, nil
}

// cleanupPresubmitTestBranch removes the presubmit test branch.
func cleanupAllPresubmitTestBranches(jirix *jiri.X, projects project.Projects) (e error) {
	printf(jirix.Stdout(), "### Cleaning up\n")
	if err := project.CleanupProjects(jirix, projects, true); err != nil {
		return err
	}
	return nil
}

// writeTestStatusFile writes the given TestResult and timestamp to a JSON file.
// This file will be collected (along with the test report xUnit file) by the
// "master" presubmit project for generating final test results message.
//
// For more details, see comments in result.go.
func writeTestStatusFile(jirix *jiri.X, result test.Result, curTimestamp int64, testName string, partIndex int) error {
	// Get the file path.
	workspace, fileName := os.Getenv("WORKSPACE"), fmt.Sprintf("status_%s.json", strings.Replace(testName, "-", "_", -1))
	statusFilePath := ""
	if workspace == "" {
		statusFilePath = filepath.Join(os.Getenv("HOME"), "tmp", testFlag, fileName)
	} else {
		statusFilePath = filepath.Join(workspace, fileName)
	}

	// Write to file.
	r := testResultInfo{
		Result:    result,
		TestName:  testName,
		Timestamp: curTimestamp,
		AxisValues: axisValuesInfo{
			Arch:      os.Getenv("ARCH"), // Architecture is stored in environment variable "ARCH"
			OS:        os.Getenv("OS"),   // OS is stored in environment variable "OS"
			PartIndex: partIndex,
		},
	}
	bytes, err := json.Marshal(r)
	if err != nil {
		return fmt.Errorf("Marshal(%v) failed: %v", r, err)
	}
	if err := jirix.NewSeq().WriteFile(statusFilePath, bytes, os.FileMode(0644)).Done(); err != nil {
		return fmt.Errorf("WriteFile(%v) failed: %v", statusFilePath, err)
	}
	return nil
}