internal/golib/go.go

// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package golib defines utilities for using the Go toolchain to build
// Vanadium binaries.
package golib

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"os/user"
	"regexp"
	"sort"
	"strings"
	"time"

	"v.io/jiri"
	"v.io/jiri/collect"
	"v.io/jiri/gitutil"
	"v.io/jiri/project"
	"v.io/jiri/runutil"
	"v.io/x/devtools/internal/buildinfo"
	"v.io/x/lib/lookpath"
	"v.io/x/lib/metadata"
	"v.io/x/lib/set"
)

// ExtraLDFlagsFlagDescription describes the --extra-ldflags flag, to be added
// to any tool that allows adding extra ldflags to those automatically
// generated.
const ExtraLDFlagsFlagDescription = `This tool sets some ldflags automatically, e.g. to set binary metadata.  The extra-ldflags are appended to the end of those automatically generated ldflags.  Note that if your go command line specifies -ldflags explicitly, it will override both the automatically generated ldflags as well as the extra-ldflags.`

var goEnvVars = map[string]bool{
	"CC":                   true,
	"CGO_ENABLED":          true,
	"CXX":                  true,
	"GOARCH":               true,
	"GOBIN":                true,
	"GOEXE":                true,
	"GOGCCFLAGS":           true,
	"GOHOSTARCH":           true,
	"GOHOSTOS":             true,
	"GOOS":                 true,
	"GOPATH":               true,
	"GORACE":               true,
	"GOROOT":               true,
	"GOTOOLDIR":            true,
	"GO15VENDOREXPERIMENT": true,
}

// PrepareGo runs recommended checks on the environment and related commands
// before execution of the Go toolchain. The Go toolchain should use the
// returned args. PrepareGo for the 'env' strips any enviornment variables
// that the go command doesn't understand.
//
// For example, it ensures that all Go files generated by the VDL compiler are
// up-to-date. It also generates flags so that build information can be embedded
// in resulting binaries.
func PrepareGo(jirix *jiri.X, env map[string]string, args []string, extraLDFlags, installSuffix string) ([]string, error) {
	switch args[0] {
	case "env":
		rargs := []string{"env"}
		for _, v := range args[1:] {
			if !goEnvVars[v] {
				fmt.Fprintf(jirix.Stdout(), "%s\n", env[v])
			} else {
				rargs = append(rargs, v)
			}
		}
		return rargs, nil
	case "build", "install":
		// Provide default ldflags to populate build info metadata in the
		// binary. Any manual specification of ldflags already in the args
		// will override this.
		var err error
		if args, err = setBuildInfoFlags(jirix, args, env, extraLDFlags, installSuffix); err != nil {
			return nil, err
		}
		fallthrough
	case "generate", "run", "test":
		// Check that all non-master branches have been merged with the
		// master branch to make sure the vdl tool is not run against
		// out-of-date code base.
		if err := reportOutdatedBranches(jirix); err != nil {
			return nil, err
		}

		// Generate vdl files, if necessary.
		if err := generateVDL(jirix, env, args[0], args[1:]); err != nil {
			return nil, err
		}
	}
	return args, nil
}

// getPlatform identifies the target platform by querying the go tool
// for the values of the GOARCH and GOOS environment variables.
func getPlatform(jirix *jiri.X, env map[string]string) (string, error) {
	goBin, err := lookpath.Look(env, "go")
	if err != nil {
		return "", err
	}
	s := jirix.NewSeq()
	var out bytes.Buffer
	if err = s.Env(env).Capture(&out, nil).Last(goBin, "env", "GOARCH"); err != nil {
		return "", err
	}
	arch := strings.TrimSpace(out.String())
	out.Reset()
	if err = s.Env(env).Capture(&out, nil).Last(goBin, "env", "GOOS"); err != nil {
		return "", err
	}
	os := strings.TrimSpace(out.String())
	return fmt.Sprintf("%s-%s", arch, os), nil
}

// setBuildInfoFlags augments the list of arguments with flags for the
// go compiler that encoded the build information expected by the
// v.io/x/lib/metadata package.
func setBuildInfoFlags(jirix *jiri.X, args []string, env map[string]string, extraLDFlags, installSuffix string) ([]string, error) {
	info := buildinfo.T{Time: time.Now()}
	// Compute the "platform" value.
	platform, err := getPlatform(jirix, env)
	if err != nil {
		return nil, err
	}
	info.Platform = platform
	// Compute the "manifest" value.
	latestManifest := jirix.UpdateHistoryLatestLink()
	manifest, err := project.ManifestFromFile(jirix, latestManifest)
	if err != nil {
		if !runutil.IsNotExist(err) {
			return nil, err
		}
		fmt.Fprintf(jirix.Stderr(), `WARNING: Could not find %s.
The contents of this file are stored as metadata in binaries the jiri
tool builds. To fix this problem, please run "jiri update".
`, latestManifest)
		manifest = &project.Manifest{}
	}

	info.Manifest = *manifest
	// Compute the "pristine" value.
	states, err := project.GetProjectStates(jirix, true)
	if err != nil {
		return nil, err
	}
	info.Pristine = true
	for _, state := range states {
		if state.CurrentBranch != "master" || state.HasUncommitted || state.HasUntracked {
			info.Pristine = false
			break
		}
	}
	// Compute the "user" value.
	if currUser, err := user.Current(); err == nil {
		info.User = currUser.Name
	}
	// Encode buildinfo as metadata and extract the appropriate ldflags.
	md, err := info.ToMetaData()
	if err != nil {
		return nil, err
	}
	ldflags := "-ldflags=" + metadata.LDFlag(md)
	if extraLDFlags != "" {
		ldflags += " " + extraLDFlags
	}
	args = append([]string{args[0], ldflags}, args[1:]...)
	if installSuffix != "" {
		args = append([]string{args[0], "-installsuffix=" + installSuffix}, args[1:]...)
	}
	return args, nil
}

// generateVDL generates VDL for the transitive Go package dependencies.
//
// Note that the vdl tool takes VDL packages as input, but we're supplying Go
// packages.  We're assuming the package paths for the VDL packages we want to
// generate have the same path names as the Go package paths.  Some of the Go
// package paths may not correspond to a valid VDL package, so we silently
// ignore these paths.
//
// It's fine if the VDL packages have dependencies not reflected in the Go
// packages; the vdl tool will compute the transitive closure of VDL package
// dependencies, as usual.
//
// TODO(toddw): Change the vdl tool to return vdl packages given the full Go
// dependencies, after vdl config files are implemented.
func generateVDL(jirix *jiri.X, env map[string]string, cmd string, args []string) error {
	// Compute which VDL-based Go packages might need to be regenerated.
	goPkgs, goFiles, goTags := processGoCmdAndArgs(cmd, args)
	goDeps, err := computeGoDeps(jirix, env, append(goPkgs, goFiles...), goTags, cmd == "test")
	if err != nil {
		return err
	}

	// Regenerate the VDL-based Go packages.
	// -ignore_unknown:  Silently ignore unknown package paths.
	vdlArgs := []string{"-ignore_unknown", "generate", "-lang=go"}
	vdlArgs = append(vdlArgs, goDeps...)
	vdlBin, err := lookpath.Look(env, "vdl")
	if err != nil {
		return err
	}
	var out bytes.Buffer
	if err := jirix.NewSeq().Env(env).Capture(&out, &out).Last(vdlBin, vdlArgs...); err != nil {
		return fmt.Errorf("failed to generate vdl: %v\n%s", err, out.String())
	}
	return nil
}

// reportOutdatedProjects checks if the currently checked out branches
// are up-to-date with respect to the local master branch. For each
// branch that is not, a notification is printed.
func reportOutdatedBranches(jirix *jiri.X) (e error) {
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}
	defer collect.Error(func() error { return jirix.NewSeq().Chdir(cwd).Done() }, &e)
	projects, err := project.LocalProjects(jirix, false)
	if err != nil {
		return err
	}
	s := jirix.NewSeq()
	for _, project := range projects {
		if err := s.Chdir(project.Path).Done(); err != nil {
			return err
		}
		switch project.Protocol {
		case "git":
			branches, _, err := gitutil.New(jirix.NewSeq()).GetBranches("--merged")
			if err != nil {
				return err
			}
			found := false
			for _, branch := range branches {
				if branch == "master" {
					found = true
					break
				}
			}
			merging, err := gitutil.New(jirix.NewSeq()).MergeInProgress()
			if err != nil {
				return err
			}
			if !found && !merging {
				fmt.Fprintf(jirix.Stderr(), "NOTE: project=%q path=%q\n", project.Name, project.Path)
				fmt.Fprintf(jirix.Stderr(), "This project is on a non-master branch that is out of date.\n")
				fmt.Fprintf(jirix.Stderr(), "Please update this branch using %q.\n", "git merge master")
				fmt.Fprintf(jirix.Stderr(), "Until then the %q tool might not function properly.\n", "jiri")
			}
		}
	}
	return nil
}

// processGoCmdAndArgs is given the cmd and args for the go tool, filters out
// flags, and returns the PACKAGES or GOFILES that were specified in args, as
// well as "foo" if -tags=foo was specified in the args.  Note that all commands
// that accept PACKAGES also accept GOFILES.
//
//   go build    [build flags]              [-o out]      [PACKAGES]
//   go generate                            [-run regexp] [PACKAGES]
//   go install  [build flags]                            [PACKAGES]
//   go run      [build flags]              [-exec prog]  [GOFILES]  [run args]
//   go test     [build flags] [test flags] [-exec prog]  [PACKAGES] [testbin flags]
//
// Sadly there's no way to do this syntactically.  It's easy for single token
// -flag and -flag=x, but non-boolean flags may be two tokens "-flag x".
//
// We keep track of all non-boolean flags F, and skip every token that starts
// with - or --, and also skip the next token if the flag is in F and isn't of
// the form -flag=x.  If we forget to update F, we'll still handle the -flag and
// -flag=x cases correctly, but we'll get "-flag x" wrong.
func processGoCmdAndArgs(cmd string, args []string) ([]string, []string, string) {
	var goTags string
	var nonBool map[string]bool
	switch cmd {
	case "build":
		nonBool = nonBoolGoBuild
	case "generate":
		nonBool = nonBoolGoGenerate
	case "install":
		nonBool = nonBoolGoInstall
	case "run":
		nonBool = nonBoolGoRun
	case "test":
		nonBool = nonBoolGoTest
	}

	// Move start to the start of PACKAGES or GOFILES, by skipping flags.
	start := 0
	for start < len(args) {
		// Handle special-case terminator --
		if args[start] == "--" {
			start++
			break
		}
		match := goFlagRE.FindStringSubmatch(args[start])
		if match == nil {
			break
		}
		// Skip this flag, and maybe skip the next token for the "-flag x" case.
		//   match[1] is the flag name
		//   match[2] is the optional "=" for the -flag=x case
		start++
		if nonBool[match[1]] && match[2] == "" {
			start++
		}
		// Grab the value of -tags, if it is specified.
		if match[1] == "tags" {
			if match[2] == "=" {
				goTags = match[3]
			} else {
				goTags = args[start-1]
			}
		}
	}

	// Move end to the end of PACKAGES or GOFILES.
	var end int
	switch cmd {
	case "test":
		// Any arg starting with - is a testbin flag.
		// https://golang.org/cmd/go/#hdr-Test_packages
		for end = start; end < len(args); end++ {
			if strings.HasPrefix(args[end], "-") {
				break
			}
		}
	case "run":
		// Go run takes gofiles, which are defined as a file ending in ".go".
		// https://golang.org/cmd/go/#hdr-Compile_and_run_Go_program
		for end = start; end < len(args); end++ {
			if !strings.HasSuffix(args[end], ".go") {
				break
			}
		}
	default:
		end = len(args)
	}

	// Decide whether these are packages or files.
	switch {
	case start == end:
		return nil, nil, goTags
	case (start < len(args) && strings.HasSuffix(args[start], ".go")):
		return nil, args[start:end], goTags
	default:
		return args[start:end], nil, goTags
	}
}

var (
	goFlagRE     = regexp.MustCompile(`^--?([^=]+)(=?)(.*)`)
	nonBoolBuild = []string{
		"p", "asmflags", "buildmode", "ccflags", "compiler", "gccgoflags", "gcflags", "installsuffix", "ldflags", "pkgdir", "tags", "toolexec",
	}
	nonBoolTest = []string{
		"bench", "benchtime", "blockprofile", "blockprofilerate", "count", "covermode", "coverpkg", "coverprofile", "cpu", "cpuprofile", "memprofile", "memprofilerate", "outputdir", "parallel", "run", "timeout", "trace",
	}
	nonBoolGoBuild    = set.StringBool.FromSlice(append(nonBoolBuild, "o"))
	nonBoolGoGenerate = set.StringBool.FromSlice([]string{"run"})
	nonBoolGoInstall  = set.StringBool.FromSlice(nonBoolBuild)
	nonBoolGoRun      = set.StringBool.FromSlice(append(nonBoolBuild, "exec"))
	nonBoolGoTest     = set.StringBool.FromSlice(append(append(nonBoolBuild, nonBoolTest...), "exec", "o"))
)

// computeGoDeps computes the transitive Go package dependencies for the given
// set of pkgs.  The strategy is to run "go list <pkgs>" with a special format
// string that dumps the specified pkgs and all deps as space / newline
// separated tokens.  The pkgs may be in any format recognized by "go list"; dir
// paths, import paths, or go files.
func computeGoDeps(jirix *jiri.X, env map[string]string, pkgs []string, tags string, test bool) ([]string, error) {
	if len(pkgs) == 0 {
		pkgs = []string{"."}
	}
	goBin, err := lookpath.Look(env, "go")
	if err != nil {
		return nil, err
	}
	if test {
		// In order to compute the test dependencies, we need to first grab the
		// direct test imports, and use the resulting set of packages to capture the
		// transitive dependencies.  We can't do this with a single run of "go
		// list", since unlike Dep, TestImports and XTestImports don't include
		// transitive dependencies.
		testDeps, err := runGoList(jirix, goBin, env, pkgs, tags, `{{join .TestImports " "}} {{join .XTestImports " "}}`)
		if err != nil {
			return nil, err
		}
		pkgs = append(pkgs, testDeps...)
	}
	return runGoList(jirix, goBin, env, pkgs, tags, `{{.ImportPath}} {{join .Deps " "}}`)
}

func runGoList(jirix *jiri.X, goBin string, env map[string]string, pkgs []string, tags, format string) ([]string, error) {
	goListArgs := []string{`list`, `-f`, format}
	if tags != "" {
		goListArgs = append(goListArgs, "-tags="+tags)
	}
	goListArgs = append(goListArgs, pkgs...)
	var stdout, stderr bytes.Buffer

	// TODO(jsimsa): Avoid buffering all of the output in memory
	// either by extending the runutil API to support piping of
	// output, or by writing the output to a temporary file
	// instead of an in-memory buffer.
	// TODO(cnicolaou): the sequence code in runutil streams using a pipe
	// internally so that could probably be taken advantage of here by having
	// stdout be a pipe that the scanner reads below.
	if err := jirix.NewSeq().Env(env).Capture(&stdout, &stderr).Last(goBin, goListArgs...); err != nil {
		return nil, fmt.Errorf("failed to compute go deps: %v\n%s\n%v", err, stderr.String(), pkgs)
	}
	scanner := bufio.NewScanner(&stdout)
	scanner.Split(bufio.ScanWords)
	depsMap := make(map[string]bool)
	for scanner.Scan() {
		// Ignore bad packages:
		//   command-line-arguments is the dummy import path for "go run".
		if dep := scanner.Text(); dep != "command-line-arguments" {
			depsMap[dep] = true
		}
	}
	if err := scanner.Err(); err != nil {
		return nil, fmt.Errorf("Scan() failed: %v", err)
	}
	deps := set.StringBool.ToSlice(depsMap)
	sort.Strings(deps)
	return deps, nil
}