blob: 298d463f7f3b1a900e2e1ca9043dafc3f5a1349b [file] [log] [blame]
// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package build implements utilities to collect VDL build information and run
// the parser and compiler.
//
// VDL Packages
//
// VDL is organized into packages, where a package is a collection of one or
// more source files. The files in a package collectively define the types,
// constants, services and errors belonging to the package; these are called
// package elements.
//
// The package elements in package P may be used in another package Q. First
// package Q must import package P, and then refer to the package elements in P.
// Imports define the package dependency graph, which must be acyclic.
//
// Build Strategy
//
// The steps to building a VDL package P:
// 1) Compute the transitive closure of P's dependencies DEPS.
// 2) Sort DEPS in dependency order.
// 3) Build each package D in DEPS.
// 3) Build package P.
//
// Building a package P requires that all elements used by P are understood,
// including elements defined outside of P. The only way for a change to
// package Q to affect the build of P is if Q is in the transitive closure of
// P's package dependencies. However there may be false positives; the change
// to Q might not actually affect P.
//
// The build process may perform more work than is strictly necessary, because
// of these false positives. However it is simple and correct.
//
// The TransitivePackages* functions implement build steps 1 and 2.
//
// The Build* functions implement build steps 3 and 4.
//
// Other functions provide related build information and utilities.
package build
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"reflect"
"regexp"
"sort"
"strings"
"v.io/v23/vdl"
"v.io/v23/vdlroot/vdltool"
"v.io/x/lib/toposort"
"v.io/x/ref/lib/vdl/build/internal/builtin_vdlroot"
"v.io/x/ref/lib/vdl/compile"
"v.io/x/ref/lib/vdl/parse"
"v.io/x/ref/lib/vdl/vdlutil"
)
const vdlrootImportPrefix = "v.io/v23/vdlroot"
// Package represents the build information for a vdl package.
type Package struct {
// Name is the name of the package, specified in the vdl files.
// E.g. "bar"
Name string
// IsRoot is true iff this package is a vdlroot standard package.
IsRoot bool
// IsBuiltIn is true iff the data for this package is built-in to the binary,
// rather than read off files from the filesystem.
IsBuiltIn bool
// Path is the package path; the path used in VDL import clauses.
// E.g. "foo/bar".
Path string
// GenPath is the package path to use for code generation. It is the same as
// Path, except for vdlroot standard packages.
// E.g. "v.io/v23/vdlroot/time"
GenPath string
// Dir is the absolute directory containing the package files.
// E.g. "/home/user/vanadium/vdl/src/foo/bar"
Dir string
// BaseFileNames is the list of sorted base vdl file names for this package.
// Join these with Dir to get absolute file names.
BaseFileNames []string
// Config is the configuration for this package, specified by an optional
// "vdl.config" file in the package directory. If no "vdl.config" file
// exists, the zero value of Config is used.
Config vdltool.Config
// OpenFilesFunc is a function that opens the files with the given fileNames,
// and returns a map from base file name to file contents.
OpenFilesFunc func(fileNames []string) (map[string]io.ReadCloser, error)
// filesToClose holds files that need to be closed.
filesToClose []io.Closer
}
// UnknownPathMode specifies the behavior when an unknown path is encountered.
type UnknownPathMode int
const (
UnknownPathIsIgnored UnknownPathMode = iota // Silently ignore unknown paths
UnknownPathIsError // Produce error for unknown paths
)
func (m UnknownPathMode) String() string {
switch m {
case UnknownPathIsIgnored:
return "UnknownPathIsIgnored"
case UnknownPathIsError:
return "UnknownPathIsError"
default:
return fmt.Sprintf("UnknownPathMode(%d)", m)
}
}
func (m UnknownPathMode) logOrErrorf(errs *vdlutil.Errors, format string, v ...interface{}) {
if m == UnknownPathIsIgnored {
vdlutil.Vlog.Printf(format, v...)
} else {
errs.Errorf(format, v...)
}
}
func pathPrefixDotOrDotDot(path string) bool {
// The point of this helper is to catch cases where the path starts with a
// . or .. element; note that ... returns false.
spath := filepath.ToSlash(path)
return path == "." || path == ".." || strings.HasPrefix(spath, "./") || strings.HasPrefix(spath, "../")
}
func ignorePathElem(elem string) bool {
return (strings.HasPrefix(elem, ".") && !pathPrefixDotOrDotDot(elem)) ||
strings.HasPrefix(elem, "_")
}
// validPackagePath returns true iff the path is valid; i.e. if none of the path
// elems is ignored.
func validPackagePath(path string) bool {
for _, elem := range strings.Split(path, "/") {
if ignorePathElem(elem) {
return false
}
}
return true
}
// validFile returns true iff the base file name is a valid vdl file.
func validFile(base string) bool {
return !ignorePathElem(base) && filepath.Ext(base) == ".vdl"
}
// New packages always start with an empty Name, which is filled in with the
// result of parse.InferPackageName.
func newPackage(path, genPath, dir string, mode UnknownPathMode, opts Opts, vdlenv *compile.Env) *Package {
pkg := &Package{
IsRoot: path != genPath,
Path: path,
GenPath: genPath,
Dir: dir,
OpenFilesFunc: openFiles,
}
if err := pkg.initBaseFileNames(); err != nil {
mode.logOrErrorf(vdlenv.Errors, "%s: bad package dir (%v)", pkg.Dir, err)
return nil
}
// TODO(toddw): Add a mechanism in vdlutil.Errors to distinguish categories of
// errors, so that it's more obvious when errors are coming from vdl.config
// files vs *.vdl files.
origErrors := vdlenv.Errors.NumErrors()
if pkg.findAndInitVDLConfig(opts, vdlenv); origErrors != vdlenv.Errors.NumErrors() {
return nil
}
return pkg
}
// initBaseFileNames initializes p.BaseFileNames from the contents of p.Dir.
func (p *Package) initBaseFileNames() error {
infos, err := ioutil.ReadDir(p.Dir)
if err != nil {
return err
}
for _, info := range infos {
if info.IsDir() {
continue
}
base, full := info.Name(), filepath.Join(p.Dir, info.Name())
if !validFile(base) {
vdlutil.Vlog.Printf("%s: ignoring file", full)
continue
}
vdlutil.Vlog.Printf("%s: adding vdl file", full)
p.BaseFileNames = append(p.BaseFileNames, base)
}
if len(p.BaseFileNames) == 0 {
return fmt.Errorf("no vdl files")
}
return nil
}
// findAndInitVDLConfig initializes p.Config based on the optional vdl.config file.
func (p *Package) findAndInitVDLConfig(opts Opts, vdlenv *compile.Env) {
name := path.Join(p.Path, opts.vdlConfigName())
data, err := os.Open(filepath.Join(p.Dir, opts.vdlConfigName()))
switch {
case os.IsNotExist(err):
return
case err != nil:
vdlenv.Errors.Errorf("%s: couldn't open (%v)", name, err)
return
}
p.initVDLConfig(name, data, vdlenv)
if err := data.Close(); err != nil {
vdlenv.Errors.Errorf("%s: couldn't close (%v)", name, err)
}
}
func (p *Package) initVDLConfig(name string, r io.Reader, vdlenv *compile.Env) {
// Build the vdl.config file with an implicit "vdltool" import. Note that the
// actual "vdltool" package should have already been populated into vdlenv.
if vdlenv.ResolvePackage("vdltool") == nil {
vdlenv.Errors.Errorf(`%s: package %q cannot specify a config file; it is a transitive dependency of "vdltool"`, name, p.Name)
return
}
BuildConfigValue(name, r, []string{"vdltool"}, vdlenv, &p.Config)
}
// OpenFiles opens all files in the package and returns a map from base file
// name to file contents. CloseFiles must be called to close the files.
func (p *Package) OpenFiles() (map[string]io.Reader, error) {
var fileNames []string
for _, baseName := range p.BaseFileNames {
fileNames = append(fileNames, filepath.Join(p.Dir, baseName))
}
files, err := p.OpenFilesFunc(fileNames)
if err != nil {
for _, c := range files {
c.Close()
}
return nil, err
}
// Convert map elem type from io.ReadCloser to io.Reader.
res := make(map[string]io.Reader, len(files))
for n, f := range files {
res[n] = f
p.filesToClose = append(p.filesToClose, f)
}
return res, nil
}
func openFiles(fileNames []string) (map[string]io.ReadCloser, error) {
files := make(map[string]io.ReadCloser, len(fileNames))
for _, fileName := range fileNames {
file, err := os.Open(fileName)
if err != nil {
for _, c := range files {
c.Close()
}
return nil, err
}
files[path.Base(fileName)] = file
}
return files, nil
}
func openBuiltInFiles(fileNames []string) (map[string]io.ReadCloser, error) {
files := make(map[string]io.ReadCloser, len(fileNames))
for _, fileName := range fileNames {
file, err := builtin_vdlroot.Asset(fileName)
if err != nil {
return nil, fmt.Errorf("%s: can't load builtin file: %v", fileName, err)
}
files[path.Base(fileName)] = ioutil.NopCloser(bytes.NewReader(file))
}
return files, nil
}
// CloseFiles closes all files returned by OpenFiles. Returns nil if all files
// were closed successfully, otherwise returns one of the errors, dropping the
// others. Regardless of whether an error is returned, Close will be called on
// all files.
func (p *Package) CloseFiles() error {
var err error
for _, c := range p.filesToClose {
if err2 := c.Close(); err == nil {
err = err2
}
}
p.filesToClose = nil
return err
}
// SrcDirs returns a list of package root source directories, based on the
// VDLPATH and VDLROOT environment variables.
//
// VDLPATH is a list of directories separated by filepath.ListSeparator;
// e.g. the separator is ":" on UNIX, and ";" on Windows. The path below each
// VDLPATH directory determines the vdl import path.
//
// See RootDir for details on VDLROOT.
func SrcDirs(errs *vdlutil.Errors) []string {
var srcDirs []string
if root := RootDir(errs); root != "" {
srcDirs = append(srcDirs, root)
}
return append(srcDirs, vdlPathSrcDirs(errs)...)
}
// RootDir returns the VDL root directory, based on the VDLROOT environment
// variable.
//
// VDLROOT is a single directory specifying the location of the standard vdl
// packages. It has the same requirements as VDLPATH components. The returned
// string may be empty, if VDLROOT is empty or unset.
func RootDir(errs *vdlutil.Errors) string {
vdlroot := os.Getenv("VDLROOT")
if vdlroot == "" {
return ""
}
abs, err := filepath.Abs(vdlroot)
if err != nil {
errs.Errorf("VDLROOT %q can't be made absolute: %v", vdlroot, err)
return ""
}
if _, err := os.Stat(abs); err != nil {
errs.Errorf("VDLROOT %q doesn't exist: %v", vdlroot, err)
return ""
}
return abs
}
func vdlPathSrcDirs(errs *vdlutil.Errors) []string {
var srcDirs []string
for _, dir := range filepath.SplitList(os.Getenv("VDLPATH")) {
if dir != "" {
abs, err := filepath.Abs(dir)
if err != nil {
errs.Errorf("VDLPATH src dir %q can't be made absolute (%v)", dir, err)
continue // keep going to collect all errors
}
srcDirs = append(srcDirs, abs)
}
}
if len(srcDirs) == 0 {
errs.Error("No src dirs; set VDLPATH to a valid value")
return nil
}
return srcDirs
}
// IsDirPath returns true iff the path is absolute, or begins with a . or
// .. element. The path denotes the package in that directory.
func IsDirPath(path string) bool {
return filepath.IsAbs(path) || pathPrefixDotOrDotDot(path)
}
// IsImportPath returns true iff !IsDirPath. The path P denotes the package in
// directory DIR/src/P, for some DIR listed in SrcDirs.
func IsImportPath(path string) bool {
return !IsDirPath(path)
}
// depSorter does the main work of collecting and sorting packages and their
// dependencies. The full syntax from the go cmdline tool is supported; we
// allow both dirs and import paths, as well as the "all" and "..." wildcards.
//
// This is slightly complicated because of dirs, and the potential for symlinks.
// E.g. let's say we have two directories, one a symlink to the other:
// /home/user/go/src/foo/bar/base
// /home/user/go/src/foo/bar2 symlink to bar
//
// The problem is that if the user has cwd pointing at one of the two "base"
// dirs and specifies a relative directory ".." it's ambiguous which absolute
// dir we'll end up with; file paths form a graph rather than a tree. For more
// details see http://plan9.bell-labs.com/sys/doc/lexnames.html
//
// This means that if the user builds a dir (rather than an import path), we
// might not be able to deduce the package path. Note that the error definition
// mechanism relies on the package path to create implicit error ids, and this
// must be known at the time the package is compiled. To handle this we call
// deducePackagePath and attempt to deduce the package path even if the user
// builds a directory, and return errors if this fails.
//
// TODO(toddw): If we care about performance we could serialize the compiled
// compile.Package information and write it out as compiler-generated artifacts,
// similar to how the regular go tool generates *.a files under the top-level
// pkg directory.
type depSorter struct {
opts Opts
rootDir string
srcDirs []string
builtInRoot map[string]*Package
pathMap map[string]*Package
dirMap map[string]*Package
sorter *toposort.Sorter
errs *vdlutil.Errors
vdlenv *compile.Env
}
func newDepSorter(opts Opts, errs *vdlutil.Errors) *depSorter {
ds := &depSorter{
opts: opts,
rootDir: RootDir(errs),
srcDirs: SrcDirs(errs),
errs: errs,
vdlenv: compile.NewEnvWithErrors(errs),
}
ds.reset()
// If VDLROOT isn't set, we must initialize the builtInRoot, to allow
// "vdltool" and its dependencies to be imported. We can't parse any config
// files for each package yet; they are initialized later.
var configFiles []string
if ds.rootDir == "" {
configFiles = ds.initBuiltInRootPackages(opts)
}
// Resolve the "vdltool" import and build all transitive packages into vdlenv.
// This special env is used when building "vdl.config" files, which have the
// implicit "vdltool" import.
if ds.resolveImportPath("vdltool", UnknownPathIsError, "<bootstrap>") == nil {
errs.Errorf(`can't resolve "vdltool" package`)
}
for _, pkg := range ds.Sort() {
BuildPackage(pkg, ds.vdlenv)
}
// We can now initialize config files for the builtin root packages.
if len(configFiles) > 0 {
ds.initBuiltInRootConfigs(configFiles)
}
// We must reset back to an empty depSorter, to ensure the transitive packages
// returned by the depSorter don't include "vdltool".
ds.reset()
return ds
}
func (ds *depSorter) reset() {
ds.pathMap = make(map[string]*Package)
ds.dirMap = make(map[string]*Package)
ds.sorter = new(toposort.Sorter)
}
func (ds *depSorter) errorf(format string, v ...interface{}) {
ds.errs.Errorf(format, v...)
}
// initBuiltInRootPackages initializes the built-in root packages, representing
// all standard VDLROOT packages, which are built-in to the binary. The package
// name and config aren't initialized yet. The name is set in addPackageAndDeps
// when the package is actually used. The config is initialized in
// initBuiltInRootConfigs, which can only happen after ds.vdlenv is populated.
// Returns a list of the vdl.config files for later processing.
func (ds *depSorter) initBuiltInRootPackages(opts Opts) []string {
root := make(map[string]*Package)
var configFiles []string
var lastDir string
// Loop through built-in vdl files to create the root package map.
fileNames := builtin_vdlroot.AssetNames()
sort.Strings(fileNames)
for _, fileName := range fileNames {
dir, base := path.Split(fileName)
dir = path.Clean(dir)
if base == opts.vdlConfigName() {
configFiles = append(configFiles, fileName)
continue
}
if !validFile(base) {
ds.errorf("%s: invalid built-in vdl file", fileName)
continue
}
vdlutil.Vlog.Printf("%s: adding built-in vdl file", fileName)
if dir != lastDir {
// Create a new built-in package and add it to the root map.
root[dir] = &Package{
IsRoot: true,
IsBuiltIn: true,
Path: dir,
GenPath: path.Join(vdlrootImportPrefix, dir),
Dir: dir,
OpenFilesFunc: openBuiltInFiles,
}
lastDir = dir
}
root[dir].BaseFileNames = append(root[dir].BaseFileNames, base)
}
vdlutil.Vlog.Printf("added builtin root %v", root)
ds.builtInRoot = root
return configFiles
}
// initBuiltInRootConfigs initializes the vdl.configs of built-in root packages.
func (ds *depSorter) initBuiltInRootConfigs(configFiles []string) {
for _, configFile := range configFiles {
configData, err := builtin_vdlroot.Asset(configFile)
if err != nil {
ds.errorf("%s: can't load builtin config file: %v", configFile, err)
continue
}
pkgPath, _ := path.Split(configFile)
pkgPath = path.Clean(pkgPath)
p := ds.builtInRoot[pkgPath]
if p == nil {
ds.errorf("%s: package %q doesn't exist in builtin root", configFile, pkgPath)
continue
}
p.initVDLConfig(configFile, bytes.NewReader(configData), ds.vdlenv)
}
}
// ResolvePath resolves path into package(s) and adds them to the sorter.
// Returns true iff path could be resolved.
func (ds *depSorter) ResolvePath(path string, mode UnknownPathMode) bool {
if path == "all" {
// Special-case "all", with the same behavior as Go.
path = "..."
}
isDirPath := IsDirPath(path)
dots := strings.Index(path, "...")
switch {
case dots >= 0:
return ds.resolveWildcardPath(isDirPath, path[:dots], path[dots:])
case isDirPath:
return ds.resolveDirPath(path, mode) != nil
default:
return ds.resolveImportPath(path, mode, "<root>") != nil
}
}
// resolveWildcardPath resolves wildcards for both dir and import paths. The
// prefix is everything before the first "...", and the suffix is everything
// including and after the first "..."; note that multiple "..." wildcards may
// occur within the suffix. Returns true iff any packages were resolved.
//
// The strategy is to compute one or more directories that contain everything
// that could possibly be matched, along with a filename pattern to match
// against. Then we walk through each directory, matching against the pattern.
func (ds *depSorter) resolveWildcardPath(isDirPath bool, prefix, suffix string) bool {
resolvedAny := false
type dirAndSrc struct {
dir, src string
}
var walkDirs []dirAndSrc // directories to walk through
var pattern string // pattern to match against, starting after root dir
if isDirPath {
// prefix and suffix are directory paths.
dir, pre := filepath.Split(prefix)
pattern = filepath.Clean(pre + suffix)
walkDirs = append(walkDirs, dirAndSrc{filepath.Clean(dir), ""})
} else {
// prefix and suffix are slash-separated import paths.
slashDir, pre := path.Split(prefix)
pattern = filepath.Clean(pre + filepath.FromSlash(suffix))
dir := filepath.FromSlash(slashDir)
for _, srcDir := range ds.srcDirs {
walkDirs = append(walkDirs, dirAndSrc{filepath.Join(srcDir, dir), srcDir})
}
// Look in our built-in vdlroot for matches against standard packages.
matcher, err := createMatcher(prefix + suffix)
if err != nil {
ds.errorf("%v", err)
return false
}
for pkgPath, pkg := range ds.builtInRoot {
if matcher.MatchString(pkgPath) {
ds.addPackageAndDeps(pkg)
resolvedAny = true
}
}
}
matcher, err := createMatcher(pattern)
if err != nil {
ds.errorf("%v", err)
return false
}
// Walk through root dirs and subdirs, looking for matches.
for _, walk := range walkDirs {
filepath.Walk(walk.dir, func(dirPath string, info os.FileInfo, err error) error {
// Ignore errors and non-directory elements.
if err != nil || !info.IsDir() {
return nil
}
// Skip the dir and subdirs if the elem should be ignored.
_, elem := filepath.Split(dirPath)
if ignorePathElem(elem) {
vdlutil.Vlog.Printf("%s: ignoring dir", dirPath)
return filepath.SkipDir
}
// Special-case to skip packages with the vdlroot import prefix. These
// packages should only appear at the root of the package path space.
if walk.src != "" {
pkgPath := strings.TrimPrefix(dirPath, walk.src)
pkgPath = strings.TrimPrefix(pkgPath, "/")
if strings.HasPrefix(pkgPath, vdlrootImportPrefix) {
return filepath.SkipDir
}
}
// Ignore the dir if it doesn't match our pattern. We still process the
// subdirs since they still might match.
//
// TODO(toddw): We could add an optimization to skip subdirs that can't
// possibly match the matcher. E.g. given pattern "a..." we can skip
// the subdirs if the dir doesn't start with "a".
matchPath := dirPath[len(walk.dir):]
if strings.HasPrefix(matchPath, pathSeparator) {
matchPath = matchPath[len(pathSeparator):]
}
if !matcher.MatchString(matchPath) {
return nil
}
// Finally resolve the dir.
if ds.resolveDirPath(dirPath, UnknownPathIsIgnored) != nil {
resolvedAny = true
}
return nil
})
}
return resolvedAny
}
const pathSeparator = string(filepath.Separator)
// createMatcher creates a regexp matcher out of the file pattern.
func createMatcher(pattern string) (*regexp.Regexp, error) {
rePat := regexp.QuoteMeta(pattern)
rePat = strings.Replace(rePat, `\.\.\.`, `.*`, -1)
// Add special-case so that x/... also matches x.
slashDotStar := regexp.QuoteMeta(pathSeparator) + ".*"
if strings.HasSuffix(rePat, slashDotStar) {
rePat = rePat[:len(rePat)-len(slashDotStar)] + "(" + slashDotStar + ")?"
}
rePat = `^` + rePat + `$`
matcher, err := regexp.Compile(rePat)
if err != nil {
return nil, fmt.Errorf("can't compile package path regexp %s: %v", rePat, err)
}
return matcher, nil
}
// resolveDirPath resolves dir into a Package. Returns the package, or nil if
// it can't be resolved.
func (ds *depSorter) resolveDirPath(dir string, mode UnknownPathMode) *Package {
// If the package already exists in our dir map, we can just return it.
absDir, err := filepath.Abs(dir)
if err != nil {
ds.errorf("%s: can't make absolute (%v)", dir, err)
}
if pkg := ds.dirMap[absDir]; pkg != nil {
return pkg
}
// Deduce the package path, and ensure it corresponds to exactly one package.
// We always deduce the package path from the package directory, even if we
// originally resolved from an import path, and thus already "know" the
// package path. This is to ensure we correctly handle vdl standard packages.
// E.g. if we're given "v.io/v23/vdlroot/vdltool" as an import path, the
// resulting package path must be "vdltool".
pkgPath, genPath, err := ds.deducePackagePath(absDir)
if err != nil {
ds.errorf("%s: can't deduce package path (%v)", absDir, err)
return nil
}
if !validPackagePath(pkgPath) {
mode.logOrErrorf(ds.errs, "%s: package path %q is invalid", absDir, pkgPath)
return nil
}
if pkg := ds.pathMap[pkgPath]; pkg != nil {
mode.logOrErrorf(ds.errs, "%s: package path %q already resolved from %s", absDir, pkgPath, pkg.Dir)
return nil
}
// Make sure the directory really exists, and add the package and deps.
fileInfo, err := os.Stat(absDir)
if err != nil {
mode.logOrErrorf(ds.errs, "%v", err)
return nil
}
if !fileInfo.IsDir() {
mode.logOrErrorf(ds.errs, "%s: package isn't a directory", absDir)
return nil
}
pkg := newPackage(pkgPath, genPath, absDir, mode, ds.opts, ds.vdlenv)
if pkg == nil {
return nil
}
ds.addPackageAndDeps(pkg)
return pkg
}
// resolveImportPath resolves pkgPath into a Package. Returns the package, or
// nil if it can't be resolved.
func (ds *depSorter) resolveImportPath(pkgPath string, mode UnknownPathMode, prefix string) *Package {
pkgPath = path.Clean(pkgPath)
if pkg := ds.pathMap[pkgPath]; pkg != nil {
return pkg
}
if !validPackagePath(pkgPath) {
mode.logOrErrorf(ds.errs, "%s: import path %q is invalid", prefix, pkgPath)
return nil
}
// Special-case to disallow packages under the vdlroot dir.
if strings.HasPrefix(pkgPath, vdlrootImportPrefix) {
mode.logOrErrorf(ds.errs, "%s: import path %q is invalid (packages under vdlroot must be specified without the vdlroot prefix)", prefix, pkgPath)
return nil
}
// Look in the builtin vdlroot first, if it has been initialized.
if pkg := ds.builtInRoot[pkgPath]; pkg != nil {
ds.addPackageAndDeps(pkg)
return pkg
}
// Look through srcDirs in-order until we find a valid package dir.
var dirs []string
for _, srcDir := range ds.srcDirs {
dir := filepath.Join(srcDir, filepath.FromSlash(pkgPath))
if pkg := ds.resolveDirPath(dir, UnknownPathIsIgnored); pkg != nil {
vdlutil.Vlog.Printf("%s: resolved import path %q", pkg.Dir, pkgPath)
return pkg
}
dirs = append(dirs, dir)
}
// We can't find a valid dir corresponding to this import path.
detail := " " + strings.Join(dirs, "\n ")
mode.logOrErrorf(ds.errs, "%s: can't resolve %q in any of:\n%s", prefix, pkgPath, detail)
return nil
}
// addPackageAndDeps adds the pkg and its dependencies to the sorter.
func (ds *depSorter) addPackageAndDeps(pkg *Package) {
vdlutil.Vlog.Printf("%s: resolved package path %q", pkg.Dir, pkg.Path)
ds.dirMap[pkg.Dir] = pkg
ds.pathMap[pkg.Path] = pkg
ds.sorter.AddNode(pkg)
pfiles := ParsePackage(pkg, parse.Opts{ImportsOnly: true}, ds.errs)
if pkg.Name == "" {
pkg.Name = parse.InferPackageName(pfiles, ds.errs)
}
for _, pf := range pfiles {
ds.addImportDeps(pkg, pf.Imports, filepath.Join(pkg.Path, pf.BaseName))
}
}
// addImportDeps adds transitive dependencies represented by imports to the
// sorter. If the pkg is non-nil, an edge is added between the pkg and its
// dependencies; otherwise each dependency is added as an independent node.
func (ds *depSorter) addImportDeps(pkg *Package, imports []*parse.Import, file string) {
for _, imp := range imports {
if dep := ds.resolveImportPath(imp.Path, UnknownPathIsError, file); dep != nil {
if pkg != nil {
ds.sorter.AddEdge(pkg, dep)
} else {
ds.sorter.AddNode(dep)
}
}
}
}
// AddConfigDeps takes a config file represented by its file name and src data,
// and adds all transitive dependencies to the sorter.
func (ds *depSorter) AddConfigDeps(fileName string, src io.Reader) {
if pconfig := parse.ParseConfig(fileName, src, parse.Opts{ImportsOnly: true}, ds.errs); pconfig != nil {
ds.addImportDeps(nil, pconfig.Imports, fileName)
}
}
// deducePackagePath deduces the package path for dir, by looking for prefix
// matches against the src dirs. The resulting package path may be incorrect
// even if no errors are reported; see the depSorter comment for details.
func (ds *depSorter) deducePackagePath(dir string) (string, string, error) {
for _, srcDir := range ds.srcDirs {
if strings.HasPrefix(dir, srcDir) {
relPath, err := filepath.Rel(srcDir, dir)
if err != nil {
return "", "", err
}
pkgPath := path.Clean(filepath.ToSlash(relPath))
genPath := pkgPath
if pre := vdlrootImportPrefix + "/"; strings.HasPrefix(pkgPath, pre) {
// The pkgPath should never include the vdlroot prefix.
pkgPath = pkgPath[len(pre):]
} else if srcDir == ds.rootDir {
// We matched against the VDLROOT dir. The genPath needs to include the
// vdlroot prefix.
genPath = path.Join(vdlrootImportPrefix, pkgPath)
}
return pkgPath, genPath, nil
}
}
return "", "", fmt.Errorf("no matching src dirs")
}
// Sort sorts all targets and returns the resulting list of Packages.
func (ds *depSorter) Sort() []*Package {
sorted, cycles := ds.sorter.Sort()
if len(cycles) > 0 {
cycleStr := toposort.DumpCycles(cycles, printPackagePath)
ds.errorf("cyclic package dependency: %v", cycleStr)
return nil
}
if len(sorted) == 0 {
return nil
}
targets := make([]*Package, len(sorted))
for ix, iface := range sorted {
targets[ix] = iface.(*Package)
}
return targets
}
func printPackagePath(v interface{}) string {
return v.(*Package).Path
}
// Opts specifies additional options for collecting build information.
type Opts struct {
// VDLConfigName specifies the name of the optional config file in each vdl
// source package. If empty we use "vdl.config" by default.
VDLConfigName string
}
func (o Opts) vdlConfigName() string {
if o.VDLConfigName != "" {
return o.VDLConfigName
}
return "vdl.config"
}
// TransitivePackages takes a list of paths, and returns the corresponding
// packages and transitive dependencies, ordered by dependency. Each path may
// either be a directory (IsDirPath) or an import (IsImportPath).
//
// A path is a pattern if it includes one or more "..." wildcards, each of which
// can match any string, including the empty string and strings containing
// slashes. Such a pattern expands to all packages found in SrcDirs with names
// matching the pattern. As a special-case, x/... matches x as well as x's
// subdirectories.
//
// The special-case "all" is a synonym for "...", and denotes all packages found
// in SrcDirs.
//
// Import path elements and file names are not allowed to begin with "." or "_";
// such paths are ignored in wildcard matches, and return errors if specified
// explicitly.
//
// The mode specifies whether we should ignore or produce errors for paths that
// don't resolve to any packages. The opts arg specifies additional options.
func TransitivePackages(paths []string, mode UnknownPathMode, opts Opts, errs *vdlutil.Errors) []*Package {
ds := newDepSorter(opts, errs)
for _, path := range paths {
if !ds.ResolvePath(path, mode) {
mode.logOrErrorf(errs, "can't resolve %q to any packages", path)
}
}
return ds.Sort()
}
// TransitivePackagesForConfig takes a config file represented by its file name
// and src data, and returns all package dependencies in transitive order.
//
// The opts arg specifies additional options.
func TransitivePackagesForConfig(fileName string, src io.Reader, opts Opts, errs *vdlutil.Errors) []*Package {
ds := newDepSorter(opts, errs)
ds.AddConfigDeps(fileName, src)
return ds.Sort()
}
// ParsePackage parses the given pkg with the given parse opts, and returns a
// slice of parsed files, sorted by name. Errors are reported in errs.
func ParsePackage(pkg *Package, opts parse.Opts, errs *vdlutil.Errors) (pfiles []*parse.File) {
vdlutil.Vlog.Printf("Parsing package %s %q, dir %s", pkg.Name, pkg.Path, pkg.Dir)
files, err := pkg.OpenFiles()
if err != nil {
errs.Errorf("can't open vdl files %v (%v)", pkg.BaseFileNames, err)
return nil
}
for filename, src := range files {
if pf := parse.ParseFile(path.Join(pkg.Path, filename), src, opts, errs); pf != nil {
pfiles = append(pfiles, pf)
}
}
sort.Sort(byBaseName(pfiles))
pkg.CloseFiles()
return
}
// byBaseName implements sort.Interface
type byBaseName []*parse.File
func (b byBaseName) Len() int { return len(b) }
func (b byBaseName) Less(i, j int) bool { return b[i].BaseName < b[j].BaseName }
func (b byBaseName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// BuildPackage parses and compiles the given pkg, updates env with the compiled
// package and returns it. Errors are reported in env.
//
// All imports that pkg depend on must have already been compiled and populated
// into env.
func BuildPackage(pkg *Package, env *compile.Env) *compile.Package {
pfiles := ParsePackage(pkg, parse.Opts{}, env.Errors)
return compile.CompilePackage(pkg.Path, pkg.GenPath, pfiles, pkg.Config, env)
}
// BuildConfig parses and compiles the given config src and returns it. Errors
// are reported in env; fileName is only used for error reporting.
//
// The implicit type is applied to the exported config const; untyped consts and
// composite literals with no explicit type assume the implicit type. Errors
// are reported if the implicit type isn't assignable from the final value. If
// the implicit type is nil, the exported config const must be explicitly typed.
//
// All packages that the config src depends on must have already been compiled
// and populated into env. The imports are injected into the parsed src,
// behaving as if the src had listed the imports explicitly.
func BuildConfig(fileName string, src io.Reader, implicit *vdl.Type, imports []string, env *compile.Env) *vdl.Value {
pconfig := parse.ParseConfig(fileName, src, parse.Opts{}, env.Errors)
if pconfig != nil {
pconfig.AddImports(imports...)
}
return compile.CompileConfig(implicit, pconfig, env)
}
// BuildConfigValue is a convenience function that runs BuildConfig, and then
// converts the result into value. The implicit type used by BuildConfig is
// inferred from the value.
func BuildConfigValue(fileName string, src io.Reader, imports []string, env *compile.Env, value interface{}) {
rv := reflect.ValueOf(value)
tt, err := vdl.TypeFromReflect(rv.Type())
if err != nil {
env.Errors.Errorf(err.Error())
return
}
if tt.Kind() == vdl.Optional {
// The value is typically a Go pointer, which translates into VDL optional.
// Remove the optional when determining the implicit type for BuildConfig.
tt = tt.Elem()
}
vconfig := BuildConfig(fileName, src, tt, imports, env)
if vconfig == nil {
return
}
if err := vdl.ConvertReflect(rv, reflect.ValueOf(vconfig)); err != nil {
env.Errors.Errorf("can't convert to %T from %v (%v)", value, vconfig, err)
return
}
}
// BuildExprs parses and compiles the given data into a slice of values. The
// input data is specified in VDL syntax, with commas separating multiple
// expressions. There must be at least one expression specified in data.
// Errors are reported in env.
//
// The given types specify the type of each returned value with the same slice
// position. If there are more types than returned values, the extra types are
// ignored. If there are fewer types than returned values, the last type is
// used for all remaining values. Nil entries in types are allowed, and
// indicate that the expression itself must be fully typed.
//
// All imports that the input data depends on must have already been compiled
// and populated into env.
func BuildExprs(data string, types []*vdl.Type, env *compile.Env) []*vdl.Value {
var values []*vdl.Value
var t *vdl.Type
for ix, pexpr := range parse.ParseExprs(data, env.Errors) {
if ix < len(types) {
t = types[ix]
}
values = append(values, compile.CompileExpr(t, pexpr, env))
}
return values
}