blob: 8d74d782d07a4f2b7c46d26cd0d6e20e7c450c4a [file] [log] [blame]
// Compiles and runs code for the Vanadium playground. Code is passed via
// os.Stdin as a JSON encoded request struct.
// NOTE(nlacasse): We use log.Panic() instead of log.Fatal() everywhere in this
// file. We do this because log.Panic calls panic(), which allows any deferred
// function to run. In particular, this will cause the mounttable and proxy
// processes to be killed in the event of a compilation error. log.Fatal, on
// the other hand, calls os.Exit(1), which does not call deferred functions,
// and will leave proxy and mounttable processes running. This is not a big
// deal for production environment, because the Docker instance gets cleaned up
// after each run, but during development and testing these extra processes can
// cause issues.
package main
import (
"encoding/json"
"flag"
"fmt"
"go/parser"
"go/token"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"time"
"v.io/core/veyron/lib/flags/consts"
"playground/lib"
"playground/lib/event"
)
var (
verbose = flag.Bool("v", true, "Whether to output debug messages.")
includeServiceOutput = flag.Bool("includeServiceOutput", false, "Whether to stream service (mounttable, wspr, proxy) output to clients.")
includeV23Env = flag.Bool("includeV23Env", false, "Whether to log the output of \"v23 env\" before compilation.")
// TODO(ivanpi): Separate out mounttable, proxy, wspr timeouts. Add compile timeout. Revise default.
runTimeout = flag.Int64("runTimeout", 3000, "Time limit for running user code, in milliseconds.")
// Sink for writing events (debug and run output) to stdout as JSON, one event per line.
out event.Sink
// Whether we have stopped execution of running files.
stopped = false
mu sync.Mutex
)
func getRunTimeout() time.Duration {
return time.Duration(*runTimeout) * time.Millisecond
}
// Type of data sent to the builder on stdin. Input should contain Files. We
// look for a file whose Name ends with .id, and parse that into credentials.
//
// TODO(ribrdb): Consider moving credentials parsing into the http server.
type request struct {
Files []*codeFile
Credentials []credentials
}
// Type of file data. Only Name and Body should be initially set. The other
// fields are added as the file is parsed.
type codeFile struct {
Name string
Body string
// Language the file is written in. Inferred from the file extension.
lang string
// Credentials to associate with the file's process.
credentials string
// The executable flag denotes whether the file should be executed as
// part of the playground run. This is currently used only for
// javascript files, and go files with package "main".
executable bool
// Name of the binary (for go files).
binaryName string
// Running cmd process for the file.
cmd *exec.Cmd
// Any subprocesses that are needed to support running the file (e.g. wspr).
subprocs []*os.Process
// The index of the file in the request.
index int
}
type exit struct {
name string
err error
}
func debug(args ...interface{}) {
event.Debug(out, args...)
}
func panicOnError(err error) {
if err != nil {
log.Panic(err)
}
}
func logV23Env() error {
if *includeV23Env {
return makeCmd("<environment>", false, "v23", "env").Run()
}
return nil
}
// All .go and .vdl files must have paths at least two directories deep,
// beginning with "src/".
func parseRequest(in io.Reader) (r request, err error) {
debug("Parsing input")
data, err := ioutil.ReadAll(in)
if err == nil {
err = json.Unmarshal(data, &r)
}
m := make(map[string]*codeFile)
for i := 0; i < len(r.Files); i++ {
f := r.Files[i]
f.index = i
if path.Ext(f.Name) == ".id" {
err = json.Unmarshal([]byte(f.Body), &r.Credentials)
if err != nil {
return
}
r.Files = append(r.Files[:i], r.Files[i+1:]...)
i--
} else {
switch path.Ext(f.Name) {
case ".js":
// JavaScript files are always executable.
f.executable = true
f.lang = "js"
case ".go":
// Go files will be marked as executable if their package name is
// "main". This happens in the "maybeSetExecutableAndBinaryName"
// function.
f.lang = "go"
case ".vdl":
f.lang = "vdl"
default:
return r, fmt.Errorf("Unknown file type: %q", f.Name)
}
basename := path.Base(f.Name)
if _, ok := m[basename]; ok {
return r, fmt.Errorf("Two files with same basename: %q", basename)
}
m[basename] = f
}
}
if len(r.Credentials) == 0 {
// Run everything with the same credentials if none are specified.
r.Credentials = append(r.Credentials, credentials{Name: "default"})
for _, f := range r.Files {
f.credentials = "default"
}
} else {
for _, creds := range r.Credentials {
for _, basename := range creds.Files {
// Check that the file associated with the credentials exists. We ignore
// cases where it doesn't because the test .id files get used for
// multiple different code files. See testdata/ids/authorized.id, for
// example.
if m[basename] != nil {
m[basename].credentials = creds.Name
}
}
}
}
return
}
func writeFiles(files []*codeFile) error {
debug("Writing files")
for _, f := range files {
if err := f.write(); err != nil {
return fmt.Errorf("Error writing %s: %v", f.Name, err)
}
}
return nil
}
// If compilation failed due to user error (bad input), returns badInput=true
// and cerr=nil. Only internal errors return non-nil cerr.
func compileFiles(files []*codeFile) (badInput bool, cerr error) {
found := make(map[string]bool)
for _, f := range files {
found[f.lang] = true
}
if !found["go"] && !found["vdl"] {
// No need to compile.
return false, nil
}
debug("Compiling files")
pwd, err := os.Getwd()
if err != nil {
return false, fmt.Errorf("Error getting current directory: %v", err)
}
srcd := filepath.Join(pwd, "src")
if err = os.Chdir(srcd); err != nil {
panicOnError(out.Write(event.New("", "stderr", ".go or .vdl files outside src/ directory.")))
return true, nil
}
os.Setenv("GOPATH", pwd+":"+os.Getenv("GOPATH"))
os.Setenv("VDLPATH", pwd+":"+os.Getenv("VDLPATH"))
// We set isService=false for compilation because "go install" only produces
// output on error, and we always want clients to see such errors.
// TODO(ivanpi): We assume *exec.ExitError results from uncompilable input
// files; other cases can result from bugs in playground backend or compiler
// itself.
if found["js"] && found["vdl"] {
debug("Generating VDL for Javascript")
err = makeCmd("<compile>", false,
"vdl", "generate", "-lang=Javascript", "-js_out_dir="+srcd, "./...").Run()
if _, ok := err.(*exec.ExitError); ok {
return true, nil
} else if err != nil {
return false, err
}
}
if found["go"] {
debug("Generating VDL for Go and compiling Go")
err = makeCmd("<compile>", false,
"v23", "go", "install", "./...").Run()
if _, ok := err.(*exec.ExitError); ok {
return true, nil
} else if err != nil {
return false, err
}
}
if err = os.Chdir(pwd); err != nil {
return false, fmt.Errorf("Error returning to parent directory: %v", err)
}
return false, nil
}
func runFiles(files []*codeFile) {
debug("Running files")
exit := make(chan exit)
running := 0
for _, f := range files {
if f.executable {
f.run(exit)
running++
}
}
timeout := time.After(getRunTimeout())
for running > 0 {
select {
case <-timeout:
panicOnError(out.Write(event.New("", "stderr", "Ran for too long; terminated.")))
stopAll(files)
case status := <-exit:
if status.err == nil {
panicOnError(out.Write(event.New(status.name, "stdout", "Exited cleanly.")))
} else {
panicOnError(out.Write(event.New(status.name, "stderr", fmt.Sprintf("Exited with error: %v", status.err))))
}
running--
stopAll(files)
}
}
}
func stopAll(files []*codeFile) {
mu.Lock()
defer mu.Unlock()
if !stopped {
stopped = true
for _, f := range files {
f.stop()
}
}
}
func (f *codeFile) maybeSetExecutableAndBinaryName() error {
debug("Parsing package from", f.Name)
file, err := parser.ParseFile(token.NewFileSet(), f.Name,
strings.NewReader(f.Body), parser.PackageClauseOnly)
if err != nil {
return err
}
pkg := file.Name.String()
if pkg == "main" {
f.executable = true
basename := path.Base(f.Name)
f.binaryName = basename[:len(basename)-len(path.Ext(basename))]
}
return nil
}
func (f *codeFile) write() error {
debug("Writing file", f.Name)
if f.lang == "go" || f.lang == "vdl" {
if err := f.maybeSetExecutableAndBinaryName(); err != nil {
return err
}
}
// Retain the original file tree structure.
if err := os.MkdirAll(path.Dir(f.Name), 0755); err != nil {
return err
}
return ioutil.WriteFile(f.Name, []byte(f.Body), 0644)
}
func (f *codeFile) startJs() error {
wsprProc, wsprPort, err := startWspr(f.Name, f.credentials, getRunTimeout())
if err != nil {
return fmt.Errorf("Error starting wspr: %v", err)
}
f.subprocs = append(f.subprocs, wsprProc)
os.Setenv("WSPR", "http://localhost:"+strconv.Itoa(wsprPort))
node := filepath.Join(os.Getenv("VANADIUM_ROOT"), "environment", "cout", "node", "bin", "node")
f.cmd = makeCmd(f.Name, false, node, f.Name)
return f.cmd.Start()
}
func (f *codeFile) startGo() error {
f.cmd = makeCmd(f.Name, false, filepath.Join("bin", f.binaryName))
if f.credentials != "" {
f.cmd.Env = append(f.cmd.Env, fmt.Sprintf("%v=%s", consts.VeyronCredentials, filepath.Join("credentials", f.credentials)))
}
return f.cmd.Start()
}
func (f *codeFile) run(ch chan exit) {
debug("Running", f.Name)
err := func() error {
mu.Lock()
defer mu.Unlock()
if stopped {
return fmt.Errorf("Execution has stopped; not running %s", f.Name)
}
switch f.lang {
case "go":
return f.startGo()
case "js":
return f.startJs()
default:
return fmt.Errorf("Cannot run file: %v", f.Name)
}
}()
if err != nil {
debug("Failed to start", f.Name, "-", err)
// Use a goroutine to avoid deadlock.
go func() {
ch <- exit{f.Name, err}
}()
return
}
// Wait for the process to exit and send result to channel.
go func() {
debug("Waiting for", f.Name)
err := f.cmd.Wait()
debug("Done waiting for", f.Name)
ch <- exit{f.Name, err}
}()
}
func (f *codeFile) stop() {
debug("Attempting to stop", f.Name)
if f.cmd == nil {
debug("Cannot stop:", f.Name, "cmd is nil")
} else if f.cmd.Process == nil {
debug("Cannot stop:", f.Name, "cmd is not nil, but cmd.Process is nil")
} else {
debug("Sending SIGTERM to", f.Name)
f.cmd.Process.Signal(syscall.SIGTERM)
}
for i, subproc := range f.subprocs {
debug("Killing subprocess", i, "for", f.Name)
subproc.Kill()
}
}
// Creates a cmd whose outputs (stdout and stderr) are streamed to stdout as
// Event objects. If you want to watch the output streams yourself, add your
// own writer(s) to the MultiWriter before starting the command.
func makeCmd(fileName string, isService bool, progName string, args ...string) *exec.Cmd {
cmd := exec.Command(progName, args...)
cmd.Env = os.Environ()
stdout, stderr := lib.NewMultiWriter(), lib.NewMultiWriter()
prefix := ""
if isService {
prefix = "svc-"
}
if !isService || *includeServiceOutput {
stdout.Add(event.NewStreamWriter(out, fileName, prefix+"stdout"))
stderr.Add(event.NewStreamWriter(out, fileName, prefix+"stderr"))
}
cmd.Stdout, cmd.Stderr = stdout, stderr
return cmd
}
func main() {
flag.Parse()
out = event.NewJsonSink(os.Stdout, !*verbose)
r, err := parseRequest(os.Stdin)
panicOnError(err)
panicOnError(createCredentials(r.Credentials))
mt, err := startMount(getRunTimeout())
panicOnError(err)
defer mt.Kill()
proxy, err := startProxy(getRunTimeout())
panicOnError(err)
defer proxy.Kill()
panicOnError(writeFiles(r.Files))
logV23Env()
badInput, err := compileFiles(r.Files)
// Panic on internal error, but not on user error.
panicOnError(err)
if badInput {
panicOnError(out.Write(event.New("<compile>", "stderr", "Compilation error.")))
return
}
runFiles(r.Files)
}