services/device/dmrun: Add ssh backend
Add an ssh backend to dmrun. It allows dmrun to be used with an
already-created vm or linux box.
Also made a few changes to how dmrun works:
- All files are now in /tmp/dmrun, so reusing a vm is easy
- Check the acl on the device manager directly, rather than
via the namespace, to work around the problem that NAT'ed
VMs somewhere other than GCE are not supported yet.
Change-Id: I263b19549ce6a968e64efb53d2a6f476a419907b
diff --git a/services/device/dmrun/backend/backend.go b/services/device/dmrun/backend/backend.go
index da8ca3e..24f3cac 100644
--- a/services/device/dmrun/backend/backend.go
+++ b/services/device/dmrun/backend/backend.go
@@ -13,25 +13,32 @@
// IP address (as a string) of the VM instance
IP() string
- // Execute a command on the VM instance
+ // Execute a command on the VM instance. The current directory will be the
+ // working directory of the VM when the command is run.
RunCommand(...string) (output []byte, err error)
- // Copy a file to the VM instance
+ // Copy a file to the working directory of VM instance. The destination is treated as a
+ // pathname relative to the workspace of the VM.
CopyFile(infile, destination string) error
// Delete the VM instance
Delete() error
+ // Provide what the user must run to run a specified command on the VM.
+ RunCommandForUser(commandPlusArgs ...string) string
+
// Provide the command that the user can use to delete a VM instance for which Delete()
// was not called
DeleteCommandForUser() string
}
func CreateCloudVM(instanceName string, options interface{}) (CloudVM, error) {
- switch options.(type) {
+ switch t := options.(type) {
default:
return nil, fmt.Errorf("Unknown options type")
case VcloudVMOptions:
- return newVcloudVM(instanceName, options.(VcloudVMOptions))
+ return newVcloudVM(instanceName, t)
+ case SSHVMOptions:
+ return newSSHVM(instanceName, t)
}
}
diff --git a/services/device/dmrun/backend/backend_ssh.go b/services/device/dmrun/backend/backend_ssh.go
new file mode 100644
index 0000000..e2e64cf
--- /dev/null
+++ b/services/device/dmrun/backend/backend_ssh.go
@@ -0,0 +1,140 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package backend
+
+import (
+ "fmt"
+ "net"
+ "os/exec"
+ "path"
+ "strings"
+)
+
+type SSHVM struct {
+ ssh, scp string // paths to the respective commands.
+ sshArgs []string // flags for ssh/scp
+ sshUserAtHost string // target of ssh
+ ip string
+ workingDir string
+ isDeleted bool
+}
+
+type SSHVMOptions struct {
+ SSHHostIP string // ip of the machine to ssh to
+ SSHUser string // username to use
+ SSHOptions []string // flags to ssh command. Use this to indicate the key file, for example
+ SSHBinary string // path to the "ssh" command
+ ScpBinary string // path to the "scp" command
+}
+
+func newSSHVM(instanceName string, opt SSHVMOptions) (vm *SSHVM, err error) {
+ // Make sure we got a valid ip
+ tmpIP := strings.TrimSpace(opt.SSHHostIP)
+ if net.ParseIP(tmpIP) == nil {
+ return nil, fmt.Errorf("IP of new ssh instance is not a valid IP address: %v", tmpIP)
+ }
+
+ g := &SSHVM{
+ ssh: "ssh",
+ scp: "scp",
+ sshArgs: opt.SSHOptions,
+ ip: tmpIP,
+ sshUserAtHost: tmpIP,
+ isDeleted: false,
+ }
+
+ if opt.SSHBinary != "" {
+ g.ssh = opt.SSHBinary
+ }
+ if opt.ScpBinary != "" {
+ g.scp = opt.ScpBinary
+ }
+ if opt.SSHUser != "" {
+ g.sshUserAtHost = fmt.Sprint(opt.SSHUser, "@", g.ip)
+ }
+
+ const workingDir = "/tmp/dmrun"
+ output, err := g.RunCommand("mkdir", workingDir)
+ if err != nil {
+ return nil, fmt.Errorf("failed to make working dir: %v (output: %s)", err, output)
+ }
+ g.workingDir = workingDir
+
+ return g, nil
+}
+
+func (g *SSHVM) Delete() error {
+ if g.isDeleted {
+ return fmt.Errorf("trying to delete a deleted SSHVM")
+ }
+
+ oldWorkingDir := g.workingDir
+ g.workingDir = ""
+ if output, err := g.RunCommand("rm", "-rf", oldWorkingDir); err != nil {
+ return fmt.Errorf("deleting working dir: %v (output: %v)", err, output)
+ }
+
+ g.isDeleted = true
+ g.ip = ""
+ return nil
+}
+
+func (g *SSHVM) Name() string {
+ return g.ip
+}
+
+func (g *SSHVM) IP() string {
+ return g.ip
+}
+
+func (g *SSHVM) RunCommand(args ...string) ([]byte, error) {
+ if g.isDeleted {
+ return nil, fmt.Errorf("RunCommand called on deleted SSHVM")
+ }
+
+ cmd := g.generateExecCmdForRun(args...)
+
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ err = fmt.Errorf("failed running [%s] on ssh VM %s", strings.Join(args, " "), g.Name())
+ }
+ return output, err
+}
+
+func (g *SSHVM) RunCommandForUser(args ...string) string {
+ if g.isDeleted {
+ return ""
+ }
+ cmd := g.generateExecCmdForRun(args...)
+
+ result := cmd.Path
+ for i := 1; i < len(cmd.Args); i++ {
+ result = fmt.Sprintf("%s %q", result, cmd.Args[i])
+ }
+ return result
+}
+
+func (g *SSHVM) generateExecCmdForRun(args ...string) *exec.Cmd {
+ return exec.Command(g.ssh, append(append(g.sshArgs, g.sshUserAtHost, "cd", g.workingDir, "&&"), args...)...)
+}
+
+func (g *SSHVM) CopyFile(infile, destination string) error {
+ if g.isDeleted {
+ return fmt.Errorf("CopyFile called on deleted SSHVM")
+ }
+
+ target := fmt.Sprint(g.sshUserAtHost, ":", path.Join(g.workingDir, destination))
+ cmd := exec.Command(g.scp, append(g.sshArgs, infile, target)...)
+
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ err = fmt.Errorf("failed copying %s to %s:%s - %v\nOutput:\n%v", infile, g.Name(), destination, err, string(output))
+ }
+ return err
+}
+
+func (g *SSHVM) DeleteCommandForUser() string {
+ return g.RunCommandForUser("rm", "-rf", g.workingDir)
+}
diff --git a/services/device/dmrun/backend/backend_vcloud.go b/services/device/dmrun/backend/backend_vcloud.go
index 38a68b4..cc01651 100644
--- a/services/device/dmrun/backend/backend_vcloud.go
+++ b/services/device/dmrun/backend/backend_vcloud.go
@@ -8,6 +8,7 @@
"fmt"
"net"
"os/exec"
+ "path"
"strings"
)
@@ -16,6 +17,7 @@
sshUser string // ssh into the VM as this user
projectArg, zoneArg string // common flags used with the vcloud command
name, ip string
+ workingDir string
isDeleted bool
}
@@ -49,6 +51,14 @@
}
g.ip = tmpIP
g.name = instanceName
+
+ const workingDir = "/tmp/dmrun"
+ output, err = g.RunCommand("mkdir", workingDir)
+ if err != nil {
+ return nil, fmt.Errorf("failed to make working dir: %v (output: %s)", err, output)
+ }
+ g.workingDir = workingDir
+
return g, nil
}
@@ -82,7 +92,7 @@
return nil, fmt.Errorf("RunCommand called on deleted VcloudVM")
}
- cmd := exec.Command(g.vcloud, append([]string{"sh", g.projectArg, g.name}, args...)...)
+ cmd := g.generateExecCmdForRun(args...)
output, err := cmd.CombinedOutput()
if err != nil {
err = fmt.Errorf("failed running [%s] on VM %s", strings.Join(args, " "), g.name)
@@ -90,12 +100,29 @@
return output, err
}
+func (g *VcloudVM) RunCommandForUser(args ...string) string {
+ if g.isDeleted {
+ return ""
+ }
+ cmd := g.generateExecCmdForRun(args...)
+
+ result := cmd.Path
+ for i := 1; i < len(cmd.Args); i++ {
+ result = fmt.Sprintf("%s %q", result, cmd.Args[i])
+ }
+ return result
+}
+
+func (g *VcloudVM) generateExecCmdForRun(args ...string) *exec.Cmd {
+ return exec.Command(g.vcloud, append([]string{"sh", g.projectArg, g.name, "cd", g.workingDir, "&&"}, args...)...)
+}
+
func (g *VcloudVM) CopyFile(infile, destination string) error {
if g.isDeleted {
return fmt.Errorf("CopyFile called on deleted VcloudVM")
}
- cmd := exec.Command("gcloud", "compute", g.projectArg, "copy-files", infile, fmt.Sprintf("%s@%s:/%s", g.sshUser, g.Name(), destination), g.zoneArg)
+ cmd := exec.Command("gcloud", "compute", g.projectArg, "copy-files", infile, fmt.Sprintf("%s@%s:/%s", g.sshUser, g.Name(), path.Join(g.workingDir, destination)), g.zoneArg)
output, err := cmd.CombinedOutput()
if err != nil {
err = fmt.Errorf("failed copying %s to %s:%s - %v\nOutput:\n%v", infile, g.name, destination, err, string(output))
diff --git a/services/device/dmrun/dmrun.go b/services/device/dmrun/dmrun.go
index 9d6081d..727b84f 100644
--- a/services/device/dmrun/dmrun.go
+++ b/services/device/dmrun/dmrun.go
@@ -20,6 +20,7 @@
import (
"archive/zip"
+ "bytes"
"flag"
"fmt"
"io"
@@ -168,9 +169,9 @@
// installArchive ships the archive to the VM instance and unpacks it.
func installArchive(archive, instance string) {
- err := vm.CopyFile(archive, "/tmp/")
+ err := vm.CopyFile(archive, "/")
dieIfErr(err, "Copying archive failed: %v", err)
- output, err := vm.RunCommand("unzip", path.Join("/tmp", filepath.Base(archive)), "-d", "/tmp/unpacked")
+ output, err := vm.RunCommand("unzip", path.Join("./", filepath.Base(archive)), "-d", "./unpacked")
dieIfErr(err, "Extracting archive failed. Output:\n%v", string(output))
}
@@ -179,9 +180,9 @@
func installDevice(instance string) (string, string) {
fmt.Println("Installing device manager...")
defer fmt.Println("Done installing device manager...")
- output, err := vm.RunCommand("V23_DEVICE_DIR=/tmp/dm", "/tmp/unpacked/devicex", "install", "/tmp/unpacked", "--single_user", "--", "--v23.tcp.address=:8151", "--deviced-port=8150", "--proxy-port=8160", "--use-pairing-token")
+ output, err := vm.RunCommand("V23_DEVICE_DIR=`pwd`/dm", "./unpacked/devicex", "install", "./unpacked", "--single_user", "--", "--v23.tcp.address=:8151", "--deviced-port=8150", "--proxy-port=8160", "--use-pairing-token")
dieIfErr(err, "Installing device manager failed. Output:\n%v", string(output))
- output, err = vm.RunCommand("V23_DEVICE_DIR=/tmp/dm", "/tmp/unpacked/devicex", "start")
+ output, err = vm.RunCommand("V23_DEVICE_DIR=`pwd`/dm", "./unpacked/devicex", "start")
dieIfErr(err, "Starting device manager failed. Output:\n%v", string(output))
// Grab the token and public key from the device manager log.
dieAfter := time.After(5 * time.Second)
@@ -196,7 +197,7 @@
} else {
firstIteration = false
}
- output, err = vm.RunCommand("cat", "/tmp/dm/dmroot/device-manager/logs/deviced.INFO")
+ output, err = vm.RunCommand("cat", "./dm/dmroot/device-manager/logs/deviced.INFO")
dieIfErr(err, "Reading device manager log failed. Output:\n%v", string(output))
pairingTokenRE := regexp.MustCompile("Device manager pairing token: (.*)")
matches := pairingTokenRE.FindSubmatch(output)
@@ -236,10 +237,11 @@
setCredentialsEnv(cmd)
output, err := cmd.CombinedOutput()
dieIfErr(err, "Claiming device manager (%v) failed. Output:\n%v", strings.Join(cmd.Args, " "), string(output))
- cmd = exec.Command(device, "acl", "get", fmt.Sprintf("/%s:8151/devmgr/device", ip))
+ cmd = exec.Command(device, "acl", "get", fmt.Sprintf("/%s:8150/device", ip))
setCredentialsEnv(cmd)
output, err = cmd.CombinedOutput()
dieIfErr(err, "Getting device manager acls (%v) failed. Output:\n%v", strings.Join(cmd.Args, " "), string(output))
+
fmt.Printf("Done claiming device manager. Device manager ACLs:\n%s", string(output))
}
@@ -251,9 +253,15 @@
cmd := exec.Command(device, args...)
setCredentialsEnv(cmd)
cmd.Env = append(cmd.Env, fmt.Sprintf("V23_NAMESPACE=/%s:8151", ip))
- output, err := cmd.CombinedOutput()
- dieIfErr(err, "Installing app (%v) failed. Output:\n%v", strings.Join(cmd.Args, " "), string(output))
+
+ // During installation, there are sometimes timeout messages on stderr even when the
+ // installation succeeds -- so we need to capture stderr separately from stdout
+ buf := new(bytes.Buffer)
+ cmd.Stderr = buf
+ output, err := cmd.Output()
+ dieIfErr(err, "Installing app (%v) failed. Output:\n%v\nStderr:\n%v", strings.Join(cmd.Args, " "), string(output), buf.String())
installationName := strings.TrimSpace(string(output))
+
fmt.Println("Installed", installationName)
return installationName
}
@@ -274,29 +282,64 @@
return instanceName
}
-func main() {
+// check flags and produce the options used to build the backend VM
+func handleFlags() (vmOpts interface{}) {
+ var sshTarget, sshOptions string
+ flag.StringVar(&sshTarget, "ssh", "", "specify [user@]ip target for ssh")
+ flag.StringVar(&sshOptions, "sshoptions", "", "flags to pass to ssh, e.g. \"-i <keyfile>\"")
flag.Parse()
- if len(flag.Args()) == 0 {
- fmt.Fprintf(os.Stderr, "Usage: %s <app> <arguments ... >\n", os.Args[0])
- os.Exit(1)
+
+ // Pick backend based on flags
+ if sshTarget != "" {
+ // Ssh backend
+ opts := backend.SSHVMOptions{}
+ targetComponents := strings.Split(sshTarget, "@")
+ switch len(targetComponents) {
+ case 1:
+ opts.SSHHostIP = targetComponents[0]
+ case 2:
+ opts.SSHUser = targetComponents[0]
+ opts.SSHHostIP = targetComponents[1]
+ default:
+ die("Unable to parse sshTarget: %s\n", sshTarget)
+ }
+ if sshOptions = strings.TrimSpace(sshOptions); sshOptions != "" {
+ opts.SSHOptions = strings.Split(sshOptions, " ")
+ }
+ vmOpts = opts
+ } else {
+ // Vcloud backend
+ vcloud = buildV23Binary(vcloudBin)
+ vmOpts = backend.VcloudVMOptions{VcloudBinary: vcloud}
}
+
+ if len(flag.Args()) == 0 {
+ die("Usage: %s [--ssh [user@]ip] [--sshoptions \"<options>\"] <app> <arguments ... >\n", os.Args[0])
+ }
+
+ return vmOpts
+}
+
+func main() {
+ vmOpts := handleFlags()
setupWorkDir()
cleanupOnDeath = func() {
os.RemoveAll(workDir)
}
defer os.RemoveAll(workDir)
- vcloud = buildV23Binary(vcloudBin)
device = buildV23Binary(deviceBin)
dmBins := buildDMBinaries()
archive := createArchive(append(dmBins, getPath(devicexRepo, devicex)))
- vmOpts := backend.VcloudVMOptions{VcloudBinary: vcloud}
+
vm, vmInstanceName, vmInstanceIP := setupInstance(vmOpts)
cleanupOnDeath = func() {
- fmt.Fprintf(os.Stderr, "Deleting VM instance ...\n")
+ fmt.Fprintf(os.Stderr, "Attempting to stop agentd/deviced ...\n")
+ vm.RunCommand("sudo", "killall", "-9", "agentd", "deviced") // errors are ignored
+ fmt.Fprintf(os.Stderr, "Cleaning up VM instance ...\n")
err := vm.Delete()
- fmt.Fprintf(os.Stderr, "Removing tmp files ...\n")
+ fmt.Fprintf(os.Stderr, "Removing local tmp files ...\n")
os.RemoveAll(workDir)
- dieIfErr(err, "Deleting VM instance failed")
+ dieIfErr(err, "Cleaning up VM instance failed")
}
installArchive(archive, vmInstanceName)
publicKey, pairingToken := installDevice(vmInstanceName)
@@ -313,6 +356,8 @@
fmt.Printf("\t${JIRI_ROOT}/release/go/bin/debug glob %s/logs/*\n", instanceName)
fmt.Println("Dump e.g. the INFO log:")
fmt.Printf("\t${JIRI_ROOT}/release/go/bin/debug logs read %s/logs/app.INFO\n", instanceName)
- fmt.Println("Clean up by deleting the VM instance:")
+
+ fmt.Println("Clean up the VM instance:")
+ fmt.Printf("\t%s\n", vm.RunCommandForUser("sudo", "killall", "-9", "agentd", "deviced"))
fmt.Printf("\t%s\n", vm.DeleteCommandForUser())
}
