services/mgmt/device/impl/app_service.go

package impl

// The app invoker is responsible for managing the state of applications on the
// device manager.  The device manager manages the applications it installs and
// runs using the following directory structure:
//
// TODO(caprita): Not all is yet implemented.
//
// <config.Root>/
//   app-<hash 1>/                  - the application dir is named using a hash of the application title
//     installation-<id 1>/         - installations are labelled with ids
//       acls/
//         data                     - the ACL data for this
//                                    installation. Controls acces to
//                                    Start, Uninstall, Update, UpdateTo
//                                    and Revert.
//         signature                - the signature for the ACLs in data
//       <status>                   - one of the values for installationState enum
//       origin                     - object name for application envelope
//       config                     - Config provided by the installer
//       packages                   - set of packages specified by the installer
//       pkg/                       - downloaded packages
//         <pkg name>
//         <pkg name>.__info
//         ...
//       <version 1 timestamp>/     - timestamp of when the version was downloaded
//         bin                      - application binary
//         previous                 - symbolic link to previous version directory
//         envelope                 - application envelope (JSON-encoded)
//         pkg/                     - the application packages
//           <pkg name>
//           <pkg name>.__info
//           ...
//       <version 2 timestamp>
//       ...
//       current                    - symbolic link to the current version
//       instances/
//         instance-<id a>/         - instances are labelled with ids
//           credentials/           - holds veyron credentials (unless running
//                                    through security agent)
//           root/                  - workspace that the instance is run from
//             packages/            - the installed packages
//               <pkg name>/
//               ...
//           logs/                  - stderr/stdout and log files generated by instance
//           info                   - metadata for the instance (such as app
//                                    cycle manager name and process id)
//           installation           - symbolic link to installation for the instance
//           version                - symbolic link to installation version for the instance
//           acls/
//             data                 - the ACLs for this instance. These
//                                    ACLs control access to Refresh,
//                                    Restart, Resume, Stop and
//                                    Suspend.
//             signature            - the signature for  these ACLs.
//           <status>               - one of the values for instanceState enum
//           systemname             - the system name used to execute this instance
//         instance-<id b>
//         ...
//     installation-<id 2>
//     ...
//   app-<hash 2>
//   ...
//
// The device manager uses the suid helper binary to invoke an application as a
// specified user.  The path to the helper is specified as config.Helper.

// When device manager starts up, it goes through all instances and resumes the
// ones that are not suspended.  If the application was still running, it
// suspends it first.  If an application fails to resume, it stays suspended.
//
// When device manager shuts down, it suspends all running instances.
//
// Start starts an instance.  Suspend kills the process but leaves the workspace
// untouched. Resume restarts the process. Stop kills the process and prevents
// future resumes (it also eventually gc's the workspace).
//
// If the process dies on its own, it stays dead and is assumed suspended.
// TODO(caprita): Later, we'll add auto-restart option.
//
// Concurrency model: installations can be created independently of one another;
// installations can be removed at any time (any running instances will be
// stopped). The first call to Uninstall will rename the installation dir as a
// first step; subsequent Uninstall's will fail. Instances can be created
// independently of one another, as long as the installation exists (if it gets
// Uninstall'ed during an instance Start, the Start may fail).
//
// The status file present in each instance is used to flag the state of the
// instance and prevent concurrent operations against the instance:
//
// - when an instance is created with Start, it is placed in state 'suspended'.
// To run the instance, Start transitions 'suspended' to 'starting' and then
// 'started' (upon success) or the instance is deleted (upon failure).
//
// - Suspend attempts to transition from 'started' to 'suspending' (if the
// instance was not in 'started' state, Suspend fails). From 'suspending', the
// instance transitions to 'suspended' upon success or back to 'started' upon
// failure.
//
// - Resume attempts to transition from 'suspended' to 'starting' (if the
// instance was not in 'suspended' state, Resume fails). From 'starting', the
// instance transitions to 'started' upon success or back to 'suspended' upon
// failure.
//
// - Stop attempts to transition from 'started' to 'stopping' and then to
// 'stopped' (upon success) or back to 'started' (upon failure); or from
// 'suspended' to 'stopped'.  If the initial state is neither 'started' or
// 'suspended', Stop fails.
//
// TODO(caprita): There is room for synergy between how device manager organizes
// its own workspace and that for the applications it runs.  In particular,
// previous, origin, and envelope could be part of a single config.  We'll
// refine that later.

import (
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"encoding/base64"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"hash/crc64"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"text/template"
	"time"

	"v.io/core/veyron2"
	"v.io/core/veyron2/context"
	"v.io/core/veyron2/ipc"
	"v.io/core/veyron2/mgmt"
	"v.io/core/veyron2/naming"
	"v.io/core/veyron2/security"
	"v.io/core/veyron2/services/mgmt/appcycle"
	"v.io/core/veyron2/services/mgmt/application"
	"v.io/core/veyron2/services/mgmt/device"
	"v.io/core/veyron2/services/security/access"
	"v.io/core/veyron2/verror"
	"v.io/core/veyron2/vlog"

	vexec "v.io/core/veyron/lib/exec"
	"v.io/core/veyron/lib/flags/consts"
	vsecurity "v.io/core/veyron/security"
	"v.io/core/veyron/security/agent"
	"v.io/core/veyron/security/agent/keymgr"
	iconfig "v.io/core/veyron/services/mgmt/device/config"
	"v.io/core/veyron/services/mgmt/lib/acls"
	libpackages "v.io/core/veyron/services/mgmt/lib/packages"
)

// instanceInfo holds state about a running instance.
type instanceInfo struct {
	AppCycleMgrName          string
	Pid                      int
	DeviceManagerPeerPattern string
	SecurityAgentHandle      []byte
}

func saveInstanceInfo(dir string, info *instanceInfo) error {
	jsonInfo, err := json.Marshal(info)
	if err != nil {
		vlog.Errorf("Marshal(%v) failed: %v", info, err)
		return verror.New(ErrOperationFailed, nil)
	}
	infoPath := filepath.Join(dir, "info")
	if err := ioutil.WriteFile(infoPath, jsonInfo, 0600); err != nil {
		vlog.Errorf("WriteFile(%v) failed: %v", infoPath, err)
		return verror.New(ErrOperationFailed, nil)
	}
	return nil
}

func loadInstanceInfo(dir string) (*instanceInfo, error) {
	infoPath := filepath.Join(dir, "info")
	info := new(instanceInfo)
	if infoBytes, err := ioutil.ReadFile(infoPath); err != nil {
		vlog.Errorf("ReadFile(%v) failed: %v", infoPath, err)
		return nil, verror.New(ErrOperationFailed, nil)
	} else if err := json.Unmarshal(infoBytes, info); err != nil {
		vlog.Errorf("Unmarshal(%v) failed: %v", infoBytes, err)
		return nil, verror.New(ErrOperationFailed, nil)
	}
	return info, nil
}

type securityAgentState struct {
	// Security agent key manager client.
	keyMgrAgent *keymgr.Agent
}

// appService implements the Device manager's Application interface.
type appService struct {
	callback *callbackState
	config   *iconfig.State
	// suffix contains the name components of the current invocation name
	// suffix.  It is used to identify an application, installation, or
	// instance.
	suffix []string
	uat    BlessingSystemAssociationStore
	locks  *acls.Locks
	// Reference to the devicemanager top-level ACL list.
	deviceACL access.TaggedACLMap
	// securityAgent holds state related to the security agent (nil if not
	// using the agent).
	securityAgent *securityAgentState
	// mtAddress is the address of the local mounttable.
	mtAddress string
	// reap is the app process monitoring subsystem.
	reap reaper
}

func saveEnvelope(dir string, envelope *application.Envelope) error {
	jsonEnvelope, err := json.Marshal(envelope)
	if err != nil {
		vlog.Errorf("Marshal(%v) failed: %v", envelope, err)
		return verror.New(ErrOperationFailed, nil)
	}
	path := filepath.Join(dir, "envelope")
	if err := ioutil.WriteFile(path, jsonEnvelope, 0600); err != nil {
		vlog.Errorf("WriteFile(%v) failed: %v", path, err)
		return verror.New(ErrOperationFailed, nil)
	}
	return nil
}

func loadEnvelope(dir string) (*application.Envelope, error) {
	path := filepath.Join(dir, "envelope")
	envelope := new(application.Envelope)
	if envelopeBytes, err := ioutil.ReadFile(path); err != nil {
		vlog.Errorf("ReadFile(%v) failed: %v", path, err)
		return nil, verror.New(ErrOperationFailed, nil)
	} else if err := json.Unmarshal(envelopeBytes, envelope); err != nil {
		vlog.Errorf("Unmarshal(%v) failed: %v", envelopeBytes, err)
		return nil, verror.New(ErrOperationFailed, nil)
	}
	return envelope, nil
}

func saveConfig(dir string, config device.Config) error {
	jsonConfig, err := json.Marshal(config)
	if err != nil {
		vlog.Errorf("Marshal(%v) failed: %v", config, err)
		return verror.New(ErrOperationFailed, nil)
	}
	path := filepath.Join(dir, "config")
	if err := ioutil.WriteFile(path, jsonConfig, 0600); err != nil {
		vlog.Errorf("WriteFile(%v) failed: %v", path, err)
		return verror.New(ErrOperationFailed, nil)
	}
	return nil
}

func loadConfig(dir string) (device.Config, error) {
	path := filepath.Join(dir, "config")
	var config device.Config
	if configBytes, err := ioutil.ReadFile(path); err != nil {
		vlog.Errorf("ReadFile(%v) failed: %v", path, err)
		return nil, verror.New(ErrOperationFailed, nil)
	} else if err := json.Unmarshal(configBytes, &config); err != nil {
		vlog.Errorf("Unmarshal(%v) failed: %v", configBytes, err)
		return nil, verror.New(ErrOperationFailed, nil)
	}
	return config, nil
}

func savePackages(dir string, packages application.Packages) error {
	jsonPackages, err := json.Marshal(packages)
	if err != nil {
		vlog.Errorf("Marshal(%v) failed: %v", packages, err)
		return verror.New(ErrOperationFailed, nil)
	}
	path := filepath.Join(dir, "packages")
	if err := ioutil.WriteFile(path, jsonPackages, 0600); err != nil {
		vlog.Errorf("WriteFile(%v) failed: %v", path, err)
		return verror.New(ErrOperationFailed, nil)
	}
	return nil
}

func loadPackages(dir string) (application.Packages, error) {
	path := filepath.Join(dir, "packages")
	var packages application.Packages
	if packagesBytes, err := ioutil.ReadFile(path); err != nil {
		vlog.Errorf("ReadFile(%v) failed: %v", path, err)
		return nil, verror.New(ErrOperationFailed, nil)
	} else if err := json.Unmarshal(packagesBytes, &packages); err != nil {
		vlog.Errorf("Unmarshal(%v) failed: %v", packagesBytes, err)
		return nil, verror.New(ErrOperationFailed, nil)
	}
	return packages, nil
}

func saveOrigin(dir, originVON string) error {
	path := filepath.Join(dir, "origin")
	if err := ioutil.WriteFile(path, []byte(originVON), 0600); err != nil {
		vlog.Errorf("WriteFile(%v) failed: %v", path, err)
		return verror.New(ErrOperationFailed, nil)
	}
	return nil
}

func loadOrigin(dir string) (string, error) {
	path := filepath.Join(dir, "origin")
	if originBytes, err := ioutil.ReadFile(path); err != nil {
		vlog.Errorf("ReadFile(%v) failed: %v", path, err)
		return "", verror.New(ErrOperationFailed, nil)
	} else {
		return string(originBytes), nil
	}
}

// generateID returns a new unique id string.  The uniqueness is based on the
// current timestamp.  Not cryptographically secure.
func generateID() string {
	timestamp := fmt.Sprintf("%v", time.Now().Format(time.RFC3339Nano))
	h := crc64.New(crc64.MakeTable(crc64.ISO))
	h.Write([]byte(timestamp))
	b := make([]byte, 8)
	binary.LittleEndian.PutUint64(b, uint64(h.Sum64()))
	return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=")
}

// generateRandomString returns a cryptographically-strong random string.
func generateRandomString() (string, error) {
	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}

// TODO(caprita): Nothing prevents different applications from sharing the same
// title, and thereby being installed in the same app dir.  Do we want to
// prevent that for the same user or across users?

// applicationDirName generates a cryptographic hash of the application title,
// to be used as a directory name for installations of the application with the
// given title.
func applicationDirName(title string) string {
	h := md5.New()
	h.Write([]byte(title))
	hash := strings.TrimRight(base64.URLEncoding.EncodeToString(h.Sum(nil)), "=")
	return "app-" + hash
}

func installationDirName(installationID string) string {
	return "installation-" + installationID
}

func instanceDirName(instanceID string) string {
	return "instance-" + instanceID
}

func mkdir(dir string) error {
	perm := os.FileMode(0700)
	if err := os.MkdirAll(dir, perm); err != nil {
		vlog.Errorf("MkdirAll(%v, %v) failed: %v", dir, perm, err)
		return err
	}
	return nil
}

func fetchAppEnvelope(ctx *context.T, origin string) (*application.Envelope, error) {
	envelope, err := fetchEnvelope(ctx, origin)
	if err != nil {
		return nil, err
	}
	if envelope.Title == application.DeviceManagerTitle {
		// Disallow device manager apps from being installed like a
		// regular app.
		return nil, verror.New(ErrInvalidOperation, ctx)
	}
	return envelope, nil
}

// newVersion sets up the directory for a new application version.
func newVersion(ctx *context.T, installationDir string, envelope *application.Envelope, oldVersionDir string) (string, error) {
	versionDir := filepath.Join(installationDir, generateVersionDirName())
	if err := mkdir(versionDir); err != nil {
		return "", verror.New(ErrOperationFailed, nil)
	}
	pkgDir := filepath.Join(versionDir, "pkg")
	if err := mkdir(pkgDir); err != nil {
		return "", verror.New(ErrOperationFailed, nil)
	}
	publisher, err := security.NewBlessings(envelope.Publisher)
	if err != nil {
		vlog.Errorf("Failed to parse publisher blessings:%v", err)
		return versionDir, verror.New(ErrOperationFailed, nil)
	}
	// TODO(caprita): Share binaries if already existing locally.
	if err := downloadBinary(ctx, publisher, &envelope.Binary, versionDir, "bin"); err != nil {
		return versionDir, err
	}
	if err := downloadPackages(ctx, publisher, envelope.Packages, pkgDir); err != nil {
		return versionDir, err
	}
	if err := saveEnvelope(versionDir, envelope); err != nil {
		return versionDir, err
	}
	if oldVersionDir != "" {
		previousLink := filepath.Join(versionDir, "previous")
		if err := os.Symlink(oldVersionDir, previousLink); err != nil {
			vlog.Errorf("Symlink(%v, %v) failed: %v", oldVersionDir, previousLink, err)
			return versionDir, verror.New(ErrOperationFailed, nil)
		}
	}
	// updateLink should be the last thing we do, after we've ensured the
	// new version is viable (currently, that just means it installs
	// properly).
	return versionDir, updateLink(versionDir, filepath.Join(installationDir, "current"))
}

func (i *appService) Install(call ipc.ServerContext, applicationVON string, config device.Config, packages application.Packages) (string, error) {
	if len(i.suffix) > 0 {
		return "", verror.New(ErrInvalidSuffix, call.Context())
	}
	ctx, cancel := context.WithTimeout(call.Context(), ipcContextLongTimeout)
	defer cancel()
	envelope, err := fetchAppEnvelope(ctx, applicationVON)
	if err != nil {
		return "", err
	}
	installationID := generateID()
	installationDir := filepath.Join(i.config.Root, applicationDirName(envelope.Title), installationDirName(installationID))
	deferrer := func() {
		cleanupDir(installationDir, "")
	}
	defer func() {
		if deferrer != nil {
			deferrer()
		}
	}()
	if _, err := newVersion(call.Context(), installationDir, envelope, ""); err != nil {
		return "", err
	}
	if newOrigin, ok := config[mgmt.AppOriginConfigKey]; ok {
		delete(config, mgmt.AppOriginConfigKey)
		applicationVON = newOrigin
	}
	if err := saveOrigin(installationDir, applicationVON); err != nil {
		return "", err
	}
	if err := saveConfig(installationDir, config); err != nil {
		return "", err
	}
	if err := savePackages(installationDir, packages); err != nil {
		return "", err
	}
	pkgDir := filepath.Join(installationDir, "pkg")
	if err := mkdir(pkgDir); err != nil {
		return "", verror.New(ErrOperationFailed, nil)
	}
	// We use a nil publisher, meaning that any signatures present in the
	// package files are not verified.
	// TODO(caprita): Issue warnings when signatures are present and ignored.
	if err := downloadPackages(call.Context(), nil, packages, pkgDir); err != nil {
		return "", err
	}
	if err := initializeInstallation(installationDir, active); err != nil {
		return "", err
	}

	// TODO(caprita,rjkroege): Should the installation ACLs really be
	// seeded with the device ACL? Instead, might want to hide the deviceACL
	// from the app?
	blessings, _ := call.RemoteBlessings().ForContext(call)
	if err := i.initializeSubACLs(call.LocalPrincipal(), installationDir, blessings, i.deviceACL.Copy()); err != nil {
		return "", err
	}
	deferrer = nil
	return naming.Join(envelope.Title, installationID), nil
}

func (*appService) Refresh(ipc.ServerContext) error {
	// TODO(jsimsa): Implement.
	return nil
}

func (*appService) Restart(ipc.ServerContext) error {
	// TODO(jsimsa): Implement.
	return nil
}

func openWriteFile(path string) (*os.File, error) {
	perm := os.FileMode(0600)
	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, perm)
	if err != nil {
		vlog.Errorf("OpenFile(%v) failed: %v", path, err)
		return nil, verror.New(ErrOperationFailed, nil)
	}
	return file, nil
}

func installationDirCore(components []string, root string) (string, error) {
	if nComponents := len(components); nComponents != 2 {
		return "", verror.New(ErrInvalidSuffix, nil)
	}
	app, installation := components[0], components[1]
	installationDir := filepath.Join(root, applicationDirName(app), installationDirName(installation))
	if _, err := os.Stat(installationDir); err != nil {
		if os.IsNotExist(err) {
			return "", verror.New(verror.NoExist, nil, naming.Join(components...))
		}
		vlog.Errorf("Stat(%v) failed: %v", installationDir, err)
		return "", verror.New(ErrOperationFailed, nil)
	}
	return installationDir, nil
}

// agentPrincipal creates a Principal backed by the given agent connection,
// taking ownership of the connection.  The returned cancel function is to be
// called when the Principal is no longer in use.
func agentPrincipal(ctx *context.T, conn *os.File) (security.Principal, func(), error) {
	agentctx, cancel := context.WithCancel(ctx)
	var err error
	if agentctx, err = veyron2.SetNewStreamManager(agentctx); err != nil {
		cancel()
		conn.Close()
		return nil, nil, err
	}
	p, err := agent.NewAgentPrincipal(agentctx, int(conn.Fd()), veyron2.GetClient(agentctx))
	if err != nil {
		cancel()
		conn.Close()
		return nil, nil, err
	}
	// conn will be closed when the connection to the agent is shut down, as
	// a result of cancel shutting down the stream manager.  No need to
	// explicitly call conn.Close() with cancel.
	return p, cancel, nil
}

// setupPrincipal sets up the instance's principal, with the right blessings.
func setupPrincipal(ctx *context.T, instanceDir, versionDir string, call ipc.ServerContext, securityAgent *securityAgentState, info *instanceInfo) error {
	var p security.Principal
	if securityAgent != nil {
		// TODO(caprita): Part of the cleanup upon destroying an
		// instance, we should tell the agent to drop the principal.
		handle, conn, err := securityAgent.keyMgrAgent.NewPrincipal(ctx, false)
		if err != nil {
			vlog.Errorf("NewPrincipal() failed %v", err)
			return verror.New(ErrOperationFailed, nil)
		}
		var cancel func()
		if p, cancel, err = agentPrincipal(ctx, conn); err != nil {
			vlog.Errorf("agentPrincipal failed: %v", err)
			return verror.New(ErrOperationFailed, nil)
		}
		defer cancel()
		info.SecurityAgentHandle = handle
		// conn will be closed when the connection to the agent is shut
		// down, as a result of cancel() shutting down the stream
		// manager.  No need to call conn.Close().
	} else {
		credentialsDir := filepath.Join(instanceDir, "credentials")
		// TODO(caprita): The app's system user id needs access to this dir.
		// Use the suidhelper to chown it.
		var err error
		if p, err = vsecurity.CreatePersistentPrincipal(credentialsDir, nil); err != nil {
			vlog.Errorf("CreatePersistentPrincipal(%v, nil) failed: %v", credentialsDir, err)
			return verror.New(ErrOperationFailed, nil)
		}
	}
	// Read the app installation version's envelope to obtain the app title.
	//
	// NOTE: we could have gotten this from the suffix as well, but the
	// format of the object name suffix may change in the future: there's no
	// guarantee it will always include the title.
	envelope, err := loadEnvelope(versionDir)
	if err != nil {
		return err
	}
	dmPrincipal := call.LocalPrincipal()
	// Take the blessings conferred upon us by the Start-er, extend them
	// with the app title.
	grantedBlessings := call.Blessings()
	if grantedBlessings == nil {
		return verror.New(ErrInvalidBlessing, nil)
	}
	// TODO(caprita): Revisit UnconstrainedUse.
	appBlessings, err := dmPrincipal.Bless(p.PublicKey(), grantedBlessings, envelope.Title, security.UnconstrainedUse())
	if err != nil {
		vlog.Errorf("Bless() failed: %v", err)
		return verror.New(ErrOperationFailed, nil)
	}
	// The blessings we extended from the blessings that the Start-er
	// granted are the default blessings for the app.
	if err := p.BlessingStore().SetDefault(appBlessings); err != nil {
		vlog.Errorf("BlessingStore.SetDefault() failed: %v", err)
		return verror.New(ErrOperationFailed, nil)
	}
	if _, err := p.BlessingStore().Set(appBlessings, security.AllPrincipals); err != nil {
		vlog.Errorf("BlessingStore.Set() failed: %v", err)
		return verror.New(ErrOperationFailed, nil)
	}
	if err := p.AddToRoots(appBlessings); err != nil {
		vlog.Errorf("AddToRoots() failed: %v", err)
		return verror.New(ErrOperationFailed, nil)
	}
	// In addition, we give the app separate blessings for the purpose of
	// communicating with the device manager.
	//
	// NOTE(caprita/ataly): Giving the app an unconstrained blessing from
	// the device manager's default presents the app with a blessing that's
	// potentially more powerful than what is strictly needed to allow
	// communication between device manager and app (which could be more
	// narrowly accomplished by using a custom-purpose self-signed blessing
	// that the device manger produces solely to talk to the app).
	//
	// TODO(caprita): Figure out if there is any feature value in providing
	// the app with a device manager-derived blessing (e.g., may the app
	// need to prove it's running on the device?).
	dmBlessings, err := dmPrincipal.Bless(p.PublicKey(), dmPrincipal.BlessingStore().Default(), "callback", security.UnconstrainedUse())
	// Put the names of the device manager's default blessings as patterns
	// for the child, so that the child uses the right blessing when talking
	// back to the device manager.
	names, _ := dmPrincipal.BlessingStore().Default().ForContext(call)
	for _, n := range names {
		if _, err := p.BlessingStore().Set(dmBlessings, security.BlessingPattern(n)); err != nil {
			vlog.Errorf("BlessingStore.Set() failed: %v", err)
			return verror.New(ErrOperationFailed, nil)
		}
	}
	// We also want to override the app cycle manager's server blessing in
	// the child (so that the device manager can send RPCs to it).  We
	// signal to the child's app manager to use a randomly generated pattern
	// to extract the right blessing to use from its store for this purpose.
	randomPattern, err := generateRandomString()
	if err != nil {
		vlog.Errorf("generateRandomString() failed: %v", err)
		return verror.New(ErrOperationFailed, nil)
	}
	if _, err := p.BlessingStore().Set(dmBlessings, security.BlessingPattern(randomPattern)); err != nil {
		vlog.Errorf("BlessingStore.Set() failed: %v", err)
		return verror.New(ErrOperationFailed, nil)
	}
	info.DeviceManagerPeerPattern = randomPattern
	if err := p.AddToRoots(dmBlessings); err != nil {
		vlog.Errorf("AddToRoots() failed: %v", err)
		return verror.New(ErrOperationFailed, nil)
	}
	return nil
}

// installationDir returns the path to the directory containing the app
// installation referred to by the invoker's suffix.  Returns an error if the
// suffix does not name an installation or if the named installation does not
// exist.
func (i *appService) installationDir() (string, error) {
	return installationDirCore(i.suffix, i.config.Root)
}

// installPackages installs all the packages for a new instance.
func installPackages(installationDir, versionDir, instanceDir string) error {
	overridePackages, err := loadPackages(installationDir)
	if err != nil {
		return err
	}
	envelope, err := loadEnvelope(versionDir)
	if err != nil {
		return err
	}
	for pkg, _ := range overridePackages {
		delete(envelope.Packages, pkg)
	}
	packagesDir := filepath.Join(instanceDir, "root", "packages")
	if err := os.MkdirAll(packagesDir, os.FileMode(0700)); err != nil {
		return err
	}
	installFrom := func(packages application.Packages, sourceDir string) error {
		// TODO(rthellend): Consider making the packages read-only and
		// sharing them between apps or instances.
		for pkg, _ := range packages {
			pkgFile := filepath.Join(sourceDir, "pkg", pkg)
			dst := filepath.Join(packagesDir, pkg)
			if err := libpackages.Install(pkgFile, dst); err != nil {
				return err
			}
		}
		return nil
	}
	if err := installFrom(envelope.Packages, versionDir); err != nil {
		return err
	}
	return installFrom(overridePackages, installationDir)
}

func (i *appService) initializeSubACLs(principal security.Principal, instanceDir string, blessings []string, acl access.TaggedACLMap) error {
	for _, b := range blessings {
		for _, tag := range access.AllTypicalTags() {
			acl.Add(security.BlessingPattern(b), string(tag))
		}
	}
	aclDir := path.Join(instanceDir, "acls")
	return i.locks.SetPathACL(principal, aclDir, acl, "")
}

// newInstance sets up the directory for a new application instance.
func (i *appService) newInstance(call ipc.ServerContext) (string, string, error) {
	installationDir, err := i.installationDir()
	if err != nil {
		return "", "", err
	}
	if !installationStateIs(installationDir, active) {
		return "", "", verror.New(ErrInvalidOperation, call.Context())
	}
	instanceID := generateID()
	instanceDir := filepath.Join(installationDir, "instances", instanceDirName(instanceID))
	if mkdir(instanceDir) != nil {
		return "", instanceID, verror.New(ErrOperationFailed, call.Context())
	}
	installationLink := filepath.Join(instanceDir, "installation")
	if err := os.Symlink(installationDir, installationLink); err != nil {
		vlog.Errorf("Symlink(%v, %v) failed: %v", installationDir, installationLink, err)
		return instanceDir, instanceID, verror.New(ErrOperationFailed, call.Context())
	}
	currLink := filepath.Join(installationDir, "current")
	versionDir, err := filepath.EvalSymlinks(currLink)
	if err != nil {
		vlog.Errorf("EvalSymlinks(%v) failed: %v", currLink, err)
		return instanceDir, instanceID, verror.New(ErrOperationFailed, call.Context())
	}
	versionLink := filepath.Join(instanceDir, "version")
	if err := os.Symlink(versionDir, versionLink); err != nil {
		vlog.Errorf("Symlink(%v, %v) failed: %v", versionDir, versionLink, err)
		return instanceDir, instanceID, verror.New(ErrOperationFailed, call.Context())
	}
	if err := installPackages(installationDir, versionDir, instanceDir); err != nil {
		vlog.Errorf("installPackages(%v, %v, %v) failed: %v", installationDir, versionDir, instanceDir, err)
		return instanceDir, instanceID, verror.New(ErrOperationFailed, call.Context())
	}
	instanceInfo := new(instanceInfo)
	if err := setupPrincipal(call.Context(), instanceDir, versionDir, call, i.securityAgent, instanceInfo); err != nil {
		return instanceDir, instanceID, err
	}
	if err := saveInstanceInfo(instanceDir, instanceInfo); err != nil {
		return instanceDir, instanceID, err
	}
	if err := initializeInstance(instanceDir, suspended); err != nil {
		return instanceDir, instanceID, err
	}

	blessings, _ := call.RemoteBlessings().ForContext(call)
	if err := i.initializeSubACLs(call.LocalPrincipal(), instanceDir, blessings, i.deviceACL.Copy()); err != nil {
		return instanceDir, instanceID, err
	}
	return instanceDir, instanceID, nil
}

func genCmd(instanceDir, helperPath, systemName string, nsRoot string) (*exec.Cmd, error) {
	versionLink := filepath.Join(instanceDir, "version")
	versionDir, err := filepath.EvalSymlinks(versionLink)
	if err != nil {
		vlog.Errorf("EvalSymlinks(%v) failed: %v", versionLink, err)
		return nil, verror.New(ErrOperationFailed, nil)
	}
	envelope, err := loadEnvelope(versionDir)
	if err != nil {
		return nil, err
	}
	binPath := filepath.Join(versionDir, "bin")
	if _, err := os.Stat(binPath); err != nil {
		vlog.Errorf("Stat(%v) failed: %v", binPath, err)
		return nil, verror.New(ErrOperationFailed, nil)
	}

	cmd := exec.Command(helperPath)

	switch yes, err := suidHelper.suidhelperEnabled(systemName, helperPath); {
	case err != nil:
		return nil, err
	case yes:
		cmd.Args = append(cmd.Args, "--username", systemName)
	case !yes:
		cmd.Args = append(cmd.Args, "--username", systemName, "--dryrun")
	}

	// Set the app's default namespace root to the local namespace.
	cmd.Env = []string{consts.NamespaceRootPrefix + "=" + nsRoot}
	cmd.Env = append(cmd.Env, envelope.Env...)
	rootDir := filepath.Join(instanceDir, "root")
	if err := mkdir(rootDir); err != nil {
		return nil, err
	}
	cmd.Dir = rootDir
	cmd.Args = append(cmd.Args, "--workspace", rootDir)

	logDir := filepath.Join(instanceDir, "logs")
	if err := mkdir(logDir); err != nil {
		return nil, err
	}
	cmd.Args = append(cmd.Args, "--logdir", logDir)
	timestamp := time.Now().UnixNano()

	stdoutLog := filepath.Join(logDir, fmt.Sprintf("STDOUT-%d", timestamp))
	if cmd.Stdout, err = openWriteFile(stdoutLog); err != nil {
		return nil, err
	}
	stderrLog := filepath.Join(logDir, fmt.Sprintf("STDERR-%d", timestamp))
	if cmd.Stderr, err = openWriteFile(stderrLog); err != nil {
		return nil, err
	}
	cmd.Args = append(cmd.Args, "--run", binPath)
	cmd.Args = append(cmd.Args, "--")

	// Args to be passed by helper to the app.
	appArgs := []string{"--log_dir=../logs"}
	appArgs = append(appArgs, envelope.Args...)

	cmd.Args = append(cmd.Args, appArgs...)
	return cmd, nil
}

func (i *appService) startCmd(instanceDir string, cmd *exec.Cmd) (int, error) {
	info, err := loadInstanceInfo(instanceDir)
	if err != nil {
		return 0, err
	}
	// Setup up the child process callback.
	callbackState := i.callback
	listener := callbackState.listenFor(mgmt.AppCycleManagerConfigKey)
	defer listener.cleanup()
	cfg := vexec.NewConfig()
	installationLink := filepath.Join(instanceDir, "installation")
	installationDir, err := filepath.EvalSymlinks(installationLink)
	if err != nil {
		vlog.Errorf("EvalSymlinks(%v) failed: %v", installationLink, err)
		return 0, verror.New(ErrOperationFailed, nil)
	}
	config, err := loadConfig(installationDir)
	if err != nil {
		return 0, err
	}
	for k, v := range config {
		cfg.Set(k, v)
	}
	cfg.Set(mgmt.ParentNameConfigKey, listener.name())
	cfg.Set(mgmt.ProtocolConfigKey, "tcp")
	cfg.Set(mgmt.AddressConfigKey, "127.0.0.1:0")
	cfg.Set(mgmt.ParentBlessingConfigKey, info.DeviceManagerPeerPattern)

	// Set up any agent-specific state.
	// NOTE(caprita): This ought to belong in genCmd.
	var agentCleaner func()
	if sa := i.securityAgent; sa != nil {
		file, err := sa.keyMgrAgent.NewConnection(info.SecurityAgentHandle)
		if err != nil {
			vlog.Errorf("NewConnection(%v) failed: %v", info.SecurityAgentHandle, err)
			return 0, err
		}
		agentCleaner = func() {
			file.Close()
		}
		// We need to account for the file descriptors corresponding to
		// std{err|out|in} as well as the implementation-specific pipes
		// that the vexec library adds to ExtraFiles during
		// handle.Start.  vexec.FileOffset properly offsets fd
		// accordingly.
		fd := len(cmd.ExtraFiles) + vexec.FileOffset
		cmd.ExtraFiles = append(cmd.ExtraFiles, file)
		cfg.Set(mgmt.SecurityAgentFDConfigKey, strconv.Itoa(fd))
	} else {
		cmd.Env = append(cmd.Env, consts.VeyronCredentials+"="+filepath.Join(instanceDir, "credentials"))
	}
	handle := vexec.NewParentHandle(cmd, vexec.ConfigOpt{cfg})
	defer func() {
		if handle != nil {
			if err := handle.Clean(); err != nil {
				vlog.Errorf("Clean() failed: %v", err)
			}
		}
	}()
	// Start the child process.
	if err := handle.Start(); err != nil {
		if agentCleaner != nil {
			agentCleaner()
		}
		vlog.Errorf("Start() failed: %v", err)
		return 0, verror.New(ErrOperationFailed, nil)
	}
	if agentCleaner != nil {
		agentCleaner()
	}

	// Wait for the child process to start.
	if err := handle.WaitForReady(childReadyTimeout); err != nil {
		vlog.Errorf("WaitForReady(%v) failed: %v", childReadyTimeout, err)
		return 0, verror.New(ErrOperationFailed, nil)
	}
	pid := handle.ChildPid()
	childName, err := listener.waitForValue(childReadyTimeout)
	if err != nil {
		return 0, verror.New(ErrOperationFailed, nil)
	}

	// Because suidhelper uses Go's in-built support for setuid forking,
	// handle.Pid() is the pid of suidhelper, not the pid of the app
	// so use the pid returned in the app's ready status.
	info.AppCycleMgrName, info.Pid = childName, pid
	if err := saveInstanceInfo(instanceDir, info); err != nil {
		return 0, err
	}
	handle = nil
	return pid, nil
}

func (i *appService) run(instanceDir, systemName string) error {
	if err := transitionInstance(instanceDir, suspended, starting); err != nil {
		return err
	}
	var pid int

	cmd, err := genCmd(instanceDir, i.config.Helper, systemName, i.mtAddress)
	if err == nil {
		pid, err = i.startCmd(instanceDir, cmd)
	}
	if err != nil {
		transitionInstance(instanceDir, starting, suspended)
		return err
	}
	if err := transitionInstance(instanceDir, starting, started); err != nil {
		return err
	}
	i.reap.startWatching(instanceDir, pid)
	return nil
}

func (i *appService) Start(call ipc.ServerContext) ([]string, error) {
	helper := i.config.Helper
	instanceDir, instanceID, err := i.newInstance(call)

	if err != nil {
		cleanupDir(instanceDir, helper)
		return nil, err
	}

	systemName := suidHelper.usernameForPrincipal(call, i.uat)
	if err := saveSystemNameForInstance(instanceDir, systemName); err != nil {
		cleanupDir(instanceDir, helper)
		return nil, err
	}

	// For now, use the namespace roots of the device manager runtime to
	// pass to the app.
	if err = i.run(instanceDir, systemName); err != nil {
		// TODO(caprita): We should call cleanupDir here, but we don't
		// in order to not lose the logs for the instance (so we can
		// debug why run failed).  Clean this up.
		// cleanupDir(instanceDir, helper)
		return nil, err
	}
	return []string{instanceID}, nil
}

// instanceDir returns the path to the directory containing the app instance
// referred to by the given suffix relative to the given root directory.
func instanceDir(root string, suffix []string) (string, error) {
	if nComponents := len(suffix); nComponents != 3 {
		return "", verror.New(ErrInvalidSuffix, nil)
	}
	app, installation, instance := suffix[0], suffix[1], suffix[2]
	instancesDir := filepath.Join(root, applicationDirName(app), installationDirName(installation), "instances")
	instanceDir := filepath.Join(instancesDir, instanceDirName(instance))
	return instanceDir, nil
}

// instanceDir returns the path to the directory containing the app instance
// referred to by the invoker's suffix, as well as the corresponding stopped
// instance dir.  Returns an error if the suffix does not name an instance.
func (i *appService) instanceDir() (string, error) {
	return instanceDir(i.config.Root, i.suffix)
}

func (i *appService) Resume(call ipc.ServerContext) error {
	instanceDir, err := i.instanceDir()
	if err != nil {
		return err
	}

	systemName := suidHelper.usernameForPrincipal(call, i.uat)
	startSystemName, err := readSystemNameForInstance(instanceDir)
	if err != nil {
		return err
	}

	if startSystemName != systemName {
		return verror.New(verror.NoAccess, call.Context(), "Not allowed to resume an application under a different system name.")
	}
	return i.run(instanceDir, systemName)
}

func stopAppRemotely(ctx *context.T, appVON string) error {
	appStub := appcycle.AppCycleClient(appVON)
	ctx, cancel := context.WithTimeout(ctx, ipcContextLongTimeout)
	defer cancel()
	stream, err := appStub.Stop(ctx)
	if err != nil {
		vlog.Errorf("%v.Stop() failed: %v", appVON, err)
		return verror.New(ErrOperationFailed, nil)
	}
	rstream := stream.RecvStream()
	for rstream.Advance() {
		vlog.VI(2).Infof("%v.Stop() task update: %v", appVON, rstream.Value())
	}
	if err := rstream.Err(); err != nil {
		vlog.Errorf("Advance() failed: %v", err)
		return verror.New(ErrOperationFailed, nil)
	}
	if err := stream.Finish(); err != nil {
		vlog.Errorf("Finish() failed: %v", err)
		return verror.New(ErrOperationFailed, nil)
	}
	return nil
}

func stop(ctx *context.T, instanceDir string, reap reaper) error {
	info, err := loadInstanceInfo(instanceDir)
	if err != nil {
		return err
	}
	err = stopAppRemotely(ctx, info.AppCycleMgrName)
	if err == nil {
		reap.stopWatching(instanceDir)
	}
	return err
}

// TODO(caprita): implement deadline for Stop.

func (i *appService) Stop(ctx ipc.ServerContext, deadline uint32) error {
	instanceDir, err := i.instanceDir()
	if err != nil {
		return err
	}
	if err := transitionInstance(instanceDir, suspended, stopped); verror.Is(err, ErrOperationFailed.ID) || err == nil {
		return err
	}
	if err := transitionInstance(instanceDir, started, stopping); err != nil {
		return err
	}
	if err := stop(ctx.Context(), instanceDir, i.reap); err != nil {
		transitionInstance(instanceDir, stopping, started)
		return err
	}
	return transitionInstance(instanceDir, stopping, stopped)
}

func (i *appService) Suspend(ctx ipc.ServerContext) error {
	instanceDir, err := i.instanceDir()
	if err != nil {
		return err
	}
	if err := transitionInstance(instanceDir, started, suspending); err != nil {
		return err
	}
	if err := stop(ctx.Context(), instanceDir, i.reap); err != nil {
		transitionInstance(instanceDir, suspending, started)
		return err
	}
	return transitionInstance(instanceDir, suspending, suspended)
}

func (i *appService) Uninstall(ipc.ServerContext) error {
	installationDir, err := i.installationDir()
	if err != nil {
		return err
	}
	return transitionInstallation(installationDir, active, uninstalled)
}

func (i *appService) Update(call ipc.ServerContext) error {
	installationDir, err := i.installationDir()
	if err != nil {
		return err
	}
	if !installationStateIs(installationDir, active) {
		return verror.New(ErrInvalidOperation, call.Context())
	}
	originVON, err := loadOrigin(installationDir)
	if err != nil {
		return err
	}
	ctx, cancel := context.WithTimeout(call.Context(), ipcContextLongTimeout)
	defer cancel()
	newEnvelope, err := fetchAppEnvelope(ctx, originVON)
	if err != nil {
		return err
	}
	currLink := filepath.Join(installationDir, "current")
	oldVersionDir, err := filepath.EvalSymlinks(currLink)
	if err != nil {
		vlog.Errorf("EvalSymlinks(%v) failed: %v", currLink, err)
		return verror.New(ErrOperationFailed, call.Context())
	}
	// NOTE(caprita): A race can occur between two competing updates, where
	// both use the old version as their baseline.  This can result in both
	// updates succeeding even if they are updating the app installation to
	// the same new envelope.  This will result in one of the updates
	// becoming the new 'current'.  Both versions will point their
	// 'previous' link to the old version.  This doesn't appear to be of
	// practical concern, so we avoid the complexity of synchronizing
	// updates.
	oldEnvelope, err := loadEnvelope(oldVersionDir)
	if err != nil {
		return err
	}
	if oldEnvelope.Title != newEnvelope.Title {
		return verror.New(ErrAppTitleMismatch, call.Context())
	}
	if reflect.DeepEqual(oldEnvelope, newEnvelope) {
		return verror.New(ErrUpdateNoOp, call.Context())
	}
	versionDir, err := newVersion(call.Context(), installationDir, newEnvelope, oldVersionDir)
	if err != nil {
		cleanupDir(versionDir, "")
		return err
	}
	return nil
}

func (*appService) UpdateTo(_ ipc.ServerContext, von string) error {
	// TODO(jsimsa): Implement.
	return nil
}

func (i *appService) Revert(ctx ipc.ServerContext) error {
	installationDir, err := i.installationDir()
	if err != nil {
		return err
	}
	if !installationStateIs(installationDir, active) {
		return verror.New(ErrInvalidOperation, ctx.Context())
	}
	// NOTE(caprita): A race can occur between an update and a revert, where
	// both use the same current version as their starting point.  This will
	// render the update inconsequential.  This doesn't appear to be of
	// practical concern, so we avoid the complexity of synchronizing
	// updates and revert operations.
	currLink := filepath.Join(installationDir, "current")
	currVersionDir, err := filepath.EvalSymlinks(currLink)
	if err != nil {
		vlog.Errorf("EvalSymlinks(%v) failed: %v", currLink, err)
		return verror.New(ErrOperationFailed, ctx.Context())
	}
	previousLink := filepath.Join(currVersionDir, "previous")
	if _, err := os.Lstat(previousLink); err != nil {
		if os.IsNotExist(err) {
			// No 'previous' link -- must be the first version.
			return verror.New(ErrUpdateNoOp, ctx.Context())
		}
		vlog.Errorf("Lstat(%v) failed: %v", previousLink, err)
		return verror.New(ErrOperationFailed, ctx.Context())
	}
	prevVersionDir, err := filepath.EvalSymlinks(previousLink)
	if err != nil {
		vlog.Errorf("EvalSymlinks(%v) failed: %v", previousLink, err)
		return verror.New(ErrOperationFailed, ctx.Context())
	}
	return updateLink(prevVersionDir, currLink)
}

type treeNode struct {
	children map[string]*treeNode
}

func newTreeNode() *treeNode {
	return &treeNode{children: make(map[string]*treeNode)}
}

func (n *treeNode) find(names []string, create bool) *treeNode {
	for {
		if len(names) == 0 {
			return n
		}
		if next, ok := n.children[names[0]]; ok {
			n = next
			names = names[1:]
			continue
		}
		if create {
			nn := newTreeNode()
			n.children[names[0]] = nn
			n = nn
			names = names[1:]
			continue
		}
		return nil
	}
}

func (i *appService) scanEnvelopes(tree *treeNode, appDir string) {
	// Find all envelopes, extract installID.
	envGlob := []string{i.config.Root, appDir, "installation-*", "*", "envelope"}
	envelopes, err := filepath.Glob(filepath.Join(envGlob...))
	if err != nil {
		vlog.Errorf("unexpected error: %v", err)
		return
	}
	for _, path := range envelopes {
		env, err := loadEnvelope(filepath.Dir(path))
		if err != nil {
			continue
		}
		relpath, _ := filepath.Rel(i.config.Root, path)
		elems := strings.Split(relpath, string(filepath.Separator))
		if len(elems) != len(envGlob)-1 {
			vlog.Errorf("unexpected number of path components: %q (%q)", elems, path)
			continue
		}
		installID := strings.TrimPrefix(elems[1], "installation-")
		tree.find([]string{env.Title, installID}, true)
	}
	return
}

func (i *appService) scanInstances(tree *treeNode) {
	if len(i.suffix) < 2 {
		return
	}
	title := i.suffix[0]
	installDir, err := installationDirCore(i.suffix[:2], i.config.Root)
	if err != nil {
		return
	}
	// Add the node corresponding to the installation itself.
	tree.find(i.suffix[:2], true)
	// Find all instances.
	infoGlob := []string{installDir, "instances", "instance-*", "info"}
	instances, err := filepath.Glob(filepath.Join(infoGlob...))
	if err != nil {
		vlog.Errorf("unexpected error: %v", err)
		return
	}
	for _, path := range instances {
		instanceDir := filepath.Dir(path)
		i.scanInstance(tree, title, instanceDir)
	}
	return
}

func (i *appService) scanInstance(tree *treeNode, title, instanceDir string) {
	if _, err := loadInstanceInfo(instanceDir); err != nil {
		return
	}
	relpath, _ := filepath.Rel(i.config.Root, instanceDir)
	elems := strings.Split(relpath, string(filepath.Separator))
	if len(elems) < 4 {
		vlog.Errorf("unexpected number of path components: %q (%q)", elems, instanceDir)
		return
	}
	installID := strings.TrimPrefix(elems[1], "installation-")
	instanceID := strings.TrimPrefix(elems[3], "instance-")
	tree.find([]string{title, installID, instanceID, "logs"}, true)
	if instanceStateIs(instanceDir, started) {
		for _, obj := range []string{"pprof", "stats"} {
			tree.find([]string{title, installID, instanceID, obj}, true)
		}
	}
}

func (i *appService) GlobChildren__(ipc.ServerContext) (<-chan string, error) {
	tree := newTreeNode()
	switch len(i.suffix) {
	case 0:
		i.scanEnvelopes(tree, "app-*")
	case 1:
		appDir := applicationDirName(i.suffix[0])
		i.scanEnvelopes(tree, appDir)
	case 2:
		i.scanInstances(tree)
	case 3:
		dir, err := i.instanceDir()
		if err != nil {
			break
		}
		i.scanInstance(tree, i.suffix[0], dir)
	default:
		return nil, verror.New(verror.NoExist, nil, i.suffix)
	}
	n := tree.find(i.suffix, false)
	if n == nil {
		return nil, verror.New(ErrInvalidSuffix, nil)
	}
	ch := make(chan string)
	go func() {
		for child, _ := range n.children {
			ch <- child
		}
		close(ch)
	}()
	return ch, nil
}

// TODO(rjkroege): Refactor to eliminate redundancy with newAppSpecificAuthorizer.
func dirFromSuffix(suffix []string, root string) (string, error) {
	if len(suffix) == 2 {
		p, err := installationDirCore(suffix, root)
		if err != nil {
			vlog.Errorf("dirFromSuffix failed: %v", err)
			return "", err
		}
		return p, nil
	} else if len(suffix) > 2 {
		p, err := instanceDir(root, suffix[0:3])
		if err != nil {
			vlog.Errorf("dirFromSuffix failed: %v", err)
			return "", err
		}
		return p, nil
	}
	return "", verror.New(ErrInvalidSuffix, nil)
}

// TODO(rjkroege): Consider maintaining an in-memory ACL cache.
func (i *appService) SetACL(ctx ipc.ServerContext, acl access.TaggedACLMap, etag string) error {
	dir, err := dirFromSuffix(i.suffix, i.config.Root)
	if err != nil {
		return err
	}
	return i.locks.SetPathACL(ctx.LocalPrincipal(), path.Join(dir, "acls"), acl, etag)
}

func (i *appService) GetACL(ctx ipc.ServerContext) (acl access.TaggedACLMap, etag string, err error) {
	dir, err := dirFromSuffix(i.suffix, i.config.Root)
	if err != nil {
		return nil, "", err
	}
	return i.locks.GetPathACL(ctx.LocalPrincipal(), path.Join(dir, "acls"))
}

func (i *appService) Debug(ctx ipc.ServerContext) (string, error) {
	switch len(i.suffix) {
	case 2:
		return i.installationDebug(ctx)
	case 3:
		return i.instanceDebug(ctx)
	default:
		return "", verror.New(ErrInvalidSuffix, nil)
	}
}

func (i *appService) installationDebug(ctx ipc.ServerContext) (string, error) {
	const installationDebug = `Installation dir: {{.InstallationDir}}

Origin: {{.Origin}}

Envelope: {{printf "%+v" .Envelope}}

Config: {{printf "%+v" .Config}}
`
	installationDebugTemplate, err := template.New("installation-debug").Parse(installationDebug)
	if err != nil {
		return "", err
	}

	installationDir, err := i.installationDir()
	if err != nil {
		return "", err
	}
	debugInfo := struct {
		InstallationDir, Origin string
		Envelope                *application.Envelope
		Config                  device.Config
	}{}
	debugInfo.InstallationDir = installationDir

	if origin, err := loadOrigin(installationDir); err != nil {
		return "", err
	} else {
		debugInfo.Origin = origin
	}

	currLink := filepath.Join(installationDir, "current")
	if envelope, err := loadEnvelope(currLink); err != nil {
		return "", err
	} else {
		debugInfo.Envelope = envelope
	}

	if config, err := loadConfig(installationDir); err != nil {
		return "", err
	} else {
		debugInfo.Config = config
	}

	var buf bytes.Buffer
	if err := installationDebugTemplate.Execute(&buf, debugInfo); err != nil {
		return "", err
	}
	return buf.String(), nil

}

func (i *appService) instanceDebug(ctx ipc.ServerContext) (string, error) {
	const instanceDebug = `Instance dir: {{.InstanceDir}}

System name / start system name: {{.SystemName}} / {{.StartSystemName}}

Cmd: {{printf "%+v" .Cmd}}

Info: {{printf "%+v" .Info}}

Principal: {{.PrincipalType}}
Public Key: {{.Principal.PublicKey}}
Blessing Store: {{.Principal.BlessingStore.DebugString}}
Roots: {{.Principal.Roots.DebugString}}
`
	instanceDebugTemplate, err := template.New("instance-debug").Parse(instanceDebug)
	if err != nil {
		return "", err
	}

	instanceDir, err := i.instanceDir()
	if err != nil {
		return "", err
	}
	debugInfo := struct {
		InstanceDir, SystemName, StartSystemName string
		Cmd                                      *exec.Cmd
		Info                                     *instanceInfo
		Principal                                security.Principal
		PrincipalType                            string
	}{}
	debugInfo.InstanceDir = instanceDir

	debugInfo.SystemName = suidHelper.usernameForPrincipal(ctx, i.uat)
	if startSystemName, err := readSystemNameForInstance(instanceDir); err != nil {
		return "", err
	} else {
		debugInfo.StartSystemName = startSystemName
	}
	if cmd, err := genCmd(instanceDir, i.config.Helper, debugInfo.SystemName, i.mtAddress); err != nil {
		return "", err
	} else {
		debugInfo.Cmd = cmd
	}
	if info, err := loadInstanceInfo(instanceDir); err != nil {
		return "", err
	} else {
		debugInfo.Info = info
	}

	if sa := i.securityAgent; sa != nil {
		file, err := sa.keyMgrAgent.NewConnection(debugInfo.Info.SecurityAgentHandle)
		if err != nil {
			vlog.Errorf("NewConnection(%v) failed: %v", debugInfo.Info.SecurityAgentHandle, err)
			return "", err
		}
		var cancel func()
		if debugInfo.Principal, cancel, err = agentPrincipal(ctx.Context(), file); err != nil {
			return "", err
		}
		defer cancel()
		debugInfo.PrincipalType = "Agent-based"
	} else {
		credentialsDir := filepath.Join(instanceDir, "credentials")
		var err error
		if debugInfo.Principal, err = vsecurity.LoadPersistentPrincipal(credentialsDir, nil); err != nil {
			return "", err
		}
		debugInfo.PrincipalType = fmt.Sprintf("Credentials dir-based (%v)", credentialsDir)
	}
	var buf bytes.Buffer
	if err := instanceDebugTemplate.Execute(&buf, debugInfo); err != nil {
		return "", err
	}
	return buf.String(), nil
}