Merge "veyron/runtimes/google/namespace: Namespace should not take a runtime parameter on New."
diff --git a/lib/appcycle/appcycle.go b/lib/appcycle/appcycle.go
index 3dd0004..c94306c 100644
--- a/lib/appcycle/appcycle.go
+++ b/lib/appcycle/appcycle.go
@@ -16,7 +16,6 @@
taskTrackers []chan<- veyron2.Task
task veyron2.Task
shutDown bool
- rt veyron2.Runtime
disp *invoker
}
@@ -24,8 +23,8 @@
ac *AppCycle
}
-func New(rt veyron2.Runtime) *AppCycle {
- ac := &AppCycle{rt: rt}
+func New() *AppCycle {
+ ac := new(AppCycle)
ac.disp = &invoker{ac}
return ac
}
diff --git a/lib/testutil/integration/util.go b/lib/testutil/integration/util.go
index 9968938..d5e2cf4 100644
--- a/lib/testutil/integration/util.go
+++ b/lib/testutil/integration/util.go
@@ -91,23 +91,29 @@
if err != nil {
return nil, err
}
+ // TODO(jsimsa): Consider using the veyron exec library to
+ // facilitate coordination and communication between the
+ // parent and the child process.
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("%q failed: %v", strings.Join(cmd.Args, " "), err)
}
- // Wait for the server to mount itself.
+ // Wait for the server to mount both its tcp and ws endpoint.
ready := make(chan struct{}, 1)
go func() {
defer outPipe.Close()
scanner := bufio.NewScanner(outPipe)
+ nmounts := 0
for scanner.Scan() {
line := scanner.Text()
if strings.Index(line, "ipc pub: mount") != -1 {
- close(ready)
- return
+ nmounts++
+ if nmounts == 2 {
+ close(ready)
+ }
}
}
if err := scanner.Err(); err != nil {
- fmt.Fprintf(os.Stderr, "Scan() failOAed: %v\n", err)
+ fmt.Fprintf(os.Stderr, "Scan() failed: %v\n", err)
}
}()
select {
@@ -115,6 +121,6 @@
return cmd.Process, nil
case <-time.After(time.Second):
cmd.Process.Kill()
- return nil, fmt.Errorf("timed out waiting for %q to mount itself", bin)
+ return nil, fmt.Errorf("timed out waiting for %q to mount itself", strings.Join(cmd.Args, " "))
}
}
diff --git a/lib/websocket/listener.go b/lib/websocket/listener.go
index 80cc980..3c4922e 100644
--- a/lib/websocket/listener.go
+++ b/lib/websocket/listener.go
@@ -152,7 +152,7 @@
continue
}
- vlog.VI(1).Info("Got a connection from %s (local address: %s)", conn.RemoteAddr(), conn.LocalAddr())
+ vlog.VI(1).Infof("Got a connection from %s (local address: %s)", conn.RemoteAddr(), conn.LocalAddr())
// Check to see if it is a regular connection or a http connection.
if magic[0] == BinaryMagicByte {
if _, err := bc.r.ReadByte(); err != nil {
diff --git a/profiles/gce/init.go b/profiles/gce/init.go
index d0f666c..49ead70 100644
--- a/profiles/gce/init.go
+++ b/profiles/gce/init.go
@@ -60,11 +60,11 @@
return "net " + p.Platform().String()
}
-func (p *profile) Init(rt veyron2.Runtime, publisher *config.Publisher) (veyron2.AppCycle, error) {
+func (p *profile) Init(veyron2.Runtime, *config.Publisher) (veyron2.AppCycle, error) {
if !gce.RunningOnGCE() {
return nil, fmt.Errorf("GCE profile used on a non-GCE system")
}
- p.ac = appcycle.New(rt)
+ p.ac = appcycle.New()
ListenSpec.Address = listenAddressFlag.String()
if ip, err := gce.ExternalIPAddress(); err != nil {
return p.ac, err
diff --git a/profiles/generic.go b/profiles/generic.go
index 7442cda..421ff22 100644
--- a/profiles/generic.go
+++ b/profiles/generic.go
@@ -50,7 +50,7 @@
func (g *generic) Init(rt veyron2.Runtime, _ *config.Publisher) (veyron2.AppCycle, error) {
rt.Logger().VI(1).Infof("%s", g)
- g.ac = appcycle.New(rt)
+ g.ac = appcycle.New()
return g.ac, nil
}
diff --git a/profiles/roaming/roaming.go b/profiles/roaming/roaming.go
index 4bf8b26..3d4d638 100644
--- a/profiles/roaming/roaming.go
+++ b/profiles/roaming/roaming.go
@@ -88,7 +88,7 @@
Proxy: lf.ListenProxy,
}
- p.ac = appcycle.New(rt)
+ p.ac = appcycle.New()
// Our address is private, so we test for running on GCE and for its
// 1:1 NAT configuration.
diff --git a/profiles/static/static.go b/profiles/static/static.go
index 86cda56..f30137c 100644
--- a/profiles/static/static.go
+++ b/profiles/static/static.go
@@ -72,7 +72,7 @@
Proxy: lf.ListenProxy,
}
- p.ac = appcycle.New(rt)
+ p.ac = appcycle.New()
// Our address is private, so we test for running on GCE and for its
// 1:1 NAT configuration. GCEPublicAddress returns a non-nil addr
diff --git a/runtimes/google/rt/signal_test.go b/runtimes/google/rt/signal_test.go
index e235e0b..03b6bf1 100644
--- a/runtimes/google/rt/signal_test.go
+++ b/runtimes/google/rt/signal_test.go
@@ -43,8 +43,8 @@
return "myprofile on " + mp.Platform().String()
}
-func (mp *myprofile) Init(rt veyron2.Runtime, _ *config.Publisher) (veyron2.AppCycle, error) {
- return appcycle.New(rt), nil
+func (mp *myprofile) Init(veyron2.Runtime, *config.Publisher) (veyron2.AppCycle, error) {
+ return appcycle.New(), nil
}
func (mp *myprofile) Cleanup() {}
diff --git a/security/agent/keymgr/client.go b/security/agent/keymgr/client.go
index 7a93f31..fa36a4c 100644
--- a/security/agent/keymgr/client.go
+++ b/security/agent/keymgr/client.go
@@ -11,7 +11,17 @@
"veyron.io/veyron/veyron/lib/unixfd"
"veyron.io/veyron/veyron/security/agent/server"
"veyron.io/veyron/veyron2/context"
- "veyron.io/veyron/veyron2/verror"
+ verror "veyron.io/veyron/veyron2/verror2"
+)
+
+const pkgPath = "veyron.io/veyron/veyron/security/agent/keymgr"
+
+// Errors
+var (
+ errInvalidResponse = verror.Register(pkgPath+".errInvalidResponse",
+ verror.NoRetry, "{1:}{2:} invalid response from agent. (expected {3} bytes, got {4})")
+ errInvalidKeyHandle = verror.Register(pkgPath+".errInvalidKeyHandle",
+ verror.NoRetry, "{1:}{2:} Invalid key handle")
)
const defaultManagerSocket = 4
@@ -42,7 +52,7 @@
// NewPrincipal creates a new principal and returns the handle and a socket serving
// the principal.
// Typically the socket will be passed to a child process using cmd.ExtraFiles.
-func (a *Agent) NewPrincipal(_ context.T, inMemory bool) (handle []byte, conn *os.File, err error) {
+func (a *Agent) NewPrincipal(ctx context.T, inMemory bool) (handle []byte, conn *os.File, err error) {
req := make([]byte, 1)
if inMemory {
req[0] = 1
@@ -61,7 +71,7 @@
}
if n != server.PrincipalHandleByteSize {
conn.Close()
- return nil, nil, verror.BadProtocolf("invalid response from agent. (expected %d bytes, got %d)", server.PrincipalHandleByteSize, n)
+ return nil, nil, verror.Make(errInvalidResponse, ctx, server.PrincipalHandleByteSize, n)
}
return buf, conn, nil
}
@@ -84,7 +94,7 @@
// Typically this will be passed to a child process using cmd.ExtraFiles.
func (a *Agent) NewConnection(handle []byte) (*os.File, error) {
if len(handle) != server.PrincipalHandleByteSize {
- return nil, verror.BadArgf("Invalid key handle")
+ return nil, verror.Make(errInvalidKeyHandle, nil)
}
a.mu.Lock()
defer a.mu.Unlock()
diff --git a/security/agent/server/server.go b/security/agent/server/server.go
index b753987..37f6970 100644
--- a/security/agent/server/server.go
+++ b/security/agent/server/server.go
@@ -22,12 +22,22 @@
"veyron.io/veyron/veyron2/options"
"veyron.io/veyron/veyron2/security"
"veyron.io/veyron/veyron2/vdl/vdlutil"
- "veyron.io/veyron/veyron2/verror"
+ verror "veyron.io/veyron/veyron2/verror2"
"veyron.io/veyron/veyron2/vlog"
)
const PrincipalHandleByteSize = sha512.Size
+const pkgPath = "veyron.io/veyron/veyron/security/agent/server"
+
+// Errors
+var (
+ errStoragePathRequired = verror.Register(pkgPath+".errStoragePathRequired",
+ verror.NoRetry, "{1:}{2:} RunKeyManager: storage path is required")
+ errNotMultiKeyMode = verror.Register(pkgPath+".errNotMultiKeyMode",
+ verror.NoRetry, "{1:}{2:} Not running in multi-key mode")
+)
+
type keyHandle [PrincipalHandleByteSize]byte
type agentd struct {
@@ -63,7 +73,7 @@
// The returned 'client' is typically passed via cmd.ExtraFiles to a child process.
func RunKeyManager(runtime veyron2.Runtime, path string, passphrase []byte) (client *os.File, err error) {
if path == "" {
- return nil, verror.BadArgf("storage path is required")
+ return nil, verror.Make(errStoragePathRequired, nil)
}
mgr := &keymgr{path: path, passphrase: passphrase, principals: make(map[keyHandle]security.Principal), runtime: runtime}
@@ -245,7 +255,7 @@
func (a keymgr) newKey(in_memory bool) (id []byte, p security.Principal, err error) {
if a.path == "" {
- return nil, nil, verror.NoAccessf("not running in multi-key mode")
+ return nil, nil, verror.Make(errNotMultiKeyMode, nil)
}
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
keyHandle, err := keyid(key)
diff --git a/services/identity/auditor/sql_database.go b/services/identity/auditor/sql_database.go
index aeb9af6..36a8edf 100644
--- a/services/identity/auditor/sql_database.go
+++ b/services/identity/auditor/sql_database.go
@@ -3,7 +3,6 @@
import (
"database/sql"
"fmt"
- _ "github.com/go-sql-driver/mysql"
"time"
"veyron.io/veyron/veyron2/vlog"
diff --git a/services/identity/identityd/main.go b/services/identity/identityd/main.go
index ccce46f..58a5a75 100644
--- a/services/identity/identityd/main.go
+++ b/services/identity/identityd/main.go
@@ -6,6 +6,7 @@
"database/sql"
"flag"
"fmt"
+ _ "github.com/go-sql-driver/mysql"
"html/template"
"io/ioutil"
"net"
diff --git a/services/mgmt/application/applicationd/test.sh b/services/mgmt/application/applicationd/test.sh
deleted file mode 100755
index 7f40da2..0000000
--- a/services/mgmt/application/applicationd/test.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-
-# Test the application repository daemon.
-#
-# This test starts an application repository daemon and uses the
-# application repository client to verify that <application>.Put(),
-# <application>.Match(), and <application>.Remove() work as expected.
-
-source "${VEYRON_ROOT}/scripts/lib/shell_test.sh"
-
-readonly WORKDIR="${shell_test_WORK_DIR}"
-
-build() {
- APPLICATIOND_BIN="$(shell_test::build_go_binary 'veyron.io/veyron/veyron/services/mgmt/application/applicationd')"
- APPLICATION_BIN="$(shell_test::build_go_binary 'veyron.io/veyron/veyron/tools/application')"
-}
-
-main() {
- cd "${WORKDIR}"
- build
-
- shell_test::setup_server_test
-
- # Start the application repository daemon.
- local -r REPO="applicationd-test-repo"
- local -r STORE=$(shell::tmp_dir)
- shell_test::start_server "${APPLICATIOND_BIN}" --name="${REPO}" --store="${STORE}" --veyron.tcp.address=127.0.0.1:0 \
- || shell_test::fail "line ${LINENO} failed to start applicationd"
-
- # Create an application envelope.
- local -r APPLICATION="${REPO}/test-application/v1"
- local -r PROFILE="test-profile"
- local -r ENVELOPE_WANT=$(shell::tmp_file)
- cat > "${ENVELOPE_WANT}" <<EOF
-{"Title":"title", "Args":[], "Binary":"foo", "Env":[]}
-EOF
- "${APPLICATION_BIN}" put "${APPLICATION}" "${PROFILE}" "${ENVELOPE_WANT}" || shell_test::fail "line ${LINENO}: 'put' failed"
-
- # Match the application envelope.
- local -r ENVELOPE_GOT=$(shell::tmp_file)
- "${APPLICATION_BIN}" match "${APPLICATION}" "${PROFILE}" | tee "${ENVELOPE_GOT}" || shell_test::fail "line ${LINENO}: 'match' failed"
- if [[ $(cmp "${ENVELOPE_WANT}" "${ENVELOPE_GOT}" &> /dev/null) ]]; then
- shell_test::fail "mismatching application envelopes"
- fi
-
- # Remove the application envelope.
- "${APPLICATION_BIN}" remove "${APPLICATION}" "${PROFILE}" || shell_test::fail "line ${LINENO}: 'remove' failed"
-
- # Check the application envelope no longer exists.
- local -r RESULT=$(shell::check_result "${APPLICATION_BIN}" match "${APPLICATION}" "${PROFILE}")
- shell_test::assert_ne "${RESULT}" "0" "${LINENO}"
-
- shell_test::pass
-}
-
-main "$@"
diff --git a/services/mgmt/application/applicationd/testdata/integration_test.go b/services/mgmt/application/applicationd/testdata/integration_test.go
new file mode 100644
index 0000000..aa0f440
--- /dev/null
+++ b/services/mgmt/application/applicationd/testdata/integration_test.go
@@ -0,0 +1,134 @@
+package integration_test
+
+import (
+ "bytes"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "veyron.io/veyron/veyron/lib/modules"
+ "veyron.io/veyron/veyron/lib/testutil/integration"
+ "veyron.io/veyron/veyron/lib/testutil/security"
+ _ "veyron.io/veyron/veyron/profiles"
+ "veyron.io/veyron/veyron2/naming"
+)
+
+var binPkgs = []string{
+ "veyron.io/veyron/veyron/services/mgmt/application/applicationd",
+ "veyron.io/veyron/veyron/tools/application",
+}
+
+func helper(t *testing.T, expectError bool, binDir, credentials, mt, cmd string, args ...string) string {
+ var out bytes.Buffer
+ args = append([]string{"-veyron.credentials=" + credentials, "-veyron.namespace.root=" + mt, cmd}, args...)
+ command := exec.Command(filepath.Join(binDir, "application"), args...)
+ command.Stdout = &out
+ command.Stderr = &out
+ err := command.Run()
+ if err != nil && !expectError {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(command.Args, " "), err, out.String())
+ }
+ if err == nil && expectError {
+ t.Fatalf("%q did not fail when it should", strings.Join(command.Args, " "))
+ }
+ return strings.TrimSpace(out.String())
+
+}
+
+func matchEnvelope(t *testing.T, expectError bool, binDir, credentials, mt, name, suffix string) string {
+ return helper(t, expectError, binDir, credentials, mt, "match", naming.Join(name, suffix), "test-profile")
+}
+
+func putEnvelope(t *testing.T, binDir, credentials, mt, name, suffix, envelope string) string {
+ return helper(t, false, binDir, credentials, mt, "put", naming.Join(name, suffix), "test-profile", envelope)
+}
+
+func removeEnvelope(t *testing.T, binDir, credentials, mt, name, suffix string) string {
+ return helper(t, false, binDir, credentials, mt, "remove", naming.Join(name, suffix), "test-profile")
+}
+
+func TestHelperProcess(t *testing.T) {
+ modules.DispatchInTest()
+}
+
+func TestApplicationRepository(t *testing.T) {
+ // Build the required binaries.
+ binDir, cleanup, err := integration.BuildPkgs(binPkgs)
+ if err != nil {
+ t.Fatalf("%v", err)
+ }
+ defer cleanup()
+
+ // Start a root mount table.
+ shell, err := modules.NewShell(nil)
+ if err != nil {
+ t.Fatalf("NewShell() failed: %v", err)
+ }
+ defer shell.Cleanup(os.Stdin, os.Stderr)
+ handle, mt, err := integration.StartRootMT(shell)
+ if err != nil {
+ t.Fatalf("%v", err)
+ }
+ defer handle.CloseStdin()
+
+ // Generate credentials.
+ root := security.NewPrincipal("root")
+ credentials := security.NewVeyronCredentials(root, "test-credentials")
+ defer os.RemoveAll(credentials)
+
+ // Start the application repository.
+ appRepoBin := filepath.Join(binDir, "applicationd")
+ appRepoName := "test-app-repo"
+ appRepoStore, err := ioutil.TempDir("", "")
+ if err != nil {
+ t.Fatalf("TempDir() failed: %v", err)
+ }
+ defer os.RemoveAll(appRepoStore)
+ args := []string{
+ "-name=" + appRepoName,
+ "-store=" + appRepoStore,
+ "-veyron.tcp.address=127.0.0.1:0",
+ "-veyron.credentials=" + credentials,
+ "-veyron.namespace.root=" + mt,
+ }
+ serverProcess, err := integration.StartServer(appRepoBin, args)
+ if err != nil {
+ t.Fatalf("%v", err)
+ }
+ defer serverProcess.Kill()
+
+ // Create an application envelope.
+ appRepoSuffix := "test-application/v1"
+ appEnvelopeFile, err := ioutil.TempFile("", "")
+ if err != nil {
+ t.Fatalf("TempFile() failed: %v", err)
+ }
+ defer appEnvelopeFile.Close()
+ defer os.Remove(appEnvelopeFile.Name())
+ wantEnvelope := `{
+ "Title": "title",
+ "Args": null,
+ "Binary": "foo",
+ "Env": null,
+ "Packages": null
+}`
+ if _, err := appEnvelopeFile.Write([]byte(wantEnvelope)); err != nil {
+ t.Fatalf("Write() failed: %v", err)
+ }
+ putEnvelope(t, binDir, credentials, mt, appRepoName, appRepoSuffix, appEnvelopeFile.Name())
+
+ // Match the application envelope.
+ gotEnvelope := matchEnvelope(t, false, binDir, credentials, mt, appRepoName, appRepoSuffix)
+ if gotEnvelope != wantEnvelope {
+ t.Fatalf("unexpected output: got %v, want %v", gotEnvelope, wantEnvelope)
+ }
+
+ // Remove the application envelope.
+ removeEnvelope(t, binDir, credentials, mt, appRepoName, appRepoSuffix)
+
+ // Check that the application envelope no longer exists.
+ matchEnvelope(t, true, binDir, credentials, mt, appRepoName, appRepoSuffix)
+}
diff --git a/services/mgmt/binary/binaryd/main.go b/services/mgmt/binary/binaryd/main.go
index 16ac319..3ac83da 100644
--- a/services/mgmt/binary/binaryd/main.go
+++ b/services/mgmt/binary/binaryd/main.go
@@ -20,9 +20,9 @@
const defaultDepth = 3
var (
- name = flag.String("name", "", "name to mount the binary repository as")
- rootFlag = flag.String("root", "", "root directory for the binary repository")
- httpAddr = flag.String("http", ":0", "TCP address on which the HTTP server runs")
+ name = flag.String("name", "", "name to mount the binary repository as")
+ rootDirFlag = flag.String("root_dir", "", "root directory for the binary repository")
+ httpAddr = flag.String("http", ":0", "TCP address on which the HTTP server runs")
)
// toIPPort tries to swap in the 'best' accessible IP for the host part of the
@@ -53,32 +53,31 @@
}
defer runtime.Cleanup()
- root, err := impl.SetupRoot(*rootFlag)
+ rootDir, err := impl.SetupRootDir(*rootDirFlag)
if err != nil {
- vlog.Errorf("SetupRoot(%q) failed: %v", *rootFlag, err)
+ vlog.Errorf("SetupRootDir(%q) failed: %v", *rootDirFlag, err)
return
}
- vlog.Infof("Binary repository rooted at %v", root)
-
- state, err := impl.NewState(root, defaultDepth)
- if err != nil {
- vlog.Errorf("NewState(%v, %v) failed: %v", root, defaultDepth, err)
- return
- }
+ vlog.Infof("Binary repository rooted at %v", rootDir)
listener, err := net.Listen("tcp", *httpAddr)
if err != nil {
vlog.Errorf("Listen(%s) failed: %v", *httpAddr, err)
os.Exit(1)
}
- vlog.Infof("Binary repository HTTP server at: %q", toIPPort(listener.Addr().String()))
+ rootURL := toIPPort(listener.Addr().String())
+ state, err := impl.NewState(rootDir, rootURL, defaultDepth)
+ if err != nil {
+ vlog.Errorf("NewState(%v, %v, %v) failed: %v", rootDir, rootURL, defaultDepth, err)
+ return
+ }
+ vlog.Infof("Binary repository HTTP server at: %q", rootURL)
go func() {
if err := http.Serve(listener, http.FileServer(impl.NewHTTPRoot(state))); err != nil {
vlog.Errorf("Serve() failed: %v", err)
os.Exit(1)
}
}()
-
server, err := runtime.NewServer()
if err != nil {
vlog.Errorf("NewServer() failed: %v", err)
diff --git a/services/mgmt/binary/binaryd/test.sh b/services/mgmt/binary/binaryd/test.sh
deleted file mode 100755
index 17347fa..0000000
--- a/services/mgmt/binary/binaryd/test.sh
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/bash
-
-# Test the binary repository daemon.
-#
-# This test starts a binary repository daemon and uses the binary
-# repository client to verify that <binary>.Upload(),
-# <binary>.Download(), and <binary>.Delete() work as expected.
-
-source "${VEYRON_ROOT}/scripts/lib/shell_test.sh"
-
-readonly WORKDIR="${shell_test_WORK_DIR}"
-
-build() {
- BINARYD_BIN="$(shell_test::build_go_binary 'veyron.io/veyron/veyron/services/mgmt/binary/binaryd')"
- BINARY_BIN="$(shell_test::build_go_binary 'veyron.io/veyron/veyron/tools/binary')"
-}
-
-main() {
- cd "${WORKDIR}"
- build
-
- shell_test::setup_server_test
-
- # Start the binary repository daemon.
- local -r REPO="binaryd-test-repo"
- shell_test::start_server "${BINARYD_BIN}" --name="${REPO}" --veyron.tcp.address=127.0.0.1:0 --http=127.0.0.1:0 \
- || shell_test::fail "line ${LINENO} failed to start binaryd"
- local -r HTTP_ADDR=$(grep 'HTTP server at: "' "${START_SERVER_LOG_FILE}" | sed -e 's/^.*HTTP server at: "//' | sed -e 's/"$//')
-
- # Create a binary file.
- local -r BINARY_SUFFIX="test-binary"
- local -r BINARY="${REPO}/${BINARY_SUFFIX}"
- local -r BINARY_FILE="${WORKDIR}/bin1"
- dd if=/dev/urandom of="${BINARY_FILE}" bs=1000000 count=16 \
- || shell_test::fail "line ${LINENO}: faile to create a random binary file"
- "${BINARY_BIN}" upload "${BINARY}" "${BINARY_FILE}" || shell_test::fail "line ${LINENO}: 'upload' failed"
-
- # Create TAR file.
- local -r TAR="${REPO}/tarobj"
- local -r TAR_FILE="${WORKDIR}/bin1.tar.gz"
- tar zcvf "${TAR_FILE}" "${BINARY_FILE}"
- "${BINARY_BIN}" upload "${TAR}" "${TAR_FILE}" || shell_test::fail "line ${LINENO}: 'upload' failed"
-
- # Download the binary file.
- local -r BINARY_FILE2="${WORKDIR}/bin2"
- "${BINARY_BIN}" download "${BINARY}" "${BINARY_FILE2}" || shell_test::fail "line ${LINENO}: 'RPC download' failed"
- if [[ $(cmp "${BINARY_FILE}" "${BINARY_FILE2}" &> /dev/null) ]]; then
- shell_test::fail "mismatching binary file downloaded via RPC"
- fi
- local -r BINARY_FILE2_INFO=$(cat "${BINARY_FILE2}.__info")
- shell_test::assert_eq "${BINARY_FILE2_INFO}" '{"Type":"application/octet-stream","Encoding":""}' "${LINENO}"
-
- # Download the tar file.
- local -r TAR_FILE2="${WORKDIR}/downloadedtar"
- "${BINARY_BIN}" download "${TAR}" "${TAR_FILE2}" || shell_test::fail "line ${LINENO}: 'RPC download' failed"
- if [[ $(cmp "${TAR_FILE}" "${TAR_FILE2}" &> /dev/null) ]]; then
- shell_test::fail "mismatching tar file downloaded via RPC"
- fi
- local -r TAR_FILE2_INFO=$(cat "${TAR_FILE2}.__info")
- shell_test::assert_eq "${TAR_FILE2_INFO}" '{"Type":"application/x-tar","Encoding":"gzip"}' "${LINENO}"
-
- local -r BINARY_FILE3="${WORKDIR}/bin3"
- curl -f -o "${BINARY_FILE3}" "http://${HTTP_ADDR}/${BINARY_SUFFIX}" || shell_test::fail "line ${LINENO}: 'HTTP download' failed"
- if [[ $(cmp "${BINARY_FILE}" "${BINARY_FILE3}" &> /dev/null) ]]; then
- shell_test::fail "mismatching binary file downloaded via HTTP"
- fi
-
- # Remove the files.
- "${BINARY_BIN}" delete "${BINARY}" || shell_test::fail "line ${LINENO}: 'delete' failed"
- "${BINARY_BIN}" delete "${TAR}" || shell_test::fail "line ${LINENO}: 'delete' failed"
-
- # Check the files no longer exist.
- local RESULT=$(shell::check_result "${BINARY_BIN}" download "${BINARY}" "${BINARY_FILE2}")
- shell_test::assert_ne "${RESULT}" "0" "${LINENO}"
- RESULT=$(shell::check_result "${BINARY_BIN}" download "${TAR}" "${TAR_FILE2}")
- shell_test::assert_ne "${RESULT}" "0" "${LINENO}"
-
- shell_test::pass
-}
-
-main "$@"
diff --git a/services/mgmt/binary/binaryd/testdata/integration_test.go b/services/mgmt/binary/binaryd/testdata/integration_test.go
new file mode 100644
index 0000000..7bd82f6
--- /dev/null
+++ b/services/mgmt/binary/binaryd/testdata/integration_test.go
@@ -0,0 +1,242 @@
+package integration_test
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "veyron.io/veyron/veyron/lib/modules"
+ "veyron.io/veyron/veyron/lib/testutil"
+ "veyron.io/veyron/veyron/lib/testutil/integration"
+ "veyron.io/veyron/veyron/lib/testutil/security"
+ _ "veyron.io/veyron/veyron/profiles"
+ "veyron.io/veyron/veyron2/naming"
+)
+
+func init() {
+ testutil.Init()
+}
+
+var binPkgs = []string{
+ "veyron.io/veyron/veyron/services/mgmt/binary/binaryd",
+ "veyron.io/veyron/veyron/tools/binary",
+}
+
+func checkFileType(t *testing.T, file, typeString string) {
+ var catOut bytes.Buffer
+ catCmd := exec.Command("cat", file+".__info")
+ catCmd.Stdout = &catOut
+ catCmd.Stderr = &catOut
+ if err := catCmd.Run(); err != nil {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(catCmd.Args, " "), err, catOut.String())
+ }
+ if got, want := strings.TrimSpace(catOut.String()), typeString; got != want {
+ t.Fatalf("unexpect file type: got %v, want %v", got, want)
+ }
+}
+
+func compareFiles(t *testing.T, f1, f2 string) {
+ var cmpOut bytes.Buffer
+ cmpCmd := exec.Command("cmp", f1, f2)
+ cmpCmd.Stdout = &cmpOut
+ cmpCmd.Stderr = &cmpOut
+ if err := cmpCmd.Run(); err != nil {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(cmpCmd.Args, " "), err, cmpOut.String())
+ }
+}
+
+func deleteFile(t *testing.T, binDir, credentials, mt, name, suffix string) {
+ var deleteOut bytes.Buffer
+ deleteArgs := []string{
+ "-veyron.credentials=" + credentials,
+ "-veyron.namespace.root=" + mt,
+ "delete", naming.Join(name, suffix),
+ }
+ deleteCmd := exec.Command(filepath.Join(binDir, "binary"), deleteArgs...)
+ deleteCmd.Stdout = &deleteOut
+ deleteCmd.Stderr = &deleteOut
+ if err := deleteCmd.Run(); err != nil {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(deleteCmd.Args, " "), err, deleteOut.String())
+ }
+}
+
+func downloadFile(t *testing.T, expectError bool, binDir, credentials, mt, name, path, suffix string) {
+ var downloadOut bytes.Buffer
+ downloadArgs := []string{
+ "-veyron.credentials=" + credentials,
+ "-veyron.namespace.root=" + mt,
+ "download", naming.Join(name, suffix), path,
+ }
+ downloadCmd := exec.Command(filepath.Join(binDir, "binary"), downloadArgs...)
+ downloadCmd.Stdout = &downloadOut
+ downloadCmd.Stderr = &downloadOut
+ err := downloadCmd.Run()
+ if err != nil && !expectError {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(downloadCmd.Args, " "), err, downloadOut.String())
+ }
+ if err == nil && expectError {
+ t.Fatalf("%q did not fail when it should", strings.Join(downloadCmd.Args, " "))
+ }
+}
+
+func downloadURL(t *testing.T, path, rootURL, suffix string) {
+ var curlOut bytes.Buffer
+ curlCmd := exec.Command("curl", "-f", "-o", path, fmt.Sprintf("%v/%v", rootURL, suffix))
+ curlCmd.Stdout = &curlOut
+ curlCmd.Stderr = &curlOut
+ if err := curlCmd.Run(); err != nil {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(curlCmd.Args, " "), err, curlOut.String())
+ }
+}
+
+func rootURL(t *testing.T, binDir, credentials, mt, name string) string {
+ var rootOut bytes.Buffer
+ rootArgs := []string{
+ "-veyron.credentials=" + credentials,
+ "-veyron.namespace.root=" + mt,
+ "url", name,
+ }
+ rootCmd := exec.Command(filepath.Join(binDir, "binary"), rootArgs...)
+ rootCmd.Stdout = &rootOut
+ rootCmd.Stderr = &rootOut
+ if err := rootCmd.Run(); err != nil {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(rootCmd.Args, " "), err, rootOut.String())
+ }
+ return strings.TrimSpace(rootOut.String())
+}
+
+func uploadFile(t *testing.T, binDir, credentials, mt, name, path, suffix string) {
+ var uploadOut bytes.Buffer
+ uploadArgs := []string{
+ "-veyron.credentials=" + credentials,
+ "-veyron.namespace.root=" + mt,
+ "upload", naming.Join(name, suffix), path,
+ }
+ uploadCmd := exec.Command(filepath.Join(binDir, "binary"), uploadArgs...)
+ uploadCmd.Stdout = &uploadOut
+ uploadCmd.Stderr = &uploadOut
+ if err := uploadCmd.Run(); err != nil {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(uploadCmd.Args, " "), err, uploadOut.String())
+ }
+}
+
+func TestHelperProcess(t *testing.T) {
+ modules.DispatchInTest()
+}
+
+func TestBinaryRepositoryIntegration(t *testing.T) {
+ // Build the required binaries.
+ binDir, cleanup, err := integration.BuildPkgs(binPkgs)
+ if err != nil {
+ t.Fatalf("%v", err)
+ }
+ defer cleanup()
+
+ // Start a root mount table.
+ shell, err := modules.NewShell(nil)
+ if err != nil {
+ t.Fatalf("NewShell() failed: %v", err)
+ }
+ defer shell.Cleanup(os.Stdin, os.Stderr)
+ handle, mt, err := integration.StartRootMT(shell)
+ if err != nil {
+ t.Fatalf("%v", err)
+ }
+ defer handle.CloseStdin()
+
+ // Generate credentials.
+ principal := security.NewPrincipal("root")
+ credentials := security.NewVeyronCredentials(principal, "test-credentials")
+ defer os.RemoveAll(credentials)
+
+ // Start the build server.
+ binaryRepoBin := filepath.Join(binDir, "binaryd")
+ binaryRepoName := "test-binary-repository"
+ args := []string{
+ "-name=" + binaryRepoName,
+ "-http=127.0.0.1:0",
+ "-veyron.tcp.address=127.0.0.1:0",
+ "-veyron.credentials=" + credentials,
+ "-veyron.namespace.root=" + mt,
+ }
+ serverProcess, err := integration.StartServer(binaryRepoBin, args)
+ if err != nil {
+ t.Fatalf("%v", err)
+ }
+ defer serverProcess.Kill()
+
+ // Upload a random binary file.
+ binFile, err := ioutil.TempFile("", "")
+ if err != nil {
+ t.Fatalf("TempFile() failed: %v", err)
+ }
+ defer binFile.Close()
+ defer os.Remove(binFile.Name())
+ if _, err := binFile.Write(testutil.RandomBytes(16 * 1000 * 1000)); err != nil {
+ t.Fatalf("Write() failed: %v", err)
+ }
+ binSuffix := "test-binary"
+ uploadFile(t, binDir, credentials, mt, binaryRepoName, binFile.Name(), binSuffix)
+
+ // Upload a compressed version of the binary file.
+ tarFile := binFile.Name() + ".tar.gz"
+ var tarOut bytes.Buffer
+ tarCmd := exec.Command("tar", "zcvf", tarFile, binFile.Name())
+ tarCmd.Stdout = &tarOut
+ tarCmd.Stderr = &tarOut
+ if err := tarCmd.Run(); err != nil {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(tarCmd.Args, " "), err, tarOut.String())
+ }
+ defer os.Remove(tarFile)
+ tarSuffix := "test-compressed-file"
+ uploadFile(t, binDir, credentials, mt, binaryRepoName, tarFile, tarSuffix)
+
+ // Download the binary file and check that it matches the
+ // original one and that it has the right file type.
+ downloadedBinFile := binFile.Name() + "-downloaded"
+ defer os.Remove(downloadedBinFile)
+ downloadFile(t, false, binDir, credentials, mt, binaryRepoName, downloadedBinFile, binSuffix)
+ compareFiles(t, binFile.Name(), downloadedBinFile)
+ checkFileType(t, downloadedBinFile, `{"Type":"application/octet-stream","Encoding":""}`)
+
+ // Download the compressed version of the binary file and
+ // check that it matches the original one and that it has the
+ // right file type.
+ downloadedTarFile := binFile.Name() + "-downloaded.tar.gz"
+ defer os.Remove(downloadedTarFile)
+ downloadFile(t, false, binDir, credentials, mt, binaryRepoName, downloadedTarFile, tarSuffix)
+ compareFiles(t, tarFile, downloadedTarFile)
+ checkFileType(t, downloadedTarFile, `{"Type":"application/x-tar","Encoding":"gzip"}`)
+
+ // Fetch the root URL of the HTTP server used by the binary
+ // repository to serve URLs.
+ root := rootURL(t, binDir, credentials, mt, binaryRepoName)
+
+ // Download the binary file using the HTTP protocol and check
+ // that it matches the original one.
+ downloadedBinFileURL := binFile.Name() + "-downloaded-url"
+ defer os.Remove(downloadedBinFileURL)
+ downloadURL(t, downloadedBinFileURL, root, binSuffix)
+ compareFiles(t, downloadedBinFile, downloadedBinFileURL)
+
+ // Download the compressed version of the binary file using
+ // the HTTP protocol and check that it matches the original
+ // one.
+ downloadedTarFileURL := binFile.Name() + "-downloaded-url.tar.gz"
+ defer os.Remove(downloadedTarFileURL)
+ downloadURL(t, downloadedTarFileURL, root, tarSuffix)
+ compareFiles(t, downloadedTarFile, downloadedTarFileURL)
+
+ // Delete the files.
+ deleteFile(t, binDir, credentials, mt, binaryRepoName, binSuffix)
+ deleteFile(t, binDir, credentials, mt, binaryRepoName, tarSuffix)
+
+ // Check the files no longer exist.
+ downloadFile(t, true, binDir, credentials, mt, binaryRepoName, downloadedBinFile, binSuffix)
+ downloadFile(t, true, binDir, credentials, mt, binaryRepoName, downloadedTarFile, tarSuffix)
+}
diff --git a/services/mgmt/binary/impl/fs_utils.go b/services/mgmt/binary/impl/fs_utils.go
index 62eaab8..3240d31 100644
--- a/services/mgmt/binary/impl/fs_utils.go
+++ b/services/mgmt/binary/impl/fs_utils.go
@@ -92,7 +92,7 @@
// createObjectNameTree returns a tree of all the valid object names in the
// repository.
func (i *binaryService) createObjectNameTree() *treeNode {
- pattern := i.state.root
+ pattern := i.state.rootDir
for d := 0; d < i.state.depth; d++ {
pattern = filepath.Join(pattern, "*")
}
diff --git a/services/mgmt/binary/impl/impl_test.go b/services/mgmt/binary/impl/impl_test.go
index 6f603e0..b336cd1 100644
--- a/services/mgmt/binary/impl/impl_test.go
+++ b/services/mgmt/binary/impl/impl_test.go
@@ -105,11 +105,11 @@
// startServer starts the binary repository server.
func startServer(t *testing.T, depth int) (repository.BinaryClientMethods, string, string, func()) {
// Setup the root of the binary repository.
- root, err := ioutil.TempDir("", veyronPrefix)
+ rootDir, err := ioutil.TempDir("", veyronPrefix)
if err != nil {
t.Fatalf("TempDir() failed: %v", err)
}
- path, perm := filepath.Join(root, VersionFile), os.FileMode(0600)
+ path, perm := filepath.Join(rootDir, VersionFile), os.FileMode(0600)
if err := ioutil.WriteFile(path, []byte(Version), perm); err != nil {
vlog.Fatalf("WriteFile(%v, %v, %v) failed: %v", path, Version, perm, err)
}
@@ -118,14 +118,14 @@
if err != nil {
t.Fatalf("NewServer() failed: %v", err)
}
- state, err := NewState(root, depth)
- if err != nil {
- t.Fatalf("NewState(%v, %v) failed: %v", root, depth, err)
- }
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
+ state, err := NewState(rootDir, listener.Addr().String(), depth)
+ if err != nil {
+ t.Fatalf("NewState(%v, %v) failed: %v", rootDir, listener.Addr().String(), depth, err)
+ }
go func() {
if err := http.Serve(listener, http.FileServer(NewHTTPRoot(state))); err != nil {
vlog.Fatalf("Serve() failed: %v", err)
@@ -148,12 +148,10 @@
t.Fatalf("Stop() failed: %v", err)
}
if err := os.RemoveAll(path); err != nil {
- t.Fatalf("Remove(%v) failed: %v", path, err)
+ t.Fatalf("RemoveAll(%v) failed: %v", path, err)
}
- // Check that any directories and files that were created to
- // represent the binary objects have been garbage collected.
- if err := os.RemoveAll(root); err != nil {
- t.Fatalf("Remove(%v) failed: %v", root, err)
+ if err := os.RemoveAll(rootDir); err != nil {
+ t.Fatalf("RemoveAll(%v) failed: %v", rootDir, err)
}
}
}
diff --git a/services/mgmt/binary/impl/service.go b/services/mgmt/binary/impl/service.go
index e88028d..6249891 100644
--- a/services/mgmt/binary/impl/service.go
+++ b/services/mgmt/binary/impl/service.go
@@ -2,10 +2,11 @@
// objects identified by object name suffixes using the local file
// system. Given an object name suffix, the implementation computes an
// MD5 hash of the suffix and generates the following path in the
-// local filesystem: /<root>/<dir_1>/.../<dir_n>/<hash>. The root and
-// the directory depth are parameters of the implementation. The
-// contents of the directory include the checksum and data for each of
-// the individual parts of the binary, and the name of the object:
+// local filesystem: /<root_dir>/<dir_1>/.../<dir_n>/<hash>. The root
+// directory and the directory depth are parameters of the
+// implementation. The contents of the directory include the checksum
+// and data for each of the individual parts of the binary, and the
+// name of the object:
//
// name
// <part_1>/checksum
@@ -156,9 +157,9 @@
}
for {
// Remove the binary and all directories on the path back to the
- // root that are left empty after the binary is removed.
+ // root directory that are left empty after the binary is removed.
path = filepath.Dir(path)
- if i.state.root == path {
+ if i.state.rootDir == path {
break
}
if err := os.Remove(path); err != nil {
@@ -204,10 +205,11 @@
return nil
}
+// TODO(jsimsa): Design and implement an access control mechanism for
+// the URL-based downloads.
func (i *binaryService) DownloadURL(ipc.ServerContext) (string, int64, error) {
vlog.Infof("%v.DownloadURL()", i.suffix)
- // TODO(jsimsa): Implement.
- return "", 0, nil
+ return i.state.rootURL + "/" + i.suffix, 0, nil
}
func (i *binaryService) Stat(ipc.ServerContext) ([]binary.PartInfo, repository.MediaInfo, error) {
diff --git a/services/mgmt/binary/impl/setup.go b/services/mgmt/binary/impl/setup.go
index 12e43fa..2f96bf5 100644
--- a/services/mgmt/binary/impl/setup.go
+++ b/services/mgmt/binary/impl/setup.go
@@ -10,9 +10,9 @@
const defaultRootPrefix = "veyron_binary_repository"
-// SetupRoot sets up the root directory if it doesn't already exist. If an
+// SetupRootDir sets up the root directory if it doesn't already exist. If an
// empty string is used as root, create a new temporary directory.
-func SetupRoot(root string) (string, error) {
+func SetupRootDir(root string) (string, error) {
if root == "" {
var err error
if root, err = ioutil.TempDir("", defaultRootPrefix); err != nil {
diff --git a/services/mgmt/binary/impl/state.go b/services/mgmt/binary/impl/state.go
index cf6c3b1..757d3a6 100644
--- a/services/mgmt/binary/impl/state.go
+++ b/services/mgmt/binary/impl/state.go
@@ -29,21 +29,24 @@
// before its performance degrades allows the binary repository to
// store 16B objects.
depth int
- // root identifies the local filesystem directory in which the
+ // rootDir identifies the local filesystem directory in which the
// binary repository stores its objects.
- root string
+ rootDir string
+ // rootURL identifies the root URL of the HTTP server serving
+ // the download URLs.
+ rootURL string
}
// NewState creates a new state object for the binary service. This
// should be passed into both NewDispatcher and NewHTTPRoot.
-func NewState(root string, depth int) (*state, error) {
+func NewState(rootDir, rootURL string, depth int) (*state, error) {
if min, max := 0, md5.Size-1; min > depth || depth > max {
return nil, fmt.Errorf("Unexpected depth, expected a value between %v and %v, got %v", min, max, depth)
}
- if _, err := os.Stat(root); err != nil {
- return nil, fmt.Errorf("Stat(%v) failed: %v", root, err)
+ if _, err := os.Stat(rootDir); err != nil {
+ return nil, fmt.Errorf("Stat(%v) failed: %v", rootDir, err)
}
- path := filepath.Join(root, VersionFile)
+ path := filepath.Join(rootDir, VersionFile)
output, err := ioutil.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("ReadFile(%v) failed: %v", path, err)
@@ -52,8 +55,9 @@
return nil, fmt.Errorf("Unexpected version: expected %v, got %v", expected, got)
}
return &state{
- depth: depth,
- root: root,
+ depth: depth,
+ rootDir: rootDir,
+ rootURL: rootURL,
}, nil
}
@@ -66,5 +70,5 @@
for j := 0; j < s.depth; j++ {
dir = filepath.Join(dir, hash[j*2:(j+1)*2])
}
- return filepath.Join(s.root, dir, hash)
+ return filepath.Join(s.rootDir, dir, hash)
}
diff --git a/services/mgmt/build/buildd/test.sh b/services/mgmt/build/buildd/test.sh
deleted file mode 100755
index 4df2905..0000000
--- a/services/mgmt/build/buildd/test.sh
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/bin/bash
-
-# Test the build server daemon.
-#
-# This test starts a build server daemon and uses the build client to
-# verify that <build>.Build() works as expected.
-
-source "${VEYRON_ROOT}/scripts/lib/shell_test.sh"
-
-readonly WORKDIR="${shell_test_WORK_DIR}"
-
-build() {
- BUILDD_BIN="$(shell_test::build_go_binary 'veyron.io/veyron/veyron/services/mgmt/build/buildd')"
- BUILD_BIN="$(shell_test::build_go_binary 'veyron.io/veyron/veyron/tools/build')"
-}
-
-main() {
- cd "${WORKDIR}"
- build
-
- shell_test::setup_server_test
-
- # Start the binary repository daemon.
- local -r SERVER="buildd-test-server"
- local GO_BIN=$(which go)
- local -r GO_ROOT=$("${GO_BIN}" env GOROOT)
- shell_test::start_server "${BUILDD_BIN}" --name="${SERVER}" --gobin="${GO_BIN}" --goroot="${GO_ROOT}" --veyron.tcp.address=127.0.0.1:0 \
- || shell_test::fail "line ${LINENO} failed to start server"
-
- # Create and build a test source file.
- local -r GO_PATH=$(shell::tmp_dir)
- local -r BIN_DIR="${GO_PATH}/bin"
- mkdir -p "${BIN_DIR}"
- local -r SRC_DIR="${GO_PATH}/src/test"
- mkdir -p "${SRC_DIR}"
- local -r SRC_FILE="${SRC_DIR}/test.go"
- cat > "${SRC_FILE}" <<EOF
-package main
-
-import "fmt"
-
-func main() {
- fmt.Printf("Hello World!\n")
-}
-EOF
- GOPATH="${GO_PATH}" GOROOT="${GO_ROOT}" TMPDIR="${BIN_DIR}" "${BUILD_BIN}" build "${SERVER}" "test" || shell_test::fail "line ${LINENO}: 'build' failed"
- if [[ ! -e "${BIN_DIR}/test" ]]; then
- shell_test::fail "test binary not found"
- fi
- local -r GOT=$("${BIN_DIR}/test")
- local -r WANT="Hello World!"
- shell_test::assert_eq "${GOT}" "${WANT}" "${LINENO}"
-
- shell_test::pass
-}
-
-main "$@"
diff --git a/services/mgmt/build/buildd/testdata/integration_test.go b/services/mgmt/build/buildd/testdata/integration_test.go
index 630a22f..cbcb948 100644
--- a/services/mgmt/build/buildd/testdata/integration_test.go
+++ b/services/mgmt/build/buildd/testdata/integration_test.go
@@ -47,7 +47,7 @@
modules.DispatchInTest()
}
-func TestBuild(t *testing.T) {
+func TestBuildServerIntegration(t *testing.T) {
// Build the required binaries.
binDir, cleanup, err := integration.BuildPkgs(binPkgs)
if err != nil {
diff --git a/services/mgmt/lib/binary/impl.go b/services/mgmt/lib/binary/impl.go
index b68fe7f..2688fbd 100644
--- a/services/mgmt/lib/binary/impl.go
+++ b/services/mgmt/lib/binary/impl.go
@@ -176,6 +176,17 @@
return nil
}
+func DownloadURL(ctx context.T, von string) (string, int64, error) {
+ ctx, cancel := ctx.WithTimeout(time.Minute)
+ defer cancel()
+ url, ttl, err := repository.BinaryClient(von).DownloadURL(ctx)
+ if err != nil {
+ vlog.Errorf("DownloadURL() failed: %v", err)
+ return "", 0, err
+ }
+ return url, ttl, nil
+}
+
func upload(ctx context.T, r io.ReadSeeker, mediaInfo repository.MediaInfo, von string) error {
client := repository.BinaryClient(von)
offset, whence := int64(0), 2
diff --git a/services/mgmt/lib/binary/impl_test.go b/services/mgmt/lib/binary/impl_test.go
index ca0a76f..216cfd1 100644
--- a/services/mgmt/lib/binary/impl_test.go
+++ b/services/mgmt/lib/binary/impl_test.go
@@ -37,11 +37,11 @@
func setupRepository(t *testing.T) (string, func()) {
// Setup the root of the binary repository.
- root, err := ioutil.TempDir("", veyronPrefix)
+ rootDir, err := ioutil.TempDir("", veyronPrefix)
if err != nil {
t.Fatalf("TempDir() failed: %v", err)
}
- path, perm := filepath.Join(root, impl.VersionFile), os.FileMode(0600)
+ path, perm := filepath.Join(rootDir, impl.VersionFile), os.FileMode(0600)
if err := ioutil.WriteFile(path, []byte(impl.Version), perm); err != nil {
vlog.Fatalf("WriteFile(%v, %v, %v) failed: %v", path, impl.Version, perm, err)
}
@@ -51,9 +51,9 @@
t.Fatalf("NewServer() failed: %v", err)
}
depth := 2
- state, err := impl.NewState(root, depth)
+ state, err := impl.NewState(rootDir, "http://test-root-url", depth)
if err != nil {
- t.Fatalf("NewState(%v, %v) failed: %v", root, depth, err)
+ t.Fatalf("NewState(%v, %v) failed: %v", rootDir, depth, err)
}
dispatcher := impl.NewDispatcher(state, nil)
endpoint, err := server.Listen(profiles.LocalListenSpec)
@@ -71,8 +71,8 @@
}
// Check that any directories and files that were created to
// represent the binary objects have been garbage collected.
- if err := os.Remove(root); err != nil {
- t.Fatalf("Remove(%v) failed: %v", root, err)
+ if err := os.RemoveAll(rootDir); err != nil {
+ t.Fatalf("Remove(%v) failed: %v", rootDir, err)
}
// Shutdown the binary repository server.
if err := server.Stop(); err != nil {
@@ -160,3 +160,17 @@
t.Errorf("Delete(%v) failed: %v", von, err)
}
}
+
+// TestDownloadURL tests the binary repository client-side library
+// DownloadURL method.
+func TestDownloadURL(t *testing.T) {
+ von, cleanup := setupRepository(t)
+ defer cleanup()
+ url, _, err := DownloadURL(runtime.NewContext(), von)
+ if err != nil {
+ t.Fatalf("DownloadURL(%v) failed: %v", von, err)
+ }
+ if got, want := url, "http://test-root-url/test"; got != want {
+ t.Fatalf("unexpect output: got %v, want %v", got, want)
+ }
+}
diff --git a/services/mgmt/node/impl/impl_test.go b/services/mgmt/node/impl/impl_test.go
index f5dd9e2..eefe69e 100644
--- a/services/mgmt/node/impl/impl_test.go
+++ b/services/mgmt/node/impl/impl_test.go
@@ -745,11 +745,11 @@
}
func startRealBinaryRepository(t *testing.T) func() {
- root, err := binaryimpl.SetupRoot("")
+ rootDir, err := binaryimpl.SetupRootDir("")
if err != nil {
- t.Fatalf("binaryimpl.SetupRoot failed: %v", err)
+ t.Fatalf("binaryimpl.SetupRootDir failed: %v", err)
}
- state, err := binaryimpl.NewState(root, 3)
+ state, err := binaryimpl.NewState(rootDir, "", 3)
if err != nil {
t.Fatalf("binaryimpl.NewState failed: %v", err)
}
@@ -774,8 +774,8 @@
if err := server.Stop(); err != nil {
t.Fatalf("server.Stop failed: %v", err)
}
- if err := os.RemoveAll(root); err != nil {
- t.Fatalf("os.RemoveAll(%q) failed: %v", root, err)
+ if err := os.RemoveAll(rootDir); err != nil {
+ t.Fatalf("os.RemoveAll(%q) failed: %v", rootDir, err)
}
}
}
diff --git a/services/mgmt/profile/impl/impl_test.go b/services/mgmt/profile/impl/impl_test.go
index 1de68d0..5517bca 100644
--- a/services/mgmt/profile/impl/impl_test.go
+++ b/services/mgmt/profile/impl/impl_test.go
@@ -45,6 +45,7 @@
if err != nil {
t.Fatalf("TempDir(%q, %q) failed: %v", dir, prefix, err)
}
+ defer os.RemoveAll(store)
dispatcher, err := NewDispatcher(store, nil)
if err != nil {
t.Fatalf("NewDispatcher() failed: %v", err)
diff --git a/services/mgmt/profile/profiled/test.sh b/services/mgmt/profile/profiled/test.sh
deleted file mode 100755
index 7022e92..0000000
--- a/services/mgmt/profile/profiled/test.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/bash
-
-# Test the profile repository daemon.
-#
-# This test starts an profile repository daemon and uses the profile
-# repository client to verify that <profile>.Put(), <profile>.Label(),
-# <profile>.Description(), <profile>.Speficiation(), and
-# <profile>.Remove() work as expected.
-
-source "${VEYRON_ROOT}/scripts/lib/shell_test.sh"
-
-readonly WORKDIR="${shell_test_WORK_DIR}"
-
-build() {
- PROFILED_BIN="$(shell_test::build_go_binary 'veyron.io/veyron/veyron/services/mgmt/profile/profiled')"
- PROFILE_BIN="$(shell_test::build_go_binary 'veyron.io/veyron/veyron/tools/profile')"
-}
-
-main() {
- local GOT OUTPUT RESULT WANT
-
- cd "${WORKDIR}"
- build
-
- shell_test::setup_server_test
-
- # Start the profile repository daemon.
- local -r REPO="profiled-test-repo"
- local -r STORE=$(shell::tmp_dir)
- shell_test::start_server "${PROFILED_BIN}" --name="${REPO}" --veyron.tcp.address=127.0.0.1:0 --store="${STORE}" \
- || shell_test::fail "line ${LINENO} failed to start server"
-
- # Create a profile.
- local -r PROFILE="${REPO}/test-profile"
- "${PROFILE_BIN}" put "${PROFILE}" || shell_test::fail "line ${LINENO}: 'put' failed"
-
- # Retrieve the profile label.
- OUTPUT=$(shell::tmp_file)
- "${PROFILE_BIN}" label "${PROFILE}" | tee "${OUTPUT}" || shell_test::fail "line ${LINENO}: 'label' failed"
- GOT=$(cat "${OUTPUT}")
- WANT="example"
- shell_test::assert_eq "${GOT}" "${WANT}" "${LINENO}"
-
- # Retrieve the profile description.
- OUTPUT=$(shell::tmp_file)
- "${PROFILE_BIN}" description "${PROFILE}" | tee "${OUTPUT}" || shell_test::fail "line ${LINENO}: 'description' failed"
- GOT=$(cat "${OUTPUT}")
- WANT="Example profile to test the profile manager implementation."
- shell_test::assert_eq "${GOT}" "${WANT}" "${LINENO}"
-
- # Retrieve the profile specification.
- OUTPUT=$(shell::tmp_file)
- "${PROFILE_BIN}" spec "${PROFILE}" | tee "${OUTPUT}" || shell_test::fail "line ${LINENO}: 'spec' failed"
- GOT=$(cat "${OUTPUT}")
- WANT='profile.Specification{Arch:"amd64", Description:"Example profile to test the profile manager implementation.", Format:"ELF", Libraries:map[profile.Library]struct {}{profile.Library{Name:"foo", MajorVersion:"1", MinorVersion:"0"}:struct {}{}}, Label:"example", OS:"linux"}'
- shell_test::assert_eq "${GOT}" "${WANT}" "${LINENO}"
-
- # Remove the profile.
- "${PROFILE_BIN}" remove "${PROFILE}" || shell_test::fail "line ${LINENO}: 'remove' failed"
-
- # Check the profile no longer exists.
- RESULT=$(shell::check_result "${PROFILE_BIN}" label "${PROFILE}")
- shell_test::assert_ne "${RESULT}" "0" "${LINENO}"
- RESULT=$(shell::check_result "${PROFILE_BIN}" description "${PROFILE}")
- shell_test::assert_ne "${RESULT}" "0" "${LINENO}"
- RESULT=$(shell::check_result "${PROFILE_BIN}" spec "${PROFILE}")
- shell_test::assert_ne "${RESULT}" "0" "${LINENO}"
-
- shell_test::pass
-}
-
-main "$@"
diff --git a/services/mgmt/profile/profiled/testdata/integration_test.go b/services/mgmt/profile/profiled/testdata/integration_test.go
new file mode 100644
index 0000000..673f07b
--- /dev/null
+++ b/services/mgmt/profile/profiled/testdata/integration_test.go
@@ -0,0 +1,155 @@
+package integration_test
+
+import (
+ "bytes"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "veyron.io/veyron/veyron/lib/modules"
+ "veyron.io/veyron/veyron/lib/testutil/integration"
+ "veyron.io/veyron/veyron/lib/testutil/security"
+ _ "veyron.io/veyron/veyron/profiles"
+ "veyron.io/veyron/veyron2/naming"
+)
+
+var binPkgs = []string{
+ "veyron.io/veyron/veyron/services/mgmt/profile/profiled",
+ "veyron.io/veyron/veyron/tools/profile",
+}
+
+func profileCommandOutput(t *testing.T, expectError bool, command, binDir, credentials, mt, name, suffix string) string {
+ var labelOut bytes.Buffer
+ labelArgs := []string{
+ "-veyron.credentials=" + credentials,
+ "-veyron.namespace.root=" + mt,
+ command, naming.Join(name, suffix),
+ }
+ labelCmd := exec.Command(filepath.Join(binDir, "profile"), labelArgs...)
+ labelCmd.Stdout = &labelOut
+ labelCmd.Stderr = &labelOut
+ err := labelCmd.Run()
+ if err != nil && !expectError {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(labelCmd.Args, " "), err, labelOut.String())
+ }
+ if err == nil && expectError {
+ t.Fatalf("%q did not fail when it should", strings.Join(labelCmd.Args, " "))
+ }
+ return strings.TrimSpace(labelOut.String())
+}
+
+func putProfile(t *testing.T, binDir, credentials, mt, name, suffix string) {
+ var putOut bytes.Buffer
+ putArgs := []string{
+ "-veyron.credentials=" + credentials,
+ "-veyron.namespace.root=" + mt,
+ "put", naming.Join(name, suffix),
+ }
+ putCmd := exec.Command(filepath.Join(binDir, "profile"), putArgs...)
+ putCmd.Stdout = &putOut
+ putCmd.Stderr = &putOut
+ if err := putCmd.Run(); err != nil {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(putCmd.Args, " "), err, putOut.String())
+ }
+}
+
+func removeProfile(t *testing.T, binDir, credentials, mt, name, suffix string) {
+ var removeOut bytes.Buffer
+ removeArgs := []string{
+ "-veyron.credentials=" + credentials,
+ "-veyron.namespace.root=" + mt,
+ "remove", naming.Join(name, suffix),
+ }
+ removeCmd := exec.Command(filepath.Join(binDir, "profile"), removeArgs...)
+ removeCmd.Stdout = &removeOut
+ removeCmd.Stderr = &removeOut
+ if err := removeCmd.Run(); err != nil {
+ t.Fatalf("%q failed: %v\n%v", strings.Join(removeCmd.Args, " "), err, removeOut.String())
+ }
+}
+
+func TestHelperProcess(t *testing.T) {
+ modules.DispatchInTest()
+}
+
+func TestProfileRepository(t *testing.T) {
+ // Build the required binaries.
+ binDir, cleanup, err := integration.BuildPkgs(binPkgs)
+ if err != nil {
+ t.Fatalf("%v", err)
+ }
+ defer cleanup()
+
+ // Start a root mount table.
+ shell, err := modules.NewShell(nil)
+ if err != nil {
+ t.Fatalf("NewShell() failed: %v", err)
+ }
+ defer shell.Cleanup(os.Stdin, os.Stderr)
+ handle, mt, err := integration.StartRootMT(shell)
+ if err != nil {
+ t.Fatalf("%v", err)
+ }
+ defer handle.CloseStdin()
+
+ // Generate credentials.
+ root := security.NewPrincipal("root")
+ credentials := security.NewVeyronCredentials(root, "test-credentials")
+ defer os.RemoveAll(credentials)
+
+ // Start the profile repository.
+ profileRepoBin := filepath.Join(binDir, "profiled")
+ profileRepoName := "test-profile-repo"
+ profileRepoStore, err := ioutil.TempDir("", "")
+ if err != nil {
+ t.Fatalf("TempDir() failed: %v", err)
+ }
+ defer os.RemoveAll(profileRepoStore)
+ args := []string{
+ "-name=" + profileRepoName, "-store=" + profileRepoStore,
+ "-veyron.tcp.address=127.0.0.1:0",
+ "-veyron.credentials=" + credentials,
+ "-veyron.namespace.root=" + mt,
+ }
+ serverProcess, err := integration.StartServer(profileRepoBin, args)
+ if err != nil {
+ t.Fatalf("%v", err)
+ }
+ defer serverProcess.Kill()
+
+ // Create a profile.
+ const profile = "test-profile"
+ putProfile(t, binDir, credentials, mt, profileRepoName, profile)
+
+ // Retrieve the profile label and check it matches the
+ // expected label.
+ profileLabel := profileCommandOutput(t, false, "label", binDir, credentials, mt, profileRepoName, profile)
+ if got, want := profileLabel, "example"; got != want {
+ t.Fatalf("unexpected output: got %v, want %v", got, want)
+ }
+
+ // Retrieve the profile description and check it matches the
+ // expected description.
+ profileDesc := profileCommandOutput(t, false, "description", binDir, credentials, mt, profileRepoName, profile)
+ if got, want := profileDesc, "Example profile to test the profile manager implementation."; got != want {
+ t.Fatalf("unexpected output: got %v, want %v", got, want)
+ }
+
+ // Retrieve the profile specification and check it matches the
+ // expected specification.
+ profileSpec := profileCommandOutput(t, false, "specification", binDir, credentials, mt, profileRepoName, profile)
+ if got, want := profileSpec, `profile.Specification{Arch:"amd64", Description:"Example profile to test the profile manager implementation.", Format:"ELF", Libraries:map[profile.Library]struct {}{profile.Library{Name:"foo", MajorVersion:"1", MinorVersion:"0"}:struct {}{}}, Label:"example", OS:"linux"}`; got != want {
+ t.Fatalf("unexpected output: got %v, want %v", got, want)
+ }
+
+ // Remove the profile.
+ removeProfile(t, binDir, credentials, mt, profileRepoName, profile)
+
+ // Check that the profile no longer exists.
+ profileCommandOutput(t, true, "label", binDir, credentials, mt, profileRepoName, profile)
+ profileCommandOutput(t, true, "description", binDir, credentials, mt, profileRepoName, profile)
+ profileCommandOutput(t, true, "specification", binDir, credentials, mt, profileRepoName, profile)
+}
diff --git a/tools/binary/impl.go b/tools/binary/impl.go
index df34d6c..54d6d25 100644
--- a/tools/binary/impl.go
+++ b/tools/binary/impl.go
@@ -10,7 +10,7 @@
var cmdDelete = &cmdline.Command{
Run: runDelete,
Name: "delete",
- Short: "Delete binary",
+ Short: "Delete a binary",
Long: "Delete connects to the binary repository and deletes the specified binary",
ArgsName: "<von>",
ArgsLong: "<von> is the veyron object name of the binary to delete",
@@ -31,7 +31,7 @@
var cmdDownload = &cmdline.Command{
Run: runDownload,
Name: "download",
- Short: "Download binary",
+ Short: "Download a binary",
Long: `
Download connects to the binary repository, downloads the specified binary, and
writes it to a file.
@@ -58,7 +58,7 @@
var cmdUpload = &cmdline.Command{
Run: runUpload,
Name: "upload",
- Short: "Upload binary",
+ Short: "Upload a binary",
Long: `
Upload connects to the binary repository and uploads the binary of the specified
file. When successful, it writes the name of the new binary to stdout.
@@ -83,6 +83,28 @@
return nil
}
+var cmdURL = &cmdline.Command{
+ Run: runURL,
+ Name: "url",
+ Short: "Fetch a download URL",
+ Long: "Connect to the binary repository and fetch the download URL for the given veyron object name.",
+ ArgsName: "<von>",
+ ArgsLong: "<von> is the veyron object name of the binary repository",
+}
+
+func runURL(cmd *cmdline.Command, args []string) error {
+ if expected, got := 1, len(args); expected != got {
+ return cmd.UsageErrorf("rooturl: incorrect number of arguments, expected %d, got %d", expected, got)
+ }
+ von := args[0]
+ url, _, err := binary.DownloadURL(runtime.NewContext(), von)
+ if err != nil {
+ return err
+ }
+ fmt.Fprintf(cmd.Stdout(), "%v\n", url)
+ return nil
+}
+
func root() *cmdline.Command {
return &cmdline.Command{
Name: "binary",
@@ -90,6 +112,6 @@
Long: `
The binary tool facilitates interaction with the veyron binary repository.
`,
- Children: []*cmdline.Command{cmdDelete, cmdDownload, cmdUpload},
+ Children: []*cmdline.Command{cmdDelete, cmdDownload, cmdUpload, cmdURL},
}
}
diff --git a/tools/binary/impl_test.go b/tools/binary/impl_test.go
index 865c35d..ba314cc 100644
--- a/tools/binary/impl_test.go
+++ b/tools/binary/impl_test.go
@@ -50,7 +50,10 @@
func (s *server) DownloadURL(ipc.ServerContext) (string, int64, error) {
vlog.Infof("DownloadURL() was called. suffix=%v", s.suffix)
- return "", 0, nil
+ if s.suffix != "" {
+ return "", 0, fmt.Errorf("non-empty suffix: %v", s.suffix)
+ }
+ return "test-download-url", 0, nil
}
func (s *server) Stat(ipc.ServerContext) ([]binary.PartInfo, repository.MediaInfo, error) {
@@ -119,19 +122,20 @@
return
}
defer stopServer(t, server)
+
// Setup the command-line.
cmd := root()
- var stdout, stderr bytes.Buffer
- cmd.Init(nil, &stdout, &stderr)
+ var out bytes.Buffer
+ cmd.Init(nil, &out, &out)
// Test the 'delete' command.
if err := cmd.Execute([]string{"delete", naming.JoinAddressName(endpoint.String(), "exists")}); err != nil {
- t.Fatalf("%v", err)
+ t.Fatalf("%v failed: %v\n%v", "delete", err, out.String())
}
- if expected, got := "Binary deleted successfully", strings.TrimSpace(stdout.String()); got != expected {
+ if expected, got := "Binary deleted successfully", strings.TrimSpace(out.String()); got != expected {
t.Errorf("Got %q, expected %q", got, expected)
}
- stdout.Reset()
+ out.Reset()
// Test the 'download' command.
dir, err := ioutil.TempDir("", "binaryimpltest")
@@ -142,9 +146,9 @@
file := path.Join(dir, "testfile")
defer os.Remove(file)
if err := cmd.Execute([]string{"download", naming.JoinAddressName(endpoint.String(), "exists"), file}); err != nil {
- t.Fatalf("%v", err)
+ t.Fatalf("%v failed: %v\n%v", "download", err, out.String())
}
- if expected, got := "Binary downloaded to file "+file, strings.TrimSpace(stdout.String()); got != expected {
+ if expected, got := "Binary downloaded to file "+file, strings.TrimSpace(out.String()); got != expected {
t.Errorf("Got %q, expected %q", got, expected)
}
buf, err := ioutil.ReadFile(file)
@@ -154,10 +158,19 @@
if expected := "HelloWorld"; string(buf) != expected {
t.Errorf("Got %q, expected %q", string(buf), expected)
}
- stdout.Reset()
+ out.Reset()
// Test the 'upload' command.
if err := cmd.Execute([]string{"upload", naming.JoinAddressName(endpoint.String(), "exists"), file}); err != nil {
- t.Fatalf("%v", err)
+ t.Fatalf("%v failed: %v\n%v", "upload", err, out.String())
+ }
+ out.Reset()
+
+ // Test the 'url' command.
+ if err := cmd.Execute([]string{"url", naming.JoinAddressName(endpoint.String(), "")}); err != nil {
+ t.Fatalf("%v failed: %v\n%v", "url", err, out.String())
+ }
+ if expected, got := "test-download-url", strings.TrimSpace(out.String()); got != expected {
+ t.Errorf("Got %q, expected %q", got, expected)
}
}
diff --git a/tools/gclogs/doc.go b/tools/gclogs/doc.go
new file mode 100644
index 0000000..302fe98
--- /dev/null
+++ b/tools/gclogs/doc.go
@@ -0,0 +1,31 @@
+// This file was auto-generated via go generate.
+// DO NOT UPDATE MANUALLY
+
+/*
+gclogs is a utility that safely deletes old log files.
+
+It looks for file names that match the format of files produced by the vlog
+package, and deletes the ones that have not changed in the amount of time
+specified by the --cutoff flag.
+
+Only files produced by the same user as the one running the gclogs command are
+considered for deletion.
+
+Usage:
+ gclogs [flags] <dir> ...
+
+<dir> ... A list of directories where to look for log files.
+
+The gclogs flags are:
+ -cutoff=24h0m0s
+ The age cut-off for a log file to be considered for garbage collection.
+ -n=false
+ If true, log files that would be deleted are shown on stdout, but not
+ actually deleted.
+ -program=.*
+ A regular expression to apply to the program part of the log file name, e.g
+ ".*test".
+ -verbose=false
+ If true, each deleted file is shown on stdout.
+*/
+package main
diff --git a/tools/gclogs/format.go b/tools/gclogs/format.go
new file mode 100644
index 0000000..502db25
--- /dev/null
+++ b/tools/gclogs/format.go
@@ -0,0 +1,69 @@
+package main
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "syscall"
+ "time"
+)
+
+var (
+ // The format of the log file names is:
+ // program.host.user.log.logger.tag.YYYYMMDD-HHmmss.pid
+ logFileRE = regexp.MustCompile(`^(.*)\.([^.]*)\.([^.]*)\.log\.([^.]*)\.([^.]*)\.(........-......)\.([0-9]*)$`)
+)
+
+type logFile struct {
+ // TODO(rthellend): Some of these fields are not used by anything yet,
+ // but they will be eventually when we need to sort the log files
+ // associated with a given instance.
+ symlink bool
+ program, host, user, logger, tag string
+ time time.Time
+ pid int
+}
+
+func parseFileInfo(dir string, fileInfo os.FileInfo) (*logFile, error) {
+ fileName := fileInfo.Name()
+ if fileInfo.Mode()&os.ModeSymlink != 0 {
+ buf := make([]byte, syscall.NAME_MAX)
+ n, err := syscall.Readlink(filepath.Join(dir, fileName), buf)
+ if err != nil {
+ return nil, err
+ }
+ linkName := string(buf[:n])
+ lf, err := parseFileName(linkName)
+ if err != nil {
+ return nil, err
+ }
+ lf.symlink = true
+ return lf, nil
+ }
+ return parseFileName(fileName)
+}
+
+func parseFileName(fileName string) (*logFile, error) {
+ if m := logFileRE.FindStringSubmatch(fileName); len(m) != 0 {
+ t, err := time.ParseInLocation("20060102-150405", m[6], time.Local)
+ if err != nil {
+ return nil, err
+ }
+ pid, err := strconv.ParseInt(m[7], 10, 32)
+ if err != nil {
+ return nil, err
+ }
+ return &logFile{
+ program: m[1],
+ host: m[2],
+ user: m[3],
+ logger: m[4],
+ tag: m[5],
+ time: t,
+ pid: int(pid),
+ }, nil
+ }
+ return nil, errors.New("not a recognized log file name")
+}
diff --git a/tools/gclogs/format_test.go b/tools/gclogs/format_test.go
new file mode 100644
index 0000000..5396894
--- /dev/null
+++ b/tools/gclogs/format_test.go
@@ -0,0 +1,97 @@
+package main
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+ "time"
+)
+
+func TestParseFileNameNoError(t *testing.T) {
+ testcases := []struct {
+ filename string
+ lf *logFile
+ }{
+ {
+ "program.host.user.log.veyron.INFO.20141204-131502.12345",
+ &logFile{false, "program", "host", "user", "veyron", "INFO", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345},
+ },
+ {
+ "prog.test.host-name.user.log.veyron.ERROR.20141204-131502.12345",
+ &logFile{false, "prog.test", "host-name", "user", "veyron", "ERROR", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345},
+ },
+ }
+ for _, tc := range testcases {
+ lf, err := parseFileName(tc.filename)
+ if err != nil {
+ t.Errorf("unexpected error for %q: %v", tc.filename, err)
+ }
+ if !reflect.DeepEqual(tc.lf, lf) {
+ t.Errorf("unexpected result: got %+v, expected %+v", lf, tc.lf)
+ }
+ }
+}
+
+func TestParseFileNameError(t *testing.T) {
+ testcases := []string{
+ "program.host.user.log.veyron.INFO.20141204-131502",
+ "prog.test.host-name.user.log.veyron.20141204-131502.12345",
+ "foo.txt",
+ }
+ for _, tc := range testcases {
+ if _, err := parseFileName(tc); err == nil {
+ t.Errorf("unexpected success for %q", tc)
+ }
+ }
+}
+
+func TestParseFileInfo(t *testing.T) {
+ tmpdir, err := ioutil.TempDir("", "parse-file-info-")
+ if err != nil {
+ t.Fatalf("ioutil.TempDir failed: %v", err)
+ }
+ defer os.RemoveAll(tmpdir)
+
+ name := "program.host.user.log.veyron.INFO.20141204-131502.12345"
+ if err := ioutil.WriteFile(filepath.Join(tmpdir, name), []byte{}, 0644); err != nil {
+ t.Fatalf("ioutil.WriteFile failed: %v", err)
+ }
+ link := "program.INFO"
+ if err := os.Symlink(name, filepath.Join(tmpdir, link)); err != nil {
+ t.Fatalf("os.Symlink failed: %v", err)
+ }
+
+ // Test regular file.
+ {
+ fi, err := os.Lstat(filepath.Join(tmpdir, name))
+ if err != nil {
+ t.Fatalf("os.Lstat failed: %v", err)
+ }
+ lf, err := parseFileInfo(tmpdir, fi)
+ if err != nil {
+ t.Errorf("parseFileInfo(%v, %v) failed: %v", tmpdir, fi, err)
+ }
+ expected := &logFile{false, "program", "host", "user", "veyron", "INFO", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345}
+ if !reflect.DeepEqual(lf, expected) {
+ t.Errorf("unexpected result: got %+v, expected %+v", lf, expected)
+ }
+ }
+
+ // Test symlink.
+ {
+ fi, err := os.Lstat(filepath.Join(tmpdir, link))
+ if err != nil {
+ t.Fatalf("os.Lstat failed: %v", err)
+ }
+ lf, err := parseFileInfo(tmpdir, fi)
+ if err != nil {
+ t.Errorf("parseFileInfo(%v, %v) failed: %v", tmpdir, fi, err)
+ }
+ expected := &logFile{true, "program", "host", "user", "veyron", "INFO", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345}
+ if !reflect.DeepEqual(lf, expected) {
+ t.Errorf("unexpected result: got %+v, expected %+v", lf, expected)
+ }
+ }
+}
diff --git a/tools/gclogs/gclogs.go b/tools/gclogs/gclogs.go
new file mode 100644
index 0000000..6fde8af
--- /dev/null
+++ b/tools/gclogs/gclogs.go
@@ -0,0 +1,157 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "os"
+ "os/user"
+ "path/filepath"
+ "regexp"
+ "time"
+
+ "veyron.io/lib/cmdline"
+)
+
+var (
+ flagCutoff time.Duration
+ flagProgname string
+ flagVerbose bool
+ flagDryrun bool
+
+ cmdGCLogs = &cmdline.Command{
+ Run: garbageCollectLogs,
+ Name: "gclogs",
+ Short: "gclogs is a utility that safely deletes old log files.",
+ Long: `
+gclogs is a utility that safely deletes old log files.
+
+It looks for file names that match the format of files produced by the vlog
+package, and deletes the ones that have not changed in the amount of time
+specified by the --cutoff flag.
+
+Only files produced by the same user as the one running the gclogs command
+are considered for deletion.
+`,
+ ArgsName: "<dir> ...",
+ ArgsLong: "<dir> ... A list of directories where to look for log files.",
+ }
+)
+
+func init() {
+ cmdGCLogs.Flags.DurationVar(&flagCutoff, "cutoff", 24*time.Hour, "The age cut-off for a log file to be considered for garbage collection.")
+ cmdGCLogs.Flags.StringVar(&flagProgname, "program", ".*", `A regular expression to apply to the program part of the log file name, e.g ".*test".`)
+ cmdGCLogs.Flags.BoolVar(&flagVerbose, "verbose", false, "If true, each deleted file is shown on stdout.")
+ cmdGCLogs.Flags.BoolVar(&flagDryrun, "n", false, "If true, log files that would be deleted are shown on stdout, but not actually deleted.")
+}
+
+func garbageCollectLogs(cmd *cmdline.Command, args []string) error {
+ if len(args) == 0 {
+ cmd.UsageErrorf("gclogs requires at least one argument")
+ }
+ timeCutoff := time.Now().Add(-flagCutoff)
+ currentUser, err := user.Current()
+ if err != nil {
+ return err
+ }
+ programRE, err := regexp.Compile(flagProgname)
+ if err != nil {
+ return err
+ }
+ var lastErr error
+ for _, logdir := range args {
+ if err := processDirectory(cmd, logdir, timeCutoff, programRE, currentUser.Username); err != nil {
+ lastErr = err
+ }
+ }
+ return lastErr
+}
+
+func processDirectory(cmd *cmdline.Command, logdir string, timeCutoff time.Time, programRE *regexp.Regexp, username string) error {
+ fmt.Fprintf(cmd.Stdout(), "Processing: %q\n", logdir)
+
+ f, err := os.Open(logdir)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ var lastErr error
+ deleted := 0
+ symlinks := []string{}
+ for {
+ fi, err := f.Readdir(100)
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ lastErr = err
+ break
+ }
+ for _, file := range fi {
+ fullname := filepath.Join(logdir, file.Name())
+ if file.IsDir() {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Skipped directory: %q\n", fullname)
+ }
+ continue
+ }
+ lf, err := parseFileInfo(logdir, file)
+ if err != nil {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Not a log file: %q\n", fullname)
+ }
+ continue
+ }
+ if lf.user != username {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Skipped log file created by other user: %q\n", fullname)
+ }
+ continue
+ }
+ if !programRE.MatchString(lf.program) {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Skipped log file doesn't match %q: %q\n", flagProgname, fullname)
+ }
+ continue
+ }
+ if lf.symlink {
+ symlinks = append(symlinks, fullname)
+ continue
+ }
+ if file.ModTime().Before(timeCutoff) {
+ if flagDryrun {
+ fmt.Fprintf(cmd.Stdout(), "Would delete %q\n", fullname)
+ continue
+ }
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Deleting %q\n", fullname)
+ }
+ if err := os.Remove(fullname); err != nil {
+ lastErr = err
+ } else {
+ deleted++
+ }
+ }
+ }
+ }
+ // Delete broken links.
+ for _, sl := range symlinks {
+ if _, err := os.Stat(sl); err != nil && os.IsNotExist(err) {
+ if flagDryrun {
+ fmt.Fprintf(cmd.Stdout(), "Would delete symlink %q\n", sl)
+ continue
+ }
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Deleting symlink %q\n", sl)
+ }
+ if err := os.Remove(sl); err != nil {
+ lastErr = err
+ } else {
+ deleted++
+ }
+ }
+
+ }
+ fmt.Fprintf(cmd.Stdout(), "Number of files deleted: %d\n", deleted)
+ return lastErr
+}
diff --git a/tools/gclogs/gclogs_test.go b/tools/gclogs/gclogs_test.go
new file mode 100644
index 0000000..f9168fe
--- /dev/null
+++ b/tools/gclogs/gclogs_test.go
@@ -0,0 +1,177 @@
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/user"
+ "path/filepath"
+ "sort"
+ "strings"
+ "testing"
+ "time"
+)
+
+func setup(t *testing.T, workdir, username string) (tmpdir string) {
+ var err error
+ tmpdir, err = ioutil.TempDir(workdir, "parse-file-info-")
+ if err != nil {
+ t.Fatalf("ioutil.TempDir failed: %v", err)
+ }
+ logfiles := []struct {
+ name string
+ link string
+ age time.Duration
+ }{
+ {"prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345", "", 4 * time.Hour},
+ {"prog1.host.%USER%.log.veyron.INFO.20141204-145040.23456", "prog1.INFO", 1 * time.Hour},
+ {"prog1.host.%USER%.log.veyron.ERROR.20141204-145040.23456", "prog1.ERROR", 1 * time.Hour},
+ {"prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456", "prog2.INFO", 4 * time.Hour},
+ {"prog3.host.otheruser.log.veyron.INFO.20141204-135040.23456", "prog3.INFO", 1 * time.Hour},
+ {"foo.txt", "", 1 * time.Hour},
+ {"bar.txt", "", 1 * time.Hour},
+ }
+ for _, l := range logfiles {
+ l.name = strings.Replace(l.name, "%USER%", username, -1)
+ filename := filepath.Join(tmpdir, l.name)
+ if err := ioutil.WriteFile(filename, []byte{}, 0644); err != nil {
+ t.Fatalf("ioutil.WriteFile failed: %v", err)
+ }
+ mtime := time.Now().Add(-l.age)
+ if err := os.Chtimes(filename, mtime, mtime); err != nil {
+ t.Fatalf("os.Chtimes failed: %v", err)
+ }
+ if l.link != "" {
+ if err := os.Symlink(l.name, filepath.Join(tmpdir, l.link)); err != nil {
+ t.Fatalf("os.Symlink failed: %v", err)
+ }
+ }
+ }
+ if err := os.Mkdir(filepath.Join(tmpdir, "subdir"), 0700); err != nil {
+ t.Fatalf("os.Mkdir failed: %v", err)
+ }
+ return
+}
+
+func TestGCLogs(t *testing.T) {
+ workdir, err := ioutil.TempDir("", "parse-file-info-")
+ if err != nil {
+ t.Fatalf("ioutil.TempDir failed: %v", err)
+ }
+ defer os.RemoveAll(workdir)
+
+ u, err := user.Current()
+ if err != nil {
+ t.Fatalf("user.Current failed: %v", err)
+ }
+
+ cmd := cmdGCLogs
+ var stdout, stderr bytes.Buffer
+ cmd.Init(nil, &stdout, &stderr)
+
+ testcases := []struct {
+ cutoff time.Duration
+ verbose bool
+ dryrun bool
+ expected []string
+ }{
+ {
+ cutoff: 6 * time.Hour,
+ verbose: false,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Number of files deleted: 0`,
+ },
+ },
+ {
+ cutoff: 2 * time.Hour,
+ verbose: false,
+ dryrun: true,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Would delete "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345"`,
+ `Would delete "%TESTDIR%/prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456"`,
+ `Number of files deleted: 0`,
+ },
+ },
+ {
+ cutoff: 2 * time.Hour,
+ verbose: false,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Number of files deleted: 3`,
+ },
+ },
+ {
+ cutoff: 2 * time.Hour,
+ verbose: true,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345"`,
+ `Deleting "%TESTDIR%/prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456"`,
+ `Deleting symlink "%TESTDIR%/prog2.INFO"`,
+ `Not a log file: "%TESTDIR%/bar.txt"`,
+ `Not a log file: "%TESTDIR%/foo.txt"`,
+ `Skipped directory: "%TESTDIR%/subdir"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.INFO"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.host.otheruser.log.veyron.INFO.20141204-135040.23456"`,
+ `Number of files deleted: 3`,
+ },
+ },
+ {
+ cutoff: time.Minute,
+ verbose: true,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.ERROR.20141204-145040.23456"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-145040.23456"`,
+ `Deleting "%TESTDIR%/prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456"`,
+ `Deleting symlink "%TESTDIR%/prog1.ERROR"`,
+ `Deleting symlink "%TESTDIR%/prog1.INFO"`,
+ `Deleting symlink "%TESTDIR%/prog2.INFO"`,
+ `Not a log file: "%TESTDIR%/bar.txt"`,
+ `Not a log file: "%TESTDIR%/foo.txt"`,
+ `Skipped directory: "%TESTDIR%/subdir"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.INFO"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.host.otheruser.log.veyron.INFO.20141204-135040.23456"`,
+ `Number of files deleted: 7`,
+ },
+ },
+ {
+ cutoff: time.Minute,
+ verbose: false,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Number of files deleted: 7`,
+ },
+ },
+ }
+ for _, tc := range testcases {
+ testdir := setup(t, workdir, u.Username)
+ cutoff := fmt.Sprintf("--cutoff=%s", tc.cutoff)
+ verbose := fmt.Sprintf("--verbose=%v", tc.verbose)
+ dryrun := fmt.Sprintf("--n=%v", tc.dryrun)
+ if err := cmd.Execute([]string{cutoff, verbose, dryrun, testdir}); err != nil {
+ t.Fatalf("%v: %v", stderr.String(), err)
+ }
+ gotsl := strings.Split(stdout.String(), "\n")
+ if len(gotsl) >= 2 {
+ sort.Strings(gotsl[1 : len(gotsl)-2])
+ }
+ got := strings.Join(gotsl, "\n")
+ expected := strings.Join(tc.expected, "\n") + "\n"
+ expected = strings.Replace(expected, "%TESTDIR%", testdir, -1)
+ expected = strings.Replace(expected, "%USER%", u.Username, -1)
+ if got != expected {
+ t.Errorf("Unexpected result for (%v, %v): got %q, expected %q", tc.cutoff, tc.verbose, got, expected)
+ }
+ stdout.Reset()
+ }
+}
diff --git a/tools/gclogs/main.go b/tools/gclogs/main.go
new file mode 100644
index 0000000..5cdbead
--- /dev/null
+++ b/tools/gclogs/main.go
@@ -0,0 +1,8 @@
+// The following enables go generate to generate the doc.go file.
+//go:generate go run $VEYRON_ROOT/veyron/go/src/veyron.io/lib/cmdline/testdata/gendoc.go . -help
+
+package main
+
+func main() {
+ cmdGCLogs.Main()
+}
diff --git a/tools/mgmt/test.sh b/tools/mgmt/test.sh
index 25ef0dd..bd4c59a 100755
--- a/tools/mgmt/test.sh
+++ b/tools/mgmt/test.sh
@@ -64,7 +64,7 @@
# Install and start node manager.
shell_test::start_server "${NMINSTALL_SCRIPT}" --single_user $(shell::tmp_dir) \
- "${BIN_STAGING_DIR}" -- || shell_test::fail "line ${LINENO} failed to start node manager"
+ "${BIN_STAGING_DIR}" -- --veyron.tcp.address=127.0.0.1:0 || shell_test::fail "line ${LINENO} failed to start node manager"
# Dump nminstall's log, just to provide visibility into its steps.
cat "${START_SERVER_LOG_FILE}"
@@ -110,7 +110,7 @@
# Upload an envelope for our test app.
local -r SAMPLE_APP_NAME="${APPLICATIOND_NAME}/testapp/v0"
local -r APP_PUBLISH_NAME="testbinaryd"
- echo "{\"Title\":\"BINARYD\", \"Args\":[\"--name=${APP_PUBLISH_NAME}\", \"--root=./binstore\"], \"Binary\":\"${SAMPLE_APP_BIN_NAME}\", \"Env\":[]}" > ./app.envelope && \
+ echo "{\"Title\":\"BINARYD\", \"Args\":[\"--name=${APP_PUBLISH_NAME}\", \"--root=./binstore\", \"--veyron.tcp.address=127.0.0.1:0\"], \"Binary\":\"${SAMPLE_APP_BIN_NAME}\", \"Env\":[]}" > ./app.envelope && \
"${APPLICATION_BIN}" put "${SAMPLE_APP_NAME}" test ./app.envelope && rm ./app.envelope
# Verify that the envelope we uploaded shows up with glob.
diff --git a/tools/profile/impl.go b/tools/profile/impl.go
index 8869787..ce7924d 100644
--- a/tools/profile/impl.go
+++ b/tools/profile/impl.go
@@ -62,7 +62,7 @@
var cmdSpecification = &cmdline.Command{
Run: runSpecification,
- Name: "spec",
+ Name: "specification",
Short: "Shows the specification of the profile.",
Long: "Shows the specification of the profile.",
ArgsName: "<profile>",
@@ -71,7 +71,7 @@
func runSpecification(cmd *cmdline.Command, args []string) error {
if expected, got := 1, len(args); expected != got {
- return cmd.UsageErrorf("spec: incorrect number of arguments, expected %d, got %d", expected, got)
+ return cmd.UsageErrorf("specification: incorrect number of arguments, expected %d, got %d", expected, got)
}
name := args[0]
p := repository.ProfileClient(name)
diff --git a/tools/profile/impl_test.go b/tools/profile/impl_test.go
index b442271..9569760 100644
--- a/tools/profile/impl_test.go
+++ b/tools/profile/impl_test.go
@@ -145,7 +145,7 @@
stdout.Reset()
// Test the 'spec' command.
- if err := cmd.Execute([]string{"spec", exists}); err != nil {
+ if err := cmd.Execute([]string{"specification", exists}); err != nil {
t.Fatalf("%v", err)
}
if expected, got := fmt.Sprintf("%#v", spec), strings.TrimSpace(stdout.String()); got != expected {
