veyron/services/mgmt: Implement optional packages
This change implements optional packages in the mgmt services. The
binary repository now saves the media type and encoding of the objects,
e.g type=application/x-tar,encoding=gzip, or type=application/zip.
The node manager uses this information to download and install the
packages requested in the application envelopes.
TODO: Test the node manager implementation. This will require either using the
real binaryd, or significantly improving the mock binaryd.
Change-Id: Idc7d0ee7663ccd1816a271289de8f61403d8dcce
diff --git a/services/mgmt/binary/binaryd/test.sh b/services/mgmt/binary/binaryd/test.sh
index 51b7201..6cfc373 100755
--- a/services/mgmt/binary/binaryd/test.sh
+++ b/services/mgmt/binary/binaryd/test.sh
@@ -30,29 +30,49 @@
# Create a binary file.
local -r BINARY_SUFFIX="test-binary"
local -r BINARY="${REPO}/${BINARY_SUFFIX}"
- local -r BINARY_FILE=$(shell::tmp_file)
+ local -r BINARY_FILE="${WORKDIR}/bin1"
dd if=/dev/urandom of="${BINARY_FILE}" bs=1000000 count=16 \
|| shell_test::fail "line ${LINENO}: faile to create a random binary file"
"${BINARY_BIN}" upload "${BINARY}" "${BINARY_FILE}" || shell_test::fail "line ${LINENO}: 'upload' failed"
+ # Create TAR file.
+ local -r TAR="${REPO}/tarobj"
+ local -r TAR_FILE="${WORKDIR}/bin1.tar.gz"
+ tar zcvf "${TAR_FILE}" "${BINARY_FILE}"
+ "${BINARY_BIN}" upload "${TAR}" "${TAR_FILE}" || shell_test::fail "line ${LINENO}: 'upload' failed"
+
# Download the binary file.
- local -r BINARY_FILE2=$(shell::tmp_file)
+ local -r BINARY_FILE2="${WORKDIR}/bin2"
"${BINARY_BIN}" download "${BINARY}" "${BINARY_FILE2}" || shell_test::fail "line ${LINENO}: 'RPC download' failed"
if [[ $(cmp "${BINARY_FILE}" "${BINARY_FILE2}" &> /dev/null) ]]; then
shell_test::fail "mismatching binary file downloaded via RPC"
fi
+ local -r BINARY_FILE2_INFO=$(cat "${BINARY_FILE2}.info")
+ shell_test::assert_eq "${BINARY_FILE2_INFO}" '{"type":"application/octet-stream"}' "${LINENO}"
- local -r BINARY_FILE3=$(shell::tmp_file)
+ # Download the tar file.
+ local -r TAR_FILE2="${WORKDIR}/downloadedtar"
+ "${BINARY_BIN}" download "${TAR}" "${TAR_FILE2}" || shell_test::fail "line ${LINENO}: 'RPC download' failed"
+ if [[ $(cmp "${TAR_FILE}" "${TAR_FILE2}" &> /dev/null) ]]; then
+ shell_test::fail "mismatching tar file downloaded via RPC"
+ fi
+ local -r TAR_FILE2_INFO=$(cat "${TAR_FILE2}.info")
+ shell_test::assert_eq "${TAR_FILE2_INFO}" '{"encoding":"gzip","type":"application/x-tar"}' "${LINENO}"
+
+ local -r BINARY_FILE3="${WORKDIR}/bin3"
curl -f -o "${BINARY_FILE3}" "http://${HTTP_ADDR}/${BINARY_SUFFIX}" || shell_test::fail "line ${LINENO}: 'HTTP download' failed"
if [[ $(cmp "${BINARY_FILE}" "${BINARY_FILE3}" &> /dev/null) ]]; then
shell_test::fail "mismatching binary file downloaded via HTTP"
fi
- # Remove the binary file.
+ # Remove the files.
"${BINARY_BIN}" delete "${BINARY}" || shell_test::fail "line ${LINENO}: 'delete' failed"
+ "${BINARY_BIN}" delete "${TAR}" || shell_test::fail "line ${LINENO}: 'delete' failed"
- # Check the binary no longer exists.
- local -r RESULT=$(shell::check_result "${BINARY_BIN}" download "${BINARY}" "${BINARY_FILE2}")
+ # Check the files no longer exist.
+ local RESULT=$(shell::check_result "${BINARY_BIN}" download "${BINARY}" "${BINARY_FILE2}")
+ shell_test::assert_ne "${RESULT}" "0" "${LINENO}"
+ RESULT=$(shell::check_result "${BINARY_BIN}" download "${TAR}" "${TAR_FILE2}")
shell_test::assert_ne "${RESULT}" "0" "${LINENO}"
shell_test::pass
diff --git a/services/mgmt/binary/impl/fs_utils.go b/services/mgmt/binary/impl/fs_utils.go
index 431f434..62eaab8 100644
--- a/services/mgmt/binary/impl/fs_utils.go
+++ b/services/mgmt/binary/impl/fs_utils.go
@@ -79,7 +79,7 @@
}
result[idx] = filepath.Join(path, partName)
} else {
- if info.Name() == "name" {
+ if info.Name() == "name" || info.Name() == "mediainfo" {
continue
}
// The only entries should correspond to the part dirs.
diff --git a/services/mgmt/binary/impl/http_test.go b/services/mgmt/binary/impl/http_test.go
index 1ca59e8..bde93b3 100644
--- a/services/mgmt/binary/impl/http_test.go
+++ b/services/mgmt/binary/impl/http_test.go
@@ -9,6 +9,7 @@
"testing"
"veyron.io/veyron/veyron2/rt"
+ "veyron.io/veyron/veyron2/services/mgmt/repository"
"veyron.io/veyron/veyron/lib/testutil"
)
@@ -27,7 +28,8 @@
size := testutil.Rand.Intn(1000*bufferLength) + 1
data[i] = testutil.RandomBytes(size)
}
- if err := binary.Create(rt.R().NewContext(), int32(length), "application/octet-stream"); err != nil {
+ mediaInfo := repository.MediaInfo{Type: "application/octet-stream"}
+ if err := binary.Create(rt.R().NewContext(), int32(length), mediaInfo); err != nil {
t.Fatalf("Create() failed: %v", err)
}
for i := 0; i < length; i++ {
diff --git a/services/mgmt/binary/impl/impl_test.go b/services/mgmt/binary/impl/impl_test.go
index ec0e151..4eaa06e 100644
--- a/services/mgmt/binary/impl/impl_test.go
+++ b/services/mgmt/binary/impl/impl_test.go
@@ -162,7 +162,7 @@
size := testutil.Rand.Intn(1000 * bufferLength)
data := testutil.RandomBytes(size)
// Test the binary repository interface.
- if err := binary.Create(rt.R().NewContext(), 1, "applicaton/octet-stream"); err != nil {
+ if err := binary.Create(rt.R().NewContext(), 1, repository.MediaInfo{Type: "application/octet-stream"}); err != nil {
t.Fatalf("Create() failed: %v", err)
}
if streamErr, err := invokeUpload(t, binary, data, 0); streamErr != nil || err != nil {
@@ -215,7 +215,7 @@
data[i] = testutil.RandomBytes(size)
}
// Test the binary repository interface.
- if err := binary.Create(rt.R().NewContext(), int32(length), "applicaton/octet-stream"); err != nil {
+ if err := binary.Create(rt.R().NewContext(), int32(length), repository.MediaInfo{Type: "application/octet-stream"}); err != nil {
t.Fatalf("Create() failed: %v", err)
}
for i := 0; i < length; i++ {
@@ -264,7 +264,7 @@
size := testutil.Rand.Intn(1000 * bufferLength)
data[i] = testutil.RandomBytes(size)
}
- if err := binary.Create(rt.R().NewContext(), int32(length), "applicaton/octet-stream"); err != nil {
+ if err := binary.Create(rt.R().NewContext(), int32(length), repository.MediaInfo{Type: "application/octet-stream"}); err != nil {
t.Fatalf("Create() failed: %v", err)
}
// Simulate a flaky upload client that keeps uploading parts until
@@ -309,10 +309,10 @@
data[i][j] = byte(testutil.Rand.Int())
}
}
- if err := binary.Create(rt.R().NewContext(), int32(length), "applicaton/octet-stream"); err != nil {
+ if err := binary.Create(rt.R().NewContext(), int32(length), repository.MediaInfo{Type: "application/octet-stream"}); err != nil {
t.Fatalf("Create() failed: %v", err)
}
- if err := binary.Create(rt.R().NewContext(), int32(length), "applicaton/octet-stream"); err == nil {
+ if err := binary.Create(rt.R().NewContext(), int32(length), repository.MediaInfo{Type: "application/octet-stream"}); err == nil {
t.Fatalf("Create() did not fail when it should have")
} else if want := verror.Exists; !verror.Is(err, want) {
t.Fatalf("Unexpected error: %v, expected error id %v", err, want)
@@ -372,7 +372,7 @@
name := naming.JoinAddressName(ep, obj)
binary := repository.BinaryClient(name)
- if err := binary.Create(rt.R().NewContext(), 1, "applicaton/octet-stream"); err != nil {
+ if err := binary.Create(rt.R().NewContext(), 1, repository.MediaInfo{Type: "application/octet-stream"}); err != nil {
t.Fatalf("Create() failed: %v", err)
}
if streamErr, err := invokeUpload(t, binary, data, 0); streamErr != nil || err != nil {
diff --git a/services/mgmt/binary/impl/service.go b/services/mgmt/binary/impl/service.go
index 44894a1..04cb34b 100644
--- a/services/mgmt/binary/impl/service.go
+++ b/services/mgmt/binary/impl/service.go
@@ -22,6 +22,7 @@
import (
"crypto/md5"
"encoding/hex"
+ "encoding/json"
"io"
"io/ioutil"
"os"
@@ -75,8 +76,8 @@
const bufferLength = 4096
-func (i *binaryService) Create(_ ipc.ServerContext, nparts int32, mediaType string) error {
- vlog.Infof("%v.Create(%v, %v)", i.suffix, nparts, mediaType)
+func (i *binaryService) Create(_ ipc.ServerContext, nparts int32, mediaInfo repository.MediaInfo) error {
+ vlog.Infof("%v.Create(%v, %v)", i.suffix, nparts, mediaInfo)
if nparts < 1 {
return errInvalidParts
}
@@ -96,6 +97,16 @@
vlog.Errorf("WriteFile(%q) failed: %v", nameFile)
return errOperationFailed
}
+ infoFile := filepath.Join(tmpDir, "mediainfo")
+ jInfo, err := json.Marshal(mediaInfo)
+ if err != nil {
+ vlog.Errorf("json.Marshal(%v) failed: %v", mediaInfo, err)
+ return errOperationFailed
+ }
+ if err := ioutil.WriteFile(infoFile, jInfo, os.FileMode(0600)); err != nil {
+ vlog.Errorf("WriteFile(%q) failed: %v", infoFile, err)
+ return errOperationFailed
+ }
for j := 0; j < int(nparts); j++ {
partPath, partPerm := generatePartPath(tmpDir, j), os.FileMode(0700)
if err := os.MkdirAll(partPath, partPerm); err != nil {
@@ -199,12 +210,12 @@
return "", 0, nil
}
-func (i *binaryService) Stat(ipc.ServerContext) ([]binary.PartInfo, string, error) {
+func (i *binaryService) Stat(ipc.ServerContext) ([]binary.PartInfo, repository.MediaInfo, error) {
vlog.Infof("%v.Stat()", i.suffix)
result := make([]binary.PartInfo, 0)
parts, err := getParts(i.path)
if err != nil {
- return []binary.PartInfo{}, "", err
+ return []binary.PartInfo{}, repository.MediaInfo{}, err
}
for _, part := range parts {
checksumFile := filepath.Join(part, checksum)
@@ -215,7 +226,7 @@
continue
}
vlog.Errorf("ReadFile(%v) failed: %v", checksumFile, err)
- return []binary.PartInfo{}, "", errOperationFailed
+ return []binary.PartInfo{}, repository.MediaInfo{}, errOperationFailed
}
dataFile := filepath.Join(part, data)
fi, err := os.Stat(dataFile)
@@ -225,12 +236,22 @@
continue
}
vlog.Errorf("Stat(%v) failed: %v", dataFile, err)
- return []binary.PartInfo{}, "", errOperationFailed
+ return []binary.PartInfo{}, repository.MediaInfo{}, errOperationFailed
}
result = append(result, binary.PartInfo{Checksum: string(bytes), Size: fi.Size()})
}
- // TODO(rthellend): Store the actual media type.
- return result, "application/octet-stream", nil
+ infoFile := filepath.Join(i.path, "mediainfo")
+ jInfo, err := ioutil.ReadFile(infoFile)
+ if err != nil {
+ vlog.Errorf("ReadFile(%q) failed: %v", infoFile)
+ return []binary.PartInfo{}, repository.MediaInfo{}, errOperationFailed
+ }
+ var mediaInfo repository.MediaInfo
+ if err := json.Unmarshal(jInfo, &mediaInfo); err != nil {
+ vlog.Errorf("json.Unmarshal(%v) failed: %v", jInfo, err)
+ return []binary.PartInfo{}, repository.MediaInfo{}, errOperationFailed
+ }
+ return result, mediaInfo, nil
}
func (i *binaryService) Upload(context repository.BinaryUploadContext, part int32) error {
diff --git a/services/mgmt/lib/binary/impl.go b/services/mgmt/lib/binary/impl.go
index a1bedb8..07bedd9 100644
--- a/services/mgmt/lib/binary/impl.go
+++ b/services/mgmt/lib/binary/impl.go
@@ -18,6 +18,8 @@
"veyron.io/veyron/veyron2/services/mgmt/repository"
"veyron.io/veyron/veyron2/verror"
"veyron.io/veyron/veyron2/vlog"
+
+ "veyron.io/veyron/veyron/services/mgmt/lib/packages"
)
var (
@@ -41,17 +43,16 @@
return nil
}
-func download(ctx context.T, w io.WriteSeeker, von string) error {
+func download(ctx context.T, w io.WriteSeeker, von string) (repository.MediaInfo, error) {
client := repository.BinaryClient(von)
- // TODO(rthellend): Use the media type.
- parts, _, err := client.Stat(ctx)
+ parts, mediaInfo, err := client.Stat(ctx)
if err != nil {
vlog.Errorf("Stat() failed: %v", err)
- return err
+ return repository.MediaInfo{}, err
}
for _, part := range parts {
if part.Checksum == binary.MissingChecksum {
- return errNotExist
+ return repository.MediaInfo{}, errNotExist
}
}
offset, whence := int64(0), 0
@@ -102,33 +103,34 @@
success = true
}
if !success {
- return errOperationFailed
+ return repository.MediaInfo{}, errOperationFailed
}
offset += part.Size
}
- return nil
+ return mediaInfo, nil
}
-func Download(ctx context.T, von string) ([]byte, error) {
+func Download(ctx context.T, von string) ([]byte, repository.MediaInfo, error) {
dir, prefix := "", ""
file, err := ioutil.TempFile(dir, prefix)
if err != nil {
vlog.Errorf("TempFile(%v, %v) failed: %v", dir, prefix, err)
- return nil, errOperationFailed
+ return nil, repository.MediaInfo{}, errOperationFailed
}
defer os.Remove(file.Name())
defer file.Close()
ctx, cancel := ctx.WithTimeout(time.Minute)
defer cancel()
- if err := download(ctx, file, von); err != nil {
- return nil, errOperationFailed
+ mediaInfo, err := download(ctx, file, von)
+ if err != nil {
+ return nil, repository.MediaInfo{}, errOperationFailed
}
bytes, err := ioutil.ReadFile(file.Name())
if err != nil {
vlog.Errorf("ReadFile(%v) failed: %v", file.Name(), err)
- return nil, errOperationFailed
+ return nil, repository.MediaInfo{}, errOperationFailed
}
- return bytes, nil
+ return bytes, mediaInfo, nil
}
func DownloadToFile(ctx context.T, von, path string) error {
@@ -141,13 +143,14 @@
defer file.Close()
ctx, cancel := ctx.WithTimeout(time.Minute)
defer cancel()
- if err := download(ctx, file, von); err != nil {
+ mediaInfo, err := download(ctx, file, von)
+ if err != nil {
if err := os.Remove(file.Name()); err != nil {
vlog.Errorf("Remove(%v) failed: %v", file.Name(), err)
}
return errOperationFailed
}
- perm := os.FileMode(0700)
+ perm := os.FileMode(0600)
if err := file.Chmod(perm); err != nil {
vlog.Errorf("Chmod(%v) failed: %v", perm, err)
if err := os.Remove(file.Name()); err != nil {
@@ -162,10 +165,17 @@
}
return errOperationFailed
}
+ if err := packages.SaveMediaInfo(path, mediaInfo); err != nil {
+ vlog.Errorf("packages.SaveMediaInfo(%v, %v) failed: %v", path, mediaInfo, err)
+ if err := os.Remove(path); err != nil {
+ vlog.Errorf("Remove(%v) failed: %v", path, err)
+ }
+ return errOperationFailed
+ }
return nil
}
-func upload(ctx context.T, r io.ReadSeeker, von string) error {
+func upload(ctx context.T, r io.ReadSeeker, mediaInfo repository.MediaInfo, von string) error {
client := repository.BinaryClient(von)
offset, whence := int64(0), 2
size, err := r.Seek(offset, whence)
@@ -174,9 +184,7 @@
return errOperationFailed
}
nparts := (size-1)/partSize + 1
- // TODO(rthellend): Determine the actual media type.
- mediaType := "application/octet-stream"
- if err := client.Create(ctx, int32(nparts), mediaType); err != nil {
+ if err := client.Create(ctx, int32(nparts), mediaInfo); err != nil {
vlog.Errorf("Create() failed: %v", err)
return err
}
@@ -258,11 +266,11 @@
return nil
}
-func Upload(ctx context.T, von string, data []byte) error {
+func Upload(ctx context.T, von string, data []byte, mediaInfo repository.MediaInfo) error {
buffer := bytes.NewReader(data)
ctx, cancel := ctx.WithTimeout(time.Minute)
defer cancel()
- return upload(ctx, buffer, von)
+ return upload(ctx, buffer, mediaInfo, von)
}
func UploadFromFile(ctx context.T, von, path string) error {
@@ -274,5 +282,6 @@
}
ctx, cancel := ctx.WithTimeout(time.Minute)
defer cancel()
- return upload(ctx, file, von)
+ mediaInfo := packages.MediaInfoForFileName(path)
+ return upload(ctx, file, mediaInfo, von)
}
diff --git a/services/mgmt/lib/binary/impl_test.go b/services/mgmt/lib/binary/impl_test.go
index ce66250..ca0a76f 100644
--- a/services/mgmt/lib/binary/impl_test.go
+++ b/services/mgmt/lib/binary/impl_test.go
@@ -5,11 +5,13 @@
"io/ioutil"
"os"
"path/filepath"
+ "reflect"
"testing"
"veyron.io/veyron/veyron2"
"veyron.io/veyron/veyron2/naming"
"veyron.io/veyron/veyron2/rt"
+ "veyron.io/veyron/veyron2/services/mgmt/repository"
"veyron.io/veyron/veyron2/vlog"
"veyron.io/veyron/veyron/lib/testutil"
@@ -85,21 +87,25 @@
von, cleanup := setupRepository(t)
defer cleanup()
data := testutil.RandomBytes(testutil.Rand.Intn(10 << 20))
- if err := Upload(runtime.NewContext(), von, data); err != nil {
+ mediaInfo := repository.MediaInfo{Type: "application/octet-stream"}
+ if err := Upload(runtime.NewContext(), von, data, mediaInfo); err != nil {
t.Fatalf("Upload(%v) failed: %v", von, err)
}
- output, err := Download(runtime.NewContext(), von)
+ output, outInfo, err := Download(runtime.NewContext(), von)
if err != nil {
t.Fatalf("Download(%v) failed: %v", von, err)
}
if bytes.Compare(data, output) != 0 {
- t.Fatalf("Data mismatch:\nexpected %v %v\ngot %v %v", len(data), data[:100], len(output), output[:100])
+ t.Errorf("Data mismatch:\nexpected %v %v\ngot %v %v", len(data), data[:100], len(output), output[:100])
}
if err := Delete(runtime.NewContext(), von); err != nil {
- t.Fatalf("Delete(%v) failed: %v", von, err)
+ t.Errorf("Delete(%v) failed: %v", von, err)
}
- if _, err := Download(runtime.NewContext(), von); err == nil {
- t.Fatalf("Download(%v) did not fail", von)
+ if _, _, err := Download(runtime.NewContext(), von); err == nil {
+ t.Errorf("Download(%v) did not fail", von)
+ }
+ if !reflect.DeepEqual(mediaInfo, outInfo) {
+ t.Errorf("unexpected media info: expected %v, got %v", mediaInfo, outInfo)
}
}
@@ -117,11 +123,15 @@
}
defer os.Remove(src.Name())
defer src.Close()
- dst, err := ioutil.TempFile(dir, prefix)
+ dstdir, err := ioutil.TempDir(dir, prefix)
if err != nil {
- t.Fatalf("TempFile(%v, %v) failed: %v", dir, prefix, err)
+ t.Fatalf("TempDir(%v, %v) failed: %v", dir, prefix, err)
}
- defer os.Remove(dst.Name())
+ defer os.RemoveAll(dstdir)
+ dst, err := ioutil.TempFile(dstdir, prefix)
+ if err != nil {
+ t.Fatalf("TempFile(%v, %v) failed: %v", dstdir, prefix, err)
+ }
defer dst.Close()
if _, err := src.Write(data); err != nil {
t.Fatalf("Write() failed: %v", err)
@@ -134,12 +144,19 @@
}
output, err := ioutil.ReadFile(dst.Name())
if err != nil {
- t.Fatalf("ReadFile(%v) failed: %v", dst.Name(), err)
+ t.Errorf("ReadFile(%v) failed: %v", dst.Name(), err)
}
if bytes.Compare(data, output) != 0 {
- t.Fatalf("Data mismatch:\nexpected %v %v\ngot %v %v", len(data), data[:100], len(output), output[:100])
+ t.Errorf("Data mismatch:\nexpected %v %v\ngot %v %v", len(data), data[:100], len(output), output[:100])
+ }
+ jMediaInfo, err := ioutil.ReadFile(dst.Name() + ".__info")
+ if err != nil {
+ t.Errorf("ReadFile(%v) failed: %v", dst.Name()+".__info", err)
+ }
+ if expected := `{"Type":"application/octet-stream","Encoding":""}`; string(jMediaInfo) != expected {
+ t.Errorf("unexpected media info: expected %q, got %q", expected, string(jMediaInfo))
}
if err := Delete(runtime.NewContext(), von); err != nil {
- t.Fatalf("Delete(%v) failed: %v", von, err)
+ t.Errorf("Delete(%v) failed: %v", von, err)
}
}
diff --git a/services/mgmt/lib/packages/packages.go b/services/mgmt/lib/packages/packages.go
new file mode 100644
index 0000000..57f9dde
--- /dev/null
+++ b/services/mgmt/lib/packages/packages.go
@@ -0,0 +1,192 @@
+// Package packages provides functionality to install ZIP and TAR packages.
+package packages
+
+import (
+ "archive/tar"
+ "archive/zip"
+ "compress/bzip2"
+ "compress/gzip"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "veyron.io/veyron/veyron2/services/mgmt/repository"
+)
+
+const defaultType = "application/octet-stream"
+
+var typemap = map[string]repository.MediaInfo{
+ ".zip": repository.MediaInfo{Type: "application/zip"},
+ ".tar": repository.MediaInfo{Type: "application/x-tar"},
+ ".tgz": repository.MediaInfo{Type: "application/x-tar", Encoding: "gzip"},
+ ".tar.gz": repository.MediaInfo{Type: "application/x-tar", Encoding: "gzip"},
+ ".tbz2": repository.MediaInfo{Type: "application/x-tar", Encoding: "bzip2"},
+ ".tb2": repository.MediaInfo{Type: "application/x-tar", Encoding: "bzip2"},
+ ".tbz": repository.MediaInfo{Type: "application/x-tar", Encoding: "bzip2"},
+ ".tar.bz2": repository.MediaInfo{Type: "application/x-tar", Encoding: "bzip2"},
+}
+
+// MediaInfoForFileName returns the MediaInfo based on the file's extension.
+func MediaInfoForFileName(fileName string) repository.MediaInfo {
+ fileName = strings.ToLower(fileName)
+ for k, v := range typemap {
+ if strings.HasSuffix(fileName, k) {
+ return v
+ }
+ }
+ return repository.MediaInfo{Type: defaultType}
+}
+
+// Install installs a package in the given directory. If the package is a TAR or
+// ZIP archive, its content is extracted in the destination directory.
+// Otherwise, the package file itself is copied to the destination directory.
+func Install(pkgFile, dir string) error {
+ mediaInfo, err := LoadMediaInfo(pkgFile)
+ if err != nil {
+ return err
+ }
+ switch mediaInfo.Type {
+ case "application/x-tar":
+ return extractTar(pkgFile, mediaInfo, dir)
+ case "application/zip":
+ return extractZip(pkgFile, dir)
+ default:
+ return fmt.Errorf("unsupported media type: %v", mediaInfo.Type)
+ }
+}
+
+// LoadMediaInfo returns the MediaInfo for the given package file.
+func LoadMediaInfo(pkgFile string) (repository.MediaInfo, error) {
+ jInfo, err := ioutil.ReadFile(pkgFile + ".__info")
+ if err != nil {
+ return repository.MediaInfo{}, err
+ }
+ var info repository.MediaInfo
+ if err := json.Unmarshal(jInfo, &info); err != nil {
+ return repository.MediaInfo{}, err
+ }
+ return info, nil
+}
+
+// SaveMediaInfo saves the media info for a package.
+func SaveMediaInfo(pkgFile string, mediaInfo repository.MediaInfo) error {
+ jInfo, err := json.Marshal(mediaInfo)
+ if err != nil {
+ return err
+ }
+ infoFile := pkgFile + ".__info"
+ if err := ioutil.WriteFile(infoFile, jInfo, os.FileMode(0600)); err != nil {
+ return err
+ }
+ return nil
+}
+
+func extractZip(zipFile, installDir string) error {
+ zr, err := zip.OpenReader(zipFile)
+ if err != nil {
+ return err
+ }
+ for _, file := range zr.File {
+ fi := file.FileInfo()
+ name := filepath.Join(installDir, file.Name)
+ if !strings.HasPrefix(name, installDir) {
+ return fmt.Errorf("failed to extract file %q outside of install directory", file.Name)
+ }
+ if fi.IsDir() {
+ if err := os.MkdirAll(name, os.FileMode(fi.Mode()&0700)); err != nil && !os.IsExist(err) {
+ return err
+ }
+ continue
+ }
+ in, err := file.Open()
+ if err != nil {
+ return err
+ }
+ parentName := filepath.Dir(name)
+ if err := os.MkdirAll(parentName, os.FileMode(0700)); err != nil {
+ return err
+ }
+ out, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY, os.FileMode(fi.Mode()&0700))
+ if err != nil {
+ in.Close()
+ return err
+ }
+ nbytes, err := io.Copy(out, in)
+ in.Close()
+ out.Close()
+ if err != nil {
+ return err
+ }
+ if nbytes != fi.Size() {
+ return fmt.Errorf("file size doesn't match for %q: %d != %d", fi.Name(), nbytes, fi.Size())
+ }
+ }
+ return nil
+}
+
+func extractTar(pkgFile string, mediaInfo repository.MediaInfo, installDir string) error {
+ f, err := os.Open(pkgFile)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ var reader io.Reader
+ switch enc := mediaInfo.Encoding; enc {
+ case "":
+ reader = f
+ case "gzip":
+ var err error
+ if reader, err = gzip.NewReader(f); err != nil {
+ return err
+ }
+ case "bzip2":
+ reader = bzip2.NewReader(f)
+ default:
+ return fmt.Errorf("unsupported encoding: %q", enc)
+ }
+
+ tr := tar.NewReader(reader)
+ for {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ name := filepath.Join(installDir, hdr.Name)
+ if !strings.HasPrefix(name, installDir) {
+ return fmt.Errorf("failed to extract file %q outside of install directory", hdr.Name)
+ }
+ // Regular file
+ if hdr.Typeflag == tar.TypeReg {
+ out, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode&0700))
+ if err != nil {
+ return err
+ }
+ nbytes, err := io.Copy(out, tr)
+ out.Close()
+ if err != nil {
+ return err
+ }
+ if nbytes != hdr.Size {
+ return fmt.Errorf("file size doesn't match for %q: %d != %d", hdr.Name, nbytes, hdr.Size)
+ }
+ continue
+ }
+ // Directory
+ if hdr.Typeflag == tar.TypeDir {
+ if err := os.Mkdir(name, os.FileMode(hdr.Mode&0700)); err != nil && !os.IsExist(err) {
+ return err
+ }
+ continue
+ }
+ // Skip unsupported types
+ // TODO(rthellend): Consider adding support for Symlink.
+ }
+}
diff --git a/services/mgmt/lib/packages/packages_test.go b/services/mgmt/lib/packages/packages_test.go
new file mode 100644
index 0000000..8182b44
--- /dev/null
+++ b/services/mgmt/lib/packages/packages_test.go
@@ -0,0 +1,242 @@
+package packages_test
+
+import (
+ "archive/tar"
+ "archive/zip"
+ "compress/gzip"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "reflect"
+ "sort"
+ "testing"
+
+ "veyron.io/veyron/veyron2/services/mgmt/repository"
+
+ "veyron.io/veyron/veyron/services/mgmt/lib/packages"
+)
+
+func TestInstall(t *testing.T) {
+ workdir, err := ioutil.TempDir("", "packages-test-")
+ if err != nil {
+ t.Fatalf("ioutil.TempDir failed: %v", err)
+ }
+ defer os.RemoveAll(workdir)
+ srcdir := filepath.Join(workdir, "src")
+ dstdir := filepath.Join(workdir, "dst")
+ createFiles(t, srcdir)
+
+ zipfile := filepath.Join(workdir, "archivezip")
+ tarfile := filepath.Join(workdir, "archivetar")
+ tgzfile := filepath.Join(workdir, "archivetgz")
+
+ makeZip(t, zipfile, srcdir)
+ makeTar(t, tarfile, srcdir)
+ doGzip(t, tarfile, tgzfile)
+
+ binfile := filepath.Join(workdir, "binfile")
+ ioutil.WriteFile(binfile, []byte("This is a binary file"), os.FileMode(0644))
+ ioutil.WriteFile(binfile+".__info", []byte(`{"type":"application/octet-stream"}`), os.FileMode(0644))
+
+ expected := []string{
+ "a perm:700",
+ "a/b perm:700",
+ "a/b/xyzzy.txt perm:600",
+ "a/bar.txt perm:600",
+ "a/foo.txt perm:600",
+ }
+ for _, file := range []string{zipfile, tarfile, tgzfile} {
+ setupDstDir(t, dstdir)
+ if err := packages.Install(file, dstdir); err != nil {
+ t.Errorf("packages.Install failed for %q: %v", file, err)
+ }
+ files := scanDir(dstdir)
+ if !reflect.DeepEqual(files, expected) {
+ t.Errorf("unexpected result for %q: Got %q, want %q", file, files, expected)
+ }
+ }
+
+ setupDstDir(t, dstdir)
+ if err := packages.Install(binfile, dstdir); err == nil {
+ t.Errorf("expected packages.Install to fail %q", binfile)
+ }
+}
+
+func TestMediaInfo(t *testing.T) {
+ testcases := []struct {
+ filename string
+ expected repository.MediaInfo
+ }{
+ {"foo.zip", repository.MediaInfo{Type: "application/zip"}},
+ {"foo.ZIP", repository.MediaInfo{Type: "application/zip"}},
+ {"foo.tar", repository.MediaInfo{Type: "application/x-tar"}},
+ {"foo.TAR", repository.MediaInfo{Type: "application/x-tar"}},
+ {"foo.tgz", repository.MediaInfo{Type: "application/x-tar", Encoding: "gzip"}},
+ {"FOO.TAR.GZ", repository.MediaInfo{Type: "application/x-tar", Encoding: "gzip"}},
+ {"foo.tbz2", repository.MediaInfo{Type: "application/x-tar", Encoding: "bzip2"}},
+ {"foo.tar.bz2", repository.MediaInfo{Type: "application/x-tar", Encoding: "bzip2"}},
+ {"foo", repository.MediaInfo{Type: "application/octet-stream"}},
+ }
+ for _, tc := range testcases {
+ if got := packages.MediaInfoForFileName(tc.filename); !reflect.DeepEqual(got, tc.expected) {
+ t.Errorf("unexpected result for %q: Got %v, want %v", tc.filename, got, tc.expected)
+ }
+ }
+}
+
+func createFiles(t *testing.T, dir string) {
+ if err := os.Mkdir(dir, os.FileMode(0755)); err != nil {
+ t.Fatalf("os.Mkdir(%q) failed: %v", dir, err)
+ }
+ dirs := []string{"a", "a/b"}
+ for _, d := range dirs {
+ fullname := filepath.Join(dir, d)
+ if err := os.Mkdir(fullname, os.FileMode(0755)); err != nil {
+ t.Fatalf("os.Mkdir(%q) failed: %v", fullname, err)
+ }
+ }
+ files := []string{"a/foo.txt", "a/bar.txt", "a/b/xyzzy.txt"}
+ for _, f := range files {
+ fullname := filepath.Join(dir, f)
+ if err := ioutil.WriteFile(fullname, []byte(f), os.FileMode(0644)); err != nil {
+ t.Fatalf("ioutil.WriteFile(%q) failed: %v", fullname, err)
+ }
+ }
+}
+
+func makeZip(t *testing.T, zipfile, dir string) {
+ z, err := os.OpenFile(zipfile, os.O_CREATE|os.O_WRONLY, os.FileMode(0644))
+ if err != nil {
+ t.Fatalf("os.OpenFile(%q) failed: %v", zipfile, err)
+ }
+ defer z.Close()
+ w := zip.NewWriter(z)
+ filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ t.Fatalf("Walk(%q) error: %v", dir, err)
+ }
+ if dir == path {
+ return nil
+ }
+ fh, err := zip.FileInfoHeader(info)
+ if err != nil {
+ t.Fatalf("FileInfoHeader failed: %v", err)
+ }
+ fh.Name, _ = filepath.Rel(dir, path)
+ hdr, err := w.CreateHeader(fh)
+ if err != nil {
+ t.Fatalf("w.CreateHeader failed: %v", err)
+ }
+ if !info.IsDir() {
+ content, err := ioutil.ReadFile(path)
+ if err != nil {
+ t.Fatalf("ioutil.ReadFile(%q) failed: %v", path, err)
+ }
+ if _, err = hdr.Write(content); err != nil {
+ t.Fatalf("hdr.Write(%q) failed: %v", content, err)
+ }
+ }
+ return nil
+ })
+ if err := w.Close(); err != nil {
+ t.Fatalf("w.Close() failed: %v", err)
+ }
+ if err := ioutil.WriteFile(zipfile+".__info", []byte(`{"type":"application/zip"}`), os.FileMode(0644)); err != nil {
+ t.Fatalf("ioutil.WriteFile() failed: %v", err)
+ }
+}
+
+func makeTar(t *testing.T, tarfile, dir string) {
+ tf, err := os.OpenFile(tarfile, os.O_CREATE|os.O_WRONLY, os.FileMode(0644))
+ if err != nil {
+ t.Fatalf("os.OpenFile(%q) failed: %v", tarfile, err)
+ }
+ defer tf.Close()
+
+ tw := tar.NewWriter(tf)
+ filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ t.Fatalf("Walk(%q) error: %v", dir, err)
+ }
+ if dir == path {
+ return nil
+ }
+ hdr, err := tar.FileInfoHeader(info, "")
+ if err != nil {
+ t.Fatalf("tar.FileInfoHeader failed: %v", err)
+ }
+ hdr.Name, _ = filepath.Rel(dir, path)
+ if err := tw.WriteHeader(hdr); err != nil {
+ t.Fatalf("tw.WriteHeader failed: %v", err)
+ }
+ if !info.IsDir() {
+ content, err := ioutil.ReadFile(path)
+ if err != nil {
+ t.Fatalf("ioutil.ReadFile(%q) failed: %v", path, err)
+ }
+ if _, err := tw.Write(content); err != nil {
+ t.Fatalf("tw.Write failed: %v", err)
+ }
+ }
+ return nil
+ })
+ if err := tw.Close(); err != nil {
+ t.Fatalf("tw.Close failed: %v", err)
+ }
+ if err := ioutil.WriteFile(tarfile+".__info", []byte(`{"type":"application/x-tar"}`), os.FileMode(0644)); err != nil {
+ t.Fatalf("ioutil.WriteFile() failed: %v", err)
+ }
+}
+
+func doGzip(t *testing.T, infile, outfile string) {
+ in, err := os.Open(infile)
+ if err != nil {
+ t.Fatalf("os.Open(%q) failed: %v", infile, err)
+ }
+ defer in.Close()
+ out, err := os.OpenFile(outfile, os.O_CREATE|os.O_WRONLY, os.FileMode(0644))
+ if err != nil {
+ t.Fatalf("os.OpenFile(%q) failed: %v", outfile, err)
+ }
+ defer out.Close()
+ writer := gzip.NewWriter(out)
+ defer writer.Close()
+ if _, err := io.Copy(writer, in); err != nil {
+ t.Fatalf("io.Copy() failed: %v", err)
+ }
+
+ info, err := packages.LoadMediaInfo(infile)
+ if err != nil {
+ t.Fatalf("LoadMediaInfo(%q) failed: %v", infile, err)
+ }
+ info.Encoding = "gzip"
+ if err := packages.SaveMediaInfo(outfile, info); err != nil {
+ t.Fatalf("SaveMediaInfo(%v) failed: %v", outfile, err)
+ }
+}
+
+func scanDir(root string) []string {
+ files := []string{}
+ filepath.Walk(root, func(path string, info os.FileInfo, _ error) error {
+ if root == path {
+ return nil
+ }
+ rel, _ := filepath.Rel(root, path)
+ perm := info.Mode() & 0700
+ files = append(files, fmt.Sprintf("%s perm:%o", rel, perm))
+ return nil
+ })
+ sort.Strings(files)
+ return files
+}
+
+func setupDstDir(t *testing.T, dst string) {
+ if err := os.RemoveAll(dst); err != nil {
+ t.Fatalf("os.RemoveAll(%q) failed: %v", dst, err)
+ }
+ if err := os.Mkdir(dst, os.FileMode(0755)); err != nil {
+ t.Fatalf("os.Mkdir(%q) failed: %v", dst, err)
+ }
+}
diff --git a/services/mgmt/node/impl/app_service.go b/services/mgmt/node/impl/app_service.go
index 794d9f0..0ad6e33 100644
--- a/services/mgmt/node/impl/app_service.go
+++ b/services/mgmt/node/impl/app_service.go
@@ -21,6 +21,10 @@
// bin - application binary
// previous - symbolic link to previous version directory
// envelope - application envelope (JSON-encoded)
+// pkg/ - the application packages
+// <pkg name>
+// <pkg name>.__info
+// ...
// <version 2 timestamp>
// ...
// current - symbolic link to the current version
@@ -29,6 +33,9 @@
// credentials/ - holds veyron credentials (unless running
// through security agent)
// root/ - workspace that the instance is run from
+// packages/ - the installed packages
+// <pkg name>/
+// ...
// logs/ - stderr/stdout and log files generated by instance
// info - metadata for the instance (such as app
// cycle manager name and process id)
@@ -136,6 +143,8 @@
vsecurity "veyron.io/veyron/veyron/security"
"veyron.io/veyron/veyron/security/agent"
"veyron.io/veyron/veyron/security/agent/keymgr"
+ libbinary "veyron.io/veyron/veyron/services/mgmt/lib/binary"
+ libpackages "veyron.io/veyron/veyron/services/mgmt/lib/packages"
iconfig "veyron.io/veyron/veyron/services/mgmt/node/config"
)
@@ -319,10 +328,25 @@
if err := mkdir(versionDir); err != nil {
return "", errOperationFailed
}
+ pkgDir := filepath.Join(versionDir, "pkg")
+ if err := mkdir(pkgDir); err != nil {
+ return "", errOperationFailed
+ }
// TODO(caprita): Share binaries if already existing locally.
if err := downloadBinary(versionDir, "bin", envelope.Binary); err != nil {
return versionDir, err
}
+ for localPkg, pkgName := range envelope.Packages {
+ if localPkg == "" || localPkg[0] == '.' || strings.Contains(localPkg, string(filepath.Separator)) {
+ vlog.Infof("invalid local package name: %q", localPkg)
+ return versionDir, errOperationFailed
+ }
+ path := filepath.Join(pkgDir, localPkg)
+ if err := libbinary.DownloadToFile(rt.R().NewContext(), pkgName, path); err != nil {
+ vlog.Infof("DownloadToFile(%q, %q) failed: %v", pkgName, path, err)
+ return versionDir, errOperationFailed
+ }
+ }
if err := saveEnvelope(versionDir, envelope); err != nil {
return versionDir, err
}
@@ -554,6 +578,31 @@
return installationDirCore(i.suffix, i.config.Root)
}
+// installPackages installs all the packages for a new instance.
+func installPackages(versionDir, instanceDir string) error {
+ envelope, err := loadEnvelope(versionDir)
+ if err != nil {
+ return err
+ }
+ packagesDir := filepath.Join(instanceDir, "root", "packages")
+ if err := os.MkdirAll(packagesDir, os.FileMode(0700)); err != nil {
+ return err
+ }
+ // TODO(rthellend): Consider making the packages read-only and sharing
+ // them between apps or instances.
+ for pkg, _ := range envelope.Packages {
+ pkgFile := filepath.Join(versionDir, "pkg", pkg)
+ dstDir := filepath.Join(packagesDir, pkg)
+ if err := os.MkdirAll(dstDir, os.FileMode(0700)); err != nil {
+ return err
+ }
+ if err := libpackages.Install(pkgFile, dstDir); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
func initializeInstanceACLs(instanceDir string, blessings []string, acl security.ACL) error {
if acl.In == nil {
// The acl.In will be empty for an unclaimed node manager. In this case,
@@ -599,6 +648,10 @@
vlog.Errorf("Symlink(%v, %v) failed: %v", versionDir, versionLink, err)
return instanceDir, instanceID, errOperationFailed
}
+ if err := installPackages(versionDir, instanceDir); err != nil {
+ vlog.Errorf("installPackages(%v, %v) failed: %v", versionDir, instanceDir, err)
+ return instanceDir, instanceID, errOperationFailed
+ }
instanceInfo := new(instanceInfo)
if err := setupPrincipal(instanceDir, versionDir, call, i.securityAgent, instanceInfo); err != nil {
return instanceDir, instanceID, err
diff --git a/services/mgmt/node/impl/mock_repo_test.go b/services/mgmt/node/impl/mock_repo_test.go
index e669595..6de199e 100644
--- a/services/mgmt/node/impl/mock_repo_test.go
+++ b/services/mgmt/node/impl/mock_repo_test.go
@@ -80,7 +80,7 @@
// BINARY REPOSITORY INTERFACE IMPLEMENTATION
-func (*brInvoker) Create(ipc.ServerContext, int32, string) error {
+func (*brInvoker) Create(ipc.ServerContext, int32, repository.MediaInfo) error {
vlog.VI(1).Infof("Create()")
return nil
}
@@ -125,16 +125,16 @@
return "", 0, nil
}
-func (*brInvoker) Stat(ipc.ServerContext) ([]binary.PartInfo, string, error) {
+func (*brInvoker) Stat(ipc.ServerContext) ([]binary.PartInfo, repository.MediaInfo, error) {
vlog.VI(1).Infof("Stat()")
h := md5.New()
bytes, err := ioutil.ReadFile(os.Args[0])
if err != nil {
- return []binary.PartInfo{}, "", errOperationFailed
+ return []binary.PartInfo{}, repository.MediaInfo{}, errOperationFailed
}
h.Write(bytes)
part := binary.PartInfo{Checksum: hex.EncodeToString(h.Sum(nil)), Size: int64(len(bytes))}
- return []binary.PartInfo{part}, "application/octet-stream", nil
+ return []binary.PartInfo{part}, repository.MediaInfo{Type: "application/octet-stream"}, nil
}
func (i *brInvoker) Upload(repository.BinaryUploadContext, int32) error {
diff --git a/services/mgmt/node/impl/util.go b/services/mgmt/node/impl/util.go
index d5ab40b..8446506 100644
--- a/services/mgmt/node/impl/util.go
+++ b/services/mgmt/node/impl/util.go
@@ -23,7 +23,7 @@
)
func downloadBinary(workspace, fileName, name string) error {
- data, err := binary.Download(rt.R().NewContext(), name)
+ data, _, err := binary.Download(rt.R().NewContext(), name)
if err != nil {
vlog.Errorf("Download(%v) failed: %v", name, err)
return errOperationFailed
diff --git a/tools/binary/impl.go b/tools/binary/impl.go
index 3874c2c..df34d6c 100644
--- a/tools/binary/impl.go
+++ b/tools/binary/impl.go
@@ -71,6 +71,7 @@
}
func runUpload(cmd *cmdline.Command, args []string) error {
+ // TODO(rthellend): Add support for creating packages on the fly.
if expected, got := 2, len(args); expected != got {
return cmd.UsageErrorf("upload: incorrect number of arguments, expected %d, got %d", expected, got)
}
diff --git a/tools/binary/impl_test.go b/tools/binary/impl_test.go
index b1f18fd..865c35d 100644
--- a/tools/binary/impl_test.go
+++ b/tools/binary/impl_test.go
@@ -27,7 +27,7 @@
suffix string
}
-func (s *server) Create(ipc.ServerContext, int32, string) error {
+func (s *server) Create(ipc.ServerContext, int32, repository.MediaInfo) error {
vlog.Infof("Create() was called. suffix=%v", s.suffix)
return nil
}
@@ -53,13 +53,13 @@
return "", 0, nil
}
-func (s *server) Stat(ipc.ServerContext) ([]binary.PartInfo, string, error) {
+func (s *server) Stat(ipc.ServerContext) ([]binary.PartInfo, repository.MediaInfo, error) {
vlog.Infof("Stat() was called. suffix=%v", s.suffix)
h := md5.New()
text := "HelloWorld"
h.Write([]byte(text))
part := binary.PartInfo{Checksum: hex.EncodeToString(h.Sum(nil)), Size: int64(len(text))}
- return []binary.PartInfo{part}, "text/plain", nil
+ return []binary.PartInfo{part}, repository.MediaInfo{Type: "text/plain"}, nil
}
func (s *server) Upload(ctx repository.BinaryUploadContext, _ int32) error {
