playground: Default bundle bootstrap into database, list, get by name.
Default examples are not fetched as static files by the client anymore;
instead, they are bundled into the database with fixed names.
The pgadmin utility has been extended to wrap pgbundle and load bundles
into the database; next step is porting pgbundle to Go and fully integrating
it into pgadmin.
Example naming and testing metadata is now read from a JSON config file
instead of hardcoded into tests/bundler.
compilerd now supports listing default examples and loading them by name.
Added tests and basic clientside support.
Fixed migration test.
Increased bundle ID entropy.
Made database handle closing more robust.
Change-Id: I5000b2d4edd0864ed468faa5244a298bccd0912d
diff --git a/client/.gitignore b/client/.gitignore
index 2d92022..59913fe 100644
--- a/client/.gitignore
+++ b/client/.gitignore
@@ -1,9 +1,5 @@
-bundles/*/*/bin
-bundles/*/*/pkg
-public/bundles
public/version
bundle.*
!browser/components/bundle.js
-bundle_*.json
node_modules
npm-debug.log
diff --git a/client/.jshintignore b/client/.jshintignore
index 13b2f5f..ea9a206 100644
--- a/client/.jshintignore
+++ b/client/.jshintignore
@@ -1,4 +1,3 @@
node_modules
build
public
-bundles
diff --git a/client/Makefile b/client/Makefile
index f8e7251..37ddc20 100644
--- a/client/Makefile
+++ b/client/Makefile
@@ -12,7 +12,7 @@
.DELETE_ON_ERROR:
.PHONY: all
-all: public/bundles public/bundle.js public/bundle.css
+all: public/bundle.js public/bundle.css
@true # silences `watch make`
.PHONY: deploy-production
@@ -29,36 +29,8 @@
public/bundle.css: stylesheets/index.css $(css_files) node_modules
bin/compile-css $< 1> $@
-# Each profile (glob file with file patterns to include) from
-# `bundles/<profile>.bundle` is applied to each example folder of the
-# form `bundles/<example>/`.
-example_profiles := $(shell find bundles -maxdepth 1 -name "*.bundle")
-example_code_bundle_dirs := $(shell find bundles -maxdepth 1 -mindepth 1 -type d)
-example_files := $(shell find bundles -mindepth 1)
-bundle_temp_file := public/bundle.json.tmp
-
-# Builds the playground bundles for each folder and profile combination.
-# This task depends on example_profiles and example_files because we want
-# to re-bundle if any of those change. However, the bundle tool works on
-# directories, so we pass in example_code_bundle_dirs as the path argument.
-# Note: Bundle correctness test was moved to builder integration test.
-# TODO(ivanpi): Move bundles out of client when bootstrap support is ready.
-public/bundles: $(example_profiles) $(example_files) | node_modules
- $(RM) -rf $@
- mkdir -p $(dir $(bundle_temp_file))
- @for profile in $(example_profiles); do \
- echo "Bundling with profile \"$${profile}\""; \
- for folder in $(example_code_bundle_dirs); do \
- pgbundle --verbose "$${profile}" "$${folder}" > $(bundle_temp_file); \
- mv -f $(bundle_temp_file) "$${folder}/bundle_$$(basename $${profile} .bundle).json"; \
- done; \
- done
- cp -r bundles $(@D)
-
node_modules: package.json
@npm prune
- # Temporary workaround: install pgbundle directly from source.
- @npm install $(V23_ROOT)/release/projects/playground/pgbundle
@npm install
@touch $@
@@ -75,8 +47,6 @@
.PHONY: distclean
distclean: clean
@npm cache clean
- @$(RM) -rf public/bundles
- @$(RM) -rf $(shell find bundles -name "bundle*.json")
.PHONY: dependency-check
dependency-check: package.json node_modules
diff --git a/client/README.md b/client/README.md
index 3bbbcb9..2b4de85 100644
--- a/client/README.md
+++ b/client/README.md
@@ -6,14 +6,14 @@
## Directory structure
-* `browser` - JS modules to be compiled/bundle via `[browserify]`.
-* `bundles` - Default playground examples. Each combination of directory and `.bundle` file forms a bundle.
-* _Makefile_ - Targets for building the client (browserifying Javascript, compile CSS, etc.).
+* `bin` - Utility scripts for building the client.
+* `browser` - JS modules to be compiled/bundled via `[browserify]`.
+* _Makefile_ - Targets for building the client (browserifying Javascript, compiling CSS, etc.).
* `node_modules` - JS dependencies created by `npm install`.
* _package.json_ - Used by `npm install` to resolve and download dependencies.
-* `public` - Static assets, including bundle targets for JS/CSS served by `make start`.
+* `public` - Static assets, including JS/CSS bundles, served by `make start`.
* `stylesheets` - CSS to be compiled to `public/bundle.css`.
-* _test.sh_ - Script testing correctness of default playground examples.
+* `test` - Client TAP unit tests.
Requires [npm] and [Node.js].
@@ -21,8 +21,8 @@
make
-The command above automatically fetches node dependencies, and builds necessary assets for the client bundles to the `public`
-directory for serving.
+The command above automatically fetches Node dependencies and builds necessary
+assets for the client bundles to the `public` directory for serving.
## Local server
diff --git a/client/browser/api/index.js b/client/browser/api/index.js
index 5c082f1..7f712d2 100644
--- a/client/browser/api/index.js
+++ b/client/browser/api/index.js
@@ -78,6 +78,9 @@
case 'read':
clone.pathname = url.resolve(clone.pathname, 'load');
break;
+ case 'list':
+ clone.pathname = url.resolve(clone.pathname, 'list');
+ break;
case 'run':
clone.pathname = url.resolve(clone.pathname, 'compile');
break;
@@ -99,28 +102,44 @@
}
};
-// Temporary way to list bundles until there is an API endpoint to hit.
API.prototype.bundles = function(callback) {
var api = this;
- // TODO(jasoncampbell): remove this list once a list API endpoint is
- // available.
- var ids = [
- '_cadcfa075a6ac6d1939d12a64ac6e57bc7256c0422fb5d0690b3d8618779565',
- '_be43fb9b2d03087dfd7c84437fd37dac7f6977d8cac330b9fce6aad94414558',
- '_5385edd72b550c57bee83b100731338c70349ac7354dc4353665a1998fa7c8c',
- '_46f8b66f0e80be00adc6222ac0235b1f8e70183daa64ec5924b14267dc6f0fd'
- ];
+ var uri = api.url({ action: 'list' });
- var workers = ids.map(createWorker);
+ request
+ .get(uri)
+ .accept('json')
+ .timeout(api.options.timeout)
+ .end(onlist);
- // Request all ids in parallel.
- parallel(workers, callback);
+ function onlist(err, res, body) {
+ if (err) {
+ return callback(err);
+ }
- function createWorker(id) {
- return worker;
+ if (! res.ok) {
+ var message = format('GET %s - %s NOT OK', uri, res.statusCode);
+ err = new Error(message);
+ return callback(err);
+ }
- function worker(cb) {
- api.get(id, cb);
+ var slugs = res.body.map(getSlugs);
+
+ function getSlugs(bundle) {
+ return bundle.slug;
+ }
+
+ var workers = slugs.map(createWorker);
+
+ // Request all ids in parallel.
+ parallel(workers, callback);
+
+ function createWorker(id) {
+ return worker;
+
+ function worker(cb) {
+ api.get(id, cb);
+ }
}
}
};
diff --git a/client/browser/api/normalize.js b/client/browser/api/normalize.js
index b2998f2..5a93a55 100644
--- a/client/browser/api/normalize.js
+++ b/client/browser/api/normalize.js
@@ -7,10 +7,10 @@
// TODO: Update once the API returns the correct data structures.
// map old data format to a new one and return a bundle state object.
function normalize(old) {
- var data = JSON.parse(old.Data);
+ var data = JSON.parse(old.data);
return {
- uuid: old.Link,
+ uuid: old.slug || old.link,
files: data.files
};
}
diff --git a/client/package.json b/client/package.json
index 8a9c96f..8a52869 100644
--- a/client/package.json
+++ b/client/package.json
@@ -38,7 +38,6 @@
"jshint": "^2.6.0",
"minimist": "^1.1.1",
"myth": "^1.4.0",
- "pgbundle": "0.0.1",
"raf": "^2.0.4",
"rework": "^1.0.1",
"rework-inherit": "^0.2.3",
diff --git a/go/src/v.io/x/playground/.gitignore b/go/src/v.io/x/playground/.gitignore
index a5f062c..38373aa 100644
--- a/go/src/v.io/x/playground/.gitignore
+++ b/go/src/v.io/x/playground/.gitignore
@@ -1,4 +1,7 @@
netrc
+node_modules
config/*.json
!config/db-*-example.json
!config/db-*-default.json
+bundles/*/*/bin
+bundles/*/*/pkg
diff --git a/go/src/v.io/x/playground/Dockerfile b/go/src/v.io/x/playground/Dockerfile
index cb701f6..36f9b6d 100644
--- a/go/src/v.io/x/playground/Dockerfile
+++ b/go/src/v.io/x/playground/Dockerfile
@@ -3,9 +3,7 @@
# Install various prereqs.
RUN apt-get update
-RUN apt-get install -y curl git nodejs npm
-# node -> nodejs needed for Ubuntu.
-RUN ln -s "$(which nodejs)" "$(dirname $(which nodejs))/node"
+RUN apt-get install -y curl git nodejs nodejs-legacy npm
# Install Go. Note, the apt-get "golang" target is too old.
RUN (cd /tmp; curl -O https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz)
diff --git a/go/src/v.io/x/playground/Makefile b/go/src/v.io/x/playground/Makefile
index 55a266a..ac5f495 100644
--- a/go/src/v.io/x/playground/Makefile
+++ b/go/src/v.io/x/playground/Makefile
@@ -47,11 +47,28 @@
--sqlconf=$< \
migrate up
+.PHONY: bootstrap
+bootstrap: config/db.json pgadmin pgbundle
+ pgadmin \
+ --sqlconf=$< \
+ bundle bootstrap
+
config/db.json:
@echo "You are missing config/db.json, create this file based"
@echo "on the appropriate examples in the config directory."
@exit 1;
+# TODO(ivanpi): Remove when pgbundle is ported to Go.
+.PHONY: pgbundle
+pgbundle:
+ PATH="$(PATH):$(V23_ROOT)/third_party/cout/node/bin" \
+ npm install --production "$(V23_ROOT)/release/projects/playground/pgbundle"
+
+.PHONY: clean
+clean:
+ @$(RM) -rf node_modules
+ @$(RM) -rf npm-debug.log
+
# Temporary workaround for linking Go tests into Jenkins.
.PHONY: test
test:
diff --git a/go/src/v.io/x/playground/README.md b/go/src/v.io/x/playground/README.md
index 88f6550..b9cfadd 100644
--- a/go/src/v.io/x/playground/README.md
+++ b/go/src/v.io/x/playground/README.md
@@ -52,7 +52,7 @@
$ GOPATH=$V23_ROOT/release/projects/playground/go v23 go install v.io/x/playground/...
-Run the compiler binary:
+Run the compilerd binary:
$ $V23_ROOT/release/projects/playground/go/bin/compilerd --listen-timeout=0 --address=localhost:8181 --origin='*'
@@ -94,6 +94,11 @@
Alternatively, make your own from example.
+The compilerd server can now be started with persistence enabled by:
+
+ $ make start
+
+
# Running tests
Make sure you have built a docker playground image, following the steps above.
@@ -158,3 +163,28 @@
the same state it was to begin with.
For more information on writing migrations, see https://github.com/rubenv/sql-migrate#writing-migrations
+
+## Bootstrapping default examples
+
+The playground client expects to find up-to-date default examples already
+present in the database to use as templates for editing. The unpacked example
+source code can be found in the `bundles` directory, described in
+`bundles/config.json`. Each example is obtained by filtering files from a
+folder according to a glob-like configuration file and bundling them into a
+JSON object that the client can parse.
+
+Bundling and loading the examples into a fresh database, as well as updating,
+is handled by the `pgadmin` tool:
+
+ $ make pgbundle
+ $ $V23_ROOT/release/projects/playground/go/bin/pgadmin -sqlconf=./config/db.json bundle bootstrap
+
+Or simply:
+
+ $ make bootstrap
+
+When adding new default examples or implementations of existing ones,
+`bundles/config.json` must also be edited to include them in bootstrapping and
+tests. For config file format documentation, see:
+
+ $ $V23_ROOT/release/projects/playground/go/bin/pgadmin bundle help bootstrap
diff --git a/go/src/v.io/x/playground/builder/builder_v23_test.go b/go/src/v.io/x/playground/builder/builder_v23_test.go
index 418782c..a450922 100644
--- a/go/src/v.io/x/playground/builder/builder_v23_test.go
+++ b/go/src/v.io/x/playground/builder/builder_v23_test.go
@@ -13,6 +13,7 @@
"strings"
"time"
+ "v.io/x/playground/lib/bundle"
_ "v.io/x/ref/runtime/factories/generic"
"v.io/x/ref/test/expect"
"v.io/x/ref/test/v23tests"
@@ -138,36 +139,32 @@
runCases("src/ids/unauthorized.id", []string{"not authorized"})
}
-// Tests that default playground examples execute successfully.
-// If any new examples are added, they should be added to bundles below.
+// Tests that default playground examples specified in `config.json` execute
+// successfully.
func V23TestPlaygroundBundles(i *v23tests.T) {
i.Pushd(i.NewTempDir(""))
defer i.Popd()
builderBin := initTest(i)
- bundlesDir := filepath.Join(playgroundRoot, "client", "bundles")
-
- // Add any new examples here.
- bundles := []string{
- "fortune",
+ bundlesDir := filepath.Join(playgroundRoot, "go", "src", "v.io", "x", "playground", "bundles")
+ bundlesCfgFile := filepath.Join(bundlesDir, "config.json")
+ bundlesCfg, err := bundle.ParseConfigFromFile(bundlesCfgFile, bundlesDir)
+ if err != nil {
+ i.Fatalf("%s: failed parsing bundle config from %q: %v", i.Caller(0), bundlesCfgFile, err)
}
- globFiles := []string{
- "go.bundle",
- "js.bundle",
- "go_js.bundle",
- "js_go.bundle",
- }
+ for _, example := range bundlesCfg.Examples {
+ i.Logf("Test example %s (%q)", example.Name, example.Path)
- for _, bundle := range bundles {
- i.Logf("Test bundle %q", bundle)
- bundlePath := filepath.Join(bundlesDir, bundle)
- for _, globFile := range globFiles {
- globFilePath := filepath.Join(bundlesDir, globFile)
- inv := runPGExample(i, builderBin, globFilePath, bundlePath, "-verbose=true", "--runTimeout=5s")
- i.Logf("glob: %q", globFile)
- // TODO(ivanpi,sadovsky): Make this "clean exit" check more robust.
- expectAndEcho(inv, "client.*Exited cleanly\\.")
+ for _, globName := range example.Globs {
+ glob, globExists := bundlesCfg.Globs[globName]
+ if !globExists {
+ i.Fatalf("%s: unknown glob %q", i.Caller(0), globName)
+ }
+
+ inv := runPGExample(i, builderBin, glob.Path, example.Path, "-verbose=true", "--runTimeout=5s")
+ i.Logf("glob: %s (%q)", globName, glob.Path)
+ expectAndEcho(inv, example.Output...)
}
}
}
diff --git a/go/src/v.io/x/playground/bundles/config.json b/go/src/v.io/x/playground/bundles/config.json
new file mode 100644
index 0000000..1487c9b
--- /dev/null
+++ b/go/src/v.io/x/playground/bundles/config.json
@@ -0,0 +1,33 @@
+{
+ "examples": [
+ {
+ "name": "fortune",
+ "path": "fortune",
+ "globs": [
+ "go",
+ "js",
+ "js-go",
+ "go-js"
+ ],
+ "output": [
+ "server.*Serving:",
+ "client.*Received:",
+ "client.*Exited cleanly\\."
+ ]
+ }
+ ],
+ "globs": {
+ "go": {
+ "path": "go.bundle"
+ },
+ "js": {
+ "path": "js.bundle"
+ },
+ "js-go": {
+ "path": "js_go.bundle"
+ },
+ "go-js": {
+ "path": "go_js.bundle"
+ }
+ }
+}
diff --git a/client/bundles/fortune/src/client/client.go b/go/src/v.io/x/playground/bundles/fortune/src/client/client.go
similarity index 100%
rename from client/bundles/fortune/src/client/client.go
rename to go/src/v.io/x/playground/bundles/fortune/src/client/client.go
diff --git a/client/bundles/fortune/src/client/client.js b/go/src/v.io/x/playground/bundles/fortune/src/client/client.js
similarity index 100%
rename from client/bundles/fortune/src/client/client.js
rename to go/src/v.io/x/playground/bundles/fortune/src/client/client.js
diff --git a/client/bundles/fortune/src/fortune/fortune.vdl b/go/src/v.io/x/playground/bundles/fortune/src/fortune/fortune.vdl
similarity index 100%
rename from client/bundles/fortune/src/fortune/fortune.vdl
rename to go/src/v.io/x/playground/bundles/fortune/src/fortune/fortune.vdl
diff --git a/client/bundles/fortune/src/server/server.go b/go/src/v.io/x/playground/bundles/fortune/src/server/server.go
similarity index 100%
rename from client/bundles/fortune/src/server/server.go
rename to go/src/v.io/x/playground/bundles/fortune/src/server/server.go
diff --git a/client/bundles/fortune/src/server/server.js b/go/src/v.io/x/playground/bundles/fortune/src/server/server.js
similarity index 100%
rename from client/bundles/fortune/src/server/server.js
rename to go/src/v.io/x/playground/bundles/fortune/src/server/server.js
diff --git a/client/bundles/go.bundle b/go/src/v.io/x/playground/bundles/go.bundle
similarity index 100%
rename from client/bundles/go.bundle
rename to go/src/v.io/x/playground/bundles/go.bundle
diff --git a/client/bundles/go_js.bundle b/go/src/v.io/x/playground/bundles/go_js.bundle
similarity index 100%
rename from client/bundles/go_js.bundle
rename to go/src/v.io/x/playground/bundles/go_js.bundle
diff --git a/client/bundles/js.bundle b/go/src/v.io/x/playground/bundles/js.bundle
similarity index 100%
rename from client/bundles/js.bundle
rename to go/src/v.io/x/playground/bundles/js.bundle
diff --git a/client/bundles/js_go.bundle b/go/src/v.io/x/playground/bundles/js_go.bundle
similarity index 100%
rename from client/bundles/js_go.bundle
rename to go/src/v.io/x/playground/bundles/js_go.bundle
diff --git a/go/src/v.io/x/playground/compilerd/main.go b/go/src/v.io/x/playground/compilerd/main.go
index 88c01e9..9e8ff6b 100644
--- a/go/src/v.io/x/playground/compilerd/main.go
+++ b/go/src/v.io/x/playground/compilerd/main.go
@@ -25,8 +25,8 @@
"time"
"v.io/x/lib/dbutil"
- "v.io/x/playground/compilerd/storage"
"v.io/x/playground/lib/log"
+ "v.io/x/playground/lib/storage"
)
func init() {
@@ -118,12 +118,14 @@
// Add routes for storage.
serveMux.HandleFunc("/load", handlerLoad)
serveMux.HandleFunc("/save", handlerSave)
+ serveMux.HandleFunc("/list", handlerListDefault)
} else {
- log.Debug("No sql config provided. Disabling /load and /save routes.")
+ log.Debug("No sql config provided. Disabling /load, /save, /list routes.")
- // Return 501 Not Implemented for the /load and /save routes.
+ // Return 501 Not Implemented for the storage routes.
serveMux.HandleFunc("/load", handlerNotImplemented)
serveMux.HandleFunc("/save", handlerNotImplemented)
+ serveMux.HandleFunc("/list", handlerNotImplemented)
}
serveMux.HandleFunc("/compile", c.handlerCompile)
diff --git a/go/src/v.io/x/playground/compilerd/storage.go b/go/src/v.io/x/playground/compilerd/storage.go
index b9f5ae1..13cf176 100644
--- a/go/src/v.io/x/playground/compilerd/storage.go
+++ b/go/src/v.io/x/playground/compilerd/storage.go
@@ -7,7 +7,10 @@
// handlerSave() handles a POST request with bundled playground source code.
// The bundle is persisted in a database and a unique ID returned.
// handlerLoad() handles a GET request with an id parameter. It returns the
-// bundle saved under the provided ID, if any.
+// bundle saved under the provided ID or slug, if any.
+// handlerListDefault() handles a GET request with no parameters. It returns
+// a list of descriptions of all default bundles. Default bundles are saved
+// using the pgadmin tool, not the HTTP API.
// The current implementation uses a MySQL-like SQL database for persistence.
package main
@@ -16,15 +19,16 @@
"encoding/json"
"fmt"
"net/http"
+ "time"
- "v.io/x/playground/compilerd/storage"
"v.io/x/playground/lib/log"
+ "v.io/x/playground/lib/storage"
)
//////////////////////////////////////////
// HTTP request handlers
-// GET request that returns the saved bundle for the given id.
+// GET request that returns the saved bundle for the given ID or slug.
func handlerLoad(w http.ResponseWriter, r *http.Request) {
if !handleCORS(w, r) {
return
@@ -34,25 +38,22 @@
if !checkGetMethod(w, r) {
return
}
- bId := r.FormValue("id")
- if bId == "" {
+ bIdOrSlug := r.FormValue("id")
+ if bIdOrSlug == "" {
storageError(w, http.StatusBadRequest, "Must specify id to load.")
return
}
- bData, err := storage.GetBundleDataByLinkId(bId)
+
+ bLink, bData, err := storage.GetBundleByLinkIdOrSlug(bIdOrSlug)
if err == storage.ErrNotFound {
storageError(w, http.StatusNotFound, "No data found for provided id.")
return
} else if err != nil {
- storageInternalError(w, "Error getting bundleLink for id ", bId, ": ", err)
+ storageInternalError(w, "Error getting bundleLink for id/slug ", bIdOrSlug, ": ", err)
return
}
- storageRespond(w, http.StatusOK, &StorageResponse{
- Link: bId,
- Data: bData.Json,
- })
- return
+ storageRespond(w, http.StatusOK, fullResponseFromLinkAndData(bLink, bData))
}
// POST request that saves the body as a new bundle and returns the bundle id.
@@ -75,32 +76,91 @@
// TODO(ivanpi): Check if bundle is parseable. Format/lint?
- bLink, bData, err := storage.StoreBundleLinkAndData(requestBody)
+ bLink, bData, err := storage.StoreBundleLinkAndData(string(requestBody))
if err != nil {
storageInternalError(w, "Error storing bundle: ", err)
return
}
- storageRespond(w, http.StatusOK, &StorageResponse{
- Link: bLink.Id,
- Data: bData.Json,
- })
+ storageRespond(w, http.StatusOK, fullResponseFromLinkAndData(bLink, bData))
+}
+
+// GET request that returns a list of default bundle descriptions.
+func handlerListDefault(w http.ResponseWriter, r *http.Request) {
+ if !handleCORS(w, r) {
+ return
+ }
+
+ // Check method. No GET parameters are currently used by /list.
+ if !checkGetMethod(w, r) {
+ return
+ }
+
+ bList, err := storage.GetDefaultBundleList()
+ if err != nil {
+ storageInternalError(w, "Error getting default bundle list: ", err)
+ return
+ }
+
+ bListResp := make([]*BundleDescResponse, 0, len(bList))
+ for _, bLink := range bList {
+ bListResp = append(bListResp, descResponseFromLink(bLink))
+ }
+
+ storageRespond(w, http.StatusOK, bListResp)
}
//////////////////////////////////////////
// Response handling
-type StorageResponse struct {
- // Error message. If empty, request was successful.
- Error string
- // Bundle ID for the saved/loaded bundle.
- Link string
- // Contents of the loaded bundle.
- Data string
+type ErrorResponse struct {
+ Error string `json:"error"`
+}
+
+type BundleDescResponse struct {
+ // Bundle ID of the saved/loaded bundle.
+ Link string `json:"link"`
+ // Slug of the saved/loaded bundle.
+ // Currently set only for most recent versions of default bundles.
+ Slug string `json:"slug,omitempty"`
+ // Creation timestamp of the loaded bundle.
+ // Since the timestamp is set by the database, /save responses omit it.
+ CreatedAt *time.Time `json:"createdAt,omitempty"`
+}
+
+type BundleFullResponse struct {
+ // Bundle description, as sent in /list response.
+ BundleDescResponse
+ // Contents of the saved/loaded bundle.
+ Data string `json:"data"`
+}
+
+func descResponseFromLink(bLink *storage.BundleLink) *BundleDescResponse {
+ return &BundleDescResponse{
+ Link: bLink.Id,
+ Slug: string(bLink.Slug),
+ CreatedAt: zeroTimeToNil(bLink.CreatedAt),
+ }
+}
+
+func fullResponseFromLinkAndData(bLink *storage.BundleLink, bData *storage.BundleData) *BundleFullResponse {
+ return &BundleFullResponse{
+ BundleDescResponse: *descResponseFromLink(bLink),
+ Data: bData.Json,
+ }
+}
+
+// Converts time to pointer, mapping zero time to nil to force it to be
+// omitted from JSON. See https://github.com/golang/go/issues/5218
+func zeroTimeToNil(t time.Time) *time.Time {
+ if t.IsZero() {
+ return nil
+ }
+ return &t
}
// Sends response to client. Request handler should exit after this call.
-func storageRespond(w http.ResponseWriter, status int, body *StorageResponse) {
+func storageRespond(w http.ResponseWriter, status int, body interface{}) {
bodyJson, _ := json.Marshal(body)
w.Header().Add("Content-Type", "application/json")
w.Header().Add("Content-Length", fmt.Sprintf("%d", len(bodyJson)))
@@ -110,7 +170,7 @@
// Sends error response with specified message to client.
func storageError(w http.ResponseWriter, status int, msg string) {
- storageRespond(w, status, &StorageResponse{
+ storageRespond(w, status, &ErrorResponse{
Error: msg,
})
}
diff --git a/go/src/v.io/x/playground/compilerd/storage/model.go b/go/src/v.io/x/playground/compilerd/storage/model.go
deleted file mode 100644
index 7e70bab..0000000
--- a/go/src/v.io/x/playground/compilerd/storage/model.go
+++ /dev/null
@@ -1,263 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//=== High-level schema ===
-// Each playground bundle is stored once as a BundleData in the bundle_data
-// table. The BundleData contains the json string corresponding to the bundle
-// files, and is indexed by the hash of the json.
-//
-// Links to a BundleData are stored in BundleLinks. There can be multiple
-// BundleLinks corresponding to a single BundleData. BundleLinks are indexed by
-// a unique id, and contain the hash of the BundleData that they correspond to.
-//
-// The additional layer of indirection provided by BundleLinks allows storing
-// identical bundles more efficiently and makes the bundle Id independent of
-// its contents, allowing implementation of change history, sharing, expiration
-// etc.
-//
-// Each bundle save request first generates and stores a new BundleLink object,
-// and will store a new BundleData only if it does not already exist in the
-// database.
-//
-// Note: If bundles larger than ~1 MiB are to be stored, the max_allowed_packed
-// SQL connection parameter must be increased.
-//
-// TODO(ivanpi): Normalize the Json (e.g. file order).
-
-package storage
-
-import (
- crand "crypto/rand"
- "database/sql"
- "encoding/binary"
- "errors"
- "fmt"
- "time"
-
- "github.com/jmoiron/sqlx"
-
- "v.io/x/playground/lib/hash"
-)
-
-var (
- // Error returned when requested item is not found in the database.
- ErrNotFound = errors.New("Not found")
-
- // Error returned when retries are exhausted.
- errTooManyRetries = errors.New("Too many retries")
-
- // Error returned from a transaction callback to trigger a rollback and
- // retry. Other errors cause a rollback and abort.
- errRetryTransaction = errors.New("Retry transaction")
-)
-
-//////////////////////////////////////////
-// SqlData type definitions
-
-type BundleData struct {
- // Raw SHA256 of the bundle contents
- Hash []byte `db:"hash"` // primary key
- // The bundle contents
- Json string `db:"json"`
-}
-
-type BundleLink struct {
- // 64-byte printable ASCII string
- Id string `db:"id"` // primary key
- // Raw SHA256 of the bundle contents
- Hash []byte `db:"hash"` // foreign key => BundleData.Hash
- // Link record creation time
- CreatedAt time.Time `db:"created_at"`
-}
-
-///////////////////////////////////////
-// DB read-only methods
-
-// TODO(nlacasse): Use prepared statements, otherwise we have an extra
-// round-trip to the db, which is slow on cloud sql.
-
-func getBundleLinkById(q sqlx.Queryer, id string) (*BundleLink, error) {
- bLink := BundleLink{}
- if err := sqlx.Get(q, &bLink, "SELECT * FROM bundle_link WHERE id=?", id); err != nil {
- if err == sql.ErrNoRows {
- err = ErrNotFound
- }
- return nil, err
- }
- return &bLink, nil
-}
-
-func getBundleDataByHash(q sqlx.Queryer, hash []byte) (*BundleData, error) {
- bData := BundleData{}
- if err := sqlx.Get(q, &bData, "SELECT * FROM bundle_data WHERE hash=?", hash); err != nil {
- if err == sql.ErrNoRows {
- err = ErrNotFound
- }
- return nil, err
- }
- return &bData, nil
-}
-
-// GetBundleDataByLinkId returns a BundleData object linked to by a BundleLink
-// with a particular id.
-// Note: This can fail if the bundle is deleted between fetching BundleLink
-// and BundleData. However, it is highly unlikely, costly to mitigate (using
-// a serializable transaction), and unimportant (error 500 instead of 404).
-func GetBundleDataByLinkId(id string) (*BundleData, error) {
- bLink, err := getBundleLinkById(dbRead, id)
- if err != nil {
- return nil, err
- }
- bData, err := getBundleDataByHash(dbRead, bLink.Hash)
- if err != nil {
- return nil, err
- }
- return bData, nil
-}
-
-////////////////////////////////////
-// DB write methods
-
-func storeBundleData(ext sqlx.Ext, bData *BundleData) error {
- _, err := sqlx.NamedExec(ext, "INSERT INTO bundle_data (hash, json) VALUES (:hash, :json)", bData)
- return err
-}
-
-func storeBundleLink(ext sqlx.Ext, bLink *BundleLink) error {
- _, err := sqlx.NamedExec(ext, "INSERT INTO bundle_link (id, hash) VALUES (:id, :hash)", bLink)
- return err
-}
-
-// StoreBundleLinkAndData creates a new bundle data for a given json byte slice
-// if one does not already exist. It will create a new bundle link pointing to
-// that data. All DB access is done in a transaction, which will retry up to 3
-// times. Both the link and the data are returned, or an error if one occured.
-func StoreBundleLinkAndData(json []byte) (bLink *BundleLink, bData *BundleData, retErr error) {
- bHashRaw := hash.Raw(json)
- bHash := bHashRaw[:]
-
- // Attempt transaction up to 3 times.
- retErr = runInTransaction(3, func(tx *sqlx.Tx) error {
- // Generate a random id for the bundle link.
- id, err := randomLink(bHash)
- if err != nil {
- return fmt.Errorf("error creating link id: %v", err)
- }
-
- // Check if bundle link with this id already exists in DB.
- if _, err := getBundleLinkById(tx, id); err == nil {
- // Bundle was found. Retry with new id.
- return errRetryTransaction
- } else if err != ErrNotFound {
- return fmt.Errorf("error checking for bundle link: %v", err)
- }
-
- // Check if bundle data with this hash already exists in DB.
- bData, err = getBundleDataByHash(tx, bHash)
- if err != nil {
- if err != ErrNotFound {
- return fmt.Errorf("error checking for bundle data: %v", err)
- }
-
- // Bundle does not exist in DB. Store it.
- bData = &BundleData{
- Hash: bHash,
- Json: string(json),
- }
- if err = storeBundleData(tx, bData); err != nil {
- return fmt.Errorf("error storing bundle data: %v", err)
- }
- }
-
- // Store the bundle link.
- bLink = &BundleLink{
- Id: id,
- Hash: bHash,
- }
- if err = storeBundleLink(tx, bLink); err != nil {
- return fmt.Errorf("error storing bundle link: %v", err)
- }
-
- return nil
- })
-
- return
-}
-
-//////////////////////////////////////////
-// Transaction support
-
-// Runs function txf inside a SQL transaction. txf should only use the database
-// handle passed to it, which shares the prepared transaction cache with the
-// original handle. If txf returns nil, the transaction is committed.
-// Otherwise, it is rolled back.
-// txf is retried at most maxRetries times, with a fresh transaction for every
-// attempt, until the commit is successful. txf should not have side effects
-// that could affect subsequent retries (apart from database operations, which
-// are rolled back).
-// If the error returned from txf is errRetryTransaction, txf is retried as if
-// the commit failed. Otherwise, txf is not retried, and RunInTransaction
-// returns the error.
-// In rare cases, txf may be retried even if the transaction was successfully
-// committed (when commit falsely returns an error). txf should be idempotent
-// or able to detect this case.
-// If maxRetries is exhausted, runInTransaction returns errTooManyRetries.
-// Nested transactions are not supported and result in undefined behaviour.
-// Inspired by https://cloud.google.com/appengine/docs/go/datastore/reference#RunInTransaction
-func runInTransaction(maxRetries int, txf func(tx *sqlx.Tx) error) error {
- for i := 0; i < maxRetries; i++ {
- err := attemptInTransaction(txf)
- if err == nil {
- return nil
- } else if err != errRetryTransaction {
- return err
- }
- }
- return errTooManyRetries
-}
-
-func attemptInTransaction(txf func(tx *sqlx.Tx) error) (rerr error) {
- tx, err := dbSeq.Beginx()
- if err != nil {
- return fmt.Errorf("Failed opening transaction: %v", err)
- }
- defer func() {
- // UPSTREAM BUG WORKAROUND: Rollback anyway to release transaction after
- // manual commit.
- //if rerr != nil {
- if true {
- // Silently ignore rollback error, we cannot do anything. Transaction
- // will timeout eventually.
- // UPSTREAM BUG: Transaction does not timeout, connection gets reused.
- // This case is unlikely, but dangerous.
- // TODO(ivanpi): Remove workaround when bug is resolved.
- _ = tx.Rollback()
- }
- }()
- // Call txf with the transaction handle - a shallow copy of the database
- // handle (sharing the mutex, database connection, queries) with the
- // transaction object added.
- if err := txf(tx); err != nil {
- return err
- }
- // UPSTREAM BUG WORKAROUND: Commit manually.
- //if err = tx.Commit(); err != nil {
- if _, err = tx.Exec("COMMIT"); err != nil {
- return errRetryTransaction
- }
- return nil
-}
-
-////////////////////////////////////////////
-// Helper methods
-
-// randomLink creates a random link id for a given hash.
-func randomLink(bHash []byte) (string, error) {
- h := make([]byte, 16, 16+len(bHash))
- err := binary.Read(crand.Reader, binary.LittleEndian, h)
- if err != nil {
- return "", fmt.Errorf("RNG failed: %v", err)
- }
- return "_" + hash.String(append(h, bHash...))[1:], nil
-}
diff --git a/go/src/v.io/x/playground/compilerd/storage/model_test.go b/go/src/v.io/x/playground/compilerd/storage/model_test.go
deleted file mode 100644
index 4b0cb4f..0000000
--- a/go/src/v.io/x/playground/compilerd/storage/model_test.go
+++ /dev/null
@@ -1,148 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Tests for the storage model.
-// These tests only test the exported API of the storage model.
-//
-// NOTE: These tests cannot be run in parallel on the same machine because they
-// interact with a fixed database on the machine.
-
-package storage_test
-
-import (
- "fmt"
- "testing"
-
- "github.com/rubenv/sql-migrate"
-
- "v.io/x/lib/dbutil"
- "v.io/x/playground/compilerd/storage"
-)
-
-var (
- dataSourceName = "playground_test@tcp(localhost:3306)/playground_test?parseTime=true"
-)
-
-// setup cleans the database, runs migrations, and connects to the database.
-// It returns a teardown function that closes the database connection.
-func setup(t *testing.T) func() {
- // Migrate down then up.
- migrations := &migrate.FileMigrationSource{
- Dir: "../../migrations",
- }
- migrate.SetTable("migrations")
-
- sqlConfig := dbutil.SqlConfig{
- DataSourceName: dataSourceName,
- TLSDisable: true,
- }
- activeSqlConfig, err := sqlConfig.Activate("")
-
- db, err := activeSqlConfig.NewSqlDBConn("SERIALIZABLE")
- if err != nil {
- t.Fatalf("Error opening database: %v", err)
- }
-
- // Remove any existing tables.
- tableNames := []string{"bundle_link", "bundle_data", "migrations"}
- for _, tableName := range tableNames {
- db.Exec("DROP TABLE " + tableName)
- }
-
- if _, err = migrate.Exec(db, "mysql", migrations, migrate.Up); err != nil {
- t.Fatalf("Error migrating up: %v", err)
- }
- if err := db.Close(); err != nil {
- t.Fatalf("db.Close() failed: %v", err)
- }
-
- // Connect to the storage.
- if err := storage.Connect(activeSqlConfig); err != nil {
- t.Fatalf("storage.Connect(%v) failed: %v", activeSqlConfig, err)
- }
-
- teardown := func() {
- if err := storage.Close(); err != nil {
- t.Fatalf("storage.Close() failed: %v", err)
- }
- }
- return teardown
-}
-
-func TestGetBundleDataByLinkId(t *testing.T) {
- defer setup(t)()
-
- // Get with a unknown id should return ErrNotFound.
- id := "foobar"
- if _, err := storage.GetBundleDataByLinkId(id); err != storage.ErrNotFound {
- t.Errorf("Expected GetBundleDataByLinkId with unknown id to return ErrNotFound, but instead got %v", err)
- }
-
- // Add a bundle.
- json := []byte("mock_json_data")
- bLink, _, err := storage.StoreBundleLinkAndData(json)
- if err != nil {
- t.Fatalf("Expected StoreBundleLinkAndData(%v) not to error, but got %v", json, err)
- }
-
- // Bundle should exist.
- gotBdata, err := storage.GetBundleDataByLinkId(bLink.Id)
- if err != nil {
- t.Errorf("Expected GetBundleDataByLinkId(%v) not to error, but got %v", bLink.Id, err)
- }
-
- // Bundle should have expected json.
- if got, want := gotBdata.Json, string(json); got != want {
- t.Errorf("Expected %v to equal %v.", got, want)
- }
-}
-
-func assertValidLinkDataPair(json string, bLink *storage.BundleLink, bData *storage.BundleData) error {
- if string(bLink.Hash) != string(bData.Hash) {
- return fmt.Errorf("Expected %v to equal %v", string(bLink.Hash), string(bData.Hash))
- }
-
- if bLink.Id == "" {
- return fmt.Errorf("Expected bundle link to have id.")
- }
-
- if bData.Json != json {
- return fmt.Errorf("Expected %v to equal %v", bData.Json, json)
- }
- return nil
-}
-
-func TestStoreBundleLinkAndData(t *testing.T) {
- defer setup(t)()
-
- mockJson := []byte("bizbaz")
-
- // Storing the json once should succeed.
- bLink1, bData1, err := storage.StoreBundleLinkAndData(mockJson)
- if err != nil {
- t.Fatalf("StoreBundleLinkAndData(%v) failed: %v", mockJson, err)
- }
- if err := assertValidLinkDataPair(string(mockJson), bLink1, bData1); err != nil {
- t.Fatalf("Got invalid link data pair: %v", err)
- }
-
- // Storing the bundle again should succeed.
- bLink2, bData2, err := storage.StoreBundleLinkAndData(mockJson)
- if err != nil {
- t.Fatalf("StoreBundleLinkAndData(%v) failed: %v", mockJson, err)
- }
- if err := assertValidLinkDataPair(string(mockJson), bLink2, bData2); err != nil {
- t.Error("Got invalid link data pair: %v", err)
- }
-
- // Bundle links should have different ids.
- if bLink1.Id == bLink2.Id {
- t.Errorf("Expected bundle links to have different ids, but got %v and %v", bLink1.Id, bLink2.Id)
- }
-
- // Bundle datas should have equal hashes.
- if want, got := string(bData1.Hash), string(bData2.Hash); want != got {
- t.Errorf("Expected bundle datas to have equal hashes, but got %v and %v", want, got)
- }
-}
diff --git a/go/src/v.io/x/playground/lib/bundle/config.go b/go/src/v.io/x/playground/lib/bundle/config.go
new file mode 100644
index 0000000..6dffe13
--- /dev/null
+++ b/go/src/v.io/x/playground/lib/bundle/config.go
@@ -0,0 +1,103 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Representation of the example bundle configuration file, parsed from JSON.
+// The configuration file specifies combinations of example folders and
+// applicable glob files for bundling default examples, as well as expected
+// output for verifying their correctness.
+
+package bundle
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "path/filepath"
+)
+
+// Description of the bundle configuration file format.
+const BundleConfigFileDescription = `File must contain a JSON object of the following form:
+ {
+ "examples": [ <Example> ... ], (array of Example objects)
+ "globs": { "<glob_name>":<Glob> ... } (map of glob names to Glob objects; glob names should be human-readable but URL-friendly)
+ }
+Example objects have the form:
+ {
+ "name": "<name>", (example names should be human-readable but URL-friendly)
+ "path": "<path/to/example/dir>", (path to directory containing files to be filtered by globs and bundled)
+ "globs": [ "<glob_name>" ... ], (names of globs to be applied to the directory; must have corresponding entries in "globs";
+ each example can be bundled into a separate bundle using one of the specified globs)
+ "output": [ "<expected_output_regex>" ... ] (expected output specification for this example, for any applicable glob;
+ each regex must match at least one output line for the test to succeed)
+ }
+Glob objects have the form:
+ {
+ "path": "<path/to/glob_file>" (path to file containing a list of glob patterns;
+ files from the example directory matching at least one pattern will be included in the bundle;
+ each glob pattern must match at least one file for the bundling to succeed)
+ }
+Non-absolute paths are interpreted relative to a configurable directory, usually the configuration file directory.`
+
+// Parsed bundle configuration. See BundleConfigFileDescription for details.
+type Config struct {
+ // List of Example folder descriptors.
+ Examples []*Example `json:"examples"`
+ // Maps glob names to Glob file descriptors.
+ Globs map[string]*Glob `json:"globs"`
+}
+
+// Represents an example folder. Each specified glob file is applied to the
+// folder to produce a separate bundle, representing different implementations
+// of the same example.
+type Example struct {
+ // Human-readable, URL-friendly name.
+ Name string `json:"name"`
+ // Path to example directory.
+ Path string `json:"path"`
+ // Names of globs to apply to the directory.
+ Globs []string `json:"globs"`
+ // Expected output regexes for testing.
+ Output []string `json:"output"`
+}
+
+// Represents a glob file for filtering bundled files.
+type Glob struct {
+ // Path to glob file.
+ Path string `json:"path"`
+}
+
+// Parses configuration from file and normalizes non-absolute paths relative to
+// baseDir. Doesn't do consistency verification.
+func ParseConfigFromFile(configPath, baseDir string) (*Config, error) {
+ cfgJson, err := ioutil.ReadFile(configPath)
+ if err != nil {
+ return nil, fmt.Errorf("failed reading bundle config from %q: %v", configPath, err)
+ }
+ var cfg Config
+ if err := json.Unmarshal(cfgJson, &cfg); err != nil {
+ return nil, fmt.Errorf("failed parsing bundle config: %v", err)
+ }
+ cfg.NormalizePaths(baseDir)
+ return &cfg, nil
+}
+
+// Canonicalizes example and glob file paths and resolves them relative to
+// baseDir.
+func (c *Config) NormalizePaths(baseDir string) {
+ for _, e := range c.Examples {
+ e.Path = normalizePath(e.Path, baseDir)
+ }
+ for _, g := range c.Globs {
+ g.Path = normalizePath(g.Path, baseDir)
+ }
+}
+
+// If path is not absolute, resolves path relative to baseDir. Otherwise,
+// canonicalizes path.
+func normalizePath(path, baseDir string) string {
+ if filepath.IsAbs(path) {
+ return filepath.Clean(path)
+ }
+ return filepath.Join(baseDir, path)
+}
diff --git a/go/src/v.io/x/playground/lib/bundle/pgbundle.go b/go/src/v.io/x/playground/lib/bundle/pgbundle.go
new file mode 100644
index 0000000..ff20260
--- /dev/null
+++ b/go/src/v.io/x/playground/lib/bundle/pgbundle.go
@@ -0,0 +1,71 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Implements bundling playground example files from a specified directory,
+// filtered using a specified glob file, into a JSON object compatible with
+// the playground client.
+// Currently implemented as a thin wrapper around the `pgbundle` executable
+// written in Node.js. Assumes that pgbundle has been installed in PATH or
+// by running `make pgbundle` in pgPackageDir.
+
+// TODO(ivanpi): Port pgbundle to Go.
+
+package bundle
+
+import (
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+)
+
+const (
+ nodeBinDir = "${V23_ROOT}/third_party/cout/node/bin"
+ pgPackageDir = "${V23_ROOT}/release/projects/playground/go/src/v.io/x/playground"
+)
+
+const BundleUsage = `
+<glob_file>: Path to file containing a list of glob patterns, one per line.
+ The bundle includes only files with path suffixes matching one
+ of the globs. Each glob must match at least one file, otherwise
+ bundling fails with a non-zero exit code.
+<root_path>: Path to directory where files matching glob patterns are taken
+ from.
+`
+
+// Bundles files from rootPath matching patterns in globFile. See BundleUsage.
+// Errors and (if verbose) informational messages are written to errout.
+func Bundle(errout io.Writer, globFile, rootPath string, verbose bool) (outBundle []byte, rerr error) {
+ nodePath, err := resolveBinary("node", nodeBinDir)
+ if err != nil {
+ return nil, err
+ }
+ pgBundlePath, err := resolveBinary("pgbundle", filepath.Join(pgPackageDir, "node_modules", ".bin"))
+ if err != nil {
+ return nil, err
+ }
+ verboseFlag := "--no-verbose"
+ if verbose {
+ verboseFlag = "--verbose"
+ }
+ cmdPGBundle := exec.Command(nodePath, pgBundlePath, verboseFlag, globFile, rootPath)
+ cmdPGBundle.Dir = os.ExpandEnv(pgPackageDir)
+ cmdPGBundle.Stderr = errout
+ return cmdPGBundle.Output()
+}
+
+// Searches for a binary with the given name in PATH. If not found, defaults to
+// the binary with the same name in defaultDir.
+func resolveBinary(binName, defaultDir string) (string, error) {
+ var errPath, errDef error
+ binPath, errPath := exec.LookPath(binName)
+ if errPath != nil {
+ binPath, errDef = exec.LookPath(filepath.Join(os.ExpandEnv(defaultDir), binName))
+ }
+ if errDef != nil {
+ return "", fmt.Errorf("failed to resolve binary %q: %v; %v", binName, errPath, errDef)
+ }
+ return binPath, nil
+}
diff --git a/go/src/v.io/x/playground/lib/errors.go b/go/src/v.io/x/playground/lib/errors.go
new file mode 100644
index 0000000..e7ab2f8
--- /dev/null
+++ b/go/src/v.io/x/playground/lib/errors.go
@@ -0,0 +1,23 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Utility functions for error handling.
+
+package lib
+
+import (
+ "fmt"
+)
+
+// MergeErrors returns nil if both errors passed to it are nil.
+// Otherwise, it concatenates non-nil error messages.
+func MergeErrors(err1, err2 error, joiner string) error {
+ if err1 == nil {
+ return err2
+ } else if err2 == nil {
+ return err1
+ } else {
+ return fmt.Errorf("%v%s%v", err1, joiner, err2)
+ }
+}
diff --git a/go/src/v.io/x/playground/compilerd/storage/db.go b/go/src/v.io/x/playground/lib/storage/db.go
similarity index 61%
rename from go/src/v.io/x/playground/compilerd/storage/db.go
rename to go/src/v.io/x/playground/lib/storage/db.go
index 099cac5..0073fad 100644
--- a/go/src/v.io/x/playground/compilerd/storage/db.go
+++ b/go/src/v.io/x/playground/lib/storage/db.go
@@ -10,6 +10,7 @@
"github.com/jmoiron/sqlx"
"v.io/x/lib/dbutil"
+ "v.io/x/playground/lib"
)
var (
@@ -24,55 +25,60 @@
// connectDb is a helper method to connect a single database with the given
// isolation parameter.
-func connectDb(sqlConfig *dbutil.ActiveSqlConfig, isolation string) (*sqlx.DB, error) {
+func connectDb(sqlConfig *dbutil.ActiveSqlConfig, isolation string) (_ *sqlx.DB, rerr error) {
// Open db connection from config,
conn, err := sqlConfig.NewSqlDBConn(isolation)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("error opening database connection: %v", err)
}
-
// Create sqlx DB.
db := sqlx.NewDb(conn, "mysql")
+ // Try to close DB on error.
+ defer func() {
+ if rerr != nil {
+ rerr = lib.MergeErrors(rerr, db.Close(), "; ")
+ }
+ }()
// Ping db to check connection.
if err := db.Ping(); err != nil {
- return nil, fmt.Errorf("Error connecting to database: %v", err)
+ return nil, fmt.Errorf("error connecting to database: %v", err)
}
+
return db, nil
}
// Connect opens 2 connections to the database, one read-only, and one
// serializable.
-func Connect(sqlConfig *dbutil.ActiveSqlConfig) (err error) {
-
+func Connect(sqlConfig *dbutil.ActiveSqlConfig) (rerr error) {
// Data writes for the schema are complex enough to require transactions with
// SERIALIZABLE isolation. However, reads do not require SERIALIZABLE. Since
// database/sql only allows setting transaction isolation per connection,
// a separate connection with only READ-COMMITTED isolation is used for reads
// to reduce lock contention and deadlock frequency.
- dbRead, err = connectDb(sqlConfig, "READ-COMMITTED")
- if err != nil {
- return err
+ dbRead, rerr = connectDb(sqlConfig, "READ-COMMITTED")
+ if rerr != nil {
+ return rerr
}
+ // dbRead is fully initialized, try to close it on subsequent error.
+ defer func() {
+ if rerr != nil {
+ rerr = lib.MergeErrors(rerr, dbRead.Close(), "; ")
+ }
+ }()
- dbSeq, err = connectDb(sqlConfig, "SERIALIZABLE")
- if err != nil {
- return err
+ dbSeq, rerr = connectDb(sqlConfig, "SERIALIZABLE")
+ if rerr != nil {
+ return rerr
}
return nil
}
-// Close closes both databases.
+// Close closes both databases. Should be called iff Connect() was successful.
func Close() error {
- if err := dbRead.Close(); err != nil {
- return err
- }
-
- if err := dbSeq.Close(); err != nil {
- return err
- }
-
- return nil
+ errRead := dbRead.Close()
+ errSeq := dbSeq.Close()
+ return lib.MergeErrors(errRead, errSeq, "; ")
}
diff --git a/go/src/v.io/x/playground/compilerd/storage/migration_test.go b/go/src/v.io/x/playground/lib/storage/migration_test.go
similarity index 82%
rename from go/src/v.io/x/playground/compilerd/storage/migration_test.go
rename to go/src/v.io/x/playground/lib/storage/migration_test.go
index 8ae0cdf..1d024a3 100644
--- a/go/src/v.io/x/playground/compilerd/storage/migration_test.go
+++ b/go/src/v.io/x/playground/lib/storage/migration_test.go
@@ -54,13 +54,13 @@
if err != nil {
t.Fatalf("Error migrating up: %v", err)
}
- fmt.Printf("Applied %v migration up.\n", up)
+ fmt.Printf("Applied %v migrations up.\n", up)
down, err := migrate.Exec(db, "mysql", migrationSource, migrate.Down)
if err != nil {
t.Fatalf("Error migrating down: %v", err)
}
- fmt.Printf("Applied %v migration down.\n", down)
+ fmt.Printf("Applied %v migrations down.\n", down)
}
// Run each migration up, down, up individually.
@@ -68,25 +68,25 @@
if err != nil {
t.Fatalf("migrationSource.FindMigrations() failed: %v", err)
}
- for i, migration := range migrations {
+ for i, _ := range migrations {
memMigrationSource := &migrate.MemoryMigrationSource{
- Migrations: []*migrate.Migration{migration},
+ Migrations: append([]*migrate.Migration(nil), migrations[:i+1]...),
}
// Migrate up.
- if _, err := migrate.Exec(db, "mysql", memMigrationSource, migrate.Up); err != nil {
+ if _, err := migrate.ExecMax(db, "mysql", memMigrationSource, migrate.Up, 1); err != nil {
t.Fatalf("Error migrating migration %v up: %v", i, err)
}
fmt.Printf("Applied migration %v up.\n", i)
// Migrate down.
- if _, err := migrate.Exec(db, "mysql", memMigrationSource, migrate.Down); err != nil {
+ if _, err := migrate.ExecMax(db, "mysql", memMigrationSource, migrate.Down, 1); err != nil {
t.Fatalf("Error migrating migration %v down: %v", i, err)
}
fmt.Printf("Applied migration %v down.\n", i)
// Migrate up.
- if _, err := migrate.Exec(db, "mysql", memMigrationSource, migrate.Up); err != nil {
+ if _, err := migrate.ExecMax(db, "mysql", memMigrationSource, migrate.Up, 1); err != nil {
t.Fatalf("Error migrating migration %v up: %v", i, err)
}
fmt.Printf("Applied migration %v up.\n", i)
diff --git a/go/src/v.io/x/playground/lib/storage/model.go b/go/src/v.io/x/playground/lib/storage/model.go
new file mode 100644
index 0000000..759ff43
--- /dev/null
+++ b/go/src/v.io/x/playground/lib/storage/model.go
@@ -0,0 +1,400 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//=== High-level schema ===
+// Each playground bundle is stored once as a BundleData in the bundle_data
+// table. The BundleData contains the json string corresponding to the bundle
+// files, and is indexed by the hash of the json.
+//
+// Links to a BundleData are stored in BundleLinks. There can be multiple
+// BundleLinks corresponding to a single BundleData. BundleLinks are indexed by
+// a unique id, and contain the hash of the BundleData that they correspond to.
+//
+// The additional layer of indirection provided by BundleLinks allows storing
+// identical bundles more efficiently and makes the bundle Id independent of
+// its contents, allowing implementation of change history, sharing, expiration
+// etc.
+//
+// Each bundle save request first generates and stores a new BundleLink object,
+// and will store a new BundleData only if it does not already exist in the
+// database.
+//
+// Note: If bundles larger than ~1 MiB are to be stored, the max_allowed_packed
+// SQL connection parameter must be increased.
+//
+// TODO(ivanpi): Normalize the Json (e.g. file order).
+
+package storage
+
+import (
+ crand "crypto/rand"
+ "database/sql"
+ "database/sql/driver"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/jmoiron/sqlx"
+
+ "v.io/x/playground/lib/hash"
+)
+
+var (
+ // Error returned when requested item is not found in the database.
+ ErrNotFound = errors.New("Not found")
+
+ // Error returned when an autogenerated ID matches an existing ID.
+ // Extremely unlikely for reasonably utilized database.
+ errIDCollision = errors.New("ID collision")
+
+ // Error returned when retries are exhausted.
+ errTooManyRetries = errors.New("Too many retries")
+
+ // Error returned from a transaction callback to trigger a rollback and
+ // retry. Other errors cause a rollback and abort.
+ errRetryTransaction = errors.New("Retry transaction")
+)
+
+//////////////////////////////////////////
+// Relational type definitions
+
+type BundleData struct {
+ // Raw SHA256 of the bundle contents
+ Hash []byte `db:"hash"` // primary key
+ // The bundle contents
+ Json string `db:"json"`
+}
+
+type BundleLink struct {
+ // 64-byte printable ASCII string
+ Id string `db:"id"` // primary key
+ // Part of the BundleLink specified by bundle author
+ BundleDesc
+ // Marks default bundles, returned by GetDefaultBundleList()
+ IsDefault bool `db:"is_default"`
+ // Raw SHA256 of the bundle contents
+ Hash []byte `db:"hash"` // foreign key => BundleData.Hash
+ // Link record creation time
+ CreatedAt time.Time `db:"created_at"`
+}
+
+type BundleDesc struct {
+ // Human-readable, URL-friendly unique name, up to 128 Unicode characters;
+ // used for newest version of default bundles (see `is_default`)
+ Slug EmptyNullString `db:"slug"`
+}
+
+//////////////////////////////////////////
+// Helper type definitions
+
+// Bundle that has not yet been saved into the database. Used as input type.
+type NewBundle struct {
+ // Part of the BundleLink specified by bundle author
+ BundleDesc
+ // The bundle contents
+ Json string `db:"json"`
+}
+
+///////////////////////////////////////
+// DB read-only methods
+
+// TODO(nlacasse): Use prepared statements, otherwise we have an extra
+// round-trip to the db, which is slow on cloud sql.
+
+func getBundleLinkById(q sqlx.Queryer, id string) (*BundleLink, error) {
+ var bLink BundleLink
+ if err := sqlx.Get(q, &bLink, "SELECT * FROM bundle_link WHERE id=?", id); err != nil {
+ if err == sql.ErrNoRows {
+ err = ErrNotFound
+ }
+ return nil, err
+ }
+ return &bLink, nil
+}
+
+// Only default bundles can be retrieved by slug for now.
+func getDefaultBundleLinkBySlug(q sqlx.Queryer, slug string) (*BundleLink, error) {
+ var bLink BundleLink
+ if err := sqlx.Get(q, &bLink, "SELECT * FROM bundle_link WHERE slug=? AND is_default", slug); err != nil {
+ if err == sql.ErrNoRows {
+ err = ErrNotFound
+ }
+ return nil, err
+ }
+ return &bLink, nil
+}
+
+func getBundleDataByHash(q sqlx.Queryer, hash []byte) (*BundleData, error) {
+ var bData BundleData
+ if err := sqlx.Get(q, &bData, "SELECT * FROM bundle_data WHERE hash=?", hash); err != nil {
+ if err == sql.ErrNoRows {
+ err = ErrNotFound
+ }
+ return nil, err
+ }
+ return &bData, nil
+}
+
+// All default bundles have non-empty slugs. Check just in case.
+func getDefaultBundleList(q sqlx.Queryer) ([]*BundleLink, error) {
+ var bLinks []*BundleLink
+ if err := sqlx.Select(q, &bLinks, "SELECT * FROM bundle_link WHERE is_default AND slug IS NOT NULL"); err != nil {
+ return nil, err
+ }
+ return bLinks, nil
+}
+
+// GetBundleByLinkIdOrSlug retrieves a BundleData object linked to by a
+// BundleLink with a particular id or slug. Id is tried first, slug if id
+// doesn't exist.
+// Note: This can fail if the bundle is deleted between fetching BundleLink
+// and BundleData. However, it is highly unlikely, costly to mitigate (using
+// a serializable transaction), and unimportant (error 500 instead of 404).
+func GetBundleByLinkIdOrSlug(idOrSlug string) (*BundleLink, *BundleData, error) {
+ bLink, err := getBundleLinkById(dbRead, idOrSlug)
+ if err == ErrNotFound {
+ bLink, err = getDefaultBundleLinkBySlug(dbRead, idOrSlug)
+ }
+ if err != nil {
+ return nil, nil, err
+ }
+ bData, err := getBundleDataByHash(dbRead, bLink.Hash)
+ if err != nil {
+ return nil, nil, err
+ }
+ return bLink, bData, nil
+}
+
+// GetDefaultBundleList retrieves a list of BundleLink objects describing
+// default bundles. All default bundles have slugs.
+func GetDefaultBundleList() ([]*BundleLink, error) {
+ return getDefaultBundleList(dbRead)
+}
+
+////////////////////////////////////
+// DB write methods
+
+func storeBundleData(ext sqlx.Ext, bData *BundleData) error {
+ _, err := sqlx.NamedExec(ext, "INSERT INTO bundle_data (hash, json) VALUES (:hash, :json)", bData)
+ return err
+}
+
+func storeBundleLink(ext sqlx.Ext, bLink *BundleLink) error {
+ _, err := sqlx.NamedExec(ext, "INSERT INTO bundle_link (id, slug, is_default, hash) VALUES (:id, :slug, :is_default, :hash)", bLink)
+ return err
+}
+
+func storeBundle(tx *sqlx.Tx, bundle *NewBundle, asDefault bool) (*BundleLink, *BundleData, error) {
+ // All default bundles must have non-empty slugs.
+ if asDefault && bundle.Slug == "" {
+ return nil, nil, fmt.Errorf("default bundle must have non-empty slug")
+ }
+
+ bHashRaw := hash.Raw([]byte(bundle.Json))
+ bHash := bHashRaw[:]
+
+ // Generate a random id for the bundle link.
+ id, err := randomLink(bHash)
+ if err != nil {
+ return nil, nil, fmt.Errorf("error creating link id: %v", err)
+ }
+
+ // Check if bundle link with this id already exists in DB.
+ if _, err = getBundleLinkById(tx, id); err == nil {
+ // Bundle was found. Return ID collision error.
+ return nil, nil, errIDCollision
+ } else if err != ErrNotFound {
+ return nil, nil, fmt.Errorf("error checking for bundle link: %v", err)
+ }
+
+ // Check if bundle data with this hash already exists in DB.
+ bData, err := getBundleDataByHash(tx, bHash)
+ if err != nil {
+ if err != ErrNotFound {
+ return nil, nil, fmt.Errorf("error checking for bundle data: %v", err)
+ }
+
+ // Bundle does not exist in DB. Store it.
+ bData = &BundleData{
+ Hash: bHash,
+ Json: bundle.Json,
+ }
+ if err = storeBundleData(tx, bData); err != nil {
+ return nil, nil, fmt.Errorf("error storing bundle data: %v", err)
+ }
+ }
+
+ // Store the bundle link.
+ bLink := &BundleLink{
+ Id: id,
+ BundleDesc: bundle.BundleDesc,
+ IsDefault: asDefault,
+ Hash: bHash,
+ }
+ if err = storeBundleLink(tx, bLink); err != nil {
+ return nil, nil, fmt.Errorf("error storing bundle link: %v", err)
+ }
+
+ return bLink, bData, nil
+}
+
+func unmarkDefaultBundles(ext sqlx.Ext) error {
+ _, err := ext.Exec("UPDATE bundle_link SET slug=NULL, is_default=false WHERE is_default")
+ if err != nil {
+ return fmt.Errorf("failed unmarking default bundles: %v", err)
+ }
+
+ return nil
+}
+
+// StoreBundleLinkAndData creates a new bundle data for a given json byte slice
+// if one does not already exist. It will create a new bundle link pointing to
+// that data. All DB access is done in a transaction, which will retry up to 3
+// times. Both the link and the data are returned, or an error if one occured.
+// Slugs are currently not allowed for user-stored bundles.
+func StoreBundleLinkAndData(json string) (bLink *BundleLink, bData *BundleData, retErr error) {
+ retErr = runInTransaction(dbSeq, 3, func(tx *sqlx.Tx) (err error) {
+ bLink, bData, err = storeBundle(tx, &NewBundle{Json: string(json)}, false)
+ if err == errIDCollision {
+ return errRetryTransaction
+ }
+ return err
+ })
+
+ return
+}
+
+// ReplaceDefaultBundles removes slugs and default flags from all existing
+// default bundles and inserts all bundles in newDefBundles as default bundles.
+// Each bundle in newDefBundles must have a unique non-empty slug.
+func ReplaceDefaultBundles(newDefBundles []*NewBundle) (retErr error) {
+ retErr = runInTransaction(dbSeq, 5, func(tx *sqlx.Tx) error {
+ if err := unmarkDefaultBundles(tx); err != nil {
+ return err
+ }
+
+ for _, bundle := range newDefBundles {
+ if _, _, err := storeBundle(tx, bundle, true); err != nil {
+ if err == errIDCollision {
+ return errRetryTransaction
+ }
+ return err
+ }
+ }
+
+ return nil
+ })
+
+ return
+}
+
+//////////////////////////////////////////
+// Transaction support
+
+// Runs function txf inside a SQL transaction. txf should only use the database
+// handle passed to it, which shares the prepared transaction cache with the
+// original handle. If txf returns nil, the transaction is committed.
+// Otherwise, it is rolled back.
+// txf is retried at most maxRetries times, with a fresh transaction for every
+// attempt, until the commit is successful. txf should not have side effects
+// that could affect subsequent retries (apart from database operations, which
+// are rolled back).
+// If the error returned from txf is errRetryTransaction, txf is retried as if
+// the commit failed. Otherwise, txf is not retried, and RunInTransaction
+// returns the error.
+// In rare cases, txf may be retried even if the transaction was successfully
+// committed (when commit falsely returns an error). txf should be idempotent
+// or able to detect this case.
+// If maxRetries is exhausted, runInTransaction returns errTooManyRetries.
+// Nested transactions are not supported and result in undefined behaviour.
+// Inspired by https://cloud.google.com/appengine/docs/go/datastore/reference#RunInTransaction
+func runInTransaction(db *sqlx.DB, maxRetries int, txf func(tx *sqlx.Tx) error) error {
+ for i := 0; i < maxRetries; i++ {
+ err := attemptInTransaction(db, txf)
+ if err == nil {
+ return nil
+ } else if err != errRetryTransaction {
+ return err
+ }
+ }
+ return errTooManyRetries
+}
+
+func attemptInTransaction(db *sqlx.DB, txf func(tx *sqlx.Tx) error) (rerr error) {
+ tx, err := db.Beginx()
+ if err != nil {
+ return fmt.Errorf("Failed opening transaction: %v", err)
+ }
+ defer func() {
+ // UPSTREAM BUG WORKAROUND: Rollback anyway to release transaction after
+ // manual commit.
+ //if rerr != nil {
+ if true {
+ // Silently ignore rollback error, we cannot do anything. Transaction
+ // will timeout eventually.
+ // UPSTREAM BUG: Transaction does not timeout, connection gets reused.
+ // This case is unlikely, but dangerous.
+ // TODO(ivanpi): Remove workaround when bug is resolved.
+ _ = tx.Rollback()
+ }
+ }()
+ // Call txf with the transaction handle - a shallow copy of the database
+ // handle (sharing the mutex, database connection, queries) with the
+ // transaction object added.
+ if err := txf(tx); err != nil {
+ return err
+ }
+ // UPSTREAM BUG WORKAROUND: Commit manually.
+ //if err = tx.Commit(); err != nil {
+ if _, err = tx.Exec("COMMIT"); err != nil {
+ return errRetryTransaction
+ }
+ return nil
+}
+
+////////////////////////////////////////////
+// SQL helper types
+
+// EmptyNullString is a convenience type mapping an empty string value to a
+// NULL value in the database and vice-versa. It is less cumbersome and
+// error-prone to use for this purpose than sql.NullString.
+// NULL values are used instead of empty strings to allow a UNIQUE index on
+// the value in MySQL with the semantics 'unique if not NULL'. MySQL doesn't
+// have support for filtered (conditional) indexes otherwise.
+type EmptyNullString string
+
+func (s *EmptyNullString) Scan(value interface{}) error {
+ var ns sql.NullString
+ if err := ns.Scan(value); err != nil {
+ return err
+ }
+ if ns.Valid {
+ *s = EmptyNullString(ns.String)
+ } else {
+ *s = ""
+ }
+ return nil
+}
+
+func (s EmptyNullString) Value() (driver.Value, error) {
+ ns := sql.NullString{
+ String: string(s),
+ Valid: (s != ""),
+ }
+ return ns.Value()
+}
+
+////////////////////////////////////////////
+// Helper methods
+
+// randomLink creates a random link id for a given hash.
+func randomLink(bHash []byte) (string, error) {
+ h := make([]byte, 32, 32+len(bHash))
+ err := binary.Read(crand.Reader, binary.LittleEndian, h)
+ if err != nil {
+ return "", fmt.Errorf("RNG failed: %v", err)
+ }
+ return "_" + hash.String(append(h, bHash...))[1:], nil
+}
diff --git a/go/src/v.io/x/playground/lib/storage/model_test.go b/go/src/v.io/x/playground/lib/storage/model_test.go
new file mode 100644
index 0000000..d01a56f
--- /dev/null
+++ b/go/src/v.io/x/playground/lib/storage/model_test.go
@@ -0,0 +1,314 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Tests for the storage model.
+// These tests only test the exported API of the storage model.
+//
+// NOTE: These tests cannot be run in parallel on the same machine because they
+// interact with a fixed database on the machine.
+
+package storage_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/rubenv/sql-migrate"
+
+ "v.io/x/lib/dbutil"
+ "v.io/x/playground/lib/storage"
+)
+
+var (
+ dataSourceName = "playground_test@tcp(localhost:3306)/playground_test?parseTime=true"
+)
+
+// setup cleans the database, runs migrations, and connects to the database.
+// It returns a teardown function that closes the database connection.
+func setup(t *testing.T) func() {
+ // Migrate down then up.
+ migrations := &migrate.FileMigrationSource{
+ Dir: "../../migrations",
+ }
+ migrate.SetTable("migrations")
+
+ sqlConfig := dbutil.SqlConfig{
+ DataSourceName: dataSourceName,
+ TLSDisable: true,
+ }
+ activeSqlConfig, err := sqlConfig.Activate("")
+
+ db, err := activeSqlConfig.NewSqlDBConn("SERIALIZABLE")
+ if err != nil {
+ t.Fatalf("Error opening database: %v", err)
+ }
+
+ // Remove any existing tables.
+ tableNames := []string{"bundle_link", "bundle_data", "migrations"}
+ for _, tableName := range tableNames {
+ db.Exec("DROP TABLE " + tableName)
+ }
+
+ if _, err = migrate.Exec(db, "mysql", migrations, migrate.Up); err != nil {
+ t.Fatalf("Error migrating up: %v", err)
+ }
+ if err := db.Close(); err != nil {
+ t.Fatalf("db.Close() failed: %v", err)
+ }
+
+ // Connect to the storage.
+ if err := storage.Connect(activeSqlConfig); err != nil {
+ t.Fatalf("storage.Connect(%v) failed: %v", activeSqlConfig, err)
+ }
+
+ teardown := func() {
+ if err := storage.Close(); err != nil {
+ t.Fatalf("storage.Close() failed: %v", err)
+ }
+ }
+ return teardown
+}
+
+func TestGetBundleDataByLinkId(t *testing.T) {
+ defer setup(t)()
+
+ // Get with a unknown id should return ErrNotFound.
+ id := "foobar"
+ if _, _, err := storage.GetBundleByLinkIdOrSlug(id); err != storage.ErrNotFound {
+ t.Errorf("Expected GetBundleByLinkIdOrSlug with unknown id to return ErrNotFound, but instead got: %v", err)
+ }
+
+ // Add a bundle.
+ json := "mock_json_data"
+ bLink, _, err := storage.StoreBundleLinkAndData(json)
+ if err != nil {
+ t.Fatalf("Expected StoreBundleLinkAndData(%v) not to error, but got: %v", json, err)
+ }
+
+ // Bundle should exist.
+ gotBLink, gotBdata, err := storage.GetBundleByLinkIdOrSlug(bLink.Id)
+ if err != nil {
+ t.Errorf("Expected GetBundleDataByLinkIdOrSlug(%v) not to error, but got: %v", bLink.Id, err)
+ }
+
+ // Bundle should have expected id.
+ if got, want := gotBLink.Id, bLink.Id; got != want {
+ t.Errorf("Expected %v to equal %v.", got, want)
+ }
+
+ // Bundle should have expected json.
+ if got, want := gotBdata.Json, string(json); got != want {
+ t.Errorf("Expected %v to equal %v.", got, want)
+ }
+}
+
+func assertValidLinkDataPair(json string, bLink *storage.BundleLink, bData *storage.BundleData) error {
+ if string(bLink.Hash) != string(bData.Hash) {
+ return fmt.Errorf("Expected %v to equal %v", string(bLink.Hash), string(bData.Hash))
+ }
+
+ if bLink.Id == "" {
+ return fmt.Errorf("Expected bundle link to have id.")
+ }
+
+ if bData.Json != json {
+ return fmt.Errorf("Expected %v to equal %v", bData.Json, json)
+ }
+ return nil
+}
+
+func TestStoreBundleLinkAndData(t *testing.T) {
+ defer setup(t)()
+
+ mockJson := "bizbaz"
+
+ // Storing the json once should succeed.
+ bLink1, bData1, err := storage.StoreBundleLinkAndData(mockJson)
+ if err != nil {
+ t.Fatalf("StoreBundleLinkAndData(%v) failed: %v", mockJson, err)
+ }
+ if err := assertValidLinkDataPair(string(mockJson), bLink1, bData1); err != nil {
+ t.Fatalf("Got invalid link data pair: %v", err)
+ }
+
+ // Storing the bundle again should succeed.
+ bLink2, bData2, err := storage.StoreBundleLinkAndData(mockJson)
+ if err != nil {
+ t.Fatalf("StoreBundleLinkAndData(%v) failed: %v", mockJson, err)
+ }
+ if err := assertValidLinkDataPair(string(mockJson), bLink2, bData2); err != nil {
+ t.Errorf("Got invalid link data pair: %v", err)
+ }
+
+ // Bundle links should have different ids.
+ if bLink1.Id == bLink2.Id {
+ t.Errorf("Expected bundle links to have different ids, but got %v and %v", bLink1.Id, bLink2.Id)
+ }
+
+ // Bundle datas should have equal hashes.
+ if want, got := string(bData1.Hash), string(bData2.Hash); want != got {
+ t.Errorf("Expected bundle datas to have equal hashes, but got %v and %v", want, got)
+ }
+}
+
+func makeMockNamedBundle(mockSlug, mockJson string) *storage.NewBundle {
+ return &storage.NewBundle{
+ BundleDesc: storage.BundleDesc{
+ Slug: storage.EmptyNullString(mockSlug),
+ },
+ Json: mockJson,
+ }
+}
+
+func expectDefaultBundles(got []*storage.BundleLink, want []*storage.NewBundle) error {
+ if len(got) != len(want) {
+ return fmt.Errorf("Expected %d, got %d bundles.", len(want), len(got))
+ }
+
+ for _, listBLink := range got {
+ // For each listed BundleLink, get corresponding BundleData.
+ gotBLink, gotBData, err := storage.GetBundleByLinkIdOrSlug(listBLink.Id)
+ if err != nil {
+ return fmt.Errorf("Expected GetBundleDataByLinkIdOrSlug(%v) not to error, but got: %v", listBLink.Id, err)
+ }
+
+ // Check that the bundle is non-anonymous and default.
+ if gotBLink.Slug == "" {
+ return fmt.Errorf("Expected bundle for %v to have non-empty slug", gotBLink.Id)
+ }
+ if !gotBLink.IsDefault {
+ return fmt.Errorf("Expected bundle %v to be marked as default", gotBLink.Slug)
+ }
+
+ // Find expected NewBundle with slug matching the retrieved bundle.
+ var original *storage.NewBundle
+ for _, newBundle := range want {
+ if newBundle.Slug == gotBLink.Slug {
+ original = newBundle
+ break
+ }
+ }
+ if original == nil {
+ return fmt.Errorf("Unexpected bundle with slug %v", gotBLink.Slug)
+ }
+
+ // Check that the retrieved bundle is valid and matches expected JSON.
+ if err := assertValidLinkDataPair(original.Json, gotBLink, gotBData); err != nil {
+ return fmt.Errorf("Got invalid link data pair: %v", err)
+ }
+ }
+ return nil
+}
+
+func expectNonDefaultBundle(bId, wantJson string) error {
+ bLink, bData, err := storage.GetBundleByLinkIdOrSlug(bId)
+ if err != nil {
+ return fmt.Errorf("GetBundleDataByLinkIdOrSlug(%v) failed: %v", bId, err)
+ }
+ if bLink.Slug != "" {
+ return fmt.Errorf("Expected anonymous bundle for %v, got slug %v", bId, bLink.Slug)
+ }
+ if bLink.IsDefault {
+ return fmt.Errorf("Expected non-default bundle for %v", bId)
+ }
+ if err := assertValidLinkDataPair(wantJson, bLink, bData); err != nil {
+ return fmt.Errorf("Got invalid link data pair: %v", err)
+ }
+ return nil
+}
+
+func TestDefaultBundles(t *testing.T) {
+ defer setup(t)()
+
+ mockSlugs := []string{"one", "two", "three", "four"}
+ mockJson := []string{"forty-two", "forty-seven", "leet"}
+
+ defBundlesA := []*storage.NewBundle{
+ makeMockNamedBundle(mockSlugs[0], mockJson[0]),
+ makeMockNamedBundle(mockSlugs[1], mockJson[0]),
+ }
+
+ // Storing default bundles should succeed.
+ if err := storage.ReplaceDefaultBundles(defBundlesA); err != nil {
+ t.Fatalf("A: ReplaceDefaultBundles(%v) failed: %v", defBundlesA, err)
+ }
+
+ // Storing a non-default bundle should succeed.
+ nondBLink, _, err := storage.StoreBundleLinkAndData(mockJson[2])
+ if err != nil {
+ t.Fatalf("StoreBundleLinkAndData(%v) failed: %v", mockJson[2], err)
+ }
+
+ defBundlesDup := []*storage.NewBundle{
+ makeMockNamedBundle(mockSlugs[1], mockJson[1]),
+ makeMockNamedBundle(mockSlugs[1], mockJson[1]),
+ }
+
+ // Trying to store default bundles with duplicate slugs should fail and be
+ // rolled back (not affect subsequent assertions).
+ if err := storage.ReplaceDefaultBundles(defBundlesDup); err == nil {
+ t.Fatalf("Dup: ReplaceDefaultBundles(%v) with duplicate slugs should have failed", defBundlesDup)
+ }
+
+ // Listing default bundles should succeed.
+ storedDefBundlesA, err := storage.GetDefaultBundleList()
+ if err != nil {
+ t.Fatalf("A: GetDefaultBundleList() failed: %v", err)
+ }
+
+ // Default bundle list should not contain the non-default bundle.
+ if err := expectDefaultBundles(storedDefBundlesA, defBundlesA); err != nil {
+ t.Errorf("A: Default bundle mismatch: %v", err)
+ }
+
+ // Non-default bundle should be untouched.
+ if err := expectNonDefaultBundle(nondBLink.Id, mockJson[2]); err != nil {
+ t.Errorf("Non-default bundle mismatch: %v", err)
+ }
+
+ defBundlesB := []*storage.NewBundle{
+ makeMockNamedBundle(mockSlugs[1], mockJson[2]),
+ makeMockNamedBundle(mockSlugs[2], mockJson[1]),
+ makeMockNamedBundle(mockSlugs[3], mockJson[0]),
+ }
+
+ // Replacing default bundles should succeed.
+ if err := storage.ReplaceDefaultBundles(defBundlesB); err != nil {
+ t.Fatalf("B: ReplaceDefaultBundles(%v) failed: %v", defBundlesB, err)
+ }
+
+ // Listing default bundles should succeed.
+ storedDefBundlesB, err := storage.GetDefaultBundleList()
+ if err != nil {
+ t.Fatalf("B: GetDefaultBundleList() failed: %v", err)
+ }
+
+ // Default bundle list should not contain the old default bundles.
+ if err := expectDefaultBundles(storedDefBundlesB, defBundlesB); err != nil {
+ t.Fatalf("B: Default bundle mismatch: %v", err)
+ }
+
+ // Non-default bundle should still be untouched.
+ if err := expectNonDefaultBundle(nondBLink.Id, mockJson[2]); err != nil {
+ t.Errorf("Non-default bundle mismatch: %v", err)
+ }
+
+ // Old default bundles should still be reachable by id.
+ if err := expectNonDefaultBundle(storedDefBundlesA[0].Id, mockJson[0]); err != nil {
+ t.Errorf("Old default bundle mismatch: %v", err)
+ }
+ // But not by slug.
+ if _, _, err := storage.GetBundleByLinkIdOrSlug(mockSlugs[0]); err != storage.ErrNotFound {
+ t.Errorf("Expected GetBundleByLinkIdOrSlug with old slug to return ErrNotFound, but instead got: %v", err)
+ }
+
+ // New bundle should be reachable by slug.
+ niBLink, niBData, err := storage.GetBundleByLinkIdOrSlug(mockSlugs[1])
+ if err != nil {
+ t.Fatalf("GetBundleDataByLinkIdOrSlug(%v) failed: %v", mockSlugs[1], err)
+ }
+ if err := assertValidLinkDataPair(mockJson[2], niBLink, niBData); err != nil {
+ t.Errorf("Got invalid link data pair: %v", err)
+ }
+}
diff --git a/go/src/v.io/x/playground/migrations/002-default-examples.sql b/go/src/v.io/x/playground/migrations/002-default-examples.sql
new file mode 100644
index 0000000..0eadbe9
--- /dev/null
+++ b/go/src/v.io/x/playground/migrations/002-default-examples.sql
@@ -0,0 +1,15 @@
+-- +migrate Up
+
+ALTER TABLE bundle_link
+ ADD COLUMN slug VARCHAR(128) NULL DEFAULT NULL AFTER id,
+ ADD COLUMN is_default BOOLEAN NOT NULL DEFAULT false AFTER slug,
+ ADD UNIQUE INDEX slug_index (slug),
+ ADD INDEX is_default_index (is_default);
+
+-- +migrate Down
+
+ALTER TABLE bundle_link
+ DROP INDEX is_default_index,
+ DROP INDEX slug_index,
+ DROP COLUMN is_default,
+ DROP COLUMN slug;
diff --git a/go/src/v.io/x/playground/pgadmin/bundle.go b/go/src/v.io/x/playground/pgadmin/bundle.go
new file mode 100644
index 0000000..7ce2de4
--- /dev/null
+++ b/go/src/v.io/x/playground/pgadmin/bundle.go
@@ -0,0 +1,168 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Bundle commands support bundling playground examples into JSON objects
+// compatible with the playground client. Glob files allow specifying file
+// subsets for different implementations of the same example. Bundles specified
+// in a configuration file can be loaded into the database as named, default
+// examples.
+
+package main
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "v.io/x/lib/cmdline"
+ "v.io/x/lib/dbutil"
+ "v.io/x/playground/lib"
+ "v.io/x/playground/lib/bundle"
+ "v.io/x/playground/lib/storage"
+)
+
+var cmdBundle = &cmdline.Command{
+ Name: "bundle",
+ Short: "Default bundle management",
+ Long: `
+Commands for bundling playground examples and loading default bundles into the
+database.
+`,
+ Children: []*cmdline.Command{cmdBundleMake, cmdBundleBootstrap},
+}
+
+var cmdBundleMake = &cmdline.Command{
+ Runner: cmdline.RunnerFunc(runBundleMake),
+ Name: "make",
+ Short: "Make a single manually specified bundle",
+ Long: `
+Bundles the example specified by <root_path>, as filtered by <glob_file>, into
+a JSON object compatible with the playground client.
+`,
+ ArgsName: "<glob_file> <root_path>",
+ ArgsLong: bundle.BundleUsage,
+}
+
+// TODO(ivanpi): Make a single bundle from config file instead of manually specified.
+// TODO(ivanpi): Add bundle metadata (title, description) via config file.
+// TODO(ivanpi): Iterate over config file, applying commands to bundles (similar to POSIX find)?
+var cmdBundleBootstrap = &cmdline.Command{
+ Runner: runWithStorage(runBundleBootstrap),
+ Name: "bootstrap",
+ Short: "Bootstrap bundles from config file into database",
+ Long: `
+Bundles all examples specified in the bundle config file and saves them as
+named default bundles into the database specified by sqlconf, replacing any
+existing default examples. Bundle slugs are '<example_name>-<glob_name>'.
+`,
+}
+
+const (
+ defaultBundleCfg = "${V23_ROOT}/release/projects/playground/go/src/v.io/x/playground/bundles/config.json"
+)
+
+var (
+ flagBundleCfgFile string
+ flagBundleDir string
+)
+
+func init() {
+ cmdBundleBootstrap.Flags.StringVar(&flagBundleCfgFile, "bundleconf", defaultBundleCfg, "Path to bundle config file. "+bundle.BundleConfigFileDescription)
+ cmdBundleBootstrap.Flags.StringVar(&flagBundleDir, "bundledir", "", "Path relative to which paths in the bundle config file are interpreted. If empty, defaults to the config file directory.")
+}
+
+// Bundles an example from the specified folder using the specified glob file.
+// TODO(ivanpi): Expose --verbose and --empty options.
+func runBundleMake(env *cmdline.Env, args []string) error {
+ if len(args) != 2 {
+ return env.UsageErrorf("exactly two arguments expected")
+ }
+ bOut, err := bundle.Bundle(env.Stderr, args[0], args[1], true)
+ if err != nil {
+ return fmt.Errorf("Bundling failed: %v", err)
+ }
+ fmt.Fprintln(env.Stdout, string(bOut))
+ return nil
+}
+
+// Returns a cmdline.RunnerFunc for loading all bundles specified in the bundle
+// config file into the database as default bundles.
+func runBundleBootstrap(env *cmdline.Env, args []string) error {
+ bundleDir := os.ExpandEnv(flagBundleDir)
+ // If bundleDir is empty, interpret paths relative to bundleCfg directory.
+ if bundleDir == "" {
+ bundleDir = filepath.Dir(os.ExpandEnv(flagBundleCfgFile))
+ }
+ bundleCfg, err := bundle.ParseConfigFromFile(os.ExpandEnv(flagBundleCfgFile), bundleDir)
+ if err != nil {
+ return fmt.Errorf("Failed parsing bundle config from %q: %v", os.ExpandEnv(flagBundleCfgFile), err)
+ }
+
+ var newDefBundles []*storage.NewBundle
+ for _, example := range bundleCfg.Examples {
+ fmt.Fprintf(env.Stdout, "Bundling example: %s (%q)\n", example.Name, example.Path)
+
+ for _, globName := range example.Globs {
+ glob, globExists := bundleCfg.Globs[globName]
+ if !globExists {
+ return fmt.Errorf("Unknown glob %q", globName)
+ }
+ fmt.Fprintf(env.Stdout, "> glob: %s (%q)\n", globName, glob.Path)
+
+ bOut, err := bundle.Bundle(env.Stderr, glob.Path, example.Path, false)
+ if err != nil {
+ return fmt.Errorf("Bundling %s with %s failed: %v", example.Name, globName, err)
+ }
+
+ // Append the bundle and metadata to new default bundles.
+ newDefBundles = append(newDefBundles, &storage.NewBundle{
+ BundleDesc: storage.BundleDesc{
+ Slug: storage.EmptyNullString(example.Name + "-" + globName),
+ },
+ Json: string(bOut),
+ })
+ }
+ }
+
+ if *flagDryRun {
+ fmt.Fprintf(env.Stdout, "Run without dry run to load %d bundles into database\n", len(newDefBundles))
+ } else {
+ // Unmark old default bundles and store new ones.
+ if err := storage.ReplaceDefaultBundles(newDefBundles); err != nil {
+ return fmt.Errorf("Failed to replace default bundles: %v", err)
+ }
+ fmt.Fprintf(env.Stdout, "Successfully loaded %d bundles into database\n", len(newDefBundles))
+ }
+ return nil
+}
+
+// runWithStorage is a wrapper method that handles opening and closing the
+// database connections used by `v.io/x/playground/lib/storage`.
+func runWithStorage(fx cmdline.RunnerFunc) cmdline.RunnerFunc {
+ return func(env *cmdline.Env, args []string) (rerr error) {
+ if *flagSQLConf == "" {
+ return env.UsageErrorf("SQL configuration file (-sqlconf) must be provided")
+ }
+
+ // Parse SQL configuration file and set up TLS.
+ dbConf, err := dbutil.ActivateSqlConfigFromFile(*flagSQLConf)
+ if err != nil {
+ return fmt.Errorf("Error parsing SQL configuration: %v", err)
+ }
+ // Connect to storage backend.
+ if err := storage.Connect(dbConf); err != nil {
+ return fmt.Errorf("Error opening database connection: %v", err)
+ }
+ // Best effort close.
+ defer func() {
+ if cerr := storage.Close(); cerr != nil {
+ cerr = fmt.Errorf("Failed closing database connection: %v", cerr)
+ rerr = lib.MergeErrors(rerr, cerr, "\n")
+ }
+ }()
+
+ // Run wrapped function.
+ return fx(env, args)
+ }
+}
diff --git a/go/src/v.io/x/playground/pgadmin/main.go b/go/src/v.io/x/playground/pgadmin/main.go
index fc50bf0..ef53569 100644
--- a/go/src/v.io/x/playground/pgadmin/main.go
+++ b/go/src/v.io/x/playground/pgadmin/main.go
@@ -22,10 +22,9 @@
Short: "Playground database management tool",
Long: `
Tool for managing the playground database and default bundles.
-Supports database schema migration.
-TODO(ivanpi): bundle bootstrap
+Supports database schema migration and loading default bundles into database.
`,
- Children: []*cmdline.Command{cmdMigrate},
+ Children: []*cmdline.Command{cmdMigrate, cmdBundle},
}
var (
diff --git a/go/src/v.io/x/playground/pgadmin/migrate.go b/go/src/v.io/x/playground/pgadmin/migrate.go
index 2cc0133..c66eae4 100644
--- a/go/src/v.io/x/playground/pgadmin/migrate.go
+++ b/go/src/v.io/x/playground/pgadmin/migrate.go
@@ -21,6 +21,7 @@
"v.io/x/lib/cmdline"
"v.io/x/lib/dbutil"
+ "v.io/x/playground/lib"
)
const mysqlWarning = `
@@ -129,12 +130,7 @@
defer func() {
if cerr := db.Close(); cerr != nil {
cerr = fmt.Errorf("Failed closing database connection: %v", cerr)
- // Merge errors.
- if rerr == nil {
- rerr = cerr
- } else {
- rerr = fmt.Errorf("%v\n%v", rerr, cerr)
- }
+ rerr = lib.MergeErrors(rerr, cerr, "\n")
}
}()
// Ping database to check connection.
diff --git a/pgbundle/Makefile b/pgbundle/Makefile
index 9b1052e..93ff2e1 100644
--- a/pgbundle/Makefile
+++ b/pgbundle/Makefile
@@ -2,6 +2,10 @@
PATH := $(PATH):$(V23_ROOT)/third_party/cout/node/bin
SHELL := /bin/bash -euo pipefail
+.DEFAULT_GOAL := node_modules
+
+.DELETE_ON_ERROR:
+
node_modules: package.json
@npm prune
@npm install
