{veyron,veyron2}: moving the store code out the main repository
The build and Go tests pass on Linux.
Minor refactoring still needed to remove the remaining dependencies on
store, but would like to get this big CL in first to unblock folks.
Once this CL is submitted, I will modify the manifest to surface the
roadmap.go.store repository under: $VEYRON_ROOT/roadmap/go/src/veyron.io/store
Change-Id: I682e846bfdf160f8b0e07d4be092132339e3114c
diff --git a/examples/bank/pbankd/main.go b/examples/bank/pbankd/main.go
index d8a7e3a..1806daa 100644
--- a/examples/bank/pbankd/main.go
+++ b/examples/bank/pbankd/main.go
@@ -39,8 +39,8 @@
// "veyron2/naming"
// "veyron2/rt"
// "veyron2/security"
-// "veyron2/storage"
-// "veyron2/storage/vstore"
+// "veyron.io/store/veyron2/storage"
+// "veyron.io/store/veyron2/storage/vstore"
// "veyron2/vlog"
// )
diff --git a/examples/boxes/android/src/boxesp2p/main.go b/examples/boxes/android/src/boxesp2p/main.go
index 24a82e6..9654184 100644
--- a/examples/boxes/android/src/boxesp2p/main.go
+++ b/examples/boxes/android/src/boxesp2p/main.go
@@ -73,7 +73,7 @@
inaming "veyron/runtimes/google/naming"
vsync "veyron/runtimes/google/vsync"
vsecurity "veyron/security"
- sstore "veyron/services/store/server"
+ sstore "veyron.io/store/veyron/services/store/server"
"veyron2"
"veyron2/context"
@@ -82,8 +82,8 @@
"veyron2/rt"
"veyron2/security"
"veyron2/services/watch/types"
- "veyron2/storage"
- "veyron2/storage/vstore"
+ "veyron.io/store/veyron2/storage"
+ "veyron.io/store/veyron2/storage/vstore"
"veyron2/vom"
)
diff --git a/examples/mdb/Makefile b/examples/mdb/Makefile
index c215080..49f36aa 100644
--- a/examples/mdb/Makefile
+++ b/examples/mdb/Makefile
@@ -1,5 +1,5 @@
build:
- ${VEYRON_ROOT}/veyron/scripts/build/go install veyron/examples/mdb/... veyron/services/mounttable/mounttabled veyron/services/store/stored veyron/tools/findunusedport veyron/tools/identity
+ ${VEYRON_ROOT}/veyron/scripts/build/go install veyron/examples/mdb/... veyron/services/mounttable/mounttabled veyron.io/store/veyron/services/store/stored veyron/tools/findunusedport veyron/tools/identity
run: build
./run.sh
diff --git a/examples/mdb/mdb_init/main.go b/examples/mdb/mdb_init/main.go
index 45e9b85..dc671eb 100644
--- a/examples/mdb/mdb_init/main.go
+++ b/examples/mdb/mdb_init/main.go
@@ -28,8 +28,8 @@
"veyron/examples/mdb/schema"
"veyron2/context"
"veyron2/rt"
- "veyron2/storage"
- "veyron2/storage/vstore"
+ "veyron.io/store/veyron2/storage"
+ "veyron.io/store/veyron2/storage/vstore"
"veyron2/vlog"
)
diff --git a/examples/mdb/schema/schema.vdl b/examples/mdb/schema/schema.vdl
index 01e96dc..d90ed86 100644
--- a/examples/mdb/schema/schema.vdl
+++ b/examples/mdb/schema/schema.vdl
@@ -1,7 +1,7 @@
package schema
import (
- "veyron2/storage"
+ "veyron.io/store/veyron2/storage"
)
// Dir is used to represent directories.
diff --git a/examples/mdb/schema/schema.vdl.go b/examples/mdb/schema/schema.vdl.go
index f270d44..6867e83 100644
--- a/examples/mdb/schema/schema.vdl.go
+++ b/examples/mdb/schema/schema.vdl.go
@@ -4,7 +4,7 @@
package schema
import (
- "veyron2/storage"
+ "veyron.io/store/veyron2/storage"
)
// Dir is used to represent directories.
diff --git a/examples/stfortune/schema/schema.vdl b/examples/stfortune/schema/schema.vdl
index c87ae0d..f23b0fc 100644
--- a/examples/stfortune/schema/schema.vdl
+++ b/examples/stfortune/schema/schema.vdl
@@ -1,7 +1,7 @@
package schema
import (
- "veyron2/storage"
+ "veyron.io/store/veyron2/storage"
)
// User contains the information corresponding to a particular UserName in the store.
diff --git a/examples/stfortune/schema/schema.vdl.go b/examples/stfortune/schema/schema.vdl.go
index 597c940..09cef86 100644
--- a/examples/stfortune/schema/schema.vdl.go
+++ b/examples/stfortune/schema/schema.vdl.go
@@ -4,7 +4,7 @@
package schema
import (
- "veyron2/storage"
+ "veyron.io/store/veyron2/storage"
)
// User contains the information corresponding to a particular UserName in the store.
diff --git a/examples/stfortune/stfortune/main.go b/examples/stfortune/stfortune/main.go
index 12ba537..18e193c 100644
--- a/examples/stfortune/stfortune/main.go
+++ b/examples/stfortune/stfortune/main.go
@@ -16,7 +16,7 @@
"os"
"strings"
"time"
- "veyron2/query"
+ "veyron.io/store/veyron2/query"
"veyron/examples/stfortune/schema"
@@ -24,8 +24,8 @@
"veyron2/naming"
"veyron2/rt"
"veyron2/services/watch/types"
- "veyron2/storage"
- "veyron2/storage/vstore"
+ "veyron.io/store/veyron2/storage"
+ "veyron.io/store/veyron2/storage/vstore"
"veyron2/vom"
)
diff --git a/examples/todos/Makefile b/examples/todos/Makefile
index cece7f9..3ddf791 100644
--- a/examples/todos/Makefile
+++ b/examples/todos/Makefile
@@ -11,7 +11,7 @@
(cd todos_appd && npm install)
buildgo:
- ${VEYRON_ROOT}/veyron/scripts/build/go install veyron/examples/todos/... veyron/services/mounttable/mounttabled veyron/services/store/stored veyron/tools/findunusedport veyron/tools/identity
+ ${VEYRON_ROOT}/veyron/scripts/build/go install veyron/examples/todos/... veyron/services/mounttable/mounttabled veyron.io/store/veyron/services/store/stored veyron/tools/findunusedport veyron/tools/identity
buildapp: node_modules
browserify -d todos_appd/browser/*.js -p [minifyify --map bundle.js.map --output ${BUNDLE_JS}.map] -o ${BUNDLE_JS}
diff --git a/examples/todos/todos_init/main.go b/examples/todos/todos_init/main.go
index ea80441..e2f9c99 100644
--- a/examples/todos/todos_init/main.go
+++ b/examples/todos/todos_init/main.go
@@ -16,8 +16,8 @@
"veyron/examples/todos/schema"
"veyron2/context"
"veyron2/rt"
- "veyron2/storage"
- "veyron2/storage/vstore"
+ "veyron.io/store/veyron2/storage"
+ "veyron.io/store/veyron2/storage/vstore"
"veyron2/vlog"
)
diff --git a/runtimes/google/ipc/full_test.go b/runtimes/google/ipc/full_test.go
index 8f3f127..4a9dc11 100644
--- a/runtimes/google/ipc/full_test.go
+++ b/runtimes/google/ipc/full_test.go
@@ -455,18 +455,18 @@
nameErr = "does not match the provided pattern"
)
var (
- // TODO(ataly, ashankar): Uncomment the following once server authorization
- // is enabled.
- // now = time.Now()
- // cavOnlyV1 = caveat.UniversalCaveat(caveat.PeerIdentity{"client/v1"})
- // cavExpired = security.ServiceCaveat{
- // Service: security.AllPrincipals,
- // Caveat: &caveat.Expiry{IssueTime: now, ExpiryTime: now},
- // }
- // clientV1ID = derive(clientID, "v1")
- // clientV2ID = derive(clientID, "v2")
- // serverV1ID = derive(serverID, "v1", cavOnlyV1)
- // serverExpiredID = derive(serverID, "expired", cavExpired)
+ // TODO(ataly, ashankar): Uncomment the following once server authorization
+ // is enabled.
+ // now = time.Now()
+ // cavOnlyV1 = caveat.UniversalCaveat(caveat.PeerIdentity{"client/v1"})
+ // cavExpired = security.ServiceCaveat{
+ // Service: security.AllPrincipals,
+ // Caveat: &caveat.Expiry{IssueTime: now, ExpiryTime: now},
+ // }
+ // clientV1ID = derive(clientID, "v1")
+ // clientV2ID = derive(clientID, "v2")
+ // serverV1ID = derive(serverID, "v1", cavOnlyV1)
+ // serverExpiredID = derive(serverID, "expired", cavExpired)
)
tests := []struct {
diff --git a/runtimes/google/vsync/dag.go b/runtimes/google/vsync/dag.go
deleted file mode 100644
index 956ff28..0000000
--- a/runtimes/google/vsync/dag.go
+++ /dev/null
@@ -1,957 +0,0 @@
-package vsync
-
-// Veyron Sync DAG (directed acyclic graph) utility functions.
-// The DAG is used to track the version history of objects in order to
-// detect and resolve conflicts (concurrent changes on different devices).
-//
-// Terminology:
-// * An object is a unique value in the Veyron Store represented by its UID.
-// * As an object mutates, its version number is updated by the Store.
-// * Each (object, version) tuple is represented by a node in the Sync DAG.
-// * The previous version of an object is its parent in the DAG, i.e. the
-// new version is derived from that parent.
-// * When there are no conflicts, the node has a single reference back to
-// a parent node.
-// * When a conflict between two concurrent object versions is resolved,
-// the new version has references back to each of the two parents to
-// indicate that it is derived from both nodes.
-// * During a sync operation from a source device to a target device, the
-// target receives a DAG fragment from the source. That fragment has to
-// be incorporated (grafted) into the target device's DAG. It may be a
-// continuation of the DAG of an object, with the attachment (graft) point
-// being the current head of DAG, in which case there are no conflicts.
-// Or the graft point(s) may be older nodes, which means the new fragment
-// is a divergence in the graph causing a conflict that must be resolved
-// in order to re-converge the two DAG fragments.
-//
-// In the diagrams below:
-// (h) represents the head node in the local device.
-// (nh) represents the new head node received from the remote device.
-// (g) represents a graft node, where new nodes attach to the existing DAG.
-// <- represents a derived-from mutation, i.e. a child-to-parent pointer
-//
-// a- No-conflict example: the new nodes (v3, v4) attach to the head node (v2).
-// In this case the new head becomes the head node, the new DAG fragment
-// being a continuation of the existing DAG.
-//
-// Before:
-// v0 <- v1 <- v2(h)
-//
-// Sync updates applied, no conflict detected:
-// v0 <- v1 <- v2(h,g) <- v3 <- v4 (nh)
-//
-// After:
-// v0 <- v1 <- v2 <- v3 <- v4 (h)
-//
-// b- Conflict example: the new nodes (v3, v4) attach to an old node (v1).
-// The current head node (v2) and the new head node (v4) are divergent
-// (concurrent) mutations that need to be resolved. The conflict
-// resolution function is passed the old head (v2), new head (v4), and
-// the common ancestor (v1) and resolves the conflict with (v5) which
-// is represented in the DAG as derived from both v2 and v4 (2 parents).
-//
-// Before:
-// v0 <- v1 <- v2(h)
-//
-// Sync updates applied, conflict detected (v2 not a graft node):
-// v0 <- v1(g) <- v2(h)
-// <- v3 <- v4 (nh)
-//
-// After, conflict resolver creates v5 having 2 parents (v2, v4):
-// v0 <- v1(g) <- v2 <------- v5(h)
-// <- v3 <- v4 <-
-//
-// Note: the DAG does not grow indefinitely. During a sync operation each
-// device learns what the other device already knows -- where it's at in
-// the version history for the objects. When a device determines that all
-// devices that sync an object (as per the definitions of replication groups
-// in the Veyron Store) have moved past some version for that object, the
-// DAG for that object can be pruned, deleting all prior (ancestor) nodes.
-//
-// The DAG DB contains three tables persisted to disk (nodes, heads, trans)
-// and three in-memory (ephemeral) maps (graft, txSet, txGC):
-// * nodes: one entry per (object, version) with references to the
-// parent node(s) it is derived from, a reference to the
-// log record identifying that change, a reference to its
-// transaction set (or NoTxID if none), and a boolean to
-// indicate whether this change was a deletion of the object.
-// * heads: one entry per object pointing to its most recent version
-// in the nodes table
-// * trans: one entry per transaction ID containing the set of objects
-// that forms the transaction and their versions.
-// * graft: during a sync operation, it tracks the nodes where the new
-// DAG fragments are attached to the existing graph for each
-// mutated object. This map is used to determine whether a
-// conflict happened for an object and, if yes, select the most
-// recent common ancestor from these graft points to use when
-// resolving the conflict. At the end of a sync operation the
-// graft map is destroyed.
-// * txSet: used to incrementally construct the transaction sets that
-// are stored in the "trans" table once all the nodes of a
-// transaction have been added. Multiple transaction sets
-// can be constructed to support the concurrency between the
-// Sync Initiator and Watcher threads.
-// * txGC: used to track the transactions impacted by objects being
-// pruned. At the end of the pruning operation the records
-// of the "trans" table are updated from the txGC information.
-//
-// Note: for regular (no-conflict) changes, a node has a reference to
-// one parent from which it was derived. When a conflict is resolved,
-// the new node has references to the two concurrent parents that triggered
-// the conflict. The states of the parents[] array are:
-// * [] The earliest/first version of an object
-// * [XYZ] Regular non-conflict version derived from XYZ
-// * [XYZ, ABC] Resolution version caused by XYZ-vs-ABC conflict
-
-import (
- "container/list"
- "errors"
- "fmt"
- "math/rand"
- "time"
-
- "veyron/services/store/raw"
-
- "veyron2/storage"
- "veyron2/vlog"
-)
-
-const (
- NoTxID = TxID(0)
-)
-
-type TxID uint64
-type dagTxMap map[storage.ID]raw.Version
-
-type dag struct {
- fname string // file pathname
- store *kvdb // underlying K/V store
- heads *kvtable // pointer to "heads" table in the store
- nodes *kvtable // pointer to "nodes" table in the store
- trans *kvtable // pointer to "trans" table in the store
- graft map[storage.ID]*graftInfo // in-memory state of DAG object grafting
- txSet map[TxID]dagTxMap // in-memory construction of transaction sets
- txGC map[TxID]dagTxMap // in-memory tracking of transaction sets to cleanup
- txGen *rand.Rand // transaction ID random number generator
-}
-
-type dagNode struct {
- Level uint64 // node distance from root
- Parents []raw.Version // references to parent versions
- Logrec string // reference to log record change
- TxID TxID // ID of a transaction set
- Deleted bool // true if the change was a delete
-}
-
-type graftInfo struct {
- newNodes map[raw.Version]struct{} // set of newly added nodes during a sync
- graftNodes map[raw.Version]uint64 // set of graft nodes and their level
- newHeads map[raw.Version]struct{} // set of candidate new head nodes
-}
-
-// openDAG opens or creates a DAG for the given filename.
-func openDAG(filename string) (*dag, error) {
- // Open the file and create it if it does not exist.
- // Also initialize the store and its tables.
- db, tbls, err := kvdbOpen(filename, []string{"heads", "nodes", "trans"})
- if err != nil {
- return nil, err
- }
-
- d := &dag{
- fname: filename,
- store: db,
- heads: tbls[0],
- nodes: tbls[1],
- trans: tbls[2],
- txGen: rand.New(rand.NewSource(time.Now().UTC().UnixNano())),
- txSet: make(map[TxID]dagTxMap),
- }
-
- d.clearGraft()
- d.clearTxGC()
-
- return d, nil
-}
-
-// close closes the DAG and invalidates its structure.
-func (d *dag) close() {
- if d.store != nil {
- d.store.close() // this also closes the tables
- }
- *d = dag{} // zero out the DAG struct
-}
-
-// flush flushes the DAG store to disk.
-func (d *dag) flush() {
- if d.store != nil {
- d.store.flush()
- }
-}
-
-// compact compacts dag's kvdb file.
-func (d *dag) compact() error {
- if d.store == nil {
- return errors.New("invalid DAG")
- }
- db, tbls, err := d.store.compact(d.fname, []string{"heads", "nodes", "trans"})
- if err != nil {
- return err
- }
- d.store = db
- d.heads = tbls[0]
- d.nodes = tbls[1]
- d.trans = tbls[2]
- return nil
-}
-
-// clearGraft clears the temporary in-memory grafting maps.
-func (d *dag) clearGraft() {
- if d.store != nil {
- d.graft = make(map[storage.ID]*graftInfo)
- }
-}
-
-// clearTxGC clears the temporary in-memory transaction garbage collection maps.
-func (d *dag) clearTxGC() {
- if d.store != nil {
- d.txGC = make(map[TxID]dagTxMap)
- }
-}
-
-// getObjectGraft returns the graft structure for an object ID.
-// The graftInfo struct for an object is ephemeral (in-memory) and it
-// tracks the following information:
-// - newNodes: the set of newly added nodes used to detect the type of
-// edges between nodes (new-node to old-node or vice versa).
-// - newHeads: the set of new candidate head nodes used to detect conflicts.
-// - graftNodes: the set of nodes used to find common ancestors between
-// conflicting nodes.
-//
-// After the received Sync logs are applied, if there are two new heads in
-// the newHeads set, there is a conflict to be resolved for this object.
-// Otherwise if there is only one head, no conflict was triggered and the
-// new head becomes the current version for the object.
-//
-// In case of conflict, the graftNodes set is used to select the common
-// ancestor to pass to the conflict resolver.
-//
-// Note: if an object's graft structure does not exist only create it
-// if the "create" parameter is set to true.
-func (d *dag) getObjectGraft(oid storage.ID, create bool) *graftInfo {
- graft := d.graft[oid]
- if graft == nil && create {
- graft = &graftInfo{
- newNodes: make(map[raw.Version]struct{}),
- graftNodes: make(map[raw.Version]uint64),
- newHeads: make(map[raw.Version]struct{}),
- }
-
- // If a current head node exists for this object, initialize
- // the set of candidate new heads to include it.
- head, err := d.getHead(oid)
- if err == nil {
- graft.newHeads[head] = struct{}{}
- }
-
- d.graft[oid] = graft
- }
- return graft
-}
-
-// addNodeTxStart generates a transaction ID and returns it to the caller.
-// The transaction ID is purely internal to the DAG. It is used to track
-// DAG nodes that are part of the same transaction.
-func (d *dag) addNodeTxStart() TxID {
- if d.store == nil {
- return NoTxID
- }
-
- // Generate a random 64-bit transaction ID different than NoTxID.
- // Also make sure the ID is not already being used.
- tid := NoTxID
- for (tid == NoTxID) || (d.txSet[tid] != nil) {
- // Generate an unsigned 64-bit random value by combining a
- // random 63-bit value and a random 1-bit value.
- tid = (TxID(d.txGen.Int63()) << 1) | TxID(d.txGen.Int63n(2))
- }
-
- // Initialize the in-memory object/version map for that transaction ID.
- d.txSet[tid] = make(dagTxMap)
-
- return tid
-}
-
-// addNodeTxEnd marks the end of a given transaction.
-// The DAG completes its internal tracking of the transaction information.
-func (d *dag) addNodeTxEnd(tid TxID) error {
- if d.store == nil {
- return errors.New("invalid DAG")
- }
- if tid == NoTxID {
- return fmt.Errorf("invalid TxID: %v", tid)
- }
-
- txMap, ok := d.txSet[tid]
- if !ok {
- return fmt.Errorf("unknown transaction ID: %v", tid)
- }
-
- if err := d.setTransaction(tid, txMap); err != nil {
- return err
- }
-
- delete(d.txSet, tid)
- return nil
-}
-
-// addNode adds a new node for an object in the DAG, linking it to its parent nodes.
-// It verifies that this node does not exist and that its parent nodes are valid.
-// It also determines the DAG level of the node from its parent nodes (max() + 1).
-//
-// If the node is due to a local change (from the Watcher API), no need to
-// update the grafting structure. Otherwise the node is due to a remote change
-// (from the Sync protocol) being grafted on the DAG:
-// - If a parent node is not new, mark it as a DAG graft point.
-// - Mark this version as a new node.
-// - Update the new head node pointer of the grafted DAG.
-//
-// If the transaction ID is set to NoTxID, this node is not part of a transaction.
-// Otherwise, track its membership in the given transaction ID.
-func (d *dag) addNode(oid storage.ID, version raw.Version, remote, deleted bool,
- parents []raw.Version, logrec string, tid TxID) error {
- if d.store == nil {
- return errors.New("invalid DAG")
- }
-
- if parents != nil {
- if len(parents) > 2 {
- return fmt.Errorf("cannot have more than 2 parents, not %d", len(parents))
- }
- if len(parents) == 0 {
- // Replace an empty array with a nil.
- parents = nil
- }
- }
-
- // The new node must not exist.
- if d.hasNode(oid, version) {
- return fmt.Errorf("node %d:%d already exists in the DAG", oid, version)
- }
-
- // A new root node (no parents) is allowed only for new objects.
- if parents == nil {
- _, err := d.getHead(oid)
- if err == nil {
- return fmt.Errorf("cannot add another root node %d:%d for this object in the DAG", oid, version)
- }
- }
-
- // For a remote change, make sure the object has a graft info entry.
- // During a sync operation, each mutated object gets new nodes added
- // in its DAG. These new nodes are either derived from nodes that
- // were previously known on this device (i.e. their parent nodes are
- // pre-existing), or they are derived from other new DAG nodes being
- // discovered during this sync (i.e. their parent nodes were also
- // just added to the DAG).
- //
- // To detect a conflict and find the most recent common ancestor to
- // pass to the conflict resolver callback, the DAG keeps track of the
- // new nodes that have old parent nodes. These old-to-new edges are
- // the points where new DAG fragments are attached (grafted) onto the
- // existing DAG. The old nodes are the "graft nodes" and they form
- // the set of possible common ancestors to use in case of conflict:
- // 1- A conflict happens when the current "head node" for an object
- // is not in the set of graft nodes. It means the object mutations
- // were not derived from what the device knows, but where divergent
- // changes from a prior point (from one of the graft nodes).
- // 2- The most recent common ancestor to use in resolving the conflict
- // is the object graft node with the deepest level (furthest from
- // the origin root node), representing the most up-to-date common
- // knowledge between this device and the divergent changes.
- //
- // Note: at the end of a sync operation between 2 devices, the whole
- // graft info is cleared (Syncd calls clearGraft()) to prepare it for
- // the new pairwise sync operation.
- graft := d.getObjectGraft(oid, remote)
-
- // Verify the parents and determine the node level.
- // Update the graft info in the DAG for this object.
- var level uint64
- for _, parent := range parents {
- node, err := d.getNode(oid, parent)
- if err != nil {
- return err
- }
- if level <= node.Level {
- level = node.Level + 1
- }
- if remote {
- // If this parent is an old node, it's a graft point in the DAG
- // and may be a common ancestor used during conflict resolution.
- if _, ok := graft.newNodes[parent]; !ok {
- graft.graftNodes[parent] = node.Level
- }
-
- // The parent nodes can no longer be candidates for new head versions.
- if _, ok := graft.newHeads[parent]; ok {
- delete(graft.newHeads, parent)
- }
- }
- }
-
- if remote {
- // This new node is a candidate for new head version.
- graft.newNodes[version] = struct{}{}
- graft.newHeads[version] = struct{}{}
- }
-
- // If this node is part of a transaction, add it to that set.
- if tid != NoTxID {
- txMap, ok := d.txSet[tid]
- if !ok {
- return fmt.Errorf("unknown transaction ID: %v", tid)
- }
-
- txMap[oid] = version
- }
-
- // Insert the new node in the kvdb.
- node := &dagNode{Level: level, Parents: parents, Logrec: logrec, TxID: tid, Deleted: deleted}
- return d.setNode(oid, version, node)
-}
-
-// hasNode returns true if the node (oid, version) exists in the DAG DB.
-func (d *dag) hasNode(oid storage.ID, version raw.Version) bool {
- if d.store == nil {
- return false
- }
- key := objNodeKey(oid, version)
- return d.nodes.hasKey(key)
-}
-
-// addParent adds to the DAG node (oid, version) linkage to this parent node.
-// If the parent linkage is due to a local change (from conflict resolution
-// by blessing an existing version), no need to update the grafting structure.
-// Otherwise a remote change (from the Sync protocol) updates the graft.
-//
-// TODO(rdaoud): recompute the levels of reachable child-nodes if the new
-// parent's level is greater or equal to the node's current level.
-func (d *dag) addParent(oid storage.ID, version, parent raw.Version, remote bool) error {
- if version == parent {
- return fmt.Errorf("addParent: object %v: node %d cannot be its own parent", oid, version)
- }
-
- node, err := d.getNode(oid, version)
- if err != nil {
- return err
- }
-
- pnode, err := d.getNode(oid, parent)
- if err != nil {
- vlog.VI(1).Infof("addParent: object %v, node %d, parent %d: parent node not found", oid, version, parent)
- return err
- }
-
- // Check if the parent is already linked to this node.
- found := false
- for i := range node.Parents {
- if node.Parents[i] == parent {
- found = true
- break
- }
- }
-
- // If the parent is not yet linked (local or remote) add it.
- if !found {
- // Make sure that adding the link does not create a cycle in the DAG.
- // This is done by verifying that the node is not an ancestor of the
- // parent that it is being linked to.
- err = d.ancestorIter(oid, pnode.Parents, func(oid storage.ID, v raw.Version, nd *dagNode) error {
- if v == version {
- return fmt.Errorf("addParent: cycle on object %v: node %d is an ancestor of parent node %d",
- oid, version, parent)
- }
- return nil
- })
- if err != nil {
- return err
- }
- node.Parents = append(node.Parents, parent)
- err = d.setNode(oid, version, node)
- if err != nil {
- return err
- }
- }
-
- // For local changes we are done, the grafting structure is not updated.
- if !remote {
- return nil
- }
-
- // If the node and its parent are new/old or old/new then add
- // the parent as a graft point (a potential common ancestor).
- graft := d.getObjectGraft(oid, true)
-
- _, nodeNew := graft.newNodes[version]
- _, parentNew := graft.newNodes[parent]
- if (nodeNew && !parentNew) || (!nodeNew && parentNew) {
- graft.graftNodes[parent] = pnode.Level
- }
-
- // The parent node can no longer be a candidate for a new head version.
- // The addParent() function only removes candidates from newHeads that
- // have become parents. It does not add the child nodes to newHeads
- // because they are not necessarily new-head candidates. If they are
- // new nodes, the addNode() function handles adding them to newHeads.
- // For old nodes, only the current head could be a candidate and it is
- // added to newHeads when the graft struct is initialized.
- if _, ok := graft.newHeads[parent]; ok {
- delete(graft.newHeads, parent)
- }
-
- return nil
-}
-
-// moveHead moves the object head node in the DAG.
-func (d *dag) moveHead(oid storage.ID, head raw.Version) error {
- if d.store == nil {
- return errors.New("invalid DAG")
- }
-
- // Verify that the node exists.
- if !d.hasNode(oid, head) {
- return fmt.Errorf("node %d:%d does not exist in the DAG", oid, head)
- }
-
- return d.setHead(oid, head)
-}
-
-// hasConflict determines if there is a conflict for this object between its
-// new and old head nodes.
-// - Yes: return (true, newHead, oldHead, ancestor)
-// - No: return (false, newHead, oldHead, NoVersion)
-// A conflict exists when there are two new-head nodes. It means the newly
-// added object versions are not derived in part from this device's current
-// knowledge. If there is a single new-head, the object changes were applied
-// without triggering a conflict.
-func (d *dag) hasConflict(oid storage.ID) (isConflict bool, newHead, oldHead, ancestor raw.Version, err error) {
- oldHead = raw.NoVersion
- newHead = raw.NoVersion
- ancestor = raw.NoVersion
- if d.store == nil {
- err = errors.New("invalid DAG")
- return
- }
-
- graft := d.graft[oid]
- if graft == nil {
- err = fmt.Errorf("node %d has no DAG graft information", oid)
- return
- }
-
- numHeads := len(graft.newHeads)
- if numHeads < 1 || numHeads > 2 {
- err = fmt.Errorf("node %d has invalid number of new head candidates %d: %v", oid, numHeads, graft.newHeads)
- return
- }
-
- // Fetch the current head for this object if it exists. The error from getHead()
- // is ignored because a newly received object is not yet known on this device and
- // will not trigger a conflict.
- oldHead, _ = d.getHead(oid)
-
- // If there is only one new head node there is no conflict.
- // The new head is that single one, even if it might also be the same old node.
- if numHeads == 1 {
- for k := range graft.newHeads {
- newHead = k
- }
- return
- }
-
- // With two candidate head nodes, the new one is the node that is
- // not the current (old) head node.
- for k := range graft.newHeads {
- if k != oldHead {
- newHead = k
- break
- }
- }
-
- // There is a conflict: the best choice ancestor is the graft point
- // node with the largest level (farthest from the root). It is
- // possible in some corner cases to have multiple graft nodes at
- // the same level. This would still be a single conflict, but the
- // multiple same-level graft points representing equivalent conflict
- // resolutions on different devices that are now merging their
- // resolutions. In such a case it does not matter which node is
- // chosen as the ancestor because the conflict resolver function
- // is assumed to be convergent. However it's nicer to make that
- // selection deterministic so all devices see the same choice.
- // For this the version number is used as a tie-breaker.
- isConflict = true
- var maxLevel uint64
- for node, level := range graft.graftNodes {
- if maxLevel < level ||
- (maxLevel == level && ancestor < node) {
- maxLevel = level
- ancestor = node
- }
- }
- return
-}
-
-// ancestorIter iterates over the DAG ancestor nodes for an object in a
-// breadth-first traversal starting from given version node(s). In its
-// traversal it invokes the callback function once for each node, passing
-// the object ID, version number and a pointer to the dagNode.
-func (d *dag) ancestorIter(oid storage.ID, startVersions []raw.Version,
- cb func(storage.ID, raw.Version, *dagNode) error) error {
- visited := make(map[raw.Version]bool)
- queue := list.New()
- for _, version := range startVersions {
- queue.PushBack(version)
- visited[version] = true
- }
-
- for queue.Len() > 0 {
- version := queue.Remove(queue.Front()).(raw.Version)
- node, err := d.getNode(oid, version)
- if err != nil {
- // Ignore it, the parent was previously pruned.
- continue
- }
- for _, parent := range node.Parents {
- if !visited[parent] {
- queue.PushBack(parent)
- visited[parent] = true
- }
- }
- if err = cb(oid, version, node); err != nil {
- return err
- }
- }
-
- return nil
-}
-
-// hasDeletedDescendant returns true if the node (oid, version) exists in the
-// DAG DB and one of its descendants is a deleted node (i.e. has its "Deleted"
-// flag set true). This means that at some object mutation after this version,
-// the object was deleted.
-func (d *dag) hasDeletedDescendant(oid storage.ID, version raw.Version) bool {
- if d.store == nil {
- return false
- }
- if !d.hasNode(oid, version) {
- return false
- }
-
- // Do a breadth-first traversal from the object's head node back to
- // the given version. Along the way, track whether a deleted node is
- // traversed. Return true only if a traversal reaches the given version
- // and had seen a deleted node along the way.
-
- // nodeStep tracks a step along a traversal. It stores the node to visit
- // when taking that step and a boolean tracking whether a deleted node
- // was seen so far along that trajectory.
- head, err := d.getHead(oid)
- if err != nil {
- return false
- }
-
- type nodeStep struct {
- node raw.Version
- deleted bool
- }
-
- visited := make(map[nodeStep]struct{})
- queue := list.New()
-
- step := nodeStep{node: head, deleted: false}
- queue.PushBack(&step)
- visited[step] = struct{}{}
-
- for queue.Len() > 0 {
- step := queue.Remove(queue.Front()).(*nodeStep)
- if step.node == version {
- if step.deleted {
- return true
- }
- continue
- }
- node, err := d.getNode(oid, step.node)
- if err != nil {
- // Ignore it, the parent was previously pruned.
- continue
- }
- nextDel := step.deleted || node.Deleted
-
- for _, parent := range node.Parents {
- nextStep := nodeStep{node: parent, deleted: nextDel}
- if _, ok := visited[nextStep]; !ok {
- queue.PushBack(&nextStep)
- visited[nextStep] = struct{}{}
- }
- }
- }
-
- return false
-}
-
-// prune trims the DAG of an object at a given version (node) by deleting
-// all its ancestor nodes, making it the new root node. For each deleted
-// node it calls the given callback function to delete its log record.
-// This function should only be called when Sync determines that all devices
-// that know about the object have gotten past this version.
-// Also track any transaction sets affected by deleting DAG objects that
-// have transaction IDs. This is later used to do garbage collection
-// on transaction sets when pruneDone() is called.
-func (d *dag) prune(oid storage.ID, version raw.Version, delLogRec func(logrec string) error) error {
- if d.store == nil {
- return errors.New("invalid DAG")
- }
-
- // Get the node at the pruning point and set its parents to nil.
- // It will become the oldest DAG node (root) for the object.
- node, err := d.getNode(oid, version)
- if err != nil {
- return err
- }
- if node.Parents == nil {
- // Nothing to do, this node is already the root.
- return nil
- }
-
- iterVersions := node.Parents
-
- node.Parents = nil
- if err = d.setNode(oid, version, node); err != nil {
- return err
- }
-
- // Delete all ancestor nodes and their log records.
- // Delete as many as possible and track the error counts.
- // Keep track of objects deleted from transaction in order
- // to cleanup transaction sets when pruneDone() is called.
- numNodeErrs, numLogErrs := 0, 0
- err = d.ancestorIter(oid, iterVersions, func(oid storage.ID, v raw.Version, node *dagNode) error {
- if tid := node.TxID; tid != NoTxID {
- if d.txGC[tid] == nil {
- d.txGC[tid] = make(dagTxMap)
- }
- d.txGC[tid][oid] = v
- }
-
- if err := delLogRec(node.Logrec); err != nil {
- numLogErrs++
- }
- if err := d.delNode(oid, v); err != nil {
- numNodeErrs++
- }
- return nil
- })
- if err != nil {
- return err
- }
- if numNodeErrs != 0 || numLogErrs != 0 {
- return fmt.Errorf("prune failed to delete %d nodes and %d log records", numNodeErrs, numLogErrs)
- }
- return nil
-}
-
-// pruneDone is called when object pruning is finished within a single pass
-// of the Sync garbage collector. It updates the transaction sets affected
-// by the objects deleted by the prune() calls.
-func (d *dag) pruneDone() error {
- if d.store == nil {
- return errors.New("invalid DAG")
- }
-
- // Update transaction sets by removing from them the objects that
- // were pruned. If the resulting set is empty, delete it.
- for tid, txMapGC := range d.txGC {
- txMap, err := d.getTransaction(tid)
- if err != nil {
- return err
- }
-
- for oid := range txMapGC {
- delete(txMap, oid)
- }
-
- if len(txMap) > 0 {
- err = d.setTransaction(tid, txMap)
- } else {
- err = d.delTransaction(tid)
- }
- if err != nil {
- return err
- }
- }
-
- d.clearTxGC()
- return nil
-}
-
-// getLogrec returns the log record information for a given object version.
-func (d *dag) getLogrec(oid storage.ID, version raw.Version) (string, error) {
- node, err := d.getNode(oid, version)
- if err != nil {
- return "", err
- }
- return node.Logrec, nil
-}
-
-// objNodeKey returns the key used to access the object node (oid, version)
-// in the DAG DB.
-func objNodeKey(oid storage.ID, version raw.Version) string {
- return fmt.Sprintf("%s:%d", oid.String(), version)
-}
-
-// setNode stores the dagNode structure for the object node (oid, version)
-// in the DAG DB.
-func (d *dag) setNode(oid storage.ID, version raw.Version, node *dagNode) error {
- if d.store == nil {
- return errors.New("invalid DAG")
- }
- key := objNodeKey(oid, version)
- return d.nodes.set(key, node)
-}
-
-// getNode retrieves the dagNode structure for the object node (oid, version)
-// from the DAG DB.
-func (d *dag) getNode(oid storage.ID, version raw.Version) (*dagNode, error) {
- if d.store == nil {
- return nil, errors.New("invalid DAG")
- }
- var node dagNode
- key := objNodeKey(oid, version)
- if err := d.nodes.get(key, &node); err != nil {
- return nil, err
- }
- return &node, nil
-}
-
-// delNode deletes the object node (oid, version) from the DAG DB.
-func (d *dag) delNode(oid storage.ID, version raw.Version) error {
- if d.store == nil {
- return errors.New("invalid DAG")
- }
- key := objNodeKey(oid, version)
- return d.nodes.del(key)
-}
-
-// objHeadKey returns the key used to access the object head in the DAG DB.
-func objHeadKey(oid storage.ID) string {
- return oid.String()
-}
-
-// setHead stores version as the object head in the DAG DB.
-func (d *dag) setHead(oid storage.ID, version raw.Version) error {
- if d.store == nil {
- return errors.New("invalid DAG")
- }
- key := objHeadKey(oid)
- return d.heads.set(key, version)
-}
-
-// getHead retrieves the object head from the DAG DB.
-func (d *dag) getHead(oid storage.ID) (raw.Version, error) {
- var version raw.Version
- if d.store == nil {
- return version, errors.New("invalid DAG")
- }
- key := objHeadKey(oid)
- err := d.heads.get(key, &version)
- if err != nil {
- version = raw.NoVersion
- }
- return version, err
-}
-
-// dagTransactionKey returns the key used to access the transaction in the DAG DB.
-func dagTransactionKey(tid TxID) string {
- return fmt.Sprintf("%v", tid)
-}
-
-// setTransaction stores the transaction object/version map in the DAG DB.
-func (d *dag) setTransaction(tid TxID, txMap dagTxMap) error {
- if d.store == nil {
- return errors.New("invalid DAG")
- }
- if tid == NoTxID {
- return fmt.Errorf("invalid TxID: %v", tid)
- }
- key := dagTransactionKey(tid)
- return d.trans.set(key, txMap)
-}
-
-// getTransaction retrieves the transaction object/version map from the DAG DB.
-func (d *dag) getTransaction(tid TxID) (dagTxMap, error) {
- if d.store == nil {
- return nil, errors.New("invalid DAG")
- }
- if tid == NoTxID {
- return nil, fmt.Errorf("invalid TxID: %v", tid)
- }
- var txMap dagTxMap
- key := dagTransactionKey(tid)
- if err := d.trans.get(key, &txMap); err != nil {
- return nil, err
- }
- return txMap, nil
-}
-
-// delTransaction deletes the transation object/version map from the DAG DB.
-func (d *dag) delTransaction(tid TxID) error {
- if d.store == nil {
- return errors.New("invalid DAG")
- }
- if tid == NoTxID {
- return fmt.Errorf("invalid TxID: %v", tid)
- }
- key := dagTransactionKey(tid)
- return d.trans.del(key)
-}
-
-// getParentMap is a testing and debug helper function that returns for
-// an object a map of all the object version in the DAG and their parents.
-// The map represents the graph of the object version history.
-func (d *dag) getParentMap(oid storage.ID) map[raw.Version][]raw.Version {
- parentMap := make(map[raw.Version][]raw.Version)
- var iterVersions []raw.Version
-
- if head, err := d.getHead(oid); err == nil {
- iterVersions = append(iterVersions, head)
- }
- if graft := d.graft[oid]; graft != nil {
- for k := range graft.newHeads {
- iterVersions = append(iterVersions, k)
- }
- }
-
- // Breadth-first traversal starting from the object head.
- d.ancestorIter(oid, iterVersions, func(oid storage.ID, v raw.Version, node *dagNode) error {
- parentMap[v] = node.Parents
- return nil
- })
-
- return parentMap
-}
-
-// getGraftNodes is a testing and debug helper function that returns for
-// an object the graft information built and used during a sync operation.
-// The newHeads map identifies the candidate head nodes based on the data
-// reported by the other device during a sync operation. The graftNodes map
-// identifies the set of old nodes where the new DAG fragments were attached
-// and their depth level in the DAG.
-func (d *dag) getGraftNodes(oid storage.ID) (map[raw.Version]struct{}, map[raw.Version]uint64) {
- if d.store != nil {
- if ginfo := d.graft[oid]; ginfo != nil {
- return ginfo.newHeads, ginfo.graftNodes
- }
- }
- return nil, nil
-}
diff --git a/runtimes/google/vsync/dag_test.go b/runtimes/google/vsync/dag_test.go
deleted file mode 100644
index ff2ebab..0000000
--- a/runtimes/google/vsync/dag_test.go
+++ /dev/null
@@ -1,1990 +0,0 @@
-package vsync
-
-// Tests for the Veyron Sync DAG component.
-
-import (
- "errors"
- "fmt"
- "os"
- "reflect"
- "testing"
- "time"
-
- "veyron/lib/testutil"
- "veyron/services/store/raw"
-
- "veyron2/storage"
-)
-
-// dagFilename generates a filename for a temporary (per unit test) DAG file.
-// Do not replace this function with TempFile because TempFile creates the new
-// file and the tests must verify that the DAG can create a non-existing file.
-func dagFilename() string {
- return fmt.Sprintf("%s/sync_dag_test_%d_%d", os.TempDir(), os.Getpid(), time.Now().UnixNano())
-}
-
-// fileSize returns the size of a file.
-func fileSize(fname string) int64 {
- finfo, err := os.Stat(fname)
- if err != nil {
- return -1
- }
- return finfo.Size()
-}
-
-// TestDAGOpen tests the creation of a DAG, closing and re-opening it. It also
-// verifies that its backing file is created and that a 2nd close is safe.
-func TestDAGOpen(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- fsize := fileSize(dagfile)
- if fsize < 0 {
- t.Fatalf("DAG file %s not created", dagfile)
- }
-
- dag.flush()
- oldfsize := fsize
- fsize = fileSize(dagfile)
- if fsize <= oldfsize {
- t.Fatalf("DAG file %s not flushed", dagfile)
- }
-
- dag.close()
-
- dag, err = openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot re-open existing DAG file %s", dagfile)
- }
-
- oldfsize = fsize
- fsize = fileSize(dagfile)
- if fsize != oldfsize {
- t.Fatalf("DAG file %s size changed across re-open", dagfile)
- }
-
- dag.close()
- dag.close() // multiple closes should be a safe NOP
-
- fsize = fileSize(dagfile)
- if fsize != oldfsize {
- t.Fatalf("DAG file %s size changed across close", dagfile)
- }
-
- // Fail opening a DAG in a non-existent directory.
- _, err = openDAG("/not/really/there/junk.dag")
- if err == nil {
- t.Fatalf("openDAG() did not fail when using a bad pathname")
- }
-}
-
-// TestInvalidDAG tests using DAG methods on an invalid (closed) DAG.
-func TestInvalidDAG(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- dag.close()
-
- oid, err := strToObjID("6789")
- if err != nil {
- t.Error(err)
- }
-
- err = dag.addNode(oid, 4, false, false, []raw.Version{2, 3}, "foobar", NoTxID)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("addNode() did not fail on a closed DAG: %v", err)
- }
-
- err = dag.moveHead(oid, 4)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("moveHead() did not fail on a closed DAG: %v", err)
- }
-
- _, _, _, _, err = dag.hasConflict(oid)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("hasConflict() did not fail on a closed DAG: %v", err)
- }
-
- _, err = dag.getLogrec(oid, 4)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("getLogrec() did not fail on a closed DAG: %v", err)
- }
-
- err = dag.prune(oid, 4, func(lr string) error {
- return nil
- })
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("prune() did not fail on a closed DAG: %v", err)
- }
-
- err = dag.pruneDone()
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("pruneDone() did not fail on a closed DAG: %v", err)
- }
-
- node := &dagNode{Level: 15, Parents: []raw.Version{444, 555}, Logrec: "logrec-23"}
- err = dag.setNode(oid, 4, node)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("setNode() did not fail on a closed DAG: %v", err)
- }
-
- _, err = dag.getNode(oid, 4)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("getNode() did not fail on a closed DAG: %v", err)
- }
-
- err = dag.delNode(oid, 4)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("delNode() did not fail on a closed DAG: %v", err)
- }
-
- err = dag.addParent(oid, 4, 2, true)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("addParent() did not fail on a closed DAG: %v", err)
- }
-
- err = dag.setHead(oid, 4)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("setHead() did not fail on a closed DAG: %v", err)
- }
-
- _, err = dag.getHead(oid)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("getHead() did not fail on a closed DAG: %v", err)
- }
-
- err = dag.compact()
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("compact() did not fail on a closed DAG: %v", err)
- }
-
- if tid := dag.addNodeTxStart(); tid != NoTxID {
- t.Errorf("addNodeTxStart() did not fail on a closed DAG: TxID %v", tid)
- }
-
- err = dag.addNodeTxEnd(1)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("addNodeTxEnd() did not fail on a closed DAG: %v", err)
- }
-
- err = dag.setTransaction(1, nil)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("setTransaction() did not fail on a closed DAG: %v", err)
- }
-
- _, err = dag.getTransaction(1)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("getTransaction() did not fail on a closed DAG: %v", err)
- }
-
- err = dag.delTransaction(1)
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("delTransaction() did not fail on a closed DAG: %v", err)
- }
-
- // These calls should be harmless NOPs.
- dag.clearGraft()
- dag.clearTxGC()
- dag.flush()
- dag.close()
- if dag.hasNode(oid, 4) {
- t.Errorf("hasNode() found an object on a closed DAG")
- }
- if dag.hasDeletedDescendant(oid, 3) {
- t.Errorf("hasDeletedDescendant() returned true on a closed DAG")
- }
- if pmap := dag.getParentMap(oid); len(pmap) != 0 {
- t.Errorf("getParentMap() found data on a closed DAG: %v", pmap)
- }
- if hmap, gmap := dag.getGraftNodes(oid); hmap != nil || gmap != nil {
- t.Errorf("getGraftNodes() found data on a closed DAG: head map: %v, graft map: %v", hmap, gmap)
- }
-}
-
-// TestSetNode tests setting and getting a DAG node across DAG open/close/reopen.
-func TestSetNode(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- version := raw.Version(0)
- oid, err := strToObjID("111")
- if err != nil {
- t.Fatal(err)
- }
-
- node, err := dag.getNode(oid, version)
- if err == nil || node != nil {
- t.Errorf("Found non-existent object %d:%d in DAG file %s: %v", oid, version, dagfile, node)
- }
-
- if dag.hasNode(oid, version) {
- t.Errorf("hasNode() found non-existent object %d:%d in DAG file %s", oid, version, dagfile)
- }
-
- if logrec, err := dag.getLogrec(oid, version); err == nil || logrec != "" {
- t.Errorf("Non-existent object %d:%d has a logrec in DAG file %s: %v", oid, version, dagfile, logrec)
- }
-
- node = &dagNode{Level: 15, Parents: []raw.Version{444, 555}, Logrec: "logrec-23"}
- if err = dag.setNode(oid, version, node); err != nil {
- t.Fatalf("Cannot set object %d:%d (%v) in DAG file %s", oid, version, node, dagfile)
- }
-
- for i := 0; i < 2; i++ {
- node2, err := dag.getNode(oid, version)
- if err != nil || node2 == nil {
- t.Errorf("Cannot find stored object %d:%d (i=%d) in DAG file %s", oid, version, i, dagfile)
- }
-
- if !dag.hasNode(oid, version) {
- t.Errorf("hasNode() did not find object %d:%d (i=%d) in DAG file %s", oid, version, i, dagfile)
- }
-
- if !reflect.DeepEqual(node, node2) {
- t.Errorf("Object %d:%d has wrong data (i=%d) in DAG file %s: %v instead of %v",
- oid, version, i, dagfile, node2, node)
- }
-
- if logrec, err := dag.getLogrec(oid, version); err != nil || logrec != "logrec-23" {
- t.Errorf("Object %d:%d has wrong logrec (i=%d) in DAG file %s: %v",
- oid, version, i, dagfile, logrec)
- }
-
- if i == 0 {
- dag.flush()
- dag.close()
- dag, err = openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot re-open DAG file %s", dagfile)
- }
- }
- }
-
- dag.close()
-}
-
-// TestDelNode tests deleting a DAG node across DAG open/close/reopen.
-func TestDelNode(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- version := raw.Version(1)
- oid, err := strToObjID("222")
- if err != nil {
- t.Fatal(err)
- }
-
- node := &dagNode{Level: 123, Parents: []raw.Version{333}, Logrec: "logrec-789"}
- if err = dag.setNode(oid, version, node); err != nil {
- t.Fatalf("Cannot set object %d:%d (%v) in DAG file %s", oid, version, node, dagfile)
- }
-
- dag.flush()
-
- err = dag.delNode(oid, version)
- if err != nil {
- t.Fatalf("Cannot delete object %d:%d in DAG file %s", oid, version, dagfile)
- }
-
- dag.flush()
-
- for i := 0; i < 2; i++ {
- node2, err := dag.getNode(oid, version)
- if err == nil || node2 != nil {
- t.Errorf("Found deleted object %d:%d (%v) (i=%d) in DAG file %s", oid, version, node2, i, dagfile)
- }
-
- if dag.hasNode(oid, version) {
- t.Errorf("hasNode() found deleted object %d:%d (i=%d) in DAG file %s", oid, version, i, dagfile)
- }
-
- if logrec, err := dag.getLogrec(oid, version); err == nil || logrec != "" {
- t.Errorf("Deleted object %d:%d (i=%d) has logrec in DAG file %s: %v", oid, version, i, dagfile, logrec)
- }
-
- if i == 0 {
- dag.close()
- dag, err = openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot re-open DAG file %s", dagfile)
- }
- }
- }
-
- dag.close()
-}
-
-// TestAddParent tests adding parents to a DAG node.
-func TestAddParent(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- version := raw.Version(7)
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- if err = dag.addParent(oid, version, 1, true); err == nil {
- t.Errorf("addParent() did not fail for an unknown object %d:%d in DAG file %s", oid, version, dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- node := &dagNode{Level: 15, Logrec: "logrec-22"}
- if err = dag.setNode(oid, version, node); err != nil {
- t.Fatalf("Cannot set object %d:%d (%v) in DAG file %s", oid, version, node, dagfile)
- }
-
- if err = dag.addParent(oid, version, version, true); err == nil {
- t.Errorf("addParent() did not fail on a self-parent for object %d:%d in DAG file %s", oid, version, dagfile)
- }
-
- for _, parent := range []raw.Version{4, 5, 6} {
- if err = dag.addParent(oid, version, parent, true); err == nil {
- t.Errorf("addParent() did not reject invalid parent %d for object %d:%d in DAG file %s",
- parent, oid, version, dagfile)
- }
-
- pnode := &dagNode{Level: 11, Logrec: fmt.Sprint("logrec-%d", parent), Parents: []raw.Version{3}}
- if err = dag.setNode(oid, parent, pnode); err != nil {
- t.Fatalf("Cannot set parent object %d:%d (%v) in DAG file %s", oid, parent, pnode, dagfile)
- }
-
- remote := parent%2 == 0
- for i := 0; i < 2; i++ {
- if err = dag.addParent(oid, version, parent, remote); err != nil {
- t.Errorf("addParent() failed on parent %d, remote %d (i=%d) for object %d:%d in DAG file %s: %v",
- parent, remote, i, oid, version, dagfile, err)
- }
- }
- }
-
- node2, err := dag.getNode(oid, version)
- if err != nil || node2 == nil {
- t.Errorf("Cannot find stored object %d:%d in DAG file %s", oid, version, dagfile)
- }
-
- expParents := []raw.Version{4, 5, 6}
- if !reflect.DeepEqual(node2.Parents, expParents) {
- t.Errorf("invalid parents for object %d:%d in DAG file %s: %v instead of %v",
- oid, version, dagfile, node2.Parents, expParents)
- }
-
- // Creating cycles should fail.
- for v := raw.Version(1); v < version; v++ {
- if err = dag.addParent(oid, v, version, false); err == nil {
- t.Errorf("addParent() failed to reject a cycle for object %d: from ancestor %d to node %d in DAG file %s",
- oid, v, version, dagfile)
- }
- }
-
- dag.close()
-}
-
-// TestSetHead tests setting and getting a DAG head node across DAG open/close/reopen.
-func TestSetHead(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- oid, err := strToObjID("333")
- if err != nil {
- t.Fatal(err)
- }
-
- version, err := dag.getHead(oid)
- if err == nil {
- t.Errorf("Found non-existent object head %d in DAG file %s: %d", oid, dagfile, version)
- }
-
- version = 555
- if err = dag.setHead(oid, version); err != nil {
- t.Fatalf("Cannot set object head %d (%d) in DAG file %s", oid, version, dagfile)
- }
-
- dag.flush()
-
- for i := 0; i < 3; i++ {
- version2, err := dag.getHead(oid)
- if err != nil {
- t.Errorf("Cannot find stored object head %d (i=%d) in DAG file %s", oid, i, dagfile)
- }
- if version != version2 {
- t.Errorf("Object %d has wrong head data (i=%d) in DAG file %s: %d instead of %d",
- oid, i, dagfile, version2, version)
- }
-
- if i == 0 {
- dag.close()
- dag, err = openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot re-open DAG file %s", dagfile)
- }
- } else if i == 1 {
- version = 888
- if err = dag.setHead(oid, version); err != nil {
- t.Fatalf("Cannot set new object head %d (%d) in DAG file %s", oid, version, dagfile)
- }
- dag.flush()
- }
- }
-
- dag.close()
-}
-
-// checkEndOfSync simulates and check the end-of-sync operations: clear the
-// node grafting metadata and verify that it is empty and that HasConflict()
-// detects this case and fails, then close the DAG.
-func checkEndOfSync(d *dag, oid storage.ID) error {
- // Clear grafting info; this happens at the end of a sync log replay.
- d.clearGraft()
-
- // There should be no grafting or transaction info, and hasConflict() should fail.
- newHeads, grafts := d.getGraftNodes(oid)
- if newHeads != nil || grafts != nil {
- return fmt.Errorf("Object %d: graft info not cleared: newHeads (%v), grafts (%v)", oid, newHeads, grafts)
- }
-
- if n := len(d.txSet); n != 0 {
- return fmt.Errorf("transaction set not empty: %d entries found", n)
- }
-
- isConflict, newHead, oldHead, ancestor, errConflict := d.hasConflict(oid)
- if errConflict == nil {
- return fmt.Errorf("Object %d: conflict did not fail: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- d.close()
- return nil
-}
-
-// TestLocalUpdates tests the sync handling of initial local updates: an object
-// is created (v0) and updated twice (v1, v2) on this device. The DAG should
-// show: v0 -> v1 -> v2 and the head should point to v2.
-func TestLocalUpdates(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must have moved to "v2" and the parent map shows the updated DAG.
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 2 {
- t.Errorf("Invalid object %d head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[raw.Version][]raw.Version{0: nil, 1: {0}, 2: {1}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Make sure an existing node cannot be added again.
- if err = dag.addNode(oid, 1, false, false, []raw.Version{0, 2}, "foobar", NoTxID); err == nil {
- t.Errorf("addNode() did not fail when given an existing node")
- }
-
- // Make sure a new node cannot have more than 2 parents.
- if err = dag.addNode(oid, 3, false, false, []raw.Version{0, 1, 2}, "foobar", NoTxID); err == nil {
- t.Errorf("addNode() did not fail when given 3 parents")
- }
-
- // Make sure a new node cannot have an invalid parent.
- if err = dag.addNode(oid, 3, false, false, []raw.Version{0, 555}, "foobar", NoTxID); err == nil {
- t.Errorf("addNode() did not fail when using an invalid parent")
- }
-
- // Make sure a new root node (no parents) cannot be added once a root exists.
- // For the parents array, check both the "nil" and the empty array as input.
- if err = dag.addNode(oid, 6789, false, false, nil, "foobar", NoTxID); err == nil {
- t.Errorf("Adding a 2nd root node (nil parents) for object %d in DAG file %s did not fail", oid, dagfile)
- }
- if err = dag.addNode(oid, 6789, false, false, []raw.Version{}, "foobar", NoTxID); err == nil {
- t.Errorf("Adding a 2nd root node (empty parents) for object %d in DAG file %s did not fail", oid, dagfile)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteUpdates tests the sync handling of initial remote updates:
-// an object is created (v0) and updated twice (v1, v2) on another device and
-// we learn about it during sync. The updated DAG should show: v0 -> v1 -> v2
-// and report no conflicts with the new head pointing at v2.
-func TestRemoteUpdates(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "remote-init-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still undefined) and the parent
- // map shows the newly grafted DAG fragment.
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e == nil {
- t.Errorf("Object %d head found in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[raw.Version][]raw.Version{0: nil, 1: {0}, 2: {1}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[raw.Version]struct{}{2: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[raw.Version]uint64{}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be no conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(!isConflict && newHead == 2 && oldHead == 0 && ancestor == 0 && errConflict == nil) {
- t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-02" {
- t.Errorf("Invalid logrec for newhead object %d:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
- }
-
- // Make sure an unknown node cannot become the new head.
- if err = dag.moveHead(oid, 55); err == nil {
- t.Errorf("moveHead() did not fail on an invalid node")
- }
-
- // Then we can move the head and clear the grafting data.
- if err = dag.moveHead(oid, newHead); err != nil {
- t.Errorf("Object %d cannot move head to %d in DAG file %s: %v", oid, newHead, dagfile, err)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteNoConflict tests sync of remote updates on top of a local initial
-// state without conflict. An object is created locally and updated twice
-// (v0 -> v1 -> v2). Another device, having gotten this info, makes 3 updates
-// on top of that (v2 -> v3 -> v4 -> v5) and sends this info in a later sync.
-// The updated DAG should show (v0 -> v1 -> v2 -> v3 -> v4 -> v5) and report
-// no conflicts with the new head pointing at v5. It should also report v2 as
-// the graft point on which the new fragment (v3 -> v4 -> v5) gets attached.
-func TestRemoteNoConflict(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-noconf-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 2 {
- t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[raw.Version][]raw.Version{0: nil, 1: {0}, 2: {1}, 3: {2}, 4: {3}, 5: {4}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[raw.Version]struct{}{5: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[raw.Version]uint64{2: 2}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be no conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(!isConflict && newHead == 5 && oldHead == 2 && ancestor == 0 && errConflict == nil) {
- t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if logrec, e := dag.getLogrec(oid, oldHead); e != nil || logrec != "logrec-02" {
- t.Errorf("Invalid logrec for oldhead object %d:%d in DAG file %s: %v", oid, oldHead, dagfile, logrec)
- }
- if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-05" {
- t.Errorf("Invalid logrec for newhead object %d:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
- }
-
- // Then we can move the head and clear the grafting data.
- if err = dag.moveHead(oid, newHead); err != nil {
- t.Errorf("Object %d cannot move head to %d in DAG file %s: %v", oid, newHead, dagfile, err)
- }
-
- // Clear the grafting data and verify that hasConflict() fails without it.
- dag.clearGraft()
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if errConflict == nil {
- t.Errorf("hasConflict() did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteConflict tests sync handling remote updates that build on the
-// local initial state and trigger a conflict. An object is created locally
-// and updated twice (v0 -> v1 -> v2). Another device, having only gotten
-// the v0 -> v1 history, makes 3 updates on top of v1 (v1 -> v3 -> v4 -> v5)
-// and sends this info during a later sync. Separately, the local device
-// makes a conflicting (concurrent) update v1 -> v2. The updated DAG should
-// show the branches: (v0 -> v1 -> v2) and (v0 -> v1 -> v3 -> v4 -> v5) and
-// report the conflict between v2 and v5 (current and new heads). It should
-// also report v1 as the graft point and the common ancestor in the conflict.
-// The conflict is resolved locally by creating v6 that is derived from both
-// v2 and v5 and it becomes the new head.
-func TestRemoteConflict(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-conf-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 2 {
- t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[raw.Version][]raw.Version{0: nil, 1: {0}, 2: {1}, 3: {1}, 4: {3}, 5: {4}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[raw.Version]struct{}{2: struct{}{}, 5: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[raw.Version]uint64{1: 1}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be a conflict between v2 and v5 with v1 as ancestor.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(isConflict && newHead == 5 && oldHead == 2 && ancestor == 1 && errConflict == nil) {
- t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if logrec, e := dag.getLogrec(oid, oldHead); e != nil || logrec != "logrec-02" {
- t.Errorf("Invalid logrec for oldhead object %d:%d in DAG file %s: %v", oid, oldHead, dagfile, logrec)
- }
- if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-05" {
- t.Errorf("Invalid logrec for newhead object %d:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
- }
- if logrec, e := dag.getLogrec(oid, ancestor); e != nil || logrec != "logrec-01" {
- t.Errorf("Invalid logrec for ancestor object %d:%d in DAG file %s: %v", oid, ancestor, dagfile, logrec)
- }
-
- // Resolve the conflict by adding a new local v6 derived from v2 and v5 (this replay moves the head).
- if err = dagReplayCommands(dag, "local-resolve-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // Verify that the head moved to v6 and the parent map shows the resolution.
- if head, e := dag.getHead(oid); e != nil || head != 6 {
- t.Errorf("Object %d has wrong head after conflict resolution in DAG file %s: %d", oid, dagfile, head)
- }
-
- exp[6] = []raw.Version{2, 5}
- pmap = dag.getParentMap(oid)
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map after conflict resolution in DAG file %s: (%v) instead of (%v)",
- oid, dagfile, pmap, exp)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteConflictTwoGrafts tests sync handling remote updates that build
-// on the local initial state and trigger a conflict with 2 graft points.
-// An object is created locally and updated twice (v0 -> v1 -> v2). Another
-// device, first learns about v0 and makes it own conflicting update v0 -> v3.
-// That remote device later learns about v1 and resolves the v1/v3 confict by
-// creating v4. Then it makes a last v4 -> v5 update -- which will conflict
-// with v2 but it doesn't know that.
-// Now the sync order is reversed and the local device learns all of what
-// happened on the remote device. The local DAG should get be augmented by
-// a subtree with 2 graft points: v0 and v1. It receives this new branch:
-// v0 -> v3 -> v4 -> v5. Note that v4 is also derived from v1 as a remote
-// conflict resolution. This should report a conflict between v2 and v5
-// (current and new heads), with v0 and v1 as graft points, and v1 as the
-// most-recent common ancestor for that conflict. The conflict is resolved
-// locally by creating v6, derived from both v2 and v5, becoming the new head.
-func TestRemoteConflictTwoGrafts(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-conf-01.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 2 {
- t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[raw.Version][]raw.Version{0: nil, 1: {0}, 2: {1}, 3: {0}, 4: {1, 3}, 5: {4}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[raw.Version]struct{}{2: struct{}{}, 5: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[raw.Version]uint64{0: 0, 1: 1}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be a conflict between v2 and v5 with v1 as ancestor.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(isConflict && newHead == 5 && oldHead == 2 && ancestor == 1 && errConflict == nil) {
- t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if logrec, e := dag.getLogrec(oid, oldHead); e != nil || logrec != "logrec-02" {
- t.Errorf("Invalid logrec for oldhead object %d:%d in DAG file %s: %v", oid, oldHead, dagfile, logrec)
- }
- if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-05" {
- t.Errorf("Invalid logrec for newhead object %d:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
- }
- if logrec, e := dag.getLogrec(oid, ancestor); e != nil || logrec != "logrec-01" {
- t.Errorf("Invalid logrec for ancestor object %d:%d in DAG file %s: %v", oid, ancestor, dagfile, logrec)
- }
-
- // Resolve the conflict by adding a new local v6 derived from v2 and v5 (this replay moves the head).
- if err = dagReplayCommands(dag, "local-resolve-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // Verify that the head moved to v6 and the parent map shows the resolution.
- if head, e := dag.getHead(oid); e != nil || head != 6 {
- t.Errorf("Object %d has wrong head after conflict resolution in DAG file %s: %d", oid, dagfile, head)
- }
-
- exp[6] = []raw.Version{2, 5}
- pmap = dag.getParentMap(oid)
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map after conflict resolution in DAG file %s: (%v) instead of (%v)",
- oid, dagfile, pmap, exp)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestAncestorIterator checks that the iterator goes over the correct set
-// of ancestor nodes for an object given a starting node. It should traverse
-// reconvergent DAG branches only visiting each ancestor once:
-// v0 -> v1 -> v2 -> v4 -> v5 -> v7 -> v8
-// |--> v3 ---| |
-// +--> v6 ---------------+
-// - Starting at v0 it should only cover v0.
-// - Starting at v2 it should only cover v0-v2.
-// - Starting at v5 it should only cover v0-v5.
-// - Starting at v8 it should cover all nodes (v0-v8).
-func TestAncestorIterator(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-01.sync"); err != nil {
- t.Fatal(err)
- }
-
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- // Loop checking the iteration behavior for different starting nodes.
- for _, start := range []raw.Version{0, 2, 5, 8} {
- visitCount := make(map[raw.Version]int)
- err = dag.ancestorIter(oid, []raw.Version{start},
- func(oid storage.ID, v raw.Version, node *dagNode) error {
- visitCount[v]++
- return nil
- })
-
- // Check that all prior nodes are visited only once.
- for i := raw.Version(0); i < (start + 1); i++ {
- if visitCount[i] != 1 {
- t.Errorf("wrong visit count for iter on object %d node %d starting from node %d: %d instead of 1",
- oid, i, start, visitCount[i])
- }
- }
- }
-
- // Make sure an error in the callback is returned through the iterator.
- cbErr := errors.New("callback error")
- err = dag.ancestorIter(oid, []raw.Version{8}, func(oid storage.ID, v raw.Version, node *dagNode) error {
- if v == 0 {
- return cbErr
- }
- return nil
- })
- if err != cbErr {
- t.Errorf("wrong error returned from callback: %v instead of %v", err, cbErr)
- }
-
- if err = checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestPruning tests sync pruning of the DAG for an object with 3 concurrent
-// updates (i.e. 2 conflict resolution convergent points). The pruning must
-// get rid of the DAG branches across the reconvergence points:
-// v0 -> v1 -> v2 -> v4 -> v5 -> v7 -> v8
-// |--> v3 ---| |
-// +--> v6 ---------------+
-// By pruning at v0, nothing is deleted.
-// Then by pruning at v1, only v0 is deleted.
-// Then by pruning at v5, v1-v4 are deleted leaving v5 and "v6 -> v7 -> v8".
-// Then by pruning at v7, v5-v6 are deleted leaving "v7 -> v8".
-// Then by pruning at v8, v7 is deleted leaving v8 as the head.
-// Then by pruning again at v8 nothing changes.
-func TestPruning(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-01.sync"); err != nil {
- t.Fatal(err)
- }
-
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- exp := map[raw.Version][]raw.Version{0: nil, 1: {0}, 2: {1}, 3: {1}, 4: {2, 3}, 5: {4}, 6: {1}, 7: {5, 6}, 8: {7}}
-
- // Loop pruning at an invalid version (333) then at v0, v5, v8 and again at v8.
- testVersions := []raw.Version{333, 0, 1, 5, 7, 8, 8}
- delCounts := []int{0, 0, 1, 4, 2, 1, 0}
-
- for i, version := range testVersions {
- del := 0
- err = dag.prune(oid, version, func(lr string) error {
- del++
- return nil
- })
-
- if i == 0 && err == nil {
- t.Errorf("pruning non-existent object %d:%d did not fail in DAG file %s", oid, version, dagfile)
- } else if i > 0 && err != nil {
- t.Errorf("pruning object %d:%d failed in DAG file %s: %v", oid, version, dagfile, err)
- }
-
- if del != delCounts[i] {
- t.Errorf("pruning object %d:%d deleted %d log records instead of %d", oid, version, del, delCounts[i])
- }
-
- if head, err := dag.getHead(oid); err != nil || head != 8 {
- t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- err = dag.pruneDone()
- if err != nil {
- t.Errorf("pruneDone() failed in DAG file %s: %v", dagfile, err)
- }
-
- // Remove pruned nodes from the expected parent map used to validate
- // and set the parents of the pruned node to nil.
- if version < 10 {
- for j := raw.Version(0); j < version; j++ {
- delete(exp, j)
- }
- exp[version] = nil
- }
-
- pmap := dag.getParentMap(oid)
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
- }
-
- if err = checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestPruningCallbackError tests sync pruning of the DAG when the callback
-// function returns an error. The pruning must try to delete as many nodes
-// and log records as possible and properly adjust the parent pointers of
-// the pruning node. The object DAG is:
-// v0 -> v1 -> v2 -> v4 -> v5 -> v7 -> v8
-// |--> v3 ---| |
-// +--> v6 ---------------+
-// By pruning at v8 and having the callback function fail for v3, all other
-// nodes must be deleted and only v8 remains as the head.
-func TestPruningCallbackError(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-01.sync"); err != nil {
- t.Fatal(err)
- }
-
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- exp := map[raw.Version][]raw.Version{8: nil}
-
- // Prune at v8 with a callback function that fails for v3.
- del, expDel := 0, 8
- version := raw.Version(8)
- err = dag.prune(oid, version, func(lr string) error {
- del++
- if lr == "logrec-03" {
- return fmt.Errorf("refuse to delete %s", lr)
- }
- return nil
- })
-
- if err == nil {
- t.Errorf("pruning object %d:%d did not fail in DAG file %s", oid, version, dagfile)
- }
- if del != expDel {
- t.Errorf("pruning object %d:%d deleted %d log records instead of %d", oid, version, del, expDel)
- }
-
- err = dag.pruneDone()
- if err != nil {
- t.Errorf("pruneDone() failed in DAG file %s: %v", dagfile, err)
- }
-
- if head, err := dag.getHead(oid); err != nil || head != 8 {
- t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- if err = checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestDAGCompact tests compacting of dag's kvdb file.
-func TestDAGCompact(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- // Put some data in "heads" table.
- headMap := make(map[storage.ID]raw.Version)
- for i := 0; i < 10; i++ {
- // Generate a random object id in [0, 1000).
- oid, err := strToObjID(fmt.Sprintf("%d", testutil.Rand.Intn(1000)))
- if err != nil {
- t.Fatal(err)
- }
- // Generate a random version number for this object.
- vers := raw.Version(testutil.Rand.Intn(5000))
-
- // Cache this <oid,version> pair to verify with getHead().
- headMap[oid] = vers
-
- if err = dag.setHead(oid, vers); err != nil {
- t.Fatalf("Cannot set object head %d (%d) in DAG file %s", oid, vers, dagfile)
- }
-
- // Flush immediately to let the kvdb file grow.
- dag.flush()
- }
-
- // Put some data in "nodes" table.
- type nodeKey struct {
- oid storage.ID
- vers raw.Version
- }
- nodeMap := make(map[nodeKey]*dagNode)
- for oid, vers := range headMap {
- // Generate a random dag node for this <oid, vers>.
- l := uint64(testutil.Rand.Intn(20))
- p1 := raw.Version(testutil.Rand.Intn(5000))
- p2 := raw.Version(testutil.Rand.Intn(5000))
- log := fmt.Sprintf("%d", testutil.Rand.Intn(1000))
- node := &dagNode{Level: l, Parents: []raw.Version{p1, p2}, Logrec: log}
-
- // Cache this <oid,version, dagNode> to verify with getNode().
- key := nodeKey{oid: oid, vers: vers}
- nodeMap[key] = node
-
- if err = dag.setNode(oid, vers, node); err != nil {
- t.Fatalf("Cannot set object %d:%d (%v) in DAG file %s", oid, vers, node, dagfile)
- }
-
- // Flush immediately to let the kvdb file grow.
- dag.flush()
- }
-
- // Get size before compaction.
- oldSize := fileSize(dagfile)
- if oldSize < 0 {
- t.Fatalf("DAG file %s not created", dagfile)
- }
-
- if err = dag.compact(); err != nil {
- t.Fatalf("Cannot compact DAG file %s", dagfile)
- }
-
- // Verify size of kvdb file is reduced.
- size := fileSize(dagfile)
- if size < 0 {
- t.Fatalf("DAG file %s not created", dagfile)
- }
- if size > oldSize {
- t.Fatalf("DAG file %s not compacted", dagfile)
- }
-
- // Check data exists after compaction.
- for oid, vers := range headMap {
- vers2, err := dag.getHead(oid)
- if err != nil {
- t.Errorf("Cannot find stored object head %d in DAG file %s", oid, dagfile)
- }
- if vers != vers2 {
- t.Errorf("Object %d has wrong head data in DAG file %s: %d instead of %d",
- oid, dagfile, vers2, vers)
- }
- }
- for key, node := range nodeMap {
- node2, err := dag.getNode(key.oid, key.vers)
- if err != nil || node2 == nil {
- t.Errorf("Cannot find stored object %d:%d in DAG file %s", key.oid, key.vers, dagfile)
- }
- if !reflect.DeepEqual(node, node2) {
- t.Errorf("Object %d:%d has wrong data in DAG file %s: %v instead of %v",
- key.oid, key.vers, dagfile, node2, node)
- }
- }
- dag.close()
-}
-
-// TestRemoteLinkedNoConflictSameHead tests sync of remote updates that contain
-// linked nodes (conflict resolution by selecting an existing version) on top of
-// a local initial state without conflict. An object is created locally and
-// updated twice (v1 -> v2 -> v3). Another device has learned about v1, created
-// (v1 -> v4), then learned about (v1 -> v2) and resolved that conflict by selecting
-// v2 over v4. Now it sends that new info (v4 and the v2/v4 link) back to the
-// original (local) device. Instead of a v3/v4 conflict, the device sees that
-// v2 was chosen over v4 and resolves it as a no-conflict case.
-func TestRemoteLinkedNoConflictSameHead(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-noconf-link-00.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still at v3) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 3 {
- t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[raw.Version][]raw.Version{1: nil, 2: {1, 4}, 3: {2}, 4: {1}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[raw.Version]struct{}{3: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[raw.Version]uint64{1: 0, 4: 1}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be no conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(!isConflict && newHead == 3 && oldHead == 3 && ancestor == raw.NoVersion && errConflict == nil) {
- t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- // Clear the grafting data and verify that hasConflict() fails without it.
- dag.clearGraft()
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if errConflict == nil {
- t.Errorf("hasConflict() did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteLinkedConflict tests sync of remote updates that contain linked
-// nodes (conflict resolution by selecting an existing version) on top of a local
-// initial state triggering a local conflict. An object is created locally and
-// updated twice (v1 -> v2 -> v3). Another device has along the way learned about v1,
-// created (v1 -> v4), then learned about (v1 -> v2) and resolved that conflict by
-// selecting v4 over v2. Now it sends that new info (v4 and the v4/v2 link) back
-// to the original (local) device. The device sees a v3/v4 conflict.
-func TestRemoteLinkedConflict(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-conf-link.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 3 {
- t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[raw.Version][]raw.Version{1: nil, 2: {1}, 3: {2}, 4: {1, 2}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[raw.Version]struct{}{3: struct{}{}, 4: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[raw.Version]uint64{1: 0, 2: 1}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be a conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(isConflict && newHead == 4 && oldHead == 3 && ancestor == 2 && errConflict == nil) {
- t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- // Clear the grafting data and verify that hasConflict() fails without it.
- dag.clearGraft()
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if errConflict == nil {
- t.Errorf("hasConflict() did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteLinkedNoConflictNewHead tests sync of remote updates that contain
-// linked nodes (conflict resolution by selecting an existing version) on top of
-// a local initial state without conflict, but moves the head node to a new one.
-// An object is created locally and updated twice (v1 -> v2 -> v3). Another device
-// has along the way learned about v1, created (v1 -> v4), then learned about
-// (v1 -> v2 -> v3) and resolved that conflict by selecting v4 over v3. Now it
-// sends that new info (v4 and the v4/v3 link) back to the original (local) device.
-// The device sees that the new head v4 is "derived" from v3 thus no conflict.
-func TestRemoteLinkedConflictNewHead(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-noconf-link-01.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 3 {
- t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[raw.Version][]raw.Version{1: nil, 2: {1}, 3: {2}, 4: {1, 3}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[raw.Version]struct{}{4: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[raw.Version]uint64{1: 0, 3: 2}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be no conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(!isConflict && newHead == 4 && oldHead == 3 && ancestor == raw.NoVersion && errConflict == nil) {
- t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- // Clear the grafting data and verify that hasConflict() fails without it.
- dag.clearGraft()
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if errConflict == nil {
- t.Errorf("hasConflict() did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteLinkedNoConflictNewHeadOvertake tests sync of remote updates that
-// contain linked nodes (conflict resolution by selecting an existing version)
-// on top of a local initial state without conflict, but moves the head node
-// to a new one that overtook the linked node.
-// An object is created locally and updated twice (v1 -> v2 -> v3). Another
-// device has along the way learned about v1, created (v1 -> v4), then learned
-// about (v1 -> v2 -> v3) and resolved that conflict by selecting v3 over v4.
-// Then it creates a new update v5 from v3 (v3 -> v5). Now it sends that new
-// info (v4, the v3/v4 link, and v5) back to the original (local) device.
-// The device sees that the new head v5 is "derived" from v3 thus no conflict.
-func TestRemoteLinkedConflictNewHeadOvertake(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-noconf-link-02.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 3 {
- t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[raw.Version][]raw.Version{1: nil, 2: {1}, 3: {2, 4}, 4: {1}, 5: {3}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[raw.Version]struct{}{5: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[raw.Version]uint64{1: 0, 3: 2, 4: 1}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be no conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(!isConflict && newHead == 5 && oldHead == 3 && ancestor == raw.NoVersion && errConflict == nil) {
- t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- // Then we can move the head and clear the grafting data.
- if err = dag.moveHead(oid, newHead); err != nil {
- t.Errorf("Object %d cannot move head to %d in DAG file %s: %v", oid, newHead, dagfile, err)
- }
-
- // Clear the grafting data and verify that hasConflict() fails without it.
- dag.clearGraft()
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if errConflict == nil {
- t.Errorf("hasConflict() did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- // Now new info comes from another device repeating the v2/v3 link.
- // Verify that it is a NOP (no changes).
- if err = dagReplayCommands(dag, "remote-noconf-link-repeat.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 5 {
- t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- newHeads, grafts = dag.getGraftNodes(oid)
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts = map[raw.Version]uint64{}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if !(!isConflict && newHead == 5 && oldHead == 5 && ancestor == raw.NoVersion && errConflict == nil) {
- t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestAddNodeTransactional tests adding multiple DAG nodes grouped within a transaction.
-func TestAddNodeTransactional(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-02.sync"); err != nil {
- t.Fatal(err)
- }
-
- oid_a, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
- oid_b, err := strToObjID("67890")
- if err != nil {
- t.Fatal(err)
- }
- oid_c, err := strToObjID("222")
- if err != nil {
- t.Fatal(err)
- }
-
- // Verify NoTxID is reported as an error.
- if err := dag.addNodeTxEnd(NoTxID); err == nil {
- t.Errorf("addNodeTxEnd() did not fail for invalid 'NoTxID' value")
- }
- if _, err := dag.getTransaction(NoTxID); err == nil {
- t.Errorf("getTransaction() did not fail for invalid 'NoTxID' value")
- }
- if err := dag.setTransaction(NoTxID, nil); err == nil {
- t.Errorf("setTransaction() did not fail for invalid 'NoTxID' value")
- }
- if err := dag.delTransaction(NoTxID); err == nil {
- t.Errorf("delTransaction() did not fail for invalid 'NoTxID' value")
- }
-
- // Mutate 2 objects within a transaction.
- tid_1 := dag.addNodeTxStart()
- if tid_1 == NoTxID {
- t.Fatal("Cannot start 1st DAG addNode() transaction")
- }
-
- txMap, ok := dag.txSet[tid_1]
- if !ok {
- t.Errorf("Transactions map for Tx ID %v not found in DAG file %s", tid_1, dagfile)
- }
- if n := len(txMap); n != 0 {
- t.Errorf("Transactions map for Tx ID %v has length %d instead of 0 in DAG file %s", tid_1, n, dagfile)
- }
-
- if err := dag.addNode(oid_a, 3, false, false, []raw.Version{2}, "logrec-a-03", tid_1); err != nil {
- t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_a, tid_1, dagfile, err)
- }
- if err := dag.addNode(oid_b, 3, false, false, []raw.Version{2}, "logrec-b-03", tid_1); err != nil {
- t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_b, tid_1, dagfile, err)
- }
-
- // At the same time mutate the 3rd object in another transaction.
- tid_2 := dag.addNodeTxStart()
- if tid_2 == NoTxID {
- t.Fatal("Cannot start 2nd DAG addNode() transaction")
- }
-
- txMap, ok = dag.txSet[tid_2]
- if !ok {
- t.Errorf("Transactions map for Tx ID %v not found in DAG file %s", tid_2, dagfile)
- }
- if n := len(txMap); n != 0 {
- t.Errorf("Transactions map for Tx ID %v has length %d instead of 0 in DAG file %s", tid_2, n, dagfile)
- }
-
- if err := dag.addNode(oid_c, 2, false, false, []raw.Version{1}, "logrec-c-02", tid_2); err != nil {
- t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_c, tid_2, dagfile, err)
- }
-
- // Verify the in-memory transaction sets constructed.
- txMap, ok = dag.txSet[tid_1]
- if !ok {
- t.Errorf("Transactions map for Tx ID %v not found in DAG file %s", tid_1, dagfile)
- }
-
- expTxMap := dagTxMap{oid_a: 3, oid_b: 3}
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Invalid transaction map for Tx ID %v in DAG file %s: %v instead of %v", tid_1, dagfile, txMap, expTxMap)
- }
-
- txMap, ok = dag.txSet[tid_2]
- if !ok {
- t.Errorf("Transactions map for Tx ID %v not found in DAG file %s", tid_2, dagfile)
- }
-
- expTxMap = dagTxMap{oid_c: 2}
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Invalid transaction map for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txMap, expTxMap)
- }
-
- // Verify failing to use a Tx ID not returned by addNodeTxStart().
- bad_tid := tid_1 + 1
- for bad_tid == NoTxID || bad_tid == tid_2 {
- bad_tid++
- }
-
- if err := dag.addNode(oid_c, 3, false, false, []raw.Version{2}, "logrec-c-03", bad_tid); err == nil {
- t.Errorf("addNode() did not fail on object %d for a bad Tx ID %v in DAG file %s", oid_c, bad_tid, dagfile)
- }
- if err := dag.addNodeTxEnd(bad_tid); err == nil {
- t.Errorf("addNodeTxEnd() did not fail for a bad Tx ID %v in DAG file %s", bad_tid, dagfile)
- }
-
- // End the 1st transaction and verify the in-memory and in-DAG data.
- if err := dag.addNodeTxEnd(tid_1); err != nil {
- t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
- }
-
- if _, ok = dag.txSet[tid_1]; ok {
- t.Errorf("Transactions map for Tx ID %v still exists in DAG file %s", tid_1, dagfile)
- }
-
- txMap, err = dag.getTransaction(tid_1)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
- }
-
- expTxMap = dagTxMap{oid_a: 3, oid_b: 3}
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Invalid transaction map from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
- tid_1, dagfile, txMap, expTxMap)
- }
-
- txMap, ok = dag.txSet[tid_2]
- if !ok {
- t.Errorf("Transactions map for Tx ID %v not found in DAG file %s", tid_2, dagfile)
- }
-
- expTxMap = dagTxMap{oid_c: 2}
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Invalid transaction map for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txMap, expTxMap)
- }
-
- // End the 2nd transaction and re-verify the in-memory and in-DAG data.
- if err := dag.addNodeTxEnd(tid_2); err != nil {
- t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
- }
-
- if _, ok = dag.txSet[tid_2]; ok {
- t.Errorf("Transactions map for Tx ID %v still exists in DAG file %s", tid_2, dagfile)
- }
-
- txMap, err = dag.getTransaction(tid_2)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
- }
-
- expTxMap = dagTxMap{oid_c: 2}
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Invalid transaction map for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txMap, expTxMap)
- }
-
- if n := len(dag.txSet); n != 0 {
- t.Errorf("Transaction sets in-memory: %d entries found, should be empty in DAG file %s", n, dagfile)
- }
-
- // Get the 3 new nodes from the DAG and verify their Tx IDs.
- node, err := dag.getNode(oid_a, 3)
- if err != nil {
- t.Errorf("Cannot find object %d:3 in DAG file %s: %v", oid_a, dagfile, err)
- }
- if node.TxID != tid_1 {
- t.Errorf("Invalid TxID for object %d:3 in DAG file %s: %v instead of %v", oid_a, dagfile, node.TxID, tid_1)
- }
- node, err = dag.getNode(oid_b, 3)
- if err != nil {
- t.Errorf("Cannot find object %d:3 in DAG file %s: %v", oid_b, dagfile, err)
- }
- if node.TxID != tid_1 {
- t.Errorf("Invalid TxID for object %d:3 in DAG file %s: %v instead of %v", oid_b, dagfile, node.TxID, tid_1)
- }
- node, err = dag.getNode(oid_c, 2)
- if err != nil {
- t.Errorf("Cannot find object %d:2 in DAG file %s: %v", oid_c, dagfile, err)
- }
- if node.TxID != tid_2 {
- t.Errorf("Invalid TxID for object %d:2 in DAG file %s: %v instead of %v", oid_c, dagfile, node.TxID, tid_2)
- }
-
- for _, oid := range []storage.ID{oid_a, oid_b, oid_c} {
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
- }
-}
-
-// TestPruningTransactions tests pruning DAG nodes grouped within transactions.
-func TestPruningTransactions(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-02.sync"); err != nil {
- t.Fatal(err)
- }
-
- oid_a, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
- oid_b, err := strToObjID("67890")
- if err != nil {
- t.Fatal(err)
- }
- oid_c, err := strToObjID("222")
- if err != nil {
- t.Fatal(err)
- }
-
- // Mutate objects in 2 transactions then add non-transactional mutations
- // to act as the pruning points. Before pruning the DAG is:
- // a1 -- a2 -- (a3) --- a4
- // b1 -- b2 -- (b3) -- (b4) -- b5
- // c1 ---------------- (c2)
- // Now by pruning at (a4, b5, c2), the new DAG should be:
- // a4
- // b5
- // (c2)
- // Transaction 1 (a3, b3) gets deleted, but transaction 2 (b4, c2) still
- // has (c2) dangling waiting for a future pruning.
- tid_1 := dag.addNodeTxStart()
- if tid_1 == NoTxID {
- t.Fatal("Cannot start 1st DAG addNode() transaction")
- }
- if err := dag.addNode(oid_a, 3, false, false, []raw.Version{2}, "logrec-a-03", tid_1); err != nil {
- t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_a, tid_1, dagfile, err)
- }
- if err := dag.addNode(oid_b, 3, false, false, []raw.Version{2}, "logrec-b-03", tid_1); err != nil {
- t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_b, tid_1, dagfile, err)
- }
- if err := dag.addNodeTxEnd(tid_1); err != nil {
- t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
- }
-
- tid_2 := dag.addNodeTxStart()
- if tid_2 == NoTxID {
- t.Fatal("Cannot start 2nd DAG addNode() transaction")
- }
- if err := dag.addNode(oid_b, 4, false, false, []raw.Version{3}, "logrec-b-04", tid_2); err != nil {
- t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_b, tid_2, dagfile, err)
- }
- if err := dag.addNode(oid_c, 2, false, false, []raw.Version{1}, "logrec-c-02", tid_2); err != nil {
- t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_c, tid_2, dagfile, err)
- }
- if err := dag.addNodeTxEnd(tid_2); err != nil {
- t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
- }
-
- if err := dag.addNode(oid_a, 4, false, false, []raw.Version{3}, "logrec-a-04", NoTxID); err != nil {
- t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_a, tid_1, dagfile, err)
- }
- if err := dag.addNode(oid_b, 5, false, false, []raw.Version{4}, "logrec-b-05", NoTxID); err != nil {
- t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_b, tid_2, dagfile, err)
- }
-
- if err = dag.moveHead(oid_a, 4); err != nil {
- t.Errorf("Object %d cannot move head in DAG file %s: %v", oid_a, dagfile, err)
- }
- if err = dag.moveHead(oid_b, 5); err != nil {
- t.Errorf("Object %d cannot move head in DAG file %s: %v", oid_b, dagfile, err)
- }
- if err = dag.moveHead(oid_c, 2); err != nil {
- t.Errorf("Object %d cannot move head in DAG file %s: %v", oid_c, dagfile, err)
- }
-
- // Verify the transaction sets.
- txMap, err := dag.getTransaction(tid_1)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
- }
-
- expTxMap := dagTxMap{oid_a: 3, oid_b: 3}
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Invalid transaction map from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
- tid_1, dagfile, txMap, expTxMap)
- }
-
- txMap, err = dag.getTransaction(tid_2)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
- }
-
- expTxMap = dagTxMap{oid_b: 4, oid_c: 2}
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Invalid transaction map for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txMap, expTxMap)
- }
-
- // Prune the 3 objects at their head nodes.
- for _, oid := range []storage.ID{oid_a, oid_b, oid_c} {
- head, err := dag.getHead(oid)
- if err != nil {
- t.Errorf("Cannot getHead() on object %d in DAG file %s: %v", oid, dagfile, err)
- }
- err = dag.prune(oid, head, func(lr string) error {
- return nil
- })
- if err != nil {
- t.Errorf("Cannot prune() on object %d in DAG file %s: %v", oid, dagfile, err)
- }
- }
-
- if err = dag.pruneDone(); err != nil {
- t.Errorf("pruneDone() failed in DAG file %s: %v", dagfile, err)
- }
-
- if n := len(dag.txGC); n != 0 {
- t.Errorf("Transaction GC map not empty after pruneDone() in DAG file %s: %d", dagfile, n)
- }
-
- // Verify that Tx-1 was deleted and Tx-2 still has c2 in it.
- txMap, err = dag.getTransaction(tid_1)
- if err == nil {
- t.Errorf("getTransaction() did not fail for Tx ID %v in DAG file %s: %v", tid_1, dagfile, txMap)
- }
-
- txMap, err = dag.getTransaction(tid_2)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
- }
-
- expTxMap = dagTxMap{oid_c: 2}
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Invalid transaction map for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txMap, expTxMap)
- }
-
- // Add c3 as a new head and prune at that point. This should GC Tx-2.
- if err := dag.addNode(oid_c, 3, false, false, []raw.Version{2}, "logrec-c-03", NoTxID); err != nil {
- t.Errorf("Cannot addNode() on object %d in DAG file %s: %v", oid_c, dagfile, err)
- }
- if err = dag.moveHead(oid_c, 3); err != nil {
- t.Errorf("Object %d cannot move head in DAG file %s: %v", oid_c, dagfile, err)
- }
-
- err = dag.prune(oid_c, 3, func(lr string) error {
- return nil
- })
- if err != nil {
- t.Errorf("Cannot prune() on object %d in DAG file %s: %v", oid_c, dagfile, err)
- }
- if err = dag.pruneDone(); err != nil {
- t.Errorf("pruneDone() #2 failed in DAG file %s: %v", dagfile, err)
- }
- if n := len(dag.txGC); n != 0 {
- t.Errorf("Transaction GC map not empty after pruneDone() in DAG file %s: %d", dagfile, n)
- }
-
- txMap, err = dag.getTransaction(tid_2)
- if err == nil {
- t.Errorf("getTransaction() did not fail for Tx ID %v in DAG file %s: %v", tid_2, dagfile, txMap)
- }
-
- for _, oid := range []storage.ID{oid_a, oid_b, oid_c} {
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
- }
-}
-
-// TestHasDeletedDescendant tests lookup of DAG deleted nodes descending from a given node.
-func TestHasDeletedDescendant(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-03.sync"); err != nil {
- t.Fatal(err)
- }
-
- oid, err := strToObjID("12345")
- if err != nil {
- t.Fatal(err)
- }
-
- // Delete node v3 to create a dangling parent link from v7 (increase code coverage).
- if err = dag.delNode(oid, 3); err != nil {
- t.Errorf("cannot delete node %d:3 in DAG file %s: %v", oid, dagfile, err)
- }
-
- type hasDelDescTest struct {
- node raw.Version
- result bool
- }
- tests := []hasDelDescTest{
- {raw.NoVersion, false},
- {999, false},
- {1, true},
- {2, true},
- {3, false},
- {4, false},
- {5, false},
- {6, false},
- {7, false},
- {8, false},
- }
-
- for _, test := range tests {
- result := dag.hasDeletedDescendant(oid, test.node)
- if result != test.result {
- t.Errorf("hasDeletedDescendant() for node %d in DAG file %s: %v instead of %v",
- test.node, dagfile, result, test.result)
- }
- }
-
- dag.close()
-}
diff --git a/runtimes/google/vsync/devtable.go b/runtimes/google/vsync/devtable.go
deleted file mode 100644
index 4a2fe71..0000000
--- a/runtimes/google/vsync/devtable.go
+++ /dev/null
@@ -1,483 +0,0 @@
-package vsync
-
-// Package vsync provides veyron sync DevTable utility functions.
-// DevTable is indexed by the device id and stores device level
-// information needed by sync. Main component of a device's info is
-// its generation vector. Generation vector is the version vector for
-// a device's store, representing all the different generations (from
-// different devices) seen by a given device. A generation represents
-// a collection of updates that originated on a device during an
-// interval of time. It serves as a checkpoint when communicating with
-// other devices. Generations do not overlap and all updates belong to
-// a generation.
-//
-// Synchronization between two devices A and B uses generation vectors
-// as follows:
-// A B
-// <== B's generation vector
-// diff(A's generation vector, B's generation vector)
-// log records of missing generations ==>
-// cache B's generation vector (for space reclamation)
-//
-// Implementation notes: DevTable is stored in a persistent K/V
-// database in the current implementation. Generation vector is
-// implemented as a map of (Device ID -> Generation ID), one entry for
-// every known device. If the generation vector contains an entry
-// (Device ID -> Generation ID), it implies that the device has
-// learned of all the generations until and including Generation
-// ID. Generation IDs start from 1. A generation ID of 0 is a
-// reserved boot strap value, and indicates the device has no updates.
-import (
- "errors"
- "sort"
- "time"
-
- "veyron2/vlog"
-)
-
-var (
- errInvalidDTab = errors.New("invalid devtable db")
-)
-
-// devInfo is the information stored per device.
-type devInfo struct {
- Vector GenVector // device generation vector.
- Ts time.Time // last communication time stamp.
-}
-
-// devTableHeader contains the header metadata.
-type devTableHeader struct {
- Resmark []byte // resume marker for watch.
- // Generation vector for space reclamation. All generations
- // less than this generation vector are deleted from storage.
- ReclaimVec GenVector
-}
-
-// devTable contains the metadata for the device table db.
-type devTable struct {
- fname string // file pathname.
- db *kvdb // underlying K/V DB.
- devices *kvtable // pointer to the "devices" table in the kvdb. Contains device info.
-
- // Key:"Head" Value:devTableHeader
- header *kvtable // pointer to the "header" table in the kvdb. Contains device table header.
- head *devTableHeader // devTable head cached in memory.
-
- s *syncd // pointer to the sync daemon object.
-}
-
-// genOrder represents a generation along with its position in the log.
-type genOrder struct {
- devID DeviceID
- genID GenID
- order uint32
-}
-
-// byOrder is used to sort the genOrder array.
-type byOrder []*genOrder
-
-func (a byOrder) Len() int {
- return len(a)
-}
-
-func (a byOrder) Swap(i, j int) {
- a[i], a[j] = a[j], a[i]
-}
-
-func (a byOrder) Less(i, j int) bool {
- return a[i].order < a[j].order
-}
-
-// openDevTable opens or creates a devTable for the given filename.
-func openDevTable(filename string, sin *syncd) (*devTable, error) {
- dtab := &devTable{
- fname: filename,
- s: sin,
- }
- // Open the file and create it if it does not exist.
- // Also initialize the kvdb and its collection.
- db, tbls, err := kvdbOpen(filename, []string{"devices", "header"})
- if err != nil {
- return nil, err
- }
-
- dtab.db = db
- dtab.devices = tbls[0]
- dtab.header = tbls[1]
-
- // Initialize local gen vector with own device id and
- // generation id of 0 if local vector doesn't exist.
- if !dtab.hasDevInfo(dtab.s.id) {
- vector := GenVector{
- dtab.s.id: 0,
- }
- if err := dtab.putGenVec(dtab.s.id, vector); err != nil {
- dtab.db.close() // this also closes the tables.
- return nil, err
- }
- }
-
- // Initialize the devTable header.
- dtab.head = &devTableHeader{
- ReclaimVec: GenVector{
- dtab.s.id: 0,
- },
- }
- // If header already exists in db, read it back from db.
- if dtab.hasHead() {
- if err := dtab.getHead(); err != nil {
- dtab.db.close() // this also closes the tables.
- return nil, err
- }
- }
-
- return dtab, nil
-}
-
-// close closes the devTable and invalidates its struct.
-func (dt *devTable) close() error {
- if dt.db == nil {
- return errInvalidDTab
- }
- // Flush the dirty data.
- if err := dt.flush(); err != nil {
- return err
- }
- dt.db.close() // this also closes the tables.
-
- *dt = devTable{} // zero out the devTable struct.
- return nil
-}
-
-// flush flushes the devTable db to storage.
-func (dt *devTable) flush() error {
- if dt.db == nil {
- return errInvalidDTab
- }
- // Set the head from memory before flushing.
- if err := dt.putHead(); err != nil {
- return err
- }
- dt.db.flush()
- return nil
-}
-
-// compact compacts the file associated with kvdb.
-func (dt *devTable) compact() error {
- if dt.db == nil {
- return errInvalidDTab
- }
- db, tbls, err := dt.db.compact(dt.fname, []string{"devices", "header"})
- if err != nil {
- return err
- }
- dt.db = db
- dt.devices = tbls[0]
- dt.header = tbls[1]
- return nil
-}
-
-// putHead puts the devTable head into the devTable db.
-func (dt *devTable) putHead() error {
- return dt.header.set("Head", dt.head)
-}
-
-// getHead gets the devTable head from the devTable db.
-func (dt *devTable) getHead() error {
- if dt.head == nil {
- return errors.New("nil devTable header")
- }
- return dt.header.get("Head", dt.head)
-}
-
-// hasHead returns true if the devTable db has a devTable head.
-func (dt *devTable) hasHead() bool {
- return dt.header.hasKey("Head")
-}
-
-// putDevInfo puts a devInfo struct in the devTable db.
-func (dt *devTable) putDevInfo(devid DeviceID, info *devInfo) error {
- if dt.db == nil {
- return errInvalidDTab
- }
- return dt.devices.set(string(devid), info)
-}
-
-// getDevInfo gets a devInfo struct from the devTable db.
-func (dt *devTable) getDevInfo(devid DeviceID) (*devInfo, error) {
- if dt.db == nil {
- return nil, errInvalidDTab
- }
- var info devInfo
- if err := dt.devices.get(string(devid), &info); err != nil {
- return nil, err
- }
- if info.Vector == nil {
- return nil, errors.New("nil genvector")
- }
- return &info, nil
-}
-
-// hasDevInfo returns true if the device (devid) has any devInfo in the devTable db.
-func (dt *devTable) hasDevInfo(devid DeviceID) bool {
- if dt.db == nil {
- return false
- }
- return dt.devices.hasKey(string(devid))
-}
-
-// putGenVec puts a generation vector in the devTable db.
-func (dt *devTable) putGenVec(devid DeviceID, v GenVector) error {
- if dt.db == nil {
- return errInvalidDTab
- }
- var info *devInfo
- if dt.hasDevInfo(devid) {
- var err error
- if info, err = dt.getDevInfo(devid); err != nil {
- return err
- }
- info.Vector = v
- } else {
- info = &devInfo{
- Vector: v,
- Ts: time.Now().UTC(),
- }
- }
- return dt.putDevInfo(devid, info)
-}
-
-// getGenVec gets a generation vector from the devTable db.
-func (dt *devTable) getGenVec(devid DeviceID) (GenVector, error) {
- if dt.db == nil {
- return nil, errInvalidDTab
- }
- info, err := dt.getDevInfo(devid)
- if err != nil {
- return nil, err
- }
- return info.Vector, nil
-}
-
-// populateGenOrderEntry populates a genOrder entry.
-func (dt *devTable) populateGenOrderEntry(e *genOrder, id DeviceID, gnum GenID) error {
- e.devID = id
- e.genID = gnum
-
- o, err := dt.s.log.getGenMetadata(id, gnum)
- if err != nil {
- return err
- }
- e.order = o.Pos
- return nil
-}
-
-// updateGeneration updates a single generation (upID, upGen) in a device's generation vector.
-func (dt *devTable) updateGeneration(key, upID DeviceID, upGen GenID) error {
- if dt.db == nil {
- return errInvalidDTab
- }
- info, err := dt.getDevInfo(key)
- if err != nil {
- return err
- }
-
- info.Vector[upID] = upGen
-
- return dt.putDevInfo(key, info)
-}
-
-// updateLocalGenVector updates local generation vector based on the remote generation vector.
-func (dt *devTable) updateLocalGenVector(local, remote GenVector) error {
- if dt.db == nil {
- return errInvalidDTab
- }
- if local == nil || remote == nil {
- return errors.New("invalid input args to function")
- }
- for rid, rgen := range remote {
- lgen, ok := local[rid]
- if !ok || lgen < rgen {
- local[rid] = rgen
- }
- }
- return nil
-}
-
-// diffGenVectors diffs generation vectors belonging to src and dest
-// and returns the generations known to src and not known to dest. In
-// addition, sync needs to maintain the order in which device
-// generations are created/received. Hence, when two generation
-// vectors are diffed, the differing generations are returned in a
-// sorted order based on their position in the src's log. genOrder
-// array consists of every generation that is missing between src and
-// dest sorted using its position in the src's log.
-// Example: Generation vector for device A (src) AVec = {A:10, B:5, C:1}
-// Generation vector for device B (dest) BVec = {A:5, B:10, D:2}
-// Missing generations in unsorted order: {A:6, A:7, A:8, A:9, A:10,
-// C:1} TODO(hpucha): Revisit for the case of a lot of generations to
-// send back (say during bootstrap).
-func (dt *devTable) diffGenVectors(srcVec, destVec GenVector) ([]*genOrder, error) {
- if dt.db == nil {
- return nil, errInvalidDTab
- }
-
- // Create an array for the generations that need to be returned.
- var gens []*genOrder
-
- // Compute missing generations for devices that are in destination and source vector.
- for devid, genid := range destVec {
- srcGenID, ok := srcVec[devid]
- // Skip since src doesn't know of this device.
- if !ok {
- continue
- }
- // Need to include all generations in the interval [genid+1, srcGenID],
- // genid+1 and srcGenID inclusive.
- // Check against reclaimVec to see if required generations are already GCed.
- // Starting gen is then max(oldGen, genid+1)
- startGen := genid + 1
- oldGen := dt.getOldestGen(devid) + 1
- if startGen < oldGen {
- vlog.VI(1).Infof("diffGenVectors:: Adjusting starting generations from %d to %d",
- startGen, oldGen)
- startGen = oldGen
- }
- for i := startGen; i <= srcGenID; i++ {
- // Populate the genorder entry.
- var entry genOrder
- if err := dt.populateGenOrderEntry(&entry, devid, i); err != nil {
- return nil, err
- }
- gens = append(gens, &entry)
- }
- }
- // Compute missing generations for devices not in destination vector but in source vector.
- for devid, genid := range srcVec {
- // Add devices destination does not know about.
- if _, ok := destVec[devid]; !ok {
- // Bootstrap generation to oldest available.
- destGenID := dt.getOldestGen(devid) + 1
- // Need to include all generations in the interval [destGenID, genid],
- // destGenID and genid inclusive.
- for i := destGenID; i <= genid; i++ {
- // Populate the genorder entry.
- var entry genOrder
- if err := dt.populateGenOrderEntry(&entry, devid, i); err != nil {
- return nil, err
- }
- gens = append(gens, &entry)
- }
- }
- }
-
- // Sort generations in log order.
- sort.Sort(byOrder(gens))
- return gens, nil
-}
-
-// getOldestGen returns the most recent gc'ed generation for the device "dev".
-func (dt *devTable) getOldestGen(dev DeviceID) GenID {
- return dt.head.ReclaimVec[dev]
-}
-
-// computeReclaimVector computes a generation vector such that the
-// generations less than or equal to those in the vector can be
-// garbage collected. Caller holds a lock on s.lock.
-//
-// Approach: For each device in the system, we compute its maximum
-// generation known to all the other devices in the system. This is a
-// O(N^2) algorithm where N is the number of devices in the system. N
-// is assumed to be small, of the order of hundreds of devices.
-func (dt *devTable) computeReclaimVector() (GenVector, error) {
- // Get local generation vector to create the set of devices in
- // the system. Local generation vector is a good bootstrap
- // device set since it contains all the devices whose log
- // records were ever stored locally.
- devSet, err := dt.getGenVec(dt.s.id)
- if err != nil {
- return nil, err
- }
-
- newReclaimVec := GenVector{}
- for devid := range devSet {
- if !dt.hasDevInfo(devid) {
- // This node knows of devid, but hasn't yet
- // contacted the device. Do not garbage
- // collect any further. For instance, when
- // node A learns of node C's generations from
- // node B, node A may not have an entry for
- // node C yet, but node C will be part of its
- // devSet.
- for dev := range devSet {
- newReclaimVec[dev] = dt.getOldestGen(dev)
- }
- return newReclaimVec, nil
- }
-
- vec, err := dt.getGenVec(devid)
- if err != nil {
- return nil, err
- }
- for dev := range devSet {
- gen1, ok := vec[dev]
- // Device "devid" does not know about device "dev".
- if !ok {
- newReclaimVec[dev] = dt.getOldestGen(dev)
- continue
- }
- gen2, ok := newReclaimVec[dev]
- if !ok || (gen1 < gen2) {
- newReclaimVec[dev] = gen1
- }
- }
- }
- return newReclaimVec, nil
-}
-
-// addDevice adds a newly learned device to the devTable state.
-func (dt *devTable) addDevice(newDev DeviceID) error {
- // Create an entry in the device table for the new device.
- vector := GenVector{
- newDev: 0,
- }
- if err := dt.putGenVec(newDev, vector); err != nil {
- return err
- }
-
- // Update local generation vector with the new device.
- local, err := dt.getDevInfo(dt.s.id)
- if err != nil {
- return err
- }
- if err := dt.updateLocalGenVector(local.Vector, vector); err != nil {
- return err
- }
- if err := dt.putDevInfo(dt.s.id, local); err != nil {
- return err
- }
- return nil
-}
-
-// updateReclaimVec updates the reclaim vector to track gc'ed generations.
-func (dt *devTable) updateReclaimVec(minGens GenVector) error {
- for dev, min := range minGens {
- gen, ok := dt.head.ReclaimVec[dev]
- if !ok {
- if min < 1 {
- vlog.Errorf("updateReclaimVec:: Received bad generation %s %d",
- dev, min)
- dt.head.ReclaimVec[dev] = 0
- } else {
- dt.head.ReclaimVec[dev] = min - 1
- }
- continue
- }
-
- // We obtained a generation that is already reclaimed.
- if min <= gen {
- return errors.New("requested gen smaller than GC'ed gen")
- }
- }
- return nil
-}
diff --git a/runtimes/google/vsync/devtable_test.go b/runtimes/google/vsync/devtable_test.go
deleted file mode 100644
index 99220d7..0000000
--- a/runtimes/google/vsync/devtable_test.go
+++ /dev/null
@@ -1,1250 +0,0 @@
-package vsync
-
-// Tests for the Veyron Sync devTable component.
-import (
- "os"
- "reflect"
- "testing"
- "time"
-)
-
-// TestDevTabStore tests creating a backing file for devTable.
-func TestDevTabStore(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- fsize := getFileSize(devfile)
- if fsize < 0 {
- t.Errorf("DevTable file %s not created", devfile)
- }
-
- if err := dtab.flush(); err != nil {
- t.Errorf("Cannot flush devTable file %s, err %v", devfile, err)
- }
-
- oldfsize := fsize
- fsize = getFileSize(devfile)
- if fsize <= oldfsize {
- t.Errorf("DevTable file %s not flushed", devfile)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-
- oldfsize = getFileSize(devfile)
-
- dtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot re-open existing devTable file %s, err %v", devfile, err)
- }
-
- fsize = getFileSize(devfile)
- if fsize != oldfsize {
- t.Errorf("DevTable file %s size changed across re-open (%d %d)", devfile, fsize, oldfsize)
- }
-
- if err := dtab.flush(); err != nil {
- t.Errorf("Cannot flush devTable file %s, err %v", devfile, err)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestInvalidDTab tests devTable methods on an invalid (closed) devTable ptr.
-func TestInvalidDTab(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-
- err = dtab.close()
- if err == nil || err != errInvalidDTab {
- t.Errorf("Close did not fail on a closed : %v", err)
- }
-
- err = dtab.flush()
- if err == nil || err != errInvalidDTab {
- t.Errorf("Flush did not fail on a closed devTable: %v", err)
- }
-
- err = dtab.compact()
- if err == nil || err != errInvalidDTab {
- t.Errorf("Compact did not fail on a closed devTable: %v", err)
- }
-
- var devid DeviceID = "VeyronPhone"
-
- err = dtab.putDevInfo(devid, &devInfo{})
- if err == nil || err != errInvalidDTab {
- t.Errorf("PutDevInfo did not fail on a closed devTable: %v", err)
- }
-
- _, err = dtab.getDevInfo(devid)
- if err == nil || err != errInvalidDTab {
- t.Errorf("GetDevInfo did not fail on a closed devTable: %v", err)
- }
-
- if dtab.hasDevInfo(devid) {
- if err == nil || err != errInvalidDTab {
- t.Errorf("HasDevInfo did not fail on a closed devTable: %v", err)
- }
- }
-
- err = dtab.putGenVec(devid, GenVector{})
- if err == nil || err != errInvalidDTab {
- t.Errorf("PutGenVec did not fail on a closed devTable: %v", err)
- }
-
- _, err = dtab.getGenVec(devid)
- if err == nil || err != errInvalidDTab {
- t.Errorf("GetGenVec did not fail on a closed devTable: %v", err)
- }
-
- err = dtab.updateGeneration(devid, devid, 0)
- if err == nil || err != errInvalidDTab {
- t.Errorf("UpdateGeneration did not fail on a closed devTable: %v", err)
- }
-
- err = dtab.updateLocalGenVector(GenVector{}, GenVector{})
- if err == nil || err != errInvalidDTab {
- t.Errorf("UpdateLocalGenVector did not fail on a closed devTable: %v", err)
- }
-
- _, err = dtab.diffGenVectors(GenVector{}, GenVector{})
- if err == nil || err != errInvalidDTab {
- t.Errorf("DiffGenVectors did not fail on a closed devTable: %v", err)
- }
-}
-
-// TestPutGetDevTableHeader tests setting and getting devTable header across devTable open/close/reopen.
-func TestPutGetDevTableHeader(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- // In memory head should be initialized.
- if dtab.head.Resmark != nil {
- t.Errorf("First time log create should reset header: %v", dtab.head.Resmark)
- }
- expVec := GenVector{dtab.s.id: 0}
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
- devfile, dtab.head.ReclaimVec, expVec)
- }
-
- // No head should be there in db.
- if err = dtab.getHead(); err == nil {
- t.Errorf("getHead() found non-existent head in devTable file %s, err %v", devfile, err)
- }
-
- if dtab.hasHead() {
- t.Errorf("hasHead() found non-existent head in devTable file %s", devfile)
- }
-
- expMark := []byte{1, 2, 3}
- expVec = GenVector{
- "VeyronTab": 30,
- "VeyronPhone": 10,
- }
- dtab.head = &devTableHeader{
- Resmark: expMark,
- ReclaimVec: expVec,
- }
-
- if err := dtab.putHead(); err != nil {
- t.Errorf("Cannot put head %v in devTable file %s, err %v", dtab.head, devfile, err)
- }
-
- // Reset values.
- dtab.head.Resmark = nil
- dtab.head.ReclaimVec = GenVector{}
-
- for i := 0; i < 2; i++ {
- if err := dtab.getHead(); err != nil {
- t.Fatalf("getHead() can not find head (i=%d) in devTable file %s, err %v", i, devfile, err)
- }
-
- if !dtab.hasHead() {
- t.Errorf("hasHead() can not find head (i=%d) in devTable file %s", i, devfile)
- }
-
- if !reflect.DeepEqual(dtab.head.Resmark, expMark) {
- t.Errorf("Data mismatch for resmark (i=%d) in devTable file %s: %v instead of %v",
- i, devfile, dtab.head.Resmark, expMark)
- }
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec (i=%d) in devTable file %s: %v instead of %v",
- i, devfile, dtab.head.ReclaimVec, expVec)
- }
-
- if i == 0 {
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
- dtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot re-open devTable file %s, err %v", devfile, err)
- }
- }
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestPersistDevTableHeader tests that devTable header is
-// automatically persisted across devTable open/close/reopen.
-func TestPersistDevTableHeader(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- // In memory head should be initialized.
- if dtab.head.Resmark != nil {
- t.Errorf("First time log create should reset header: %v", dtab.head.Resmark)
- }
- expVec := GenVector{dtab.s.id: 0}
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
- devfile, dtab.head.ReclaimVec, expVec)
- }
-
- expMark := []byte{0, 2, 255}
- expVec = GenVector{
- "VeyronTab": 100,
- "VeyronPhone": 10000,
- }
- dtab.head = &devTableHeader{
- Resmark: expMark,
- ReclaimVec: expVec,
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-
- dtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- // In memory head should be initialized from db.
- if !reflect.DeepEqual(dtab.head.Resmark, expMark) {
- t.Errorf("Data mismatch for resmark in devTable file %s: %v instead of %v",
- devfile, dtab.head.Resmark, expMark)
- }
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
- devfile, dtab.head.ReclaimVec, expVec)
- }
-
- expMark = []byte{60, 180, 7}
- expVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 1987,
- }
- dtab.head = &devTableHeader{
- Resmark: expMark,
- ReclaimVec: expVec,
- }
-
- if err := dtab.flush(); err != nil {
- t.Errorf("Cannot flush devTable file %s, err %v", devfile, err)
- }
-
- // Reset values.
- dtab.head.Resmark = nil
- dtab.head.ReclaimVec = GenVector{}
-
- if err := dtab.getHead(); err != nil {
- t.Fatalf("getHead() can not find head in devTable file %s, err %v", devfile, err)
- }
-
- // In memory head should be initialized from db.
- if !reflect.DeepEqual(dtab.head.Resmark, expMark) {
- t.Errorf("Data mismatch for resmark in devTable file %s: %v instead of %v",
- devfile, dtab.head.Resmark, expMark)
- }
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
- devfile, dtab.head.ReclaimVec, expVec)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestPutGetDevInfo tests setting and getting devInfo across devTable open/close/reopen.
-func TestPutGetDevInfo(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- var devid DeviceID = "VeyronTab"
-
- info, err := dtab.getDevInfo(devid)
- if err == nil || info != nil {
- t.Errorf("GetDevInfo() found non-existent object %s in devTable file %s: %v, err %v",
- devid, devfile, info, err)
- }
-
- if dtab.hasDevInfo(devid) {
- t.Errorf("HasDevInfo() found non-existent object %s in devTable file %s",
- devid, devfile)
- }
-
- v := GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 10,
- }
-
- info = &devInfo{
- Vector: v,
- Ts: time.Now().UTC(),
- }
-
- if err := dtab.putDevInfo(devid, info); err != nil {
- t.Errorf("Cannot put object %s (%v) in devTable file %s, err %v", devid, info, devfile, err)
- }
-
- for i := 0; i < 2; i++ {
- curInfo, err := dtab.getDevInfo(devid)
- if err != nil || curInfo == nil {
- t.Fatalf("GetDevInfo() can not find object %s (i=%d) in devTable file %s: %v, err: %v",
- devid, i, devfile, curInfo, err)
- }
-
- if !dtab.hasDevInfo(devid) {
- t.Errorf("HasDevInfo() can not find object %s (i=%d) in devTable file %s",
- devid, i, devfile)
- }
-
- if !reflect.DeepEqual(info, curInfo) {
- t.Errorf("Data mismatch for object %s (i=%d) in devTable file %s: %v instead of %v",
- devid, i, devfile, curInfo, info)
- }
-
- if i == 0 {
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
- dtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot re-open devTable file %s, err %v", devfile, err)
- }
- }
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestPutGetGenVec tests setting and getting generation vector across dtab open/close/reopen.
-func TestPutGetGenVec(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- local := GenVector{
- "VeyronPhone": 0,
- }
-
- var devid DeviceID = "VeyronTab"
- vec, err := dtab.getGenVec(devid)
- if err == nil || vec != nil {
- t.Errorf("GetGenVec() found non-existent object %s in devTable file %s: %v, err %v",
- devid, devfile, vec, err)
- }
-
- v := GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
-
- if err := dtab.putGenVec(devid, v); err != nil {
- t.Errorf("Cannot put object %s (%v) in devTable file %s, err %v", devid, v, devfile, err)
- }
-
- for i := 0; i < 2; i++ {
- // Check for devid.
- curVec, err := dtab.getGenVec(devid)
- if err != nil || curVec == nil {
- t.Fatalf("GetGenVec() can not find object %s (i=%d) in devTable file %s, err %v",
- devid, i, devfile, err)
- }
-
- if !reflect.DeepEqual(v, curVec) {
- t.Errorf("Data mismatch for object %s (i=%d) in devTable file %s: %v instead of %v",
- devid, i, devfile, curVec, v)
- }
-
- // Check for s.id.
- curLocal, err := dtab.getGenVec(s.id)
- if err != nil || curLocal == nil {
- t.Fatalf("GetGenVec() can not find object %s (i=%d) in devTable file %s: %v, err: %v",
- s.id, i, devfile, curLocal, err)
- }
-
- if !reflect.DeepEqual(local, curLocal) {
- t.Errorf("Data mismatch for object %s (i=%d) in devTable file %s: %v instead of %v",
- s.id, i, devfile, curLocal, local)
- }
-
- if i == 0 {
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
- dtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot re-open devTable file %s, err %v", devfile, err)
- }
- }
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestUpdateGeneration tests updating a generation.
-func TestUpdateGeneration(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- var devid DeviceID = "VeyronTab"
- err = dtab.updateGeneration(devid, devid, 10)
- if err == nil {
- t.Errorf("UpdateGeneration() found non-existent object %s in devTable file %s, err %v",
- devid, devfile, err)
- }
- v := GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
-
- if err := dtab.putGenVec(devid, v); err != nil {
- t.Errorf("Cannot put object %s (%v) in devTable file %s, err %v", devid, v, devfile, err)
- }
- err = dtab.updateGeneration(devid, devid, 10)
- if err != nil {
- t.Errorf("UpdateGeneration() failed for %s in devTable file %s with error %v",
- devid, devfile, err)
- }
- err = dtab.updateGeneration(devid, "VeyronLaptop", 18)
- if err != nil {
- t.Errorf("UpdateGeneration() failed for %s in devTable file %s with error %v",
- devid, devfile, err)
- }
- curVec, err := dtab.getGenVec(devid)
- if err != nil || curVec == nil {
- t.Fatalf("GetGenVec() can not find object %s in devTable file %s, err %v",
- devid, devfile, err)
- }
- vExp := GenVector{
- "VeyronTab": 10,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 18,
- }
-
- if !reflect.DeepEqual(curVec, vExp) {
- t.Errorf("Data mismatch for object %s in devTable file %s: %v instead of %v",
- devid, devfile, v, vExp)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestUpdateLocalGenVector tests updating a gen vector.
-func TestUpdateLocalGenVector(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- // Test nil args.
- if err := dtab.updateLocalGenVector(nil, nil); err == nil {
- t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
- devfile, err)
- }
-
- // Nothing to update.
- local := GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 1,
- }
- remote := GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 1,
- }
- if err := dtab.updateLocalGenVector(local, remote); err != nil {
- t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
- devfile, err)
- }
-
- if !reflect.DeepEqual(local, remote) {
- t.Errorf("Data mismatch for object %v instead of %v",
- local, remote)
- }
-
- // local is missing a generation.
- local = GenVector{
- "VeyronPhone": 1,
- }
- if err := dtab.updateLocalGenVector(local, remote); err != nil {
- t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
- devfile, err)
- }
- if !reflect.DeepEqual(local, remote) {
- t.Errorf("Data mismatch for object %v instead of %v",
- local, remote)
- }
-
- // local is stale compared to remote.
- local = GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 0,
- }
- remote = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 0,
- "VeyronLaptop": 2,
- }
- if err := dtab.updateLocalGenVector(local, remote); err != nil {
- t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
- devfile, err)
- }
- if !reflect.DeepEqual(local, remote) {
- t.Errorf("Data mismatch for object %v instead of %v",
- local, remote)
- }
-
- // local is partially stale.
- local = GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 0,
- "VeyronDesktop": 20,
- }
- remote = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronLaptop": 2,
- }
- localExp := GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
- if err := dtab.updateLocalGenVector(local, remote); err != nil {
- t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
- devfile, err)
- }
- if !reflect.DeepEqual(local, localExp) {
- t.Errorf("Data mismatch for object %v instead of %v",
- local, localExp)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestDiffGenVectors tests diffing gen vectors.
-func TestDiffGenVectors(t *testing.T) {
- logOrder := []DeviceID{"VeyronTab", "VeyronPhone", "VeyronDesktop", "VeyronLaptop"}
- var expGens []*genOrder
-
- // set reclaimVec such that it doesn't affect diffs.
- reclaimVec := GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 0,
- "VeyronDesktop": 0,
- "VeyronLaptop": 0,
- }
-
- // src and dest are identical vectors.
- vec := GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
- setupAndTestDiff(t, vec, vec, reclaimVec, logOrder, expGens)
-
- // src has no updates.
- srcVec := GenVector{
- "VeyronTab": 0,
- }
- remoteVec := GenVector{
- "VeyronTab": 5,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 8,
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, []DeviceID{}, expGens)
-
- // src and remote have no updates.
- srcVec = GenVector{
- "VeyronTab": 0,
- }
- remoteVec = GenVector{
- "VeyronTab": 0,
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, []DeviceID{}, expGens)
-
- // set reclaimVec such that it doesn't affect diffs.
- reclaimVec = GenVector{
- "VeyronTab": 0,
- }
-
- // src is staler than remote.
- srcVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
- remoteVec = GenVector{
- "VeyronTab": 5,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 8,
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
-
- // src is fresher than remote.
- srcVec = GenVector{
- "VeyronTab": 5,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
- remoteVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
- expGens = make([]*genOrder, 4)
- for i := 0; i < 4; i++ {
- expGens[i] = &genOrder{
- devID: "VeyronTab",
- genID: GenID(i + 2),
- order: uint32(i + 1),
- }
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
-
- // src is fresher than remote in all but one device.
- srcVec = GenVector{
- "VeyronTab": 5,
- "VeyronPhone": 10,
- "VeyronDesktop": 22,
- "VeyronLaptop": 2,
- }
- remoteVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- "VeyronCloud": 40,
- }
- expGens = make([]*genOrder, 6)
- for i := 0; i < 6; i++ {
- switch {
- case i < 4:
- expGens[i] = &genOrder{
- devID: "VeyronTab",
- genID: GenID(i + 2),
- order: uint32(i + 1),
- }
- default:
- expGens[i] = &genOrder{
- devID: "VeyronDesktop",
- genID: GenID(i - 4 + 21),
- order: uint32(i - 4 + 35),
- }
- }
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
-
- // src is fresher than dest, scramble log order.
- o := []DeviceID{"VeyronTab", "VeyronLaptop", "VeyronPhone", "VeyronDesktop"}
- srcVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 2,
- "VeyronDesktop": 3,
- "VeyronLaptop": 4,
- }
- remoteVec = GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 2,
- "VeyronDesktop": 0,
- }
- expGens = make([]*genOrder, 8)
- for i := 0; i < 8; i++ {
- switch {
- case i < 1:
- expGens[i] = &genOrder{
- devID: "VeyronTab",
- genID: GenID(i + 1),
- order: uint32(i),
- }
- case i >= 1 && i < 5:
- expGens[i] = &genOrder{
- devID: "VeyronLaptop",
- genID: GenID(i),
- order: uint32(i),
- }
- default:
- expGens[i] = &genOrder{
- devID: "VeyronDesktop",
- genID: GenID(i - 4),
- order: uint32(i - 5 + 7),
- }
- }
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, o, expGens)
-
- // remote has no updates.
- srcVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 2,
- "VeyronDesktop": 3,
- "VeyronLaptop": 4,
- }
- remoteVec = GenVector{
- "VeyronPhone": 0,
- }
- expGens = make([]*genOrder, 10)
- for i := 0; i < 10; i++ {
- switch {
- case i < 1:
- expGens[i] = &genOrder{
- devID: "VeyronTab",
- genID: GenID(i + 1),
- order: uint32(i),
- }
- case i >= 1 && i < 3:
- expGens[i] = &genOrder{
- devID: "VeyronPhone",
- genID: GenID(i),
- order: uint32(i),
- }
- case i >= 3 && i < 6:
- expGens[i] = &genOrder{
- devID: "VeyronDesktop",
- genID: GenID(i - 2),
- order: uint32(i),
- }
- default:
- expGens[i] = &genOrder{
- devID: "VeyronLaptop",
- genID: GenID(i - 5),
- order: uint32(i),
- }
- }
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
-
- // Test with reclaimVec fast-fwded.
- reclaimVec = GenVector{
- "VeyronPhone": 1,
- "VeyronLaptop": 2,
- }
- srcVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 2,
- "VeyronDesktop": 3,
- "VeyronLaptop": 4,
- }
- remoteVec = GenVector{
- "VeyronPhone": 0,
- }
- expGens = make([]*genOrder, 7)
- for i := 0; i < 7; i++ {
- switch {
- case i < 1:
- expGens[i] = &genOrder{
- devID: "VeyronTab",
- genID: GenID(i + 1),
- order: uint32(i),
- }
- case i == 1:
- expGens[i] = &genOrder{
- devID: "VeyronPhone",
- genID: GenID(i + 1),
- order: uint32(i + 1),
- }
- case i >= 2 && i < 5:
- expGens[i] = &genOrder{
- devID: "VeyronDesktop",
- genID: GenID(i - 1),
- order: uint32(i + 1),
- }
- default:
- expGens[i] = &genOrder{
- devID: "VeyronLaptop",
- genID: GenID(i - 2),
- order: uint32(i + 3),
- }
- }
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
-}
-
-// setupAndTestDiff is an utility function to test diffing generation vectors.
-func setupAndTestDiff(t *testing.T, srcVec, remoteVec, reclaimVec GenVector, logOrder []DeviceID, expGens []*genOrder) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- logfile := getFileName()
- defer os.Remove(logfile)
-
- var srcid DeviceID = "VeyronTab"
- var destid DeviceID = "VeyronPhone"
-
- var err error
- s := &syncd{id: srcid}
- s.log, err = openILog(logfile, s)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
- dtab.head.ReclaimVec = reclaimVec
-
- // Populate generations in log order.
- var order uint32
- for _, k := range logOrder {
- v, ok := (srcVec)[k]
- if !ok {
- t.Errorf("Cannot find key %s in srcVec %v", k, srcVec)
- }
- for i := GenID(1); i <= v; i++ {
- val := &genMetadata{Pos: order}
- if err := dtab.s.log.putGenMetadata(k, i, val); err != nil {
- t.Errorf("Cannot put object %s:%d in log file %s, err %v", k, v, logfile, err)
- }
- order++
- }
- }
- gens, err := dtab.diffGenVectors(srcVec, remoteVec)
- if err != nil {
- t.Fatalf("DiffGenVectors() failed src: %s %v dest: %s %v in devTable file %s, err %v",
- srcid, srcVec, destid, remoteVec, devfile, err)
- }
-
- if !reflect.DeepEqual(gens, expGens) {
- t.Fatalf("Data mismatch for genorder %v instead of %v, src %v dest %v reclaim %v",
- gens, expGens, srcVec, remoteVec, reclaimVec)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestGetOldestGen tests obtaining generations from reclaimVec.
-func TestGetOldestGen(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- var srcid DeviceID = "VeyronTab"
- s := &syncd{id: srcid}
- var err error
- s.devtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- if s.devtab.getOldestGen(srcid) != 0 {
- t.Errorf("Cannot get generation for device %s in devTable file %s",
- srcid, devfile)
- }
-
- var destid DeviceID = "VeyronPhone"
- if s.devtab.getOldestGen(destid) != 0 {
- t.Errorf("Cannot get generation for device %s in devTable file %s",
- destid, devfile)
- }
-
- s.devtab.head.ReclaimVec[srcid] = 10
- if s.devtab.getOldestGen(srcid) != 10 {
- t.Errorf("Cannot get generation for device %s in devTable file %s",
- srcid, devfile)
- }
- if s.devtab.getOldestGen(destid) != 0 {
- t.Errorf("Cannot get generation for device %s in devTable file %s",
- destid, devfile)
- }
-
- if err := s.devtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestComputeReclaimVector tests reclaim vector computation.
-func TestComputeReclaimVector(t *testing.T) {
- devArr := []DeviceID{"VeyronTab", "VeyronPhone", "VeyronDesktop", "VeyronLaptop"}
- genVecArr := make([]GenVector, 4)
-
- // All devices are up-to-date.
- genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
- genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
- genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
- genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
- setupAndTestReclaimVector(t, devArr, genVecArr, nil, genVecArr[0])
-
- // Every device is missing at least one other device. Not possible to gc.
- genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
- genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronLaptop": 4}
- genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3}
- genVecArr[3] = GenVector{"VeyronDesktop": 3, "VeyronLaptop": 4}
- expReclaimVec := GenVector{"VeyronTab": 0, "VeyronPhone": 0, "VeyronDesktop": 0, "VeyronLaptop": 0}
- setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
-
- // All devices know at least one generation from other devices.
- genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
- genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 2, "VeyronLaptop": 2}
- genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 3, "VeyronLaptop": 1}
- genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 1, "VeyronLaptop": 4}
- expReclaimVec = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 1, "VeyronLaptop": 1}
- setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
-
- // One device is missing from one other device.
- genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
- genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 2}
- genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 3, "VeyronLaptop": 1}
- genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 1, "VeyronLaptop": 4}
- expReclaimVec = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 1, "VeyronLaptop": 0}
- setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
-
- // All devices know at least "n" generations from other devices.
- var n GenID = 10
- genVecArr[0] = GenVector{"VeyronTab": n + 10, "VeyronPhone": n,
- "VeyronDesktop": n + 8, "VeyronLaptop": n + 4}
- genVecArr[1] = GenVector{"VeyronTab": n + 6, "VeyronPhone": n + 10,
- "VeyronDesktop": n, "VeyronLaptop": n + 3}
- genVecArr[2] = GenVector{"VeyronTab": n, "VeyronPhone": n + 2,
- "VeyronDesktop": n + 10, "VeyronLaptop": n}
- genVecArr[3] = GenVector{"VeyronTab": n + 7, "VeyronPhone": n + 1,
- "VeyronDesktop": n + 5, "VeyronLaptop": n + 10}
- expReclaimVec = GenVector{"VeyronTab": n, "VeyronPhone": n, "VeyronDesktop": n, "VeyronLaptop": n}
- setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
-
- // Never contacted a device.
- devArr = []DeviceID{"VeyronTab", "VeyronDesktop", "VeyronLaptop"}
- genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
- genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
- genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
- expReclaimVec = GenVector{"VeyronTab": 0, "VeyronPhone": 0, "VeyronDesktop": 0, "VeyronLaptop": 0}
- setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
-
- // Start from existing reclaim vector.
- devArr = []DeviceID{"VeyronTab", "VeyronPhone", "VeyronDesktop", "VeyronLaptop"}
- reclaimVec := GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
- genVecArr[0] = GenVector{"VeyronTab": 6, "VeyronPhone": 6, "VeyronDesktop": 6, "VeyronLaptop": 6}
- genVecArr[1] = GenVector{"VeyronTab": 6, "VeyronPhone": 6, "VeyronDesktop": 3, "VeyronLaptop": 6}
- genVecArr[2] = GenVector{"VeyronTab": 6, "VeyronPhone": 6, "VeyronDesktop": 6, "VeyronLaptop": 4}
- genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 6, "VeyronLaptop": 6}
-
- setupAndTestReclaimVector(t, devArr, genVecArr, reclaimVec, reclaimVec)
-}
-
-// setupAndTestReclaimVector is an utility function to test reclaim vector computation.
-func setupAndTestReclaimVector(t *testing.T, devArr []DeviceID, genVecArr []GenVector, reclaimStart, expReclaimVec GenVector) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronTab"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
- if reclaimStart != nil {
- dtab.head.ReclaimVec = reclaimStart
- }
-
- for i := range devArr {
- if err := dtab.putGenVec(devArr[i], genVecArr[i]); err != nil {
- t.Errorf("Cannot put object %s (%v) in devTable file %s, err %v",
- devArr[i], genVecArr[i], devfile, err)
- }
- }
-
- reclaimVec, err := dtab.computeReclaimVector()
- if err != nil {
- t.Fatalf("computeReclaimVector() failed devices: %v, vectors: %v in devTable file %s, err %v",
- devArr, genVecArr, devfile, err)
- }
-
- if !reflect.DeepEqual(reclaimVec, expReclaimVec) {
- t.Fatalf("Data mismatch for reclaimVec %v instead of %v",
- reclaimVec, expReclaimVec)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestAddDevice tests adding a device to the devTable.
-func TestAddDevice(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- var dev DeviceID = "VeyronLaptop"
- if err := dtab.addDevice(dev); err != nil {
- t.Fatalf("Cannot add new device in devTable file %s, err %v", devfile, err)
- }
-
- vec, err := dtab.getGenVec(dev)
- if err != nil || vec == nil {
- t.Fatalf("GetGenVec() can not find object %s in devTable file %s, err %v",
- dev, devfile, err)
- }
- expVec := GenVector{dev: 0}
- if !reflect.DeepEqual(vec, expVec) {
- t.Errorf("Data mismatch for object %s in devTable file %s: %v instead of %v",
- dev, devfile, vec, expVec)
- }
-
- vec, err = dtab.getGenVec(dtab.s.id)
- if err != nil || vec == nil {
- t.Fatalf("GetGenVec() can not find object %s in devTable file %s, err %v",
- dtab.s.id, devfile, err)
- }
- expVec = GenVector{dtab.s.id: 0, dev: 0}
- if !reflect.DeepEqual(vec, expVec) {
- t.Errorf("Data mismatch for object %s in devTable file %s: %v instead of %v",
- dtab.s.id, devfile, vec, expVec)
- }
-
- expVec = GenVector{dtab.s.id: 10, "VeyronDesktop": 40, dev: 80}
- if err := dtab.putGenVec(dtab.s.id, expVec); err != nil {
- t.Fatalf("PutGenVec() can not put object %s in devTable file %s, err %v",
- dtab.s.id, devfile, err)
- }
- dev = "VeyronTab"
- if err := dtab.addDevice(dev); err != nil {
- t.Fatalf("Cannot add new device in devTable file %s, err %v", devfile, err)
- }
- expVec[dev] = 0
-
- vec, err = dtab.getGenVec(dtab.s.id)
- if err != nil || vec == nil {
- t.Fatalf("GetGenVec() can not find object %s in devTable file %s, err %v",
- dtab.s.id, devfile, err)
- }
- if !reflect.DeepEqual(vec, expVec) {
- t.Errorf("Data mismatch for object %s in devTable file %s: %v instead of %v",
- dtab.s.id, devfile, vec, expVec)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestUpdateReclaimVec tests updating the reclaim vector.
-func TestUpdateReclaimVec(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- minGens := GenVector{"VeyronTab": 1, "VeyronDesktop": 3, "VeyronLaptop": 4}
- if err := dtab.updateReclaimVec(minGens); err != nil {
- t.Fatalf("Cannot update reclaimvec in devTable file %s, err %v", devfile, err)
- }
- expVec := GenVector{dtab.s.id: 0, "VeyronTab": 0, "VeyronDesktop": 2, "VeyronLaptop": 3}
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
- devfile, dtab.head.ReclaimVec, expVec)
- }
-
- dtab.head.ReclaimVec[DeviceID("VeyronTab")] = 4
- minGens = GenVector{"VeyronTab": 1}
- if err := dtab.updateReclaimVec(minGens); err == nil {
- t.Fatalf("Update reclaimvec didn't fail in devTable file %s", devfile)
- }
-
- minGens = GenVector{"VeyronTab": 5}
- if err := dtab.updateReclaimVec(minGens); err != nil {
- t.Fatalf("Cannot update reclaimvec in devTable file %s, err %v", devfile, err)
- }
- expVec = GenVector{dtab.s.id: 0, "VeyronTab": 4, "VeyronDesktop": 2, "VeyronLaptop": 3}
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
- devfile, dtab.head.ReclaimVec, expVec)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestDevTableCompact tests compacting of devtable's kvdb file.
-func TestDevTableCompact(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- // Put some data in "devices" table.
- devices := []DeviceID{"Phone", "Tab", "Laptop", "Desktop"}
- for i, dev := range devices {
- v := GenVector{}
- for j, d := range devices {
- v[d] = GenID(i*100 + j)
- }
- if err := dtab.putGenVec(dev, v); err != nil {
- t.Errorf("Cannot put object %s (%v) in devTable file %s, err %v", dev, v, devfile, err)
- }
- if err := dtab.flush(); err != nil {
- t.Errorf("Cannot flush devTable file %s, err %v", devfile, err)
- }
- }
-
- // Put some data in "header" table.
- expMark := []byte{1, 2, 3}
- expVec := GenVector{
- "VeyronTab": 30,
- "VeyronPhone": 10,
- }
- dtab.head = &devTableHeader{
- Resmark: expMark,
- ReclaimVec: expVec,
- }
- if err := dtab.flush(); err != nil {
- t.Errorf("Cannot flush devTable file %s, err %v", devfile, err)
- }
-
- // Get size before compaction.
- oldSize := getFileSize(devfile)
- if oldSize < 0 {
- t.Fatalf("DevTable file %s not created", devfile)
- }
-
- if err := dtab.compact(); err != nil {
- t.Errorf("Cannot compact devTable file %s, err %v", devfile, err)
- }
-
- // Verify size of kvdb file is reduced.
- size := getFileSize(devfile)
- if size < 0 {
- t.Fatalf("DevTable file %s not created", devfile)
- }
- if size > oldSize {
- t.Fatalf("DevTable file %s not compacted", devfile)
- }
-
- // Check data exists after compaction.
- for i, dev := range devices {
- exp := GenVector{}
- for j, d := range devices {
- exp[d] = GenID(i*100 + j)
- }
- curVec, err := dtab.getGenVec(dev)
- if err != nil || curVec == nil {
- t.Fatalf("GetGenVec() can not find object %s in devTable file %s, err %v",
- dev, devfile, err)
- }
- if !reflect.DeepEqual(exp, curVec) {
- t.Errorf("Data mismatch for object %s in devTable file %s: %v instead of %v",
- dev, devfile, curVec, exp)
- }
- }
-
- dtab.head.Resmark = nil
- dtab.head.ReclaimVec = GenVector{}
- if err := dtab.getHead(); err != nil {
- t.Fatalf("getHead() can not find head in devTable file %s, err %v", devfile, err)
- }
- if !reflect.DeepEqual(dtab.head.Resmark, expMark) {
- t.Errorf("Data mismatch for resmark in devTable file %s: %v instead of %v",
- devfile, dtab.head.Resmark, expMark)
- }
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
- devfile, dtab.head.ReclaimVec, expVec)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
diff --git a/runtimes/google/vsync/doc.go b/runtimes/google/vsync/doc.go
deleted file mode 100644
index 4fd8a17..0000000
--- a/runtimes/google/vsync/doc.go
+++ /dev/null
@@ -1,4 +0,0 @@
-// Package vsync provides a cross-device replication and synchronization
-// service for data in the Veyron Store. See document below:
-// https://docs.google.com/a/google.com/document/d/16ioEAI65GV9-pL8M9bnVgR21xQgQ3ioVQrWXKWqqQMk/
-package vsync
diff --git a/runtimes/google/vsync/gc.go b/runtimes/google/vsync/gc.go
deleted file mode 100644
index dedd575..0000000
--- a/runtimes/google/vsync/gc.go
+++ /dev/null
@@ -1,516 +0,0 @@
-package vsync
-
-// Garbage collection (GC) in sync reclaims space occupied by sync's
-// data structures when possible. For its operation, sync keeps every
-// version of every object produced locally and remotely in its dag
-// and log data structures. Keeping these versions indefinitely is not
-// feasible given space constraints on any device. Thus to reclaim
-// space, a GC thread periodically checks to see if any state can be
-// deleted. GC looks for generations that every device in the system
-// already knows about and deletes state belonging to those
-// generations. Since every device in the system already knows about
-// these generations, it is safe to delete them. Newly added devices
-// will only get state starting from the generations not yet
-// reclaimed. Policies are needed to handle devices that were part of
-// the system, but are no longer available. Such devices will prevent
-// GC from moving forward since they will not request new generations.
-//
-// GC in sync happens in 3 stages:
-// ** reclamation phase
-// ** object pruning phase
-// ** online consistency check phase
-//
-// Reclamation phase: GC learns of the state of the other devices when
-// it talks to those devices to obtain missing updates
-// (initiator). The generation vectors of these devices are stored in
-// the device table. In the reclamation phase, we go through the
-// generation vectors of all the devices and compute the maximum
-// generation of each device known to every other device. This maximum
-// generation for each device is stored in the reclaim generation
-// vector. We then iterate through each generation between the old
-// reclaim vector to the new reclaim vector, and create for each
-// object belonging to those generations, the history of versions that
-// will be reclaimed and the most recent version that can be
-// reclaimed.
-//
-// Object pruning phase: In this phase, for an object marked for
-// reclamation, we prune its dag starting from the most recent version
-// that is being reclaimed and delete all the versions that are
-// older. As we prune the dag, we also delete the corresponding log
-// records and update the generation metadata. Note that since the
-// actual deletes proceed object by object, the generations will start
-// to have missing log records, and we use the generation metadata to
-// ensure that the generation deletes are tracked accurately. Thus,
-// the decision of what to prune happens top down using generation
-// information, while the actual pruning happens bottom up from the
-// dag. Pruning bottom up ensures that the object dags are consistent.
-//
-// Online consistency check phase: GC stages need write access to the
-// sync data structures since they perform delete operations. Hence,
-// GC is executed under a write lock and excludes other goroutines in
-// syncd. In order to control the impact of GC on foreground
-// operations, GC is designed to be incremental in its execution. Once
-// objects are marked for deletion, only a small batch of objects are
-// pruned and persisted and the lock is released. Thus objects are
-// incrementally deleted, a small batch every
-// garbageCollectInterval. To persist the changes from a round of GC,
-// we immediately persist the new reclaim vector. For the batch of
-// objects gc'ed in a round, we also persist their deletions. However,
-// if the system restarts or crashes when all the dirty objects from a
-// round of GC are not processed, there will be state from generations
-// older than the reclaim vector still persisted in kvdb. Since the
-// reclaim vector has already been advanced, this state cannot be
-// detected, resulting in leakage of space. To prevent this, we could
-// have persisted the GC state to support restartability. However, to
-// keep GC light weight, we went with the approach of not persisting
-// the transient GC state but lazily performing a consistency check on
-// kvdb to detect dangling records. Online consistency check phase
-// performs this checking. It checks every generation older than the
-// reclaim vector snapshotted at bootstrap to see if it has any state
-// left over in kvdb. If it finds dangling state, it marks the
-// corresponding objects as dirty for pruning. This consistency check
-// happens only once upon reboot. Once all generations lower than the
-// reclaim vector snapshot are verified, this phase is a noop. Once
-// again, to limit the overhead of this phase, it processes only a
-// small batch of generations in each round of GC invocation.
-//
-// Finally, the underlying kvdb store persists state by writing to a
-// log file upon flush. Thus, as we continue to flush to kvdb, the log
-// file will keep growing. In addition, upon restart, this log file
-// must be processed to reconstruct the kvdb state. To keep this log
-// file from becoming large, we need to periodically compact kvdb.
-import (
- "errors"
- "fmt"
- "time"
-
- "veyron/services/store/raw"
-
- "veyron2/storage"
- "veyron2/vlog"
-)
-
-var (
- // garbage collection (space reclamation) is invoked every
- // garbageCollectInterval.
- garbageCollectInterval = 3600 * time.Second
-
- // strictCheck when enabled performs strict checking of every
- // log record being deleted to confirm that it should be in
- // fact deleted.
- // TODO(hpucha): Support strictCheck in the presence
- // of Link log records.
- strictCheck = false
-
- // Every compactCount iterations of garbage collection, kvdb
- // is compacted. This value has performance implications as
- // kvdb compaction is expensive.
- compactCount = 100
-
- // Batch size for the number of objects that are garbage
- // collected every gc iteration. This value impacts the
- // amount of time gc is running. GC holds a write lock on the
- // data structures, blocking out all other operations in the
- // system while it is running.
- objBatchSize = 20
-
- // Batch size for the number of generations that are verified
- // every gc iteration.
- genBatchSize = 100
-
- // Errors.
- errBadMetadata = errors.New("bad metadata")
-)
-
-// objGCState tracks the per-object GC state.
-// "version" is the most recent version of the object that can be
-// pruned (and hence all older versions can be pruned as well).
-//
-// "pos" is used to compute the most recent version of the object
-// among all the versions that can be pruned (version with the highest
-// pos is the most recent). "version" of the object belongs to a
-// generation. "pos" is the position of that generation in the local
-// log.
-type objGCState struct {
- version raw.Version
- pos uint32
-}
-
-// objVersHist tracks all the versions of the object that need to be
-// gc'ed when strictCheck is enabled.
-type objVersHist struct {
- versions map[raw.Version]struct{}
-}
-
-// syncGC contains the metadata and state for the Sync GC thread.
-type syncGC struct {
- // syncd is a pointer to the Syncd instance owning this GC.
- syncd *syncd
-
- // checkConsistency controls whether online consistency checks are run.
- checkConsistency bool
-
- // reclaimSnap is the snapshot of the reclaim vector at startup.
- reclaimSnap GenVector
-
- // pruneObjects holds the per-object state for garbage collection.
- pruneObjects map[storage.ID]*objGCState
-
- // verifyPruneMap holds the per-object version history to verify GC operations
- // on an object. It is used when strictCheck is enabled.
- verifyPruneMap map[storage.ID]*objVersHist
-}
-
-// newGC creates a new syncGC instance attached to the given Syncd instance.
-func newGC(syncd *syncd) *syncGC {
- g := &syncGC{
- syncd: syncd,
- checkConsistency: true,
- reclaimSnap: GenVector{},
- pruneObjects: make(map[storage.ID]*objGCState),
- }
-
- if strictCheck {
- g.verifyPruneMap = make(map[storage.ID]*objVersHist)
- }
-
- // Take a snapshot (copy) of the reclaim vector at startup.
- for dev, gnum := range syncd.devtab.head.ReclaimVec {
- g.reclaimSnap[dev] = gnum
- }
-
- return g
-}
-
-// garbageCollect wakes up every garbageCollectInterval to check if it
-// can reclaim space.
-func (g *syncGC) garbageCollect() {
- gcIters := 0
- ticker := time.NewTicker(garbageCollectInterval)
- for {
- select {
- case <-g.syncd.closed:
- ticker.Stop()
- g.syncd.pending.Done()
- return
- case <-ticker.C:
- gcIters++
- if gcIters == compactCount {
- gcIters = 0
- g.doGC(true)
- } else {
- g.doGC(false)
- }
- }
- }
-}
-
-// doGC performs the actual garbage collection steps.
-// If "compact" is true, also compact the Sync DBs.
-func (g *syncGC) doGC(compact bool) {
- vlog.VI(1).Infof("doGC:: Started at %v", time.Now().UTC())
-
- g.syncd.lock.Lock()
- defer g.syncd.lock.Unlock()
-
- if err := g.onlineConsistencyCheck(); err != nil {
- vlog.Fatalf("onlineConsistencyCheck:: failed with err %v", err)
- }
-
- if err := g.reclaimSpace(); err != nil {
- vlog.Fatalf("reclaimSpace:: failed with err %v", err)
- }
- // TODO(hpucha): flush devtable state.
-
- if err := g.pruneObjectBatch(); err != nil {
- vlog.Fatalf("pruneObjectBatch:: failed with err %v", err)
- }
- // TODO(hpucha): flush log and dag state.
-
- if compact {
- if err := g.compactDB(); err != nil {
- vlog.Fatalf("compactDB:: failed with err %v", err)
- }
- }
-}
-
-// onlineConsistencyCheck checks if generations lower than the
-// ReclaimVec (snapshotted at startup) are deleted from the log and
-// dag data structures. It is needed to prevent space leaks when the
-// system crashes while pruning a batch of objects. GC state is not
-// aggressively persisted to make it efficient. Instead, upon reboot,
-// onlineConsistencyCheck executes incrementally checking all the
-// generations lower than the ReclaimVec snap to ensure that they are
-// deleted. Each iteration of onlineConsistencyCheck checks a small
-// batch of generations. Once all generations below the ReclaimVec
-// snap are verified once, onlineConsistencyCheck is a noop.
-func (g *syncGC) onlineConsistencyCheck() error {
- vlog.VI(1).Infof("onlineConsistencyCheck:: called with %v", g.checkConsistency)
- if !g.checkConsistency {
- return nil
- }
-
- vlog.VI(2).Infof("onlineConsistencyCheck:: reclaimSnap is %v", g.reclaimSnap)
- genCount := 0
- for dev, gen := range g.reclaimSnap {
- if gen == 0 {
- continue
- }
- for i := gen; i > 0; i-- {
- if genCount == genBatchSize {
- g.reclaimSnap[dev] = i
- return nil
- }
- if g.syncd.log.hasGenMetadata(dev, i) {
- if err := g.garbageCollectGeneration(dev, i); err != nil {
- return err
- }
- }
-
- genCount++
- }
- g.reclaimSnap[dev] = 0
- }
-
- // Done with all the generations of all devices. Consistency
- // check is no longer needed.
- g.checkConsistency = false
- vlog.VI(1).Infof("onlineConsistencyCheck:: exited with %v", g.checkConsistency)
- return nil
-}
-
-// garbageCollectGeneration garbage collects any existing log records
-// for a particular generation.
-func (g *syncGC) garbageCollectGeneration(devid DeviceID, gnum GenID) error {
- vlog.VI(2).Infof("garbageCollectGeneration:: processing generation %s:%d", devid, gnum)
- // Bootstrap generation for a device. Nothing to GC.
- if gnum == 0 {
- return nil
- }
- gen, err := g.syncd.log.getGenMetadata(devid, gnum)
- if err != nil {
- return err
- }
- if gen.Count <= 0 {
- return errBadMetadata
- }
-
- var count uint64
- // Check for log records for this generation.
- for l := LSN(0); l <= gen.MaxLSN; l++ {
- if !g.syncd.log.hasLogRec(devid, gnum, l) {
- continue
- }
-
- count++
- rec, err := g.syncd.log.getLogRec(devid, gnum, l)
- if err != nil {
- return err
- }
-
- if rec.RecType == LinkRec {
- // For a link log record, gc it right away.
- g.dagPruneCallBack(logRecKey(devid, gnum, l))
- continue
- }
-
- // Insert the object in this log record to the prune
- // map if needed.
- // If this object does not exist, create it.
- // If the object exists, update the object version to
- // prune at if the current gen is greater than the gen
- // in the prune map or if the gen is the same but the
- // current lsn is greater than the previous lsn.
- gcState, ok := g.pruneObjects[rec.ObjID]
- if !ok || gcState.pos <= gen.Pos {
- if !ok {
- gcState = &objGCState{}
- g.pruneObjects[rec.ObjID] = gcState
- }
- gcState.pos = gen.Pos
- gcState.version = rec.CurVers
- vlog.VI(2).Infof("Replacing for obj %v pos %d version %d",
- rec.ObjID, gcState.pos, gcState.version)
- }
-
- // When strictCheck is enabled, track object's version
- // history so that we can check against the versions
- // being deleted.
- if strictCheck {
- objHist, ok := g.verifyPruneMap[rec.ObjID]
- if !ok {
- objHist = &objVersHist{
- versions: make(map[raw.Version]struct{}),
- }
- g.verifyPruneMap[rec.ObjID] = objHist
- }
- // Add this version to the versions that need to be pruned.
- objHist.versions[rec.CurVers] = struct{}{}
- }
- }
-
- if count != gen.Count {
- return errors.New("incorrect number of log records")
- }
-
- return nil
-}
-
-// reclaimSpace performs periodic space reclamation by deleting
-// generations known to all devices.
-//
-// Approach: For each device in the system, we compute its maximum
-// generation known to all the other devices in the system. We then
-// delete all log and dag records below this generation. This is a
-// O(N^2) algorithm where N is the number of devices in the system.
-func (g *syncGC) reclaimSpace() error {
- newReclaimVec, err := g.syncd.devtab.computeReclaimVector()
- if err != nil {
- return err
- }
-
- vlog.VI(1).Infof("reclaimSpace:: reclaimVectors: new %v old %v",
- newReclaimVec, g.syncd.devtab.head.ReclaimVec)
- // Clean up generations from reclaimVec+1 to newReclaimVec.
- for dev, high := range newReclaimVec {
- low := g.syncd.devtab.getOldestGen(dev)
-
- // Garbage collect from low+1 to high.
- for i := GenID(low + 1); i <= high; i++ {
- if err := g.garbageCollectGeneration(dev, i); err != nil {
- return err
- }
- }
- }
-
- // Update reclaimVec.
- g.syncd.devtab.head.ReclaimVec = newReclaimVec
- return nil
-}
-
-// pruneObjectBatch processes a batch of objects to be pruned from log
-// and dag.
-func (g *syncGC) pruneObjectBatch() error {
- vlog.VI(1).Infof("pruneObjectBatch:: Called at %v", time.Now().UTC())
- count := 0
- for obj, gcState := range g.pruneObjects {
- if count == objBatchSize {
- return nil
- }
- vlog.VI(1).Infof("pruneObjectBatch:: Pruning obj %v at version %v", obj, gcState.version)
- // Call dag prune on this object.
- if err := g.syncd.dag.prune(obj, gcState.version, g.dagPruneCallBack); err != nil {
- return err
- }
-
- if strictCheck {
- // Ensure that all but one version in the object version history are gc'ed.
- objHist, ok := g.verifyPruneMap[obj]
- if !ok {
- return fmt.Errorf("missing object in verification map %v", obj)
- }
- if len(objHist.versions) != 1 {
- return fmt.Errorf("leftover/no versions %d", len(objHist.versions))
- }
- for v, _ := range objHist.versions {
- if v != gcState.version {
- return fmt.Errorf("leftover version %d %v", v, obj)
- }
- }
- }
-
- delete(g.pruneObjects, obj)
- count++
- }
-
- return (g.syncd.dag.pruneDone())
-}
-
-// dagPruneCallBack deletes the log record associated with the dag
-// node being pruned and updates the generation metadata for the
-// generation that this log record belongs to.
-func (g *syncGC) dagPruneCallBack(logKey string) error {
- dev, gnum, lsn, err := splitLogRecKey(logKey)
- if err != nil {
- return err
- }
- vlog.VI(2).Infof("dagPruneCallBack:: called for key %s (%s %d %d)", logKey, dev, gnum, lsn)
-
- // Check if the log record being deleted is correct as per GC state.
- oldestGen := g.syncd.devtab.getOldestGen(dev)
- if gnum > oldestGen {
- vlog.VI(2).Infof("gnum is %d oldest is %d", gnum, oldestGen)
- return errors.New("deleting incorrect log record")
- }
-
- if !g.syncd.log.hasLogRec(dev, gnum, lsn) {
- return errors.New("missing log record")
- }
-
- if strictCheck {
- rec, err := g.syncd.log.getLogRec(dev, gnum, lsn)
- if err != nil {
- return err
- }
- if rec.RecType == NodeRec {
- objHist, ok := g.verifyPruneMap[rec.ObjID]
- if !ok {
- return errors.New("obj not found in verifyMap")
- }
- _, found := objHist.versions[rec.CurVers]
- // If online consistency check is in progress, we
- // cannot strictly verify all the versions to be
- // deleted, and we ignore the failure to find a
- // version.
- if found {
- delete(objHist.versions, rec.CurVers)
- } else if !g.checkConsistency {
- return errors.New("verification failed")
- }
- }
- }
-
- if err := g.syncd.log.delLogRec(dev, gnum, lsn); err != nil {
- return err
- }
-
- // Update generation metadata.
- gen, err := g.syncd.log.getGenMetadata(dev, gnum)
- if err != nil {
- return err
- }
- if gen.Count <= 0 {
- return errBadMetadata
- }
- gen.Count--
- if gen.Count == 0 {
- if err := g.syncd.log.delGenMetadata(dev, gnum); err != nil {
- return err
- }
- } else {
- if err := g.syncd.log.putGenMetadata(dev, gnum, gen); err != nil {
- return err
- }
- }
- return nil
-}
-
-// compactDB compacts the underlying kvdb store for all data structures.
-func (g *syncGC) compactDB() error {
- vlog.VI(1).Infof("compactDB:: Compacting DBs")
- if err := g.syncd.log.compact(); err != nil {
- return err
- }
- if err := g.syncd.devtab.compact(); err != nil {
- return err
- }
- if err := g.syncd.dag.compact(); err != nil {
- return err
- }
- return nil
-}
-
-// deleteDevice takes care of permanently deleting a device from its sync peers.
-// TODO(hpucha): to be implemented.
-func (g *syncGC) deleteDevice() {
-}
diff --git a/runtimes/google/vsync/gc_test.go b/runtimes/google/vsync/gc_test.go
deleted file mode 100644
index a0d2643..0000000
--- a/runtimes/google/vsync/gc_test.go
+++ /dev/null
@@ -1,1064 +0,0 @@
-package vsync
-
-// Tests for sync garbage collection.
-import (
- "os"
- "reflect"
- "testing"
-
- _ "veyron/lib/testutil"
- "veyron/services/store/raw"
-
- "veyron2/storage"
-)
-
-// TestGCOnlineConsistencyCheck tests the online consistency check in GC.
-func TestGCOnlineConsistencyCheck(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-1obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
-
- if err := s.hdlGC.onlineConsistencyCheck(); err != nil {
- t.Fatalf("onlineConsistencyCheck failed for test %s, err %v", testFile, err)
- }
- // No objects should be marked for GC.
- if s.hdlGC.checkConsistency {
- t.Errorf("onlineConsistencyCheck didn't finish in test %s", testFile)
- }
- if len(s.hdlGC.pruneObjects) > 0 {
- t.Errorf("onlineConsistencyCheck created unexpected objects in test %s, map: %v",
- testFile, s.hdlGC.pruneObjects)
- }
- if strictCheck && len(s.hdlGC.verifyPruneMap) > 0 {
- t.Errorf("onlineConsistencyCheck created unexpected objects in test %s, map: %v",
- testFile, s.hdlGC.verifyPruneMap)
- }
-
- // Fast-forward the reclaimSnap.
- s.hdlGC.reclaimSnap = GenVector{"A": 2, "B": 1, "C": 2}
-
- if err := s.hdlGC.onlineConsistencyCheck(); err != nil {
- t.Fatalf("onlineConsistencyCheck failed for test %s, err %v", testFile, err)
- }
- // Nothing should change since ock is false.
- if len(s.hdlGC.pruneObjects) > 0 {
- t.Errorf("onlineConsistencyCheck created unexpected objects in test %s, map: %v",
- testFile, s.hdlGC.pruneObjects)
- }
- if strictCheck && len(s.hdlGC.verifyPruneMap) > 0 {
- t.Errorf("onlineConsistencyCheck created unexpected objects in test %s, map: %v",
- testFile, s.hdlGC.verifyPruneMap)
- }
- expVec := GenVector{"A": 2, "B": 1, "C": 2}
- // Ensuring reclaimSnap didn't get modified in onlineConsistencyCheck().
- if !reflect.DeepEqual(expVec, s.hdlGC.reclaimSnap) {
- t.Errorf("Data mismatch for reclaimSnap: %v instead of %v", s.hdlGC.reclaimSnap, expVec)
- }
-
- s.hdlGC.checkConsistency = true
- genBatchSize = 0
- if err := s.hdlGC.onlineConsistencyCheck(); err != nil {
- t.Fatalf("onlineConsistencyCheck failed for test %s, err %v", testFile, err)
- }
- // Nothing should change since genBatchSize is 0.
- if len(s.hdlGC.pruneObjects) > 0 {
- t.Errorf("onlineConsistencyCheck created unexpected objects in test %s, map: %v",
- testFile, s.hdlGC.pruneObjects)
- }
- if strictCheck && len(s.hdlGC.verifyPruneMap) > 0 {
- t.Errorf("onlineConsistencyCheck created unexpected objects in test %s, map: %v",
- testFile, s.hdlGC.verifyPruneMap)
- }
- if !reflect.DeepEqual(expVec, s.hdlGC.reclaimSnap) {
- t.Errorf("Data mismatch for reclaimSnap: %v instead of %v", s.hdlGC.reclaimSnap, expVec)
- }
-
- // Test batching.
- genBatchSize = 1
- if err := s.hdlGC.onlineConsistencyCheck(); err != nil {
- t.Fatalf("onlineConsistencyCheck failed for test %s, err %v", testFile, err)
- }
- if !s.hdlGC.checkConsistency {
- t.Errorf("onlineConsistencyCheck finished in test %s", testFile)
- }
- total := (expVec[DeviceID("A")] - s.hdlGC.reclaimSnap[DeviceID("A")]) +
- (expVec[DeviceID("B")] - s.hdlGC.reclaimSnap[DeviceID("B")]) +
- (expVec[DeviceID("C")] - s.hdlGC.reclaimSnap[DeviceID("C")])
- if total != 1 {
- t.Errorf("onlineConsistencyCheck failed in test %s, %v", testFile, s.hdlGC.reclaimSnap)
- }
-
- genBatchSize = 4
- if err := s.hdlGC.onlineConsistencyCheck(); err != nil {
- t.Fatalf("onlineConsistencyCheck failed for test %s, err %v", testFile, err)
- }
-
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- expMap := make(map[storage.ID]*objGCState)
- expMap[objid] = &objGCState{pos: 4, version: 4}
- if !reflect.DeepEqual(expMap, s.hdlGC.pruneObjects) {
- t.Errorf("Data mismatch for pruneObjects map in vsyncd: %v instead of %v",
- s.hdlGC.pruneObjects[objid], expMap[objid])
- }
- expMap1 := make(map[storage.ID]*objVersHist)
- if strictCheck {
- expMap1[objid] = &objVersHist{versions: make(map[raw.Version]struct{})}
- for i := 0; i < 5; i++ {
- expMap1[objid].versions[raw.Version(i)] = struct{}{}
- }
- if !reflect.DeepEqual(expMap1, s.hdlGC.verifyPruneMap) {
- t.Errorf("Data mismatch for verifyPruneMap: %v instead of %v", s.hdlGC.verifyPruneMap, expMap1)
- }
- }
- expVec = GenVector{"A": 0, "B": 0, "C": 0}
- if !reflect.DeepEqual(expVec, s.hdlGC.reclaimSnap) {
- t.Errorf("Data mismatch for reclaimSnap: %v instead of %v", s.hdlGC.reclaimSnap, expVec)
- }
- if s.hdlGC.checkConsistency {
- t.Errorf("onlineConsistencyCheck didn't finish in test %s", testFile)
- }
-}
-
-// TestGCGeneration tests the garbage collection of a generation.
-func TestGCGeneration(t *testing.T) {
- // Run the test for both values of strictCheck flag.
- flags := []bool{true, false}
- for _, val := range flags {
- strictCheck = val
- setupGarbageCollectGeneration(t)
- }
-}
-
-// setupGarbageCollectGeneration performs the setup to test garbage collection of a generation.
-func setupGarbageCollectGeneration(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-1obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
-
- // Test GenID of 0.
- if err := s.hdlGC.garbageCollectGeneration(DeviceID("A"), 0); err != nil {
- t.Errorf("garbageCollectGeneration failed for test %s, dev A gnum 0, err %v\n", testFile, err)
- }
-
- // Test a non-existent generation.
- if err := s.hdlGC.garbageCollectGeneration(DeviceID("A"), 10); err == nil {
- t.Errorf("garbageCollectGeneration failed for test %s, dev A gnum 10\n", testFile)
- }
-
- if err := s.hdlGC.garbageCollectGeneration(DeviceID("A"), 2); err != nil {
- t.Errorf("garbageCollectGeneration failed for test %s, dev A gnum 2, err %v\n", testFile, err)
- }
-
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
-
- expMap := make(map[storage.ID]*objGCState)
- expMap[objid] = &objGCState{pos: 3, version: 3}
- if !reflect.DeepEqual(expMap, s.hdlGC.pruneObjects) {
- t.Errorf("Data mismatch for pruneObjects map: %v instead of %v", s.hdlGC.pruneObjects, expMap)
- }
-
- expMap1 := make(map[storage.ID]*objVersHist)
- if strictCheck {
- expMap1[objid] = &objVersHist{versions: make(map[raw.Version]struct{})}
- expMap1[objid].versions[raw.Version(3)] = struct{}{}
- if !reflect.DeepEqual(expMap1, s.hdlGC.verifyPruneMap) {
- t.Errorf("Data mismatch for verifyPruneMap: %v instead of %v", s.hdlGC.verifyPruneMap, expMap1)
- }
- }
-
- // Test GC'ing a generation lower than A:2.
- if err := s.hdlGC.garbageCollectGeneration(DeviceID("A"), 1); err != nil {
- t.Errorf("garbageCollectGeneration failed for test %s, dev A gnum 1, err %v\n", testFile, err)
- }
- if !reflect.DeepEqual(expMap, s.hdlGC.pruneObjects) {
- t.Errorf("Data mismatch for pruneObjects map: %v instead of %v", s.hdlGC.pruneObjects, expMap)
- }
-
- if strictCheck {
- expMap1[objid].versions[raw.Version(2)] = struct{}{}
- if !reflect.DeepEqual(expMap1, s.hdlGC.verifyPruneMap) {
- t.Errorf("Data mismatch for verifyPruneMap: %v instead of %v", s.hdlGC.verifyPruneMap, expMap1)
- }
- }
-
- // Test GC'ing a generation higher than A:2.
- if err := s.hdlGC.garbageCollectGeneration(DeviceID("B"), 3); err != nil {
- t.Errorf("garbageCollectGeneration failed for test %s, dev B gnum 3, err %v\n", testFile, err)
- }
- expMap[objid].pos = 6
- expMap[objid].version = 6
- if !reflect.DeepEqual(expMap, s.hdlGC.pruneObjects) {
- t.Errorf("Data mismatch for pruneObjects map: %v instead of %v", s.hdlGC.pruneObjects, expMap)
- }
-
- if strictCheck {
- expMap1[objid].versions[raw.Version(6)] = struct{}{}
- if !reflect.DeepEqual(expMap1, s.hdlGC.verifyPruneMap) {
- t.Errorf("Data mismatch for verifyPruneMap: %v instead of %v",
- s.hdlGC.verifyPruneMap[objid], expMap1[objid])
- }
- }
-}
-
-// TestGCReclaimSpace tests the space reclamation algorithm in GC.
-func TestGCReclaimSpace(t *testing.T) {
- // Run the tests for both values of strictCheck flag.
- flags := []bool{true, false}
- for _, val := range flags {
- strictCheck = val
- setupGCReclaimSpace1Obj(t)
- setupGCReclaimSpace3Objs(t)
- }
-}
-
-// setupGCReclaimSpace performs the setup to test space reclamation for a scenario with 1 object.
-func setupGCReclaimSpace1Obj(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-1obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
-
- if err := s.hdlGC.reclaimSpace(); err != nil {
- t.Errorf("reclaimSpace failed for test %s, err %v\n", testFile, err)
- }
-
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
-
- expMap := make(map[storage.ID]*objGCState)
- expMap[objid] = &objGCState{pos: 4, version: 4}
- if !reflect.DeepEqual(expMap, s.hdlGC.pruneObjects) {
- t.Errorf("Data mismatch for pruneObjects map: %v instead of %v", s.hdlGC.pruneObjects[objid], expMap[objid])
- }
-
- expMap1 := make(map[storage.ID]*objVersHist)
- expMap1[objid] = &objVersHist{versions: make(map[raw.Version]struct{})}
- if strictCheck {
- for i := 0; i < 5; i++ {
- expMap1[objid].versions[raw.Version(i)] = struct{}{}
- }
- if !reflect.DeepEqual(expMap1, s.hdlGC.verifyPruneMap) {
- t.Errorf("Data mismatch for verifyPruneMap: %v instead of %v",
- s.hdlGC.verifyPruneMap[objid], expMap1[objid])
- }
- }
- expVec := GenVector{"A": 2, "B": 1, "C": 2}
- if !reflect.DeepEqual(expVec, s.devtab.head.ReclaimVec) {
- t.Errorf("Data mismatch for reclaimVec: %v instead of %v",
- s.devtab.head.ReclaimVec, expVec)
- }
-
- // Allow for GCing B:3 incrementally.
- cVec := GenVector{"A": 2, "B": 3, "C": 2}
- if err := s.devtab.putGenVec(DeviceID("C"), cVec); err != nil {
- t.Errorf("putGenVec failed for test %s, err %v\n", testFile, err)
- }
- if err := s.hdlGC.reclaimSpace(); err != nil {
- t.Errorf("reclaimSpace failed for test %s, err %v\n", testFile, err)
- }
- expMap[objid] = &objGCState{pos: 6, version: 6}
- if !reflect.DeepEqual(expMap, s.hdlGC.pruneObjects) {
- t.Errorf("Data mismatch for pruneObjects map: %v instead of %v", s.hdlGC.pruneObjects[objid], expMap[objid])
- }
- if strictCheck {
- expMap1[objid].versions[raw.Version(5)] = struct{}{}
- expMap1[objid].versions[raw.Version(6)] = struct{}{}
- if !reflect.DeepEqual(expMap1, s.hdlGC.verifyPruneMap) {
- t.Errorf("Data mismatch for verifyPruneMap: %v instead of %v",
- s.hdlGC.verifyPruneMap[objid], expMap[objid])
- }
- }
- if !reflect.DeepEqual(cVec, s.devtab.head.ReclaimVec) {
- t.Errorf("Data mismatch for reclaimVec: %v instead of %v",
- s.devtab.head.ReclaimVec, cVec)
- }
-}
-
-// setupGCReclaimSpace3Objs performs the setup to test space reclamation for a scenario with 3 objects.
-func setupGCReclaimSpace3Objs(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-3obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
-
- if err := s.hdlGC.reclaimSpace(); err != nil {
- t.Errorf("reclaimSpace failed for test %s, err %v\n", testFile, err)
- }
-
- expMap := make(map[storage.ID]*objGCState)
- expMap1 := make(map[storage.ID]*objVersHist)
-
- obj1, err := strToObjID("123")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- expMap[obj1] = &objGCState{pos: 8, version: 6}
- expMap1[obj1] = &objVersHist{versions: make(map[raw.Version]struct{})}
- for i := 1; i < 7; i++ {
- expMap1[obj1].versions[raw.Version(i)] = struct{}{}
- }
-
- obj2, err := strToObjID("456")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- expMap[obj2] = &objGCState{pos: 10, version: 7}
- expMap1[obj2] = &objVersHist{versions: make(map[raw.Version]struct{})}
- for i := 1; i < 6; i++ {
- expMap1[obj2].versions[raw.Version(i)] = struct{}{}
- }
- expMap1[obj2].versions[raw.Version(7)] = struct{}{}
-
- obj3, err := strToObjID("789")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- expMap[obj3] = &objGCState{pos: 8, version: 4}
- expMap1[obj3] = &objVersHist{versions: make(map[raw.Version]struct{})}
- for i := 1; i < 5; i++ {
- expMap1[obj3].versions[raw.Version(i)] = struct{}{}
- }
-
- if !reflect.DeepEqual(expMap, s.hdlGC.pruneObjects) {
- t.Errorf("Data mismatch for pruneObjects map: %v instead of %v", s.hdlGC.pruneObjects, expMap)
- }
- if strictCheck {
- if !reflect.DeepEqual(expMap1, s.hdlGC.verifyPruneMap) {
- t.Errorf("Data mismatch for verifyPruneMap: %v instead of %v", s.hdlGC.verifyPruneMap, expMap1)
- }
- }
- expVec := GenVector{"A": 4, "B": 3, "C": 4}
- if !reflect.DeepEqual(expVec, s.devtab.head.ReclaimVec) {
- t.Errorf("Data mismatch for reclaimVec: %v instead of %v",
- s.devtab.head.ReclaimVec, expVec)
- }
-
- // Advance GC by one more generation.
- expVec[DeviceID("A")] = 5
- expVec[DeviceID("C")] = 5
- if err := s.devtab.putGenVec(DeviceID("C"), expVec); err != nil {
- t.Errorf("putGenVec failed for test %s, err %v\n", testFile, err)
- }
- if err := s.devtab.putGenVec(DeviceID("B"), expVec); err != nil {
- t.Errorf("putGenVec failed for test %s, err %v\n", testFile, err)
- }
-
- if err := s.hdlGC.reclaimSpace(); err != nil {
- t.Errorf("reclaimSpace failed for test %s, err %v\n", testFile, err)
- }
-
- expMap[obj1] = &objGCState{pos: 12, version: 9}
- for i := 7; i < 10; i++ {
- expMap1[obj1].versions[raw.Version(i)] = struct{}{}
- }
- expMap[obj2] = &objGCState{pos: 12, version: 8}
- for i := 6; i < 9; i++ {
- expMap1[obj2].versions[raw.Version(i)] = struct{}{}
- }
- expMap[obj3] = &objGCState{pos: 12, version: 6}
- for i := 5; i < 7; i++ {
- expMap1[obj3].versions[raw.Version(i)] = struct{}{}
- }
-
- if !reflect.DeepEqual(expMap, s.hdlGC.pruneObjects) {
- t.Errorf("Data mismatch for pruneObjects map: %v instead of %v", s.hdlGC.pruneObjects, expMap)
- }
- if strictCheck {
- if !reflect.DeepEqual(expMap1, s.hdlGC.verifyPruneMap) {
- t.Errorf("Data mismatch for verifyPruneMap: %v instead of %v", s.hdlGC.verifyPruneMap, expMap1)
- }
- }
- expVec = GenVector{"A": 5, "B": 3, "C": 5}
- if !reflect.DeepEqual(expVec, s.devtab.head.ReclaimVec) {
- t.Errorf("Data mismatch for reclaimVec: %v instead of %v",
- s.devtab.head.ReclaimVec, expVec)
- }
-}
-
-// TestGCDAGPruneCallBack tests the callback function called by dag prune.
-func TestGCDAGPruneCallBack(t *testing.T) {
- // Run the tests for both values of strictCheck flag.
- flags := []bool{true, false}
- for _, val := range flags {
- strictCheck = val
- setupGCDAGPruneCallBack(t)
- setupGCDAGPruneCallBackStrict(t)
- setupGCDAGPruneCBPartGen(t)
- }
-}
-
-// setupGCDAGPruneCallBack performs the setup to test the callback function given to dag prune.
-func setupGCDAGPruneCallBack(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- if err := s.hdlGC.dagPruneCallBack("A:1:0"); err == nil {
- t.Errorf("dagPruneCallBack error check failed\n")
- }
-
- testFile := "test-1obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
- s.devtab.head.ReclaimVec = GenVector{"A": 2, "B": 1, "C": 2}
-
- // Call should succeed irrespective of strictCheck since "ock" is true.
- if strictCheck {
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- s.hdlGC.verifyPruneMap[objid] = &objVersHist{
- versions: make(map[raw.Version]struct{}),
- }
- }
- if err := s.hdlGC.dagPruneCallBack("A:1:0"); err != nil {
- t.Errorf("dagPruneCallBack failed for test %s, err %v\n", testFile, err)
- }
-
- // Calling the same key after success should fail.
- if err := s.hdlGC.dagPruneCallBack("A:1:0"); err == nil {
- t.Errorf("dagPruneCallBack failed for test %s, err %v\n", testFile, err)
- }
-
- if s.log.hasLogRec(DeviceID("A"), GenID(1), LSN(0)) {
- t.Errorf("Log record still exists for test %s\n", testFile)
- }
- if s.log.hasGenMetadata(DeviceID("A"), GenID(1)) {
- t.Errorf("Gen metadata still exists for test %s\n", testFile)
- }
-}
-
-// setupGCDAGPruneCallBackStrict performs the setup to test the
-// callback function called by dag prune when strictCheck is true.
-func setupGCDAGPruneCallBackStrict(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-1obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
- s.devtab.head.ReclaimVec = GenVector{"A": 2, "B": 1, "C": 2}
- if !strictCheck {
- return
- }
-
- s.hdlGC.checkConsistency = false
- if err := s.hdlGC.dagPruneCallBack("A:1:0"); err == nil {
- t.Errorf("dagPruneCallBack should have failed for test %s\n", testFile)
- }
-
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- s.hdlGC.verifyPruneMap[objid] = &objVersHist{
- versions: make(map[raw.Version]struct{}),
- }
- s.hdlGC.verifyPruneMap[objid].versions[raw.Version(2)] = struct{}{}
- if err := s.hdlGC.dagPruneCallBack("A:1:0"); err != nil {
- t.Errorf("dagPruneCallBack failed for test %s, err %v\n", testFile, err)
- }
-
- // Calling the same key after success should fail.
- if err := s.hdlGC.dagPruneCallBack("A:1:0"); err == nil {
- t.Errorf("dagPruneCallBack should have failed for test %s, err %v\n", testFile, err)
- }
- if s.log.hasLogRec(DeviceID("A"), GenID(1), LSN(0)) {
- t.Errorf("Log record still exists for test %s\n", testFile)
- }
- if s.log.hasGenMetadata(DeviceID("A"), GenID(1)) {
- t.Errorf("Gen metadata still exists for test %s\n", testFile)
- }
- if len(s.hdlGC.verifyPruneMap[objid].versions) > 0 {
- t.Errorf("Unexpected object version in test %s, map: %v",
- testFile, s.hdlGC.verifyPruneMap[objid])
- }
-}
-
-// setupGCDAGPruneCBPartGen performs the setup to test the callback
-// function called by dag prune when only one entry from a generation
-// (partial gen) is pruned.
-func setupGCDAGPruneCBPartGen(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-3obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
- s.devtab.head.ReclaimVec = GenVector{"A": 4, "B": 3, "C": 4}
- s.hdlGC.checkConsistency = false
- if strictCheck {
- objid, err := strToObjID("789")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- s.hdlGC.verifyPruneMap[objid] = &objVersHist{
- versions: make(map[raw.Version]struct{}),
- }
- s.hdlGC.verifyPruneMap[objid].versions[raw.Version(4)] = struct{}{}
- }
-
- // Before pruning.
- expGen := &genMetadata{Pos: 8, Count: 3, MaxLSN: 2}
- gen, err := s.log.getGenMetadata(DeviceID("A"), GenID(3))
- if err != nil {
- t.Errorf("getGenMetadata failed for test %s, err %v\n", testFile, err)
- }
- if !reflect.DeepEqual(expGen, gen) {
- t.Errorf("Data mismatch for genMetadata: %v instead of %v",
- gen, expGen)
- }
-
- if err := s.hdlGC.dagPruneCallBack("A:3:2"); err != nil {
- t.Errorf("dagPruneCallBack failed for test %s, err %v\n", testFile, err)
- }
-
- if s.log.hasLogRec(DeviceID("A"), GenID(3), LSN(2)) {
- t.Errorf("Log record still exists for test %s\n", testFile)
- }
- if !s.log.hasGenMetadata(DeviceID("A"), GenID(3)) {
- t.Errorf("Gen metadata still exists for test %s\n", testFile)
- }
- expGen = &genMetadata{Pos: 8, Count: 2, MaxLSN: 2}
- gen, err = s.log.getGenMetadata(DeviceID("A"), GenID(3))
- if err != nil {
- t.Errorf("getGenMetadata failed for test %s, err %v\n", testFile, err)
- }
- if !reflect.DeepEqual(expGen, gen) {
- t.Errorf("Data mismatch for genMetadata: %v instead of %v",
- gen, expGen)
- }
-
- if strictCheck {
- objid, err := strToObjID("123")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- s.hdlGC.verifyPruneMap[objid] = &objVersHist{
- versions: make(map[raw.Version]struct{}),
- }
- s.hdlGC.verifyPruneMap[objid].versions[raw.Version(6)] = struct{}{}
- }
- if err := s.hdlGC.dagPruneCallBack("A:3:0"); err != nil {
- t.Errorf("dagPruneCallBack failed for test %s, err %v\n", testFile, err)
- }
- expGen = &genMetadata{Pos: 8, Count: 1, MaxLSN: 2}
- gen, err = s.log.getGenMetadata(DeviceID("A"), GenID(3))
- if err != nil {
- t.Errorf("getGenMetadata failed for test %s, err %v\n", testFile, err)
- }
- if !reflect.DeepEqual(expGen, gen) {
- t.Errorf("Data mismatch for genMetadata: %v instead of %v",
- gen, expGen)
- }
-}
-
-// TestGCPruning tests the object pruning in GC.
-func TestGCPruning(t *testing.T) {
- // Run the tests for both values of strictCheck flag.
- flags := []bool{true, false}
- for _, val := range flags {
- strictCheck = val
- setupGCPruneObject(t)
- setupGCPruneObjectBatching(t)
- setupGCPrune3Objects(t)
- }
-}
-
-// setupGCPruneObject performs the setup to test pruning an object.
-func setupGCPruneObject(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-1obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
- if err := s.hdlGC.reclaimSpace(); err != nil {
- t.Errorf("reclaimSpace failed for test %s, err %v\n", testFile, err)
- }
-
- s.hdlGC.checkConsistency = false
- objBatchSize = 0
- if err := s.hdlGC.pruneObjectBatch(); err != nil {
- t.Errorf("pruneObjectBatch failed for test %s, err %v\n", testFile, err)
- }
- if len(s.hdlGC.pruneObjects) != 1 {
- t.Errorf("pruneObjectBatch deleted object in test %s, map: %v", testFile, s.hdlGC.pruneObjects)
- }
-
- objBatchSize = 1
- if err := s.hdlGC.pruneObjectBatch(); err != nil {
- t.Errorf("pruneObjectBatch failed for test %s, err %v\n", testFile, err)
- }
-
- if len(s.hdlGC.pruneObjects) > 0 {
- t.Errorf("pruneObjectBatch left unexpected objects in test %s, map: %v", testFile, s.hdlGC.pruneObjects)
- }
-
- // Generations that should have been deleted.
- expVec := GenVector{"A": 2, "B": 1, "C": 1}
- for dev, gen := range expVec {
- for i := GenID(1); i <= gen; i++ {
- if s.log.hasGenMetadata(dev, i) {
- t.Errorf("pruneObjectBatch left unexpected generation in test %s, %s %d",
- testFile, dev, i)
- }
- if s.log.hasLogRec(dev, i, 0) {
- t.Errorf("pruneObjectBatch left unexpected logrec in test %s, %s %d 0",
- testFile, dev, i)
- }
- }
- }
-
- // Generations that should remain.
- devArr := []DeviceID{"B", "B", "C"}
- genArr := []GenID{2, 3, 2}
- for pos, dev := range devArr {
- if _, err := s.log.getGenMetadata(dev, genArr[pos]); err != nil {
- t.Errorf("pruneObjectBatch didn't find expected generation in test %s, %s %d",
- testFile, dev, genArr[pos])
- }
- if _, err := s.log.getLogRec(dev, genArr[pos], 0); err != nil {
- t.Errorf("pruneObjectBatch didn't find expected logrec in test %s, %s %d 0",
- testFile, dev, genArr[pos])
- }
- }
-
- // Verify DAG state.
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- if head, err := s.dag.getHead(objid); err != nil || head != 6 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-}
-
-// setupGCPruneObjectBatching performs the setup to test batching while pruning objects.
-func setupGCPruneObjectBatching(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-3obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
- if err := s.hdlGC.reclaimSpace(); err != nil {
- t.Errorf("reclaimSpace failed for test %s, err %v\n", testFile, err)
- }
-
- s.hdlGC.checkConsistency = false
- objBatchSize = 1
- if err := s.hdlGC.pruneObjectBatch(); err != nil {
- t.Errorf("pruneObjectBatch failed for test %s, err %v\n", testFile, err)
- }
- if len(s.hdlGC.pruneObjects) != 2 {
- t.Errorf("pruneObjectBatch didn't remove expected objects in test %s, map: %v", testFile, s.hdlGC.pruneObjects)
- }
-
- // Add a spurious version to the version history and verify error under strictCheck.
- if strictCheck {
- for _, obj := range s.hdlGC.verifyPruneMap {
- obj.versions[80] = struct{}{}
- }
- if err := s.hdlGC.pruneObjectBatch(); err == nil {
- t.Errorf("pruneObjectBatch didn't fail for test %s, err %v\n", testFile, err)
- }
- }
-}
-
-// TestGCPruneObjCheckError checks the error path in pruneObjectBatch under strictCheck.
-func TestGCPruneObjCheckError(t *testing.T) {
- if !strictCheck {
- return
- }
-
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-3obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
- if err := s.hdlGC.reclaimSpace(); err != nil {
- t.Errorf("reclaimSpace failed for test %s, err %v\n", testFile, err)
- }
-
- s.hdlGC.checkConsistency = false
- objBatchSize = 1
-
- // Remove the prune point, add a spurious version.
- for id, obj := range s.hdlGC.verifyPruneMap {
- v := s.hdlGC.pruneObjects[id].version
- obj.versions[80] = struct{}{}
- delete(obj.versions, v)
- }
-
- if err = s.hdlGC.pruneObjectBatch(); err == nil {
- t.Errorf("pruneObjectBatch didn't fail for test %s, err %v\n", testFile, err)
- }
-}
-
-// setupGCPrune3Objects performs the setup to test pruning in a 3 object scenario.
-func setupGCPrune3Objects(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-3obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
- if err := s.hdlGC.reclaimSpace(); err != nil {
- t.Errorf("reclaimSpace failed for test %s, err %v\n", testFile, err)
- }
-
- objBatchSize = 5
- s.hdlGC.checkConsistency = false
- gen, err := s.log.getGenMetadata("A", 3)
- if err != nil {
- t.Errorf("getGenMetadata failed for test %s, err %v\n", testFile, err)
- }
- if gen.Count != 3 {
- t.Errorf("GenMetadata has incorrect value for test %s\n", testFile)
- }
- if err := s.hdlGC.pruneObjectBatch(); err != nil {
- t.Errorf("pruneObjectBatch failed for test %s, err %v\n", testFile, err)
- }
- if len(s.hdlGC.pruneObjects) > 0 {
- t.Errorf("pruneObjectBatch didn't remove expected objects in test %s, map: %v", testFile, s.hdlGC.pruneObjects)
- }
- // Generations that should have been deleted.
- expVec := GenVector{"A": 2, "B": 3, "C": 4}
- for dev, gnum := range expVec {
- for i := GenID(1); i <= gnum; i++ {
- if s.log.hasGenMetadata(dev, i) {
- t.Errorf("pruneObjectBatch left unexpected generation in test %s, %s %d",
- testFile, dev, i)
- }
- // Check the first log record.
- if s.log.hasLogRec(dev, i, 0) {
- t.Errorf("pruneObjectBatch left unexpected logrec in test %s, %s %d 0",
- testFile, dev, i)
- }
- }
- }
- // Check the partial generation.
- if gen, err = s.log.getGenMetadata("A", 3); err != nil {
- t.Errorf("getGenMetadata failed for test %s, err %v\n", testFile, err)
- }
- if gen.Count != 2 {
- t.Errorf("GenMetadata has incorrect value for test %s\n", testFile)
- }
- // Verify DAG state.
- objArr := []string{"123", "456", "789"}
- heads := []raw.Version{10, 8, 6}
- for pos, o := range objArr {
- objid, err := strToObjID(o)
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- if head, err := s.dag.getHead(objid); err != nil || head != heads[pos] {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
- }
-}
-
-// TestGCStages tests the interactions across the different stages in GC.
-func TestGCStages(t *testing.T) {
- // Run the tests for both values of strictCheck flag.
- flags := []bool{true, false}
- for _, val := range flags {
- strictCheck = val
- setupGCReclaimAndOnlineCk(t)
- setupGCReclaimAndOnlineCkIncr(t)
- setupGCOnlineCkAndPrune(t)
- }
-}
-
-// setupGCReclaimAndOnlineCk performs the setup to test interaction between reclaimSpace and consistency check.
-func setupGCReclaimAndOnlineCk(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-3obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
- if err := s.hdlGC.reclaimSpace(); err != nil {
- t.Errorf("reclaimSpace failed for test %s, err %v\n", testFile, err)
- }
- objBatchSize = 1
- if err := s.hdlGC.pruneObjectBatch(); err != nil {
- t.Errorf("pruneObjectBatch failed for test %s, err %v\n", testFile, err)
- }
-
- // Clean up state to simulate a reboot. Given that 1 object
- // is already GC'ed, there are now partial generations left in
- // the state.
- for obj, _ := range s.hdlGC.pruneObjects {
- delete(s.hdlGC.pruneObjects, obj)
- delete(s.hdlGC.verifyPruneMap, obj)
- }
- s.hdlGC.reclaimSnap = GenVector{"A": 4, "B": 3, "C": 4}
- genBatchSize = 1
- for s.hdlGC.checkConsistency == true {
- if err := s.hdlGC.onlineConsistencyCheck(); err != nil {
- t.Fatalf("onlineConsistencyCheck failed for test %s, err %v", testFile, err)
- }
- }
-
- // Since dag prunes everything older than a version, all 3
- // objects show up once again in the pruneObjects map.
- if len(s.hdlGC.pruneObjects) != 3 {
- t.Errorf("onlineConsistencyCheck didn't add objects in test %s, map: %v", testFile, s.hdlGC.pruneObjects)
- }
-
- objBatchSize = 3
- if err := s.hdlGC.pruneObjectBatch(); err != nil {
- t.Errorf("pruneObjectBatch failed for test %s, err %v\n", testFile, err)
- }
- if len(s.hdlGC.pruneObjects) > 0 {
- t.Errorf("pruneObjectBatch didn't remove expected objects in test %s, map: %v", testFile, s.hdlGC.pruneObjects)
- }
- // Generations that should have been deleted.
- expVec := GenVector{"A": 2, "B": 3, "C": 4}
- for dev, gnum := range expVec {
- for i := GenID(1); i <= gnum; i++ {
- if s.log.hasGenMetadata(dev, i) {
- t.Errorf("pruneObjectBatch left unexpected generation in test %s, %s %d",
- testFile, dev, i)
- }
- // Check the first log record.
- if s.log.hasLogRec(dev, i, 0) {
- t.Errorf("pruneObjectBatch left unexpected logrec in test %s, %s %d 0",
- testFile, dev, i)
- }
- }
- }
- // Check the partial generation.
- gen, err := s.log.getGenMetadata("A", 3)
- if err != nil {
- t.Errorf("getGenMetadata failed for test %s, err %v\n", testFile, err)
- }
- if gen.Count != 2 {
- t.Errorf("GenMetadata has incorrect value for test %s\n", testFile)
- }
- // Verify DAG state.
- objArr := []string{"123", "456", "789"}
- heads := []raw.Version{10, 8, 6}
- for pos, o := range objArr {
- objid, err := strToObjID(o)
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- if head, err := s.dag.getHead(objid); err != nil || head != heads[pos] {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
- }
-}
-
-// setupGCReclaimAndOnlineCkIncr tests interaction between reclaimSpace
-// and consistency check when both are running one after the other
-// incrementally.
-func setupGCReclaimAndOnlineCkIncr(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-1obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
- // Fast-forward the reclaimSnap and ReclaimVec.
- s.hdlGC.reclaimSnap = GenVector{"A": 2, "B": 1, "C": 2}
- s.devtab.head.ReclaimVec = GenVector{"A": 2, "B": 1, "C": 2}
-
- genBatchSize = 1
- if err := s.hdlGC.onlineConsistencyCheck(); err != nil {
- t.Fatalf("onlineConsistencyCheck failed for test %s, err %v", testFile, err)
- }
- objBatchSize = 1
- if err := s.hdlGC.pruneObjectBatch(); err != nil {
- t.Errorf("pruneObjectBatch failed for test %s, err %v\n", testFile, err)
- }
-
- cVec := GenVector{"A": 2, "B": 3, "C": 2}
- if err := s.devtab.putGenVec(DeviceID("C"), cVec); err != nil {
- t.Errorf("putGenVec failed for test %s, err %v\n", testFile, err)
- }
- if err := s.hdlGC.reclaimSpace(); err != nil {
- t.Errorf("reclaimSpace failed for test %s, err %v\n", testFile, err)
- }
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- expMap := make(map[storage.ID]*objGCState)
- expMap[objid] = &objGCState{pos: 6, version: 6}
- if !reflect.DeepEqual(expMap, s.hdlGC.pruneObjects) {
- t.Errorf("Data mismatch for pruneObjects map: %v instead of %v", s.hdlGC.pruneObjects[objid], expMap[objid])
- }
- objBatchSize = 1
- if err := s.hdlGC.pruneObjectBatch(); err != nil {
- t.Errorf("pruneObjectBatch failed for test %s, err %v\n", testFile, err)
- }
- for s.hdlGC.checkConsistency == true {
- if err := s.hdlGC.onlineConsistencyCheck(); err != nil {
- t.Fatalf("onlineConsistencyCheck failed for test %s, err %v", testFile, err)
- }
- }
- if len(s.hdlGC.pruneObjects) > 0 {
- t.Errorf("onlineConsistencyCheck created an object in test %s, map: %v", testFile, s.hdlGC.pruneObjects)
- }
-}
-
-// setupGCOnlineCkAndPrune performs the setup to test interaction
-// between consistency check and object pruning (when both are
-// incremental).
-func setupGCOnlineCkAndPrune(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- s := NewSyncd("", "", "A", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- testFile := "test-3obj.gc.sync"
- if err := vsyncInitState(s, testFile); err != nil {
- t.Fatal(err)
- }
- if err := s.hdlGC.reclaimSpace(); err != nil {
- t.Errorf("reclaimSpace failed for test %s, err %v\n", testFile, err)
- }
- objBatchSize = 1
- if err := s.hdlGC.pruneObjectBatch(); err != nil {
- t.Errorf("pruneObjectBatch failed for test %s, err %v\n", testFile, err)
- }
-
- // Clean up state to simulate a reboot. Given that 1 object
- // is already GC'ed, there are now partial generations left in
- // the state.
- for obj, _ := range s.hdlGC.pruneObjects {
- delete(s.hdlGC.pruneObjects, obj)
- delete(s.hdlGC.verifyPruneMap, obj)
- }
- s.hdlGC.reclaimSnap = GenVector{"A": 4, "B": 3, "C": 4}
- genBatchSize = 3
- objBatchSize = 3
- for s.hdlGC.checkConsistency == true {
- if err := s.hdlGC.onlineConsistencyCheck(); err != nil {
- t.Fatalf("onlineConsistencyCheck failed for test %s, err %v", testFile, err)
- }
- if err := s.hdlGC.pruneObjectBatch(); err != nil {
- t.Errorf("pruneObjectBatch failed for test %s, err %v\n", testFile, err)
- }
- }
- if len(s.hdlGC.pruneObjects) > 0 {
- t.Errorf("pruneObjectBatch didn't remove expected objects in test %s, map: %v", testFile, s.hdlGC.pruneObjects)
- }
-}
diff --git a/runtimes/google/vsync/ilog.go b/runtimes/google/vsync/ilog.go
deleted file mode 100644
index 764b6e5..0000000
--- a/runtimes/google/vsync/ilog.go
+++ /dev/null
@@ -1,466 +0,0 @@
-package vsync
-
-// Package vsync provides veyron sync ILog utility functions. ILog
-// (Indexed Log) provides log functionality with indexing support.
-// ILog stores log records that are locally generated or obtained over
-// the network. Indexing is needed since sync needs to selectively
-// retrieve log records that belong to a particular device and
-// generation during synchronization.
-//
-// When a device receives a request to send log records, it first
-// computes the missing generations between itself and the incoming
-// request. It then sends all the log records belonging to each
-// missing generation. A device that receives log records over the
-// network replays all the records received from another device in a
-// single batch. Each replayed log record adds a new version to the
-// dag of the object contained in the log record. At the end of
-// replaying all the log records, conflict detection and resolution is
-// carried out for all the objects learned during this
-// iteration. Conflict detection and resolution is carried out after a
-// batch of log records are replayed, instead of incrementally after
-// each record is replayed, to avoid repeating conflict resolution
-// already performed by other devices.
-//
-// New log records are created when objects in the local store are
-// created/updated. Local log records are also replayed to keep the
-// per-object dags consistent with the local store state.
-//
-// Implementation notes: ILog records are stored in a persistent K/V
-// database in the current implementation. ILog db consists of 3
-// tables:
-// ** records: table consists of all the log records indexed
-// by deviceid:genid:lsn referring to the device that creates the log
-// record, the generation on the device the log record is part of, and
-// its sequence number in that generation. Note that lsn in each
-// generation starts from 0 and genid starts from 1.
-// ** gens: table consists of the generation metadata for each
-// generation, and is indexed by deviceid:genid.
-// ** head: table consists of the log header.
-import (
- "errors"
- "fmt"
- "strconv"
- "strings"
-
- "veyron/services/store/raw"
-
- "veyron2/storage"
- "veyron2/vlog"
-)
-
-var (
- errNoUpdates = errors.New("no new local updates")
- errInvalidLog = errors.New("invalid log db")
-)
-
-// iLogHeader contains the log header metadata.
-type iLogHeader struct {
- Curgen GenID // generation id for a device's current generation.
- Curlsn LSN // log sequence number for a device's current generation.
- Curorder uint32 // position in log for the next generation.
-}
-
-// genMetadata contains the metadata for a generation.
-type genMetadata struct {
- // All generations stored in the log are ordered wrt each
- // other and this order needs to be preserved.
- // Position of this generation in the log.
- Pos uint32
-
- // Number of log records in this generation still stored in the log.
- // This count is used during garbage collection.
- Count uint64
-
- // Maximum LSN that was part of this generation.
- // This is useful during garbage collection to track any unclaimed log records.
- MaxLSN LSN
-}
-
-// iLog contains the metadata for the ILog db.
-type iLog struct {
- fname string // file pathname.
- db *kvdb // underlying k/v db.
-
- // Key:deviceid-genid-lsn Value:LogRecord
- records *kvtable // pointer to the "records" table in the kvdb. Contains log records.
-
- // Key:deviceid-genid Value:genMetadata
- gens *kvtable // pointer to the "gens" table in the kvdb. Contains generation metadata for each generation.
-
- // Key:"Head" Value:iLogHeader
- header *kvtable // pointer to the "header" table in the kvdb. Contains logheader.
-
- head *iLogHeader // log head cached in memory.
-
- s *syncd // pointer to the sync daemon object.
-}
-
-// openILog opens or creates a ILog for the given filename.
-func openILog(filename string, sin *syncd) (*iLog, error) {
- ilog := &iLog{
- fname: filename,
- head: nil,
- s: sin,
- }
- // Open the file and create it if it does not exist.
- // Also initialize the kvdb and its three collections.
- db, tbls, err := kvdbOpen(filename, []string{"records", "gens", "header"})
- if err != nil {
- return nil, err
- }
-
- ilog.db = db
- ilog.records = tbls[0]
- ilog.gens = tbls[1]
- ilog.header = tbls[2]
-
- // Initialize the log header.
- // First generation to be created is generation 1. A generation of 0
- // represents no updates on the device.
- ilog.head = &iLogHeader{
- Curgen: 1,
- }
- // If header already exists in db, read it back from db.
- if ilog.hasHead() {
- if err := ilog.getHead(); err != nil {
- ilog.db.close() // this also closes the tables.
- return nil, err
- }
- }
- return ilog, nil
-}
-
-// close closes the ILog and invalidate its struct.
-func (l *iLog) close() error {
- if l.db == nil {
- return errInvalidLog
- }
- // Flush the dirty data.
- if err := l.flush(); err != nil {
- return err
- }
-
- l.db.close() // this also closes the tables.
-
- *l = iLog{} // zero out the ILog struct.
- return nil
-}
-
-// flush flushes the ILog db to disk.
-func (l *iLog) flush() error {
- if l.db == nil {
- return errInvalidLog
- }
- // Set the head from memory before flushing.
- if err := l.putHead(); err != nil {
- return err
- }
-
- l.db.flush()
-
- return nil
-}
-
-// compact compacts the file associated with kvdb.
-func (l *iLog) compact() error {
- if l.db == nil {
- return errInvalidLog
- }
- db, tbls, err := l.db.compact(l.fname, []string{"records", "gens", "header"})
- if err != nil {
- return err
- }
- l.db = db
- l.records = tbls[0]
- l.gens = tbls[1]
- l.header = tbls[2]
- return nil
-}
-
-// putHead puts the log head into the ILog db.
-func (l *iLog) putHead() error {
- return l.header.set("Head", l.head)
-}
-
-// getHead gets the log head from the ILog db.
-func (l *iLog) getHead() error {
- if l.head == nil {
- return errors.New("nil log header")
- }
- err := l.header.get("Head", l.head)
- return err
-}
-
-// hasHead returns true if the ILog db has a log head.
-func (l *iLog) hasHead() bool {
- return l.header.hasKey("Head")
-}
-
-// logRecKey creates a key for a log record.
-func logRecKey(devid DeviceID, gnum GenID, lsn LSN) string {
- return fmt.Sprintf("%s:%d:%d", devid, uint64(gnum), uint64(lsn))
-}
-
-// splitLogRecKey splits a : separated logrec key into its components.
-func splitLogRecKey(key string) (DeviceID, GenID, LSN, error) {
- args := strings.Split(key, ":")
- if len(args) != 3 {
- return "", 0, 0, fmt.Errorf("bad logrec key %s", key)
- }
- gnum, _ := strconv.ParseUint(args[1], 10, 64)
- lsn, _ := strconv.ParseUint(args[2], 10, 64)
- return DeviceID(args[0]), GenID(gnum), LSN(lsn), nil
-}
-
-// putLogRec puts the log record into the ILog db.
-func (l *iLog) putLogRec(rec *LogRec) (string, error) {
- if l.db == nil {
- return "", errInvalidLog
- }
- key := logRecKey(rec.DevID, rec.GNum, rec.LSN)
- return key, l.records.set(key, rec)
-}
-
-// getLogRec gets the log record from the ILog db.
-func (l *iLog) getLogRec(devid DeviceID, gnum GenID, lsn LSN) (*LogRec, error) {
- if l.db == nil {
- return nil, errInvalidLog
- }
- key := logRecKey(devid, gnum, lsn)
- var rec LogRec
- if err := l.records.get(key, &rec); err != nil {
- return nil, err
- }
- return &rec, nil
-}
-
-// hasLogRec returns true if the ILog db has a log record matching (devid, gnum, lsn).
-func (l *iLog) hasLogRec(devid DeviceID, gnum GenID, lsn LSN) bool {
- if l.db == nil {
- return false
- }
- key := logRecKey(devid, gnum, lsn)
- return l.records.hasKey(key)
-}
-
-// delLogRec deletes the log record matching (devid, gnum, lsn) from the ILog db.
-func (l *iLog) delLogRec(devid DeviceID, gnum GenID, lsn LSN) error {
- if l.db == nil {
- return errInvalidLog
- }
- key := logRecKey(devid, gnum, lsn)
- return l.records.del(key)
-}
-
-// generationKey creates a key for a generation.
-func generationKey(devid DeviceID, gnum GenID) string {
- return fmt.Sprintf("%s:%d", devid, gnum)
-}
-
-// splitGenerationKey splits a : separated logrec key into its components.
-func splitGenerationKey(key string) (DeviceID, GenID, error) {
- args := strings.Split(key, ":")
- if len(args) != 2 {
- return "", 0, fmt.Errorf("bad generation key %s", key)
- }
- gnum, _ := strconv.ParseUint(args[1], 10, 64)
- return DeviceID(args[0]), GenID(gnum), nil
-}
-
-// putGenMetadata puts the metadata of the generation (devid, gnum) into the ILog db.
-func (l *iLog) putGenMetadata(devid DeviceID, gnum GenID, val *genMetadata) error {
- key := generationKey(devid, gnum)
- return l.gens.set(key, val)
-}
-
-// getGenMetadata gets the metadata of the generation (devid, gnum) from the ILog db.
-func (l *iLog) getGenMetadata(devid DeviceID, gnum GenID) (*genMetadata, error) {
- if l.db == nil {
- return nil, errInvalidLog
- }
- key := generationKey(devid, gnum)
- var val genMetadata
- if err := l.gens.get(key, &val); err != nil {
- return nil, err
- }
- return &val, nil
-}
-
-// hasGenMetadata returns true if the ILog db has the generation (devid, gnum).
-func (l *iLog) hasGenMetadata(devid DeviceID, gnum GenID) bool {
- key := generationKey(devid, gnum)
- return l.gens.hasKey(key)
-}
-
-// delGenMetadata deletes the generation (devid, gnum) metadata from the ILog db.
-func (l *iLog) delGenMetadata(devid DeviceID, gnum GenID) error {
- if l.db == nil {
- return errInvalidLog
- }
- key := generationKey(devid, gnum)
- return l.gens.del(key)
-}
-
-// createLocalLogRec creates a new local log record of type NodeRec.
-func (l *iLog) createLocalLogRec(obj storage.ID, vers raw.Version, par []raw.Version, val *LogValue) (*LogRec, error) {
- rec := &LogRec{
- DevID: l.s.id,
- GNum: l.head.Curgen,
- LSN: l.head.Curlsn,
- RecType: NodeRec,
-
- ObjID: obj,
- CurVers: vers,
- Parents: par,
- Value: *val,
- }
-
- // Increment the LSN for the local log.
- l.head.Curlsn++
-
- return rec, nil
-}
-
-// createLocalLinkLogRec creates a new local log record of type LinkRec.
-func (l *iLog) createLocalLinkLogRec(obj storage.ID, vers, par raw.Version) (*LogRec, error) {
- rec := &LogRec{
- DevID: l.s.id,
- GNum: l.head.Curgen,
- LSN: l.head.Curlsn,
- RecType: LinkRec,
-
- ObjID: obj,
- CurVers: vers,
- Parents: []raw.Version{par},
- }
-
- // Increment the LSN for the local log.
- l.head.Curlsn++
-
- return rec, nil
-}
-
-// createRemoteGeneration adds a new remote generation.
-func (l *iLog) createRemoteGeneration(dev DeviceID, gnum GenID, gen *genMetadata) error {
- if l.db == nil {
- return errInvalidLog
- }
-
- if gen.Count != uint64(gen.MaxLSN+1) {
- return errors.New("mismatch in count and lsn")
- }
-
- gen.Pos = l.head.Curorder
- l.head.Curorder++
-
- return l.putGenMetadata(dev, gnum, gen)
-}
-
-// createLocalGeneration creates a new local generation.
-// createLocalGeneration is currently called when there is an incoming GetDeltas request.
-func (l *iLog) createLocalGeneration() (GenID, error) {
- if l.db == nil {
- return 0, errInvalidLog
- }
-
- g := l.head.Curgen
-
- // If there are no updates, there will be no new generation.
- if l.head.Curlsn == 0 {
- return g - 1, errNoUpdates
- }
-
- // Add the current generation to the db.
- val := &genMetadata{
- Pos: l.head.Curorder,
- Count: uint64(l.head.Curlsn),
- MaxLSN: l.head.Curlsn - 1,
- }
- err := l.putGenMetadata(l.s.id, g, val)
-
- vlog.VI(2).Infof("createLocalGeneration:: created gen %d %v", g, val)
- // Move to the next generation irrespective of err.
- l.head.Curorder++
- l.head.Curgen++
- l.head.Curlsn = 0
-
- return g, err
-}
-
-// processWatchRecord processes new object versions obtained from the local store.
-func (l *iLog) processWatchRecord(objID storage.ID, vers, parent raw.Version, val *LogValue, txID TxID) error {
- if l.db == nil {
- return errInvalidLog
- }
-
- vlog.VI(2).Infof("processWatchRecord:: adding object %v %v", objID, vers)
-
- if vers != raw.NoVersion {
- // Check if the object's vers already exists in the DAG.
- if l.s.dag.hasNode(objID, vers) {
- return nil
- }
- } else {
- // Check if the parent version has a deleted
- // descendant already in the DAG.
- if l.s.dag.hasDeletedDescendant(objID, parent) {
- return nil
- }
- }
-
- var pars []raw.Version
- if parent != raw.NoVersion {
- pars = []raw.Version{parent}
- }
-
- // If the current version is a deletion, generate a new version number.
- if val.Delete {
- if vers != raw.NoVersion {
- return fmt.Errorf("deleted vers is %v", vers)
- }
- vers = raw.NewVersion()
- val.Mutation.Version = vers
- }
-
- // Create a log record from Watch's Change Record.
- rec, err := l.createLocalLogRec(objID, vers, pars, val)
- if err != nil {
- return err
- }
-
- // Insert the new log record into the log.
- logKey, err := l.putLogRec(rec)
- if err != nil {
- return err
- }
-
- // Insert the new log record into dag.
- if err = l.s.dag.addNode(rec.ObjID, rec.CurVers, false, val.Delete, rec.Parents, logKey, txID); err != nil {
- return err
- }
-
- // Move the head.
- if err := l.s.dag.moveHead(rec.ObjID, rec.CurVers); err != nil {
- return err
- }
-
- return nil
-}
-
-// dumpILog dumps the ILog data structure.
-func (l *iLog) dumpILog() {
- fmt.Println("In-memory Header")
- fmt.Println("Current generation: ", l.head.Curgen, l.head.Curlsn, " Current order: ",
- l.head.Curorder)
-
- fmt.Println("================================================")
-
- fmt.Println("In-DB Header")
- head := &iLogHeader{}
- if err := l.header.get("Head", head); err != nil {
- fmt.Println("Couldn't access in DB header")
- } else {
- fmt.Println("Current generation: ", head.Curgen, head.Curlsn, " Current order: ", head.Curorder)
- }
- fmt.Println("================================================")
-}
diff --git a/runtimes/google/vsync/ilog_test.go b/runtimes/google/vsync/ilog_test.go
deleted file mode 100644
index ea1ba6e..0000000
--- a/runtimes/google/vsync/ilog_test.go
+++ /dev/null
@@ -1,933 +0,0 @@
-package vsync
-
-// Tests for the Veyron Sync ILog component.
-import (
- "os"
- "reflect"
- "testing"
-
- "veyron/services/store/raw"
-
- "veyron2/storage"
-)
-
-// TestILogStore tests creating a backing file for ILog.
-func TestILogStore(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new ILog file %s, err %v", logfile, err)
- }
-
- fsize := getFileSize(logfile)
- if fsize < 0 {
- t.Errorf("Log file %s not created", logfile)
- }
-
- if err := log.flush(); err != nil {
- t.Errorf("Cannot flush ILog file %s, err %v", logfile, err)
- }
-
- oldfsize := fsize
- fsize = getFileSize(logfile)
- if fsize <= oldfsize {
- t.Errorf("Log file %s not flushed", logfile)
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close ILog file %s, err %v", logfile)
- }
-
- oldfsize = getFileSize(logfile)
-
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open existing log file %s, err %v", logfile)
- }
-
- fsize = getFileSize(logfile)
- if fsize != oldfsize {
- t.Errorf("Log file %s size changed across re-open (%d %d)", logfile, fsize, oldfsize)
- }
-
- if err := log.flush(); err != nil {
- t.Errorf("Cannot flush ILog file %s, err %v", logfile)
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close ILog file %s, err %v", logfile)
- }
-}
-
-// TestInvalidLog tests log methods on an invalid (closed) log ptr.
-func TestInvalidLog(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new ILog file %s, err %v", logfile, err)
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close ILog file %s, err %v", logfile, err)
- }
-
- err = log.close()
- if err == nil || err != errInvalidLog {
- t.Errorf("Close did not fail on a closed log: %v", err)
- }
-
- err = log.flush()
- if err == nil || err != errInvalidLog {
- t.Errorf("Flush did not fail on a closed log: %v", err)
- }
-
- err = log.compact()
- if err == nil || err != errInvalidLog {
- t.Errorf("Compact did not fail on a closed log: %v", err)
- }
-
- _, err = log.putLogRec(&LogRec{})
- if err == nil || err != errInvalidLog {
- t.Errorf("PutLogRec did not fail on a closed log: %v", err)
- }
-
- var devid DeviceID = "VeyronPhone"
- var gnum GenID = 1
- var lsn LSN
-
- _, err = log.getLogRec(devid, gnum, lsn)
- if err == nil || err != errInvalidLog {
- t.Errorf("GetLogRec did not fail on a closed log: %v", err)
- }
-
- if log.hasLogRec(devid, gnum, lsn) {
- if err == nil || err != errInvalidLog {
- t.Errorf("HasLogRec did not fail on a closed log: %v", err)
- }
- }
-
- err = log.delLogRec(devid, gnum, lsn)
- if err == nil || err != errInvalidLog {
- t.Errorf("DelLogRec did not fail on a closed log: %v", err)
- }
-
- _, err = log.getGenMetadata(devid, gnum)
- if err == nil || err != errInvalidLog {
- t.Errorf("GetGenMetadata did not fail on a closed log: %v", err)
- }
-
- err = log.delGenMetadata(devid, gnum)
- if err == nil || err != errInvalidLog {
- t.Errorf("DelGenMetadata did not fail on a closed log: %v", err)
- }
-
- err = log.createRemoteGeneration(devid, gnum, &genMetadata{})
- if err == nil || err != errInvalidLog {
- t.Errorf("CreateRemoteGeneration did not fail on a closed log: %v", err)
- }
-
- _, err = log.createLocalGeneration()
- if err == nil || err != errInvalidLog {
- t.Errorf("CreateLocalGeneration did not fail on a closed log: %v", err)
- }
-
- err = log.processWatchRecord(storage.NewID(), 2, raw.Version(999), &LogValue{}, NoTxID)
- if err == nil || err != errInvalidLog {
- t.Errorf("ProcessWatchRecord did not fail on a closed log: %v", err)
- }
-}
-
-// TestPutGetLogHeader tests setting and getting log header across log open/close/reopen.
-func TestPutGetLogHeader(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- // In memory head should be initialized.
- if log.head.Curgen != 1 || log.head.Curlsn != 0 || log.head.Curorder != 0 {
- t.Errorf("First time log create should reset header")
- }
-
- // No head should be there in db.
- if err = log.getHead(); err == nil {
- t.Errorf("getHead() found non-existent head in log file %s, err %v", logfile, err)
- }
-
- if log.hasHead() {
- t.Errorf("hasHead() found non-existent head in log file %s", logfile)
- }
-
- log.head = &iLogHeader{
- Curgen: 10,
- Curlsn: 100,
- Curorder: 1000,
- }
-
- if err := log.putHead(); err != nil {
- t.Errorf("Cannot put head %v in log file %s, err %v", log.head, logfile, err)
- }
-
- // Reset values.
- log.head.Curgen = 0
- log.head.Curlsn = 0
- log.head.Curorder = 0
-
- for i := 0; i < 2; i++ {
- if err := log.getHead(); err != nil {
- t.Fatalf("getHead() can not find head (i=%d) in log file %s, err %v", i, logfile, err)
- }
-
- if !log.hasHead() {
- t.Errorf("hasHead() can not find head (i=%d) in log file %s", i, logfile)
- }
-
- if log.head.Curgen != 10 && log.head.Curlsn != 100 && log.head.Curorder != 1000 {
- t.Errorf("Data mismatch for head (i=%d) in log file %s: %v",
- i, logfile, log.head)
- }
-
- if i == 0 {
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- }
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestPersistLogHeader tests that log header is automatically persisted across log open/close/reopen.
-func TestPersistLogHeader(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- // In memory head should be initialized.
- if log.head.Curgen != 1 || log.head.Curlsn != 0 || log.head.Curorder != 0 {
- t.Errorf("First time log create should reset header")
- }
-
- log.head = &iLogHeader{
- Curgen: 10,
- Curlsn: 100,
- Curorder: 1000,
- }
-
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- // In memory head should be initialized from db.
- if log.head.Curgen != 10 && log.head.Curlsn != 100 && log.head.Curorder != 1000 {
- t.Errorf("Data mismatch for head in log file %s: %v", logfile, log.head)
- }
-
- log.head = &iLogHeader{
- Curgen: 1000,
- Curlsn: 10,
- Curorder: 100,
- }
-
- if err := log.flush(); err != nil {
- t.Errorf("Cannot flush ILog file %s, err %v", logfile, err)
- }
-
- // Reset values.
- log.head.Curgen = 0
- log.head.Curlsn = 0
- log.head.Curorder = 0
-
- if err := log.getHead(); err != nil {
- t.Fatalf("getHead() can not find head in log file %s, err %v", logfile, err)
- }
-
- // In memory head should be initialized from db.
- if log.head.Curgen != 1000 && log.head.Curlsn != 10 && log.head.Curorder != 100 {
- t.Errorf("Data mismatch for head in log file %s: %v", logfile, log.head)
- }
-
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestPutGetLogRec tests setting and getting a log record across log open/close/reopen.
-func TestPutGetLogRec(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- var devid DeviceID = "VeyronTab"
- var gnum GenID = 100
- var lsn LSN = 1000
-
- rec, err := log.getLogRec(devid, gnum, lsn)
- if err == nil || rec != nil {
- t.Errorf("GetLogRec() found non-existent object %s:%d:%d in log file %s: %v, err %v",
- devid, gnum, lsn, logfile, rec, err)
- }
-
- if log.hasLogRec(devid, gnum, lsn) {
- t.Errorf("HasLogRec() found non-existent object %s:%d:%d in log file %s",
- devid, gnum, lsn, logfile)
- }
-
- objID := storage.NewID()
- rec = &LogRec{
- DevID: devid,
- GNum: gnum,
- LSN: lsn,
- ObjID: objID,
- CurVers: 2,
- Parents: []raw.Version{0, 1},
- Value: LogValue{},
- }
-
- if _, err := log.putLogRec(rec); err != nil {
- t.Errorf("Cannot put object %s:%d:%d (%v) in log file %s, err %v", devid, gnum, lsn, rec, logfile, err)
- }
-
- for i := 0; i < 2; i++ {
- curRec, err := log.getLogRec(devid, gnum, lsn)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:%d:%d (i=%d) in log file %s, err %v",
- devid, gnum, lsn, i, logfile, err)
- }
-
- if !log.hasLogRec(devid, gnum, lsn) {
- t.Errorf("HasLogRec() can not find object %s:%d:%d (i=%d) in log file %s",
- devid, gnum, lsn, i, logfile)
- }
-
- if !reflect.DeepEqual(rec, curRec) {
- t.Errorf("Data mismatch for object %s:%d:%d (i=%d) in log file %s: %v instead of %v",
- devid, gnum, lsn, i, logfile, curRec, rec)
- }
-
- if i == 0 {
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- }
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestDelLogRec tests deleting a log record across log open/close/reopen.
-func TestDelLogRec(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- var devid DeviceID = "VeyronTab"
- var gnum GenID = 100
- var lsn LSN = 1000
-
- objID := storage.NewID()
- rec := &LogRec{
- DevID: devid,
- GNum: gnum,
- LSN: lsn,
- ObjID: objID,
- CurVers: 2,
- Parents: []raw.Version{0, 1},
- Value: LogValue{},
- }
-
- if _, err := log.putLogRec(rec); err != nil {
- t.Errorf("Cannot put object %s:%d:%d (%v) in log file %s, err %v", devid, gnum, lsn, rec, logfile, err)
- }
-
- curRec, err := log.getLogRec(devid, gnum, lsn)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:%d:%d in log file %s, err %v",
- devid, gnum, lsn, logfile, err)
- }
-
- if err := log.delLogRec(devid, gnum, lsn); err != nil {
- t.Fatalf("DelLogRec() can not delete object %s:%d:%d in log file %s, err %v",
- devid, gnum, lsn, logfile, err)
- }
-
- for i := 0; i < 2; i++ {
- curRec, err = log.getLogRec(devid, gnum, lsn)
- if err == nil || curRec != nil {
- t.Fatalf("GetLogRec() finds deleted object %s:%d:%d (i=%d) in log file %s, err %v",
- devid, gnum, lsn, i, logfile, err)
- }
-
- if log.hasLogRec(devid, gnum, lsn) {
- t.Errorf("HasLogRec() finds deleted object %s:%d:%d (i=%d) in log file %s",
- devid, gnum, lsn, i, logfile)
- }
-
- if i == 0 {
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- }
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-
-}
-
-// TestPutGetGenMetadata tests setting and getting generation metadata across log open/close/reopen.
-func TestPutGetGenMetadata(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- var devid DeviceID = "VeyronTab"
- var gnum GenID = 100
-
- val, err := log.getGenMetadata(devid, gnum)
- if err == nil || val != nil {
- t.Errorf("GetGenMetadata() found non-existent object %s:%d in log file %s: %v, err %v",
- devid, gnum, logfile, val, err)
- }
-
- if log.hasGenMetadata(devid, gnum) {
- t.Errorf("hasGenMetadata() found non-existent object %s:%d in log file %s",
- devid, gnum, logfile)
- }
-
- val = &genMetadata{Pos: 40, Count: 100, MaxLSN: 99}
- if err := log.putGenMetadata(devid, gnum, val); err != nil {
- t.Errorf("Cannot put object %s:%d in log file %s, err %v", devid, gnum, logfile, err)
- }
-
- for i := 0; i < 2; i++ {
- curVal, err := log.getGenMetadata(devid, gnum)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%d (i=%d) in log file %s, err %v",
- devid, gnum, i, logfile, err)
- }
-
- if !log.hasGenMetadata(devid, gnum) {
- t.Errorf("hasGenMetadata() can not find object %s:%d (i=%d) in log file %s",
- devid, gnum, i, logfile)
- }
-
- if !reflect.DeepEqual(val, curVal) {
- t.Errorf("Data mismatch for object %s:%d (i=%d) in log file %s: %v instead of %v",
- devid, gnum, i, logfile, curVal, val)
- }
-
- if i == 0 {
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- }
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestDelGenMetadata tests deleting generation metadata across log open/close/reopen.
-func TestDelGenMetadata(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- var devid DeviceID = "VeyronTab"
- var gnum GenID = 100
-
- val := &genMetadata{Pos: 40, Count: 100, MaxLSN: 99}
- if err := log.putGenMetadata(devid, gnum, val); err != nil {
- t.Errorf("Cannot put object %s:%d in log file %s, err %v", devid, gnum, logfile, err)
- }
-
- curVal, err := log.getGenMetadata(devid, gnum)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%d in log file %s, err %v",
- devid, gnum, logfile, err)
- }
-
- if err := log.delGenMetadata(devid, gnum); err != nil {
- t.Fatalf("DelGenMetadata() can not delete object %s:%d in log file %s, err %v",
- devid, gnum, logfile, err)
- }
-
- for i := 0; i < 2; i++ {
- curVal, err := log.getGenMetadata(devid, gnum)
- if err == nil || curVal != nil {
- t.Fatalf("GetGenMetadata() finds deleted object %s:%d (i=%d) in log file %s, err %v",
- devid, gnum, i, logfile, err)
- }
-
- if log.hasGenMetadata(devid, gnum) {
- t.Errorf("hasGenMetadata() finds deleted object %s:%d (i=%d) in log file %s",
- devid, gnum, i, logfile)
- }
-
- if i == 0 {
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- }
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestPersistLogState tests that generation metadata and record state
-// is persisted across log open/close/reopen.
-func TestPersistLogState(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- var devid DeviceID = "VeyronTab"
-
- // Add several generations.
- for i := uint32(0); i < 10; i++ {
- val := &genMetadata{Pos: i}
- if err := log.putGenMetadata(devid, GenID(i+10), val); err != nil {
- t.Errorf("Cannot put object %s:%d in log file %s, err %v", devid, i, logfile, err)
- }
- }
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- for i := uint32(0); i < 10; i++ {
- curVal, err := log.getGenMetadata(devid, GenID(i+10))
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%d in log file %s, err %v",
- devid, i, logfile, err)
- }
- if curVal.Pos != i {
- t.Errorf("Data mismatch for object %s:%d in log file %s: %v",
- devid, i, logfile, curVal)
- }
- // Should safely overwrite the same keys.
- curVal.Pos = i + 10
- if err := log.putGenMetadata(devid, GenID(i+10), curVal); err != nil {
- t.Errorf("Cannot put object %s:%d in log file %s, err %v", devid, i, logfile, err)
- }
- }
- for i := uint32(0); i < 10; i++ {
- curVal, err := log.getGenMetadata(devid, GenID(i+10))
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%d in log file %s, err %v",
- devid, i, logfile, err)
- }
- if curVal.Pos != (i + 10) {
- t.Errorf("Data mismatch for object %s:%d in log file %s: %v, err %v",
- devid, i, logfile, curVal, err)
- }
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// fillFakeLogRecords fills fake log records for testing purposes.
-func (l *iLog) fillFakeLogRecords() {
- const num = 10
- var parvers []raw.Version
- id := storage.NewID()
- for i := int(0); i < num; i++ {
- // Create a local log record.
- curvers := raw.Version(i)
- rec, err := l.createLocalLogRec(id, curvers, parvers, &LogValue{})
- if err != nil {
- return
- }
- // Insert the new log record into the log.
- _, err = l.putLogRec(rec)
- if err != nil {
- return
- }
- parvers = []raw.Version{curvers}
- }
-}
-
-// TestCreateGeneration tests that local log records and local
-// generations are created uniquely and remote generations are
-// correctly inserted in log order.
-func TestCreateGeneration(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- s := &syncd{id: "VeyronTab"}
- log, err := openILog(logfile, s)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- if g, err := log.createLocalGeneration(); err != errNoUpdates {
- t.Errorf("Should not find local updates gen %d with error %v", g, err)
- }
-
- const num = 10
- var parvers []raw.Version
- id := storage.NewID()
- for i := int(0); i < num; i++ {
- // Create a local log record.
- curvers := raw.Version(i)
- rec, err := log.createLocalLogRec(id, curvers, parvers, &LogValue{})
- if err != nil {
- t.Fatalf("Cannot create local log rec ObjID: %v Current: %s Parents: %v Error: %v",
- id, curvers, parvers, err)
- }
-
- temprec := &LogRec{
- DevID: log.s.id,
- GNum: GenID(1),
- LSN: LSN(i),
- ObjID: id,
- CurVers: curvers,
- Parents: parvers,
- Value: LogValue{},
- }
- // Verify that the log record has the right values.
- if !reflect.DeepEqual(rec, temprec) {
- t.Errorf("Data mismtach in log record %v instead of %v", rec, temprec)
- }
-
- // Insert the new log record into the log.
- _, err = log.putLogRec(rec)
- if err != nil {
- t.Errorf("Cannot put log record:: failed with err %v", err)
- }
-
- parvers = []raw.Version{curvers}
- }
-
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-
- log, err = openILog(logfile, s)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- if log.head.Curgen != 1 || log.head.Curlsn != num {
- t.Errorf("Data mismatch in log header %v", log.head)
- }
-
- g, err := log.createLocalGeneration()
- if g != 1 || err != nil {
- t.Errorf("Could not create local generation gen %d with error %v", g, err)
- }
- curVal, err := log.getGenMetadata(log.s.id, g)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%d in log file %s, err %v",
- log.s.id, g, logfile, err)
- }
- expVal := &genMetadata{Pos: 0, Count: 10, MaxLSN: 9}
- if !reflect.DeepEqual(expVal, curVal) {
- t.Errorf("Data mismatch for object %s:%d in log file %s: %v instead of %v",
- log.s.id, g, logfile, curVal, expVal)
- }
- if log.head.Curgen != 2 || log.head.Curlsn != 0 || log.head.Curorder != 1 {
- t.Errorf("Data mismatch in log header %v", log.head)
- }
-
- if g, err := log.createLocalGeneration(); err != errNoUpdates {
- t.Errorf("Should not find local updates gen %d with error %v", g, err)
- }
-
- // Populate one more generation.
- log.fillFakeLogRecords()
-
- g, err = log.createLocalGeneration()
- if g != 2 || err != nil {
- t.Errorf("Could not create local generation gen %d with error %v", g, err)
- }
-
- // Create a remote generation.
- expGen := &genMetadata{Count: 1, MaxLSN: 50}
- if err = log.createRemoteGeneration("VeyronPhone", 1, expGen); err == nil {
- t.Errorf("Remote generation create should have failed", g, err)
- }
- expGen.MaxLSN = 0
- if err = log.createRemoteGeneration("VeyronPhone", 1, expGen); err != nil {
- t.Errorf("createRemoteGeneration failed with err %v", err)
- }
- if expGen.Pos != 2 {
- t.Errorf("createRemoteGeneration created incorrect log order %d", expGen.Pos)
- }
-
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-
- // Reopen the log and ensure that all log records for generations exist.
- // Also ensure that generation metadata is accurate.
- log, err = openILog(logfile, s)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- for g := GenID(1); g < 3; g++ {
- // Check all log records.
- for i := LSN(0); i < 10; i++ {
- curRec, err := log.getLogRec(log.s.id, g, i)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:%d:%d in log file %s, err %v",
- log.s.id, g, i, logfile, err)
- }
- }
- // Check generation metadata.
- curVal, err := log.getGenMetadata(log.s.id, g)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%d in log file %s, err %v",
- log.s.id, g, logfile, err)
- }
- expVal.Pos = uint32(g - 1)
- if !reflect.DeepEqual(expVal, curVal) {
- t.Errorf("Data mismatch for object %s:%d in log file %s: %v instead of %v",
- log.s.id, g, logfile, curVal, expVal)
- }
- }
-
- // Check remote generation metadata.
- curVal, err = log.getGenMetadata("VeyronPhone", 1)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object in log file %s, err %v",
- logfile, err)
- }
- if !reflect.DeepEqual(expGen, curVal) {
- t.Errorf("Data mismatch for object in log file %s: %v instead of %v",
- logfile, curVal, expGen)
- }
- if log.head.Curgen != 3 || log.head.Curlsn != 0 || log.head.Curorder != 3 {
- t.Errorf("Data mismatch in log header %v", log.head)
- }
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestProcessWatchRecord tests that local updates are correctly handled.
-// Commands are in file testdata/local-init-00.log.sync.
-func TestProcessWatchRecord(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- dagfile := getFileName()
- defer os.Remove(dagfile)
-
- var err error
- s := &syncd{id: "VeyronTab"}
- if s.dag, err = openDAG(dagfile); err != nil {
- t.Fatalf("Cannot open new dag file %s, err %v", logfile, err)
- }
- log, err := openILog(logfile, s)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- if _, err = logReplayCommands(log, "local-init-00.log.sync"); err != nil {
- t.Error(err)
- }
-
- if log.head.Curgen != 1 || log.head.Curlsn != 3 || log.head.Curorder != 0 {
- t.Errorf("Data mismatch in log header %v", log.head)
- }
-
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
-
- // Check all log records.
- for i := LSN(0); i < 3; i++ {
- curRec, err := log.getLogRec(log.s.id, GenID(1), i)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:%d:%d in log file %s, err %v",
- log.s.id, 0, i, logfile, err)
- }
-
- if curRec.ObjID != objid {
- t.Errorf("Data mismatch in log record %v", curRec)
- }
- }
-
- // Verify DAG state.
- if head, err := log.s.dag.getHead(objid); err != nil || head != 3 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-
- s.dag.flush()
- s.dag.close()
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestILogCompact tests compacting of ilog's kvdb file.
-func TestILogCompact(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- s := &syncd{id: "VeyronTab"}
- log, err := openILog(logfile, s)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- // Put some data in "records" table.
- log.fillFakeLogRecords()
- if err := log.flush(); err != nil {
- t.Errorf("Cannot flush ILog file %s, err %v", logfile, err)
- }
-
- // Put some data in "gens" table.
- for i := uint32(0); i < 10; i++ {
- val := &genMetadata{Pos: i, Count: uint64(i + 100)}
- if err := log.putGenMetadata(s.id, GenID(i+10), val); err != nil {
- t.Errorf("Cannot put object %s:%d in log file %s, err %v", s.id, i, logfile, err)
- }
- // Flush immediately to let the kvdb file grow.
- if err := log.flush(); err != nil {
- t.Errorf("Cannot flush ILog file %s, err %v", logfile, err)
- }
- }
-
- // Put some data in "header" table.
- log.head = &iLogHeader{
- Curgen: 1000,
- Curlsn: 10,
- Curorder: 100,
- }
- if err := log.flush(); err != nil {
- t.Errorf("Cannot flush ILog file %s, err %v", logfile, err)
- }
-
- // Get size before compaction.
- oldSize := getFileSize(logfile)
- if oldSize < 0 {
- t.Fatalf("Log file %s not created", logfile)
- }
-
- if err := log.compact(); err != nil {
- t.Errorf("Cannot compact ILog file %s, err %v", logfile, err)
- }
-
- // Verify size of kvdb file is reduced.
- size := getFileSize(logfile)
- if size < 0 {
- t.Fatalf("ILog file %s not created", logfile)
- }
- if size > oldSize {
- t.Fatalf("ILog file %s not compacted", logfile)
- }
-
- // Check data exists after compaction.
- for i := LSN(0); i < 10; i++ {
- curRec, err := log.getLogRec(log.s.id, GenID(1), i)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:1:%d in log file %s, err %v",
- log.s.id, i, logfile, err)
- }
- if curRec.CurVers != raw.Version(i) {
- t.Errorf("Data mismatch for logrec %s:1:%d in log file %s: %v",
- log.s.id, i, logfile, curRec)
- }
- }
-
- for i := uint32(0); i < 10; i++ {
- curVal, err := log.getGenMetadata(log.s.id, GenID(i+10))
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%d in log file %s, err %v",
- log.s.id, i, logfile, err)
- }
- if curVal.Pos != i || curVal.Count != uint64(i+100) {
- t.Errorf("Data mismatch for object %s:%d in log file %s: %v",
- log.s.id, i, logfile, curVal)
- }
- }
-
- log.head.Curgen = 0
- log.head.Curlsn = 0
- log.head.Curorder = 0
-
- if err := log.getHead(); err != nil {
- t.Fatalf("getHead() can not find head in log file %s, err %v", logfile, err)
- }
-
- if log.head.Curgen != 1000 && log.head.Curlsn != 10 && log.head.Curorder != 100 {
- t.Errorf("Data mismatch for head in log file %s: %v", logfile, log.head)
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
diff --git a/runtimes/google/vsync/initiator.go b/runtimes/google/vsync/initiator.go
deleted file mode 100644
index b1d39e0..0000000
--- a/runtimes/google/vsync/initiator.go
+++ /dev/null
@@ -1,736 +0,0 @@
-package vsync
-
-import (
- "errors"
- "fmt"
- "math/rand"
- "strings"
- "time"
-
- "veyron/services/store/raw"
-
- "veyron2/context"
- "veyron2/naming"
- "veyron2/rt"
- "veyron2/storage"
- "veyron2/vlog"
-)
-
-// Policies to pick a peer to sync with.
-const (
- // Picks a peer at random from the available set.
- selectRandom = iota
-
- // TODO(hpucha): implement other policies.
- // Picks a peer with most differing generations.
- selectMostDiff
-
- // Picks a peer that was synced with the furthest in the past.
- selectOldest
-)
-
-// Policies for conflict resolution.
-const (
- // Resolves conflicts by picking the mutation with the most recent timestamp.
- useTime = iota
-
- // TODO(hpucha): implement other policies.
- // Resolves conflicts by using the app conflict resolver callbacks via store.
- useCallback
-)
-
-var (
- // peerSyncInterval is the duration between two consecutive
- // sync events. In every sync event, the initiator contacts
- // one of its peers to obtain any pending updates.
- peerSyncInterval = 50 * time.Millisecond
-
- // peerSelectionPolicy is the policy used to select a peer when
- // the initiator gets a chance to sync.
- peerSelectionPolicy = selectRandom
-
- // conflictResolutionPolicy is the policy used to resolve conflicts.
- conflictResolutionPolicy = useTime
-
- errNoUsefulPeer = errors.New("no useful peer to contact")
-)
-
-// syncInitiator contains the metadata and state for the initiator thread.
-type syncInitiator struct {
- syncd *syncd
-
- // State to contact peers periodically and get deltas.
- // TODO(hpucha): This is an initial version with command line arguments.
- // Next steps are to tie this up into mount table and auto-discover neighbors.
- neighbors []string
- neighborIDs []string
-
- updObjects map[storage.ID]*objConflictState
-}
-
-// objConflictState contains the conflict state for objects that are
-// updated during an initiator run.
-type objConflictState struct {
- isConflict bool
- newHead raw.Version
- oldHead raw.Version
- ancestor raw.Version
- resolvVal *LogValue
-}
-
-// newInitiator creates a new initiator instance attached to the given syncd instance.
-func newInitiator(syncd *syncd, peerEndpoints, peerDeviceIDs string, syncTick time.Duration) *syncInitiator {
- i := &syncInitiator{syncd: syncd,
- updObjects: make(map[storage.ID]*objConflictState),
- }
-
- // Bootstrap my peer list.
- if peerEndpoints != "" || peerDeviceIDs != "" {
- i.neighbors = strings.Split(peerEndpoints, ",")
- i.neighborIDs = strings.Split(peerDeviceIDs, ",")
- if len(i.neighbors) != len(i.neighborIDs) {
- vlog.Fatalf("newInitiator: Mismatch between number of endpoints and IDs")
- }
-
- // Neighbor IDs must be distinct and different from my ID.
- neighborIDs := make(map[string]struct{})
- for _, nID := range i.neighborIDs {
- if DeviceID(nID) == i.syncd.id {
- vlog.Fatalf("newInitiator: neighboor ID %v cannot be the same as my ID %v", nID, i.syncd.id)
- }
- if _, ok := neighborIDs[nID]; ok {
- vlog.Fatalf("newInitiator: neighboor ID %v is duplicated", nID)
- }
- neighborIDs[nID] = struct{}{}
- }
- }
-
- // Override the default peerSyncInterval value if syncTick is specified.
- if syncTick > 0 {
- peerSyncInterval = syncTick
- }
-
- vlog.VI(1).Infof("newInitiator: My device ID: %s", i.syncd.id)
- vlog.VI(1).Infof("newInitiator: Peer endpoints: %v", i.neighbors)
- vlog.VI(1).Infof("newInitiator: Peer IDs: %v", i.neighborIDs)
- vlog.VI(1).Infof("newInitiator: Sync interval: %v", peerSyncInterval)
-
- return i
-}
-
-// contactPeers wakes up every peerSyncInterval to contact peers and get deltas from them.
-func (i *syncInitiator) contactPeers() {
- ticker := time.NewTicker(peerSyncInterval)
- for {
- select {
- case <-i.syncd.closed:
- ticker.Stop()
- i.syncd.pending.Done()
- return
- case <-ticker.C:
- }
-
- id, ep, err := i.pickPeer()
- if err != nil {
- continue
- }
-
- // Freeze the most recent batch of local changes
- // before fetching remote changes from a peer.
- //
- // We only allow an initiator to create new local
- // generations (not responders/watcher) in order to
- // maintain a static baseline for the duration of a
- // sync. This addresses the following race condition:
- // If we allow responders to create new local
- // generations while the initiator is in progress,
- // they may beat the initiator and send these new
- // generations to remote devices. These remote
- // devices in turn can send these generations back to
- // the initiator in progress which was started with
- // older generation information.
- local, err := i.updateLocalGeneration()
- if err != nil {
- vlog.Fatalf("contactPeers:: error updating local generation: err %v", err)
- }
-
- i.getDeltasFromPeer(id, ep, local)
- }
-}
-
-// pickPeer picks a sync endpoint in the neighborhood to sync with.
-func (i *syncInitiator) pickPeer() (string, string, error) {
- switch peerSelectionPolicy {
- case selectRandom:
- // Pick a neighbor at random.
- if i.neighbors == nil {
- return "", "", errNoUsefulPeer
- }
- ind := rand.Intn(len(i.neighbors))
- return i.neighborIDs[ind], i.neighbors[ind], nil
- default:
- return "", "", fmt.Errorf("unknown peer selection policy")
- }
-}
-
-// updateLocalGeneration creates a new local generation if needed and
-// returns the newest local generation vector.
-func (i *syncInitiator) updateLocalGeneration() (GenVector, error) {
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.Lock()
- defer i.syncd.lock.Unlock()
-
- // Create a new local generation if there are any local updates.
- gen, err := i.syncd.log.createLocalGeneration()
- if err == errNoUpdates {
- vlog.VI(1).Infof("createLocalGeneration:: No new updates. Local at %d", gen)
- return i.syncd.devtab.getGenVec(i.syncd.id)
- }
- if err != nil {
- return GenVector{}, err
- }
-
- vlog.VI(2).Infof("updateLocalGeneration:: created gen %d", gen)
- // Update local generation vector in devTable.
- if err = i.syncd.devtab.updateGeneration(i.syncd.id, i.syncd.id, gen); err != nil {
- return GenVector{}, err
- }
- return i.syncd.devtab.getGenVec(i.syncd.id)
-}
-
-// getDeltasFromPeer contacts the specified endpoint to obtain deltas wrt its current generation vector.
-func (i *syncInitiator) getDeltasFromPeer(dID, ep string, local GenVector) {
- ctx, cancel := rt.R().NewContext().WithTimeout(time.Minute)
- defer cancel()
-
- vlog.VI(1).Infof("GetDeltasFromPeer:: From server %s with DeviceID %s at %v", ep, dID, time.Now().UTC())
-
- // Construct a new stub that binds to peer endpoint.
- c, err := BindSync(naming.JoinAddressName(ep, "sync"))
- if err != nil {
- vlog.Errorf("GetDeltasFromPeer:: error binding to server: err %v", err)
- return
- }
-
- vlog.VI(1).Infof("GetDeltasFromPeer:: Sending local information: %v", local)
-
- // Issue a GetDeltas() rpc.
- stream, err := c.GetDeltas(ctx, local, i.syncd.id)
- if err != nil {
- vlog.Errorf("GetDeltasFromPeer:: error getting deltas: err %v", err)
- return
- }
-
- minGens, err := i.processLogStream(stream)
- if err != nil {
- vlog.Fatalf("GetDeltasFromPeer:: error processing logs: err %v", err)
- }
-
- remote, err := stream.Finish()
- if err != nil {
- vlog.Fatalf("GetDeltasFromPeer:: finish failed with err %v", err)
- }
-
- if err := i.processUpdatedObjects(ctx, local, minGens, remote, DeviceID(dID)); err != nil {
- vlog.Fatalf("GetDeltasFromPeer:: error processing objects: err %v", err)
- }
-
- vlog.VI(1).Infof("GetDeltasFromPeer:: Local vector %v", local)
- vlog.VI(1).Infof("GetDeltasFromPeer:: Remote vector %v", remote)
-}
-
-// processLogStream replays an entire log stream spanning multiple
-// generations from different devices received from a single GetDeltas
-// call. It does not perform any conflict resolution during replay.
-// This avoids resolving conflicts that have already been resolved by
-// other devices.
-func (i *syncInitiator) processLogStream(stream SyncGetDeltasCall) (GenVector, error) {
- // Map to track new generations received in the Call reply.
- // TODO(hpucha): If needed, this can be optimized under the
- // assumption that an entire generation is received
- // sequentially. We can then parse a generation at a time.
- newGens := make(map[string]*genMetadata)
- // Array to track order of arrival for the generations.
- // We need to preserve this order.
- var orderGens []string
- // Compute the minimum generation for every device in this set.
- minGens := GenVector{}
-
- curTx := NoTxID
- rStream := stream.RecvStream()
- for rStream.Advance() {
- rec := rStream.Value()
-
- // Begin a new transaction if needed.
- if curTx == NoTxID && rec.Value.Continued {
- curTx = i.syncd.dag.addNodeTxStart()
- vlog.VI(2).Infof("processLogStream:: Begin Tx %v", curTx)
- }
-
- if err := i.insertRecInLogAndDag(&rec, curTx); err != nil {
- return GenVector{}, err
- }
-
- // End the previous transaction if any.
- if curTx != NoTxID && !rec.Value.Continued {
- if err := i.syncd.dag.addNodeTxEnd(curTx); err != nil {
- return GenVector{}, err
- }
- vlog.VI(2).Infof("processLogStream:: End Tx %v", curTx)
- curTx = NoTxID
- }
-
- // Mark object dirty.
- i.updObjects[rec.ObjID] = &objConflictState{}
-
- // Populate the generation metadata.
- genKey := generationKey(rec.DevID, rec.GNum)
- if gen, ok := newGens[genKey]; !ok {
- // New generation in the stream.
- orderGens = append(orderGens, genKey)
- newGens[genKey] = &genMetadata{
- Count: 1,
- MaxLSN: rec.LSN,
- }
- g, ok := minGens[rec.DevID]
- if !ok || g > rec.GNum {
- minGens[rec.DevID] = rec.GNum
- }
- } else {
- gen.Count++
- if rec.LSN > gen.MaxLSN {
- gen.MaxLSN = rec.LSN
- }
- }
- }
-
- if err := rStream.Err(); err != nil {
- return GenVector{}, err
- }
- if curTx != NoTxID {
- return GenVector{}, fmt.Errorf("incomplete transaction in a generation")
- }
- if err := i.createGenMetadataBatch(newGens, orderGens); err != nil {
- return GenVector{}, err
- }
-
- return minGens, nil
-}
-
-// insertLogAndDag adds a new log record to log and dag data structures.
-func (i *syncInitiator) insertRecInLogAndDag(rec *LogRec, txID TxID) error {
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.Lock()
- defer i.syncd.lock.Unlock()
-
- logKey, err := i.syncd.log.putLogRec(rec)
- if err != nil {
- return err
- }
-
- vlog.VI(2).Infof("insertRecInLogAndDag:: Adding log record %v, Tx %v", rec, txID)
- switch rec.RecType {
- case NodeRec:
- return i.syncd.dag.addNode(rec.ObjID, rec.CurVers, true, rec.Value.Delete, rec.Parents, logKey, txID)
- case LinkRec:
- return i.syncd.dag.addParent(rec.ObjID, rec.CurVers, rec.Parents[0], true)
- default:
- return fmt.Errorf("unknown log record type")
- }
-}
-
-// createGenMetadataBatch inserts a batch of generations into the log.
-func (i *syncInitiator) createGenMetadataBatch(newGens map[string]*genMetadata, orderGens []string) error {
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.Lock()
- defer i.syncd.lock.Unlock()
-
- for _, key := range orderGens {
- gen := newGens[key]
- // Insert the generation metadata.
- dev, gnum, err := splitGenerationKey(key)
- if err != nil {
- return err
- }
- if err := i.syncd.log.createRemoteGeneration(dev, gnum, gen); err != nil {
- return err
- }
- }
-
- return nil
-}
-
-// processUpdatedObjects processes all the updates received by the
-// initiator, one object at a time. For each updated object, we first
-// check if the object has any conflicts, resulting in three
-// possibilities:
-//
-// * There is no conflict, and no updates are needed to the store
-// (isConflict=false, newHead == oldHead). All changes received convey
-// information that still keeps the local head as the most recent
-// version. This occurs when conflicts are resolved by blessings.
-//
-// * There is no conflict, but a remote version is discovered that
-// builds on the local head (isConflict=false, newHead != oldHead). In
-// this case, we generate a store mutation to simply update the store
-// to the latest value.
-//
-// * There is a conflict and we call into the app or the system to
-// resolve the conflict, resulting in three possibilties: (a) conflict
-// was resolved by blessing the local version. In this case, store
-// need not be updated, but a link is added to record the
-// blessing. (b) conflict was resolved by blessing the remote
-// version. In this case, store is updated with the remote version and
-// a link is added as well. (c) conflict was resolved by generating a
-// new store mutation. In this case, store is updated with the new
-// version.
-//
-// We then put all these mutations in the store. If the put succeeds,
-// we update the log and dag state suitably (move the head ptr of the
-// object in the dag to the latest version, and create a new log
-// record reflecting conflict resolution if any). Puts to store can
-// fail since preconditions on the objects may have been violated. In
-// this case, we wait to get the latest versions of objects from the
-// store, and recheck if the object has any conflicts and repeat the
-// above steps, until put to store succeeds.
-func (i *syncInitiator) processUpdatedObjects(ctx context.T, local, minGens, remote GenVector, dID DeviceID) error {
- for {
- if err := i.detectConflicts(); err != nil {
- return err
- }
-
- if err := i.resolveConflicts(); err != nil {
- return err
- }
-
- err := i.updateStoreAndSync(ctx, local, minGens, remote, dID)
- if err == nil {
- break
- }
-
- vlog.Errorf("PutMutations failed %v. Will retry", err)
- // TODO(hpucha): Sleeping and retrying is a temporary
- // solution. Next iteration will have coordination
- // with watch thread to intelligently retry. Hence
- // this value is not a config param.
- time.Sleep(1 * time.Second)
- }
-
- // Remove any pending state.
- i.updObjects = make(map[storage.ID]*objConflictState)
- i.syncd.dag.clearGraft()
- return nil
-}
-
-// detectConflicts iterates through all the updated objects to detect
-// conflicts.
-func (i *syncInitiator) detectConflicts() error {
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.RLock()
- defer i.syncd.lock.RUnlock()
-
- for obj, st := range i.updObjects {
- // Check if object has a conflict.
- var err error
- st.isConflict, st.newHead, st.oldHead, st.ancestor, err = i.syncd.dag.hasConflict(obj)
- vlog.VI(2).Infof("detectConflicts:: object %v state %v err %v",
- obj, st, err)
- if err != nil {
- return err
- }
- }
- return nil
-}
-
-// resolveConflicts resolves conflicts for updated objects. Conflicts
-// may be resolved by adding new versions or blessing either the local
-// or the remote version.
-func (i *syncInitiator) resolveConflicts() error {
- switch conflictResolutionPolicy {
- case useTime:
- if err := i.resolveConflictsByTime(); err != nil {
- return err
- }
- default:
- return fmt.Errorf("unknown conflict resolution policy")
- }
- return nil
-}
-
-// resolveConflictsByTime resolves conflicts using the timestamps
-// of the conflicting mutations. It picks a mutation with the larger
-// timestamp, i.e. the most recent update. If the timestamps are equal,
-// it uses the mutation version numbers as a tie-breaker, picking the
-// mutation with the larger version.
-//
-// TODO(hpucha): Based on a few more policies, reconsider nesting
-// order of the conflict resolution loop and switch-on-policy.
-func (i *syncInitiator) resolveConflictsByTime() error {
- for obj, st := range i.updObjects {
- if !st.isConflict {
- continue
- }
-
- versions := make([]raw.Version, 3)
- versions[0] = st.oldHead
- versions[1] = st.newHead
- versions[2] = st.ancestor
-
- lrecs, err := i.getLogRecsBatch(obj, versions)
- if err != nil {
- return err
- }
-
- res := 0
- switch {
- case lrecs[0].Value.SyncTime > lrecs[1].Value.SyncTime:
- res = 0
- case lrecs[0].Value.SyncTime < lrecs[1].Value.SyncTime:
- res = 1
- case lrecs[0].Value.Mutation.Version > lrecs[1].Value.Mutation.Version:
- res = 0
- case lrecs[0].Value.Mutation.Version < lrecs[1].Value.Mutation.Version:
- res = 1
- }
-
- // Instead of creating a new version that resolves the
- // conflict, we are blessing an existing version as
- // the conflict resolution.
- st.resolvVal = &lrecs[res].Value
- }
-
- return nil
-}
-
-// getLogRecsBatch gets the log records for an array of versions.
-func (i *syncInitiator) getLogRecsBatch(obj storage.ID, versions []raw.Version) ([]*LogRec, error) {
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.RLock()
- defer i.syncd.lock.RUnlock()
-
- lrecs := make([]*LogRec, len(versions))
- var err error
- for p, v := range versions {
- lrecs[p], err = i.getLogRec(obj, v)
- if err != nil {
- return nil, err
- }
- }
- return lrecs, nil
-}
-
-// updateStoreAndSync updates the store, and if that is successful,
-// updates log and dag data structures.
-func (i *syncInitiator) updateStoreAndSync(ctx context.T, local, minGens, remote GenVector, dID DeviceID) error {
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.Lock()
- defer i.syncd.lock.Unlock()
-
- var m []raw.Mutation
- for obj, st := range i.updObjects {
- if !st.isConflict {
- rec, err := i.getLogRec(obj, st.newHead)
- if err != nil {
- return err
- }
- st.resolvVal = &rec.Value
- // Sanity check.
- if st.resolvVal.Mutation.Version != st.newHead {
- return fmt.Errorf("bad mutation %d %d",
- st.resolvVal.Mutation.Version, st.newHead)
- }
- }
-
- // If the local version is picked, no further updates
- // to the store are needed. If the remote version is
- // picked, we put it in the store.
- if st.resolvVal.Mutation.Version != st.oldHead {
- st.resolvVal.Mutation.PriorVersion = st.oldHead
-
- // Convert resolvVal.Mutation into a mutation for the Store.
- stMutation, err := i.storeMutation(obj, st.resolvVal)
- if err != nil {
- return err
- }
-
- vlog.VI(2).Infof("updateStoreAndSync:: Try to append mutation %v (%v) for obj %v (nh %v, oh %v)",
- st.resolvVal.Mutation, stMutation, obj, st.newHead, st.oldHead)
-
- // Append to mutations, skipping a delete following a delete mutation.
- if stMutation.Version != raw.NoVersion ||
- stMutation.PriorVersion != raw.NoVersion {
- vlog.VI(2).Infof("updateStoreAndSync:: appending mutation %v for obj %v",
- stMutation, obj)
- m = append(m, stMutation)
- }
- }
- }
-
- // TODO(hpucha): We will hold the lock across PutMutations rpc
- // to prevent a race with watcher. The next iteration will
- // clean up this coordination.
- if store := i.syncd.store; store != nil && len(m) > 0 {
- stream, err := store.PutMutations(ctx)
- if err != nil {
- vlog.Errorf("updateStoreAndSync:: putmutations err %v", err)
- return err
- }
- sender := stream.SendStream()
- for i := range m {
- if err := sender.Send(m[i]); err != nil {
- vlog.Errorf("updateStoreAndSync:: send err %v", err)
- return err
- }
- }
- if err := sender.Close(); err != nil {
- vlog.Errorf("updateStoreAndSync:: closesend err %v", err)
- return err
- }
- if err := stream.Finish(); err != nil {
- vlog.Errorf("updateStoreAndSync:: finish err %v", err)
- return err
- }
- }
-
- vlog.VI(2).Infof("updateStoreAndSync:: putmutations succeeded")
- if err := i.updateLogAndDag(); err != nil {
- return err
- }
-
- if err := i.updateGenVecs(local, minGens, remote, DeviceID(dID)); err != nil {
- return err
- }
-
- return nil
-}
-
-// storeMutation converts a resolved mutation generated by syncd to
-// one that can be sent to the store. To send to the store, it
-// converts the version numbers corresponding to object deletions to
-// raw.NoVersion when required.
-func (i *syncInitiator) storeMutation(obj storage.ID, resolvVal *LogValue) (raw.Mutation, error) {
- curDelete := resolvVal.Delete
- priorDelete := false
- if resolvVal.Mutation.PriorVersion != raw.NoVersion {
- oldRec, err := i.getLogRec(obj, resolvVal.Mutation.PriorVersion)
- if err != nil {
- return raw.Mutation{}, err
- }
- priorDelete = oldRec.Value.Delete
- }
-
- // Current version and prior versions are not deletes.
- if !curDelete && !priorDelete {
- return resolvVal.Mutation, nil
- }
-
- // Creating a new copy of the mutation to adjust version
- // numbers when handling deletions.
- stMutation := resolvVal.Mutation
- // Adjust the current version if this a deletion.
- if curDelete {
- stMutation.Version = raw.NoVersion
- }
- // Adjust the prior version if it is a deletion.
- if priorDelete {
- stMutation.PriorVersion = raw.NoVersion
- }
-
- return stMutation, nil
-}
-
-// getLogRec returns the log record corresponding to a given object and its version.
-func (i *syncInitiator) getLogRec(obj storage.ID, vers raw.Version) (*LogRec, error) {
- logKey, err := i.syncd.dag.getLogrec(obj, vers)
- if err != nil {
- return nil, err
- }
- dev, gen, lsn, err := splitLogRecKey(logKey)
- if err != nil {
- return nil, err
- }
- rec, err := i.syncd.log.getLogRec(dev, gen, lsn)
- if err != nil {
- return nil, err
- }
- return rec, nil
-}
-
-// updateLogAndDag updates the log and dag data structures on a successful store put.
-func (i *syncInitiator) updateLogAndDag() error {
- for obj, st := range i.updObjects {
- if st.isConflict {
- // Object had a conflict, which was resolved successfully.
- // Put is successful, create a log record.
- var err error
- var rec *LogRec
-
- switch {
- case st.resolvVal.Mutation.Version == st.oldHead:
- // Local version was blessed as the conflict resolution.
- rec, err = i.syncd.log.createLocalLinkLogRec(obj, st.oldHead, st.newHead)
- case st.resolvVal.Mutation.Version == st.newHead:
- // Remote version was blessed as the conflict resolution.
- rec, err = i.syncd.log.createLocalLinkLogRec(obj, st.newHead, st.oldHead)
- default:
- // New version was created to resolve the conflict.
- parents := []raw.Version{st.newHead, st.oldHead}
- rec, err = i.syncd.log.createLocalLogRec(obj, st.resolvVal.Mutation.Version, parents, st.resolvVal)
-
- }
- if err != nil {
- return err
- }
- logKey, err := i.syncd.log.putLogRec(rec)
- if err != nil {
- return err
- }
- // Add a new DAG node.
- switch rec.RecType {
- case NodeRec:
- // TODO(hpucha): addNode operations arising out of conflict resolution
- // may need to be part of a transaction when app-driven resolution
- // is introduced.
- err = i.syncd.dag.addNode(obj, rec.CurVers, false, rec.Value.Delete, rec.Parents, logKey, NoTxID)
- case LinkRec:
- err = i.syncd.dag.addParent(obj, rec.CurVers, rec.Parents[0], false)
- default:
- return fmt.Errorf("unknown log record type")
- }
- if err != nil {
- return err
- }
- }
-
- // Move the head. This should be idempotent. We may
- // move head to the local head in some cases.
- if err := i.syncd.dag.moveHead(obj, st.resolvVal.Mutation.Version); err != nil {
- return err
- }
- }
- return nil
-}
-
-// updateGenVecs updates local, reclaim and remote vectors at the end of an initiator cycle.
-func (i *syncInitiator) updateGenVecs(local, minGens, remote GenVector, dID DeviceID) error {
- // Update the local gen vector and put it in kvdb only if we have new updates.
- if len(i.updObjects) > 0 {
- if err := i.syncd.devtab.updateLocalGenVector(local, remote); err != nil {
- return err
- }
-
- if err := i.syncd.devtab.putGenVec(i.syncd.id, local); err != nil {
- return err
- }
-
- if err := i.syncd.devtab.updateReclaimVec(minGens); err != nil {
- return err
- }
- }
-
- // Cache the remote generation vector for space reclamation.
- if err := i.syncd.devtab.putGenVec(dID, remote); err != nil {
- return err
- }
- return nil
-}
diff --git a/runtimes/google/vsync/initiator_test.go b/runtimes/google/vsync/initiator_test.go
deleted file mode 100644
index d2668e5..0000000
--- a/runtimes/google/vsync/initiator_test.go
+++ /dev/null
@@ -1,1330 +0,0 @@
-package vsync
-
-// Tests for sync initiator.
-import (
- "fmt"
- "os"
- "reflect"
- "testing"
- "time"
-
- "veyron/services/store/raw"
-
- "veyron2/storage"
-)
-
-// TestGetLogRec tests getting a log record from kvdb based on object id and version.
-func TestGetLogRec(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- s.lock.Lock()
- defer s.lock.Unlock()
- defer s.Close()
- defer os.RemoveAll(dir)
-
- // Create some data.
- objID := storage.NewID()
- expRec := &LogRec{
- DevID: "VeyronTab",
- GNum: 50,
- LSN: 100,
- ObjID: objID,
- CurVers: 20,
- Value: LogValue{Mutation: raw.Mutation{Version: 20}},
- }
- if _, err := s.hdlInitiator.getLogRec(objID, expRec.CurVers); err == nil {
- t.Errorf("GetLogRec didn't fail")
- }
- logKey, err := s.log.putLogRec(expRec)
- if err != nil {
- t.Errorf("PutLogRec failed with err %v", err)
- }
- if _, err := s.hdlInitiator.getLogRec(objID, expRec.CurVers); err == nil {
- t.Errorf("GetLogRec didn't fail")
- }
- if err = s.dag.addNode(objID, expRec.CurVers, false, false, expRec.Parents, logKey, NoTxID); err != nil {
- t.Errorf("AddNode failed with err %v", err)
- }
- curRec, err := s.hdlInitiator.getLogRec(objID, expRec.CurVers)
- if err != nil {
- t.Errorf("GetLogRec failed with err %v", err)
- }
- if !reflect.DeepEqual(curRec, expRec) {
- t.Errorf("Data mismatch for %v instead of %v",
- curRec, expRec)
- }
-}
-
-// TestResolveConflictByTime tests the timestamp-based conflict resolution policy.
-func TestResolveConflictByTime(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- objID := storage.NewID()
- s.hdlInitiator.updObjects[objID] = &objConflictState{
- isConflict: true,
- oldHead: 40,
- newHead: 20,
- ancestor: 10,
- }
- versions := []raw.Version{10, 40, 20}
- for _, v := range versions {
- expRec := &LogRec{
- DevID: "VeyronTab",
- GNum: GenID(50 + v),
- LSN: LSN(100 + v),
- ObjID: objID,
- CurVers: v,
- Value: LogValue{Mutation: raw.Mutation{Version: v, PriorVersion: 500 + v}, SyncTime: int64(v)},
- }
- logKey, err := s.log.putLogRec(expRec)
- if err != nil {
- t.Errorf("PutLogRec failed with err %v", err)
- }
- if err = s.dag.addNode(objID, expRec.CurVers, false, false, expRec.Parents, logKey, NoTxID); err != nil {
- t.Errorf("AddNode failed with err %v", err)
- }
- }
-
- if err := s.hdlInitiator.resolveConflictsByTime(); err != nil {
- t.Errorf("ResolveConflictsByTime failed with err %v", err)
- }
- if s.hdlInitiator.updObjects[objID].resolvVal.Mutation.PriorVersion != 540 {
- t.Errorf("Data mismatch for resolution %v", s.hdlInitiator.updObjects[objID].resolvVal)
- }
-}
-
-// TODO(hpucha): Add more tests around retrying failed puts in the next pass (processUpdatedObjects).
-// TestLogStreamRemoteOnly tests processing of a remote log stream. Commands are in file
-// testdata/remote-init-00.log.sync.
-func TestLogStreamRemoteOnly(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- stream, err := createReplayStream("remote-init-00.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
-
- // Check minGens.
- expVec := GenVector{"VeyronPhone": 1}
- if !reflect.DeepEqual(expVec, minGens) {
- t.Errorf("Data mismatch for minGens: %v instead of %v",
- minGens, expVec)
- }
-
- // Check generation metadata.
- curVal, err := s.log.getGenMetadata("VeyronPhone", 1)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object in log file err %v", err)
- }
- expVal := &genMetadata{Pos: 0, Count: 3, MaxLSN: 2}
- if !reflect.DeepEqual(expVal, curVal) {
- t.Errorf("Data mismatch for generation metadata: %v instead of %v",
- curVal, expVal)
- }
-
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- // Check all log records.
- for i := LSN(0); i < 3; i++ {
- curRec, err := s.log.getLogRec("VeyronPhone", GenID(1), i)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %d in log file err %v",
- i, err)
- }
- if curRec.ObjID != objid {
- t.Errorf("Data mismatch in log record %v", curRec)
- }
- // Verify DAG state.
- if _, err := s.dag.getNode(objid, raw.Version(i+1)); err != nil {
- t.Errorf("GetNode() can not find object %d %d in DAG, err %v", objid, i, err)
- }
- }
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- if len(s.hdlInitiator.updObjects) != 1 {
- t.Errorf("Unexpected number of updated objects %d", len(s.hdlInitiator.updObjects))
- }
- st := s.hdlInitiator.updObjects[objid]
- if st.isConflict {
- t.Errorf("Detected a conflict %v", st)
- }
- if st.newHead != 3 || st.oldHead != raw.NoVersion {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
- if st.resolvVal.Mutation.PriorVersion != raw.NoVersion || st.resolvVal.Mutation.Version != 3 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 0 || s.log.head.Curorder != 1 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
- // Verify DAG state.
- if head, err := s.dag.getHead(objid); err != nil || head != 3 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-}
-
-// TestLogStreamGCedRemote tests that a remote log stream can be
-// correctly applied when its generations don't start at 1 and have
-// been GC'ed already. Commands are in file
-// testdata/remote-init-01.log.sync.
-func TestLogStreamGCedRemote(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- stream, err := createReplayStream("remote-init-01.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
-
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
-
- // Check minGens.
- expVec := GenVector{"VeyronPhone": 5}
- if !reflect.DeepEqual(expVec, minGens) {
- t.Errorf("Data mismatch for minGens: %v instead of %v",
- minGens, expVec)
- }
-
- // Check generation metadata.
- curVal, err := s.log.getGenMetadata("VeyronPhone", 5)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object in log file err %v", err)
- }
- expVal := &genMetadata{Pos: 0, Count: 3, MaxLSN: 2}
- if !reflect.DeepEqual(expVal, curVal) {
- t.Errorf("Data mismatch for generation metadata: %v instead of %v",
- curVal, expVal)
- }
-
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- // Check all log records.
- for i := LSN(0); i < 3; i++ {
- curRec, err := s.log.getLogRec("VeyronPhone", GenID(5), i)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %d in log file err %v",
- i, err)
- }
- if curRec.ObjID != objid {
- t.Errorf("Data mismatch in log record %v", curRec)
- }
- // Verify DAG state.
- if _, err := s.dag.getNode(objid, raw.Version(i+1)); err != nil {
- t.Errorf("GetNode() can not find object %d %d in DAG, err %v", objid, i, err)
- }
- }
-
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- if len(s.hdlInitiator.updObjects) != 1 {
- t.Errorf("Unexpected number of updated objects %d", len(s.hdlInitiator.updObjects))
- }
- st := s.hdlInitiator.updObjects[objid]
- if st.isConflict {
- t.Errorf("Detected a conflict %v", st)
- }
- if st.newHead != 3 || st.oldHead != raw.NoVersion {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
- if st.resolvVal.Mutation.PriorVersion != raw.NoVersion || st.resolvVal.Mutation.Version != 3 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 0 || s.log.head.Curorder != 1 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
- // Verify DAG state.
- if head, err := s.dag.getHead(objid); err != nil || head != 3 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-}
-
-// TestLogStreamRemoteWithTx tests processing of a remote log stream
-// that contains transactions. Commands are in file
-// testdata/remote-init-02.log.sync.
-func TestLogStreamRemoteWithTx(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- stream, err := createReplayStream("remote-init-02.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
-
- // Check minGens.
- expVec := GenVector{"VeyronPhone": 1, "VeyronTab": 1}
- if !reflect.DeepEqual(expVec, minGens) {
- t.Errorf("Data mismatch for minGens: %v instead of %v",
- minGens, expVec)
- }
-
- // Verify transaction state.
- objs := []string{"123", "456", "789"}
- objids := make([]storage.ID, 3)
- maxVers := []raw.Version{3, 2, 4}
- txVers := map[string]struct{}{
- "123-2": struct{}{},
- "123-3": struct{}{},
- "456-1": struct{}{},
- "456-2": struct{}{},
- "789-1": struct{}{},
- }
- for pos, o := range objs {
- var err error
- objids[pos], err = strToObjID(o)
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- for i := raw.Version(1); i <= raw.Version(maxVers[pos]); i++ {
- node, err := s.dag.getNode(objids[pos], i)
- if err != nil {
- t.Errorf("cannot find dag node for object %d %v: %s", objids[pos], i, err)
- }
- key := fmt.Sprintf("%s-%d", objs[pos], i)
- _, ok := txVers[key]
- if !ok && node.TxID != NoTxID {
- t.Errorf("expecting NoTxID, found txid %v for object %d:%v", node.TxID, objids[pos], i)
- }
- if ok && node.TxID == NoTxID {
- t.Errorf("expecting non nil txid for object %d:%v", objids[pos], i)
- }
- }
- }
-
- // Verify transaction state for the first transaction.
- node, err := s.dag.getNode(objids[0], raw.Version(2))
- if err != nil {
- t.Errorf("cannot find dag node for object %d v1: %s", objids[0], err)
- }
- if node.TxID == NoTxID {
- t.Errorf("expecting non nil txid for object %d:v1", objids[0])
- }
- txMap, err := s.dag.getTransaction(node.TxID)
- if err != nil {
- t.Errorf("cannot find transaction for id %v: %s", node.TxID, err)
- }
- expTxMap := dagTxMap{
- objids[0]: raw.Version(2),
- objids[1]: raw.Version(1),
- objids[2]: raw.Version(1),
- }
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Data mismatch for txid %v txmap %v instead of %v",
- node.TxID, txMap, expTxMap)
- }
-
- // Verify transaction state for the second transaction.
- node, err = s.dag.getNode(objids[0], raw.Version(3))
- if err != nil {
- t.Errorf("cannot find dag node for object %d v1: %s", objids[0], err)
- }
- if node.TxID == NoTxID {
- t.Errorf("expecting non nil txid for object %d:v1", objids[0])
- }
- txMap, err = s.dag.getTransaction(node.TxID)
- if err != nil {
- t.Errorf("cannot find transaction for id %v: %s", node.TxID, err)
- }
- expTxMap = dagTxMap{
- objids[0]: raw.Version(3),
- objids[1]: raw.Version(2),
- }
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Data mismatch for txid %v txmap %v instead of %v",
- node.TxID, txMap, expTxMap)
- }
-
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- if len(s.hdlInitiator.updObjects) != 3 {
- t.Errorf("Unexpected number of updated objects %d", len(s.hdlInitiator.updObjects))
- }
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
-}
-
-// TestLogStreamRemoteWithDel tests processing of a remote log stream
-// that contains object deletion. Commands are in file
-// testdata/remote-init-03.log.sync.
-func TestLogStreamRemoteWithDel(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- stream, err := createReplayStream("remote-init-03.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
-
- // Check minGens.
- expVec := GenVector{"VeyronPhone": 1}
- if !reflect.DeepEqual(expVec, minGens) {
- t.Errorf("Data mismatch for minGens: %v instead of %v",
- minGens, expVec)
- }
-
- // Check generation metadata.
- curVal, err := s.log.getGenMetadata("VeyronPhone", 1)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object in log file err %v", err)
- }
- expVal := &genMetadata{Pos: 0, Count: 3, MaxLSN: 2}
- if !reflect.DeepEqual(expVal, curVal) {
- t.Errorf("Data mismatch for generation metadata: %v instead of %v",
- curVal, expVal)
- }
-
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- // Check all log records.
- for i := LSN(0); i < 3; i++ {
- curRec, err := s.log.getLogRec("VeyronPhone", GenID(1), i)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %d in log file err %v",
- i, err)
- }
- if curRec.ObjID != objid {
- t.Errorf("Data mismatch in log record %v", curRec)
- }
- // Verify DAG state.
- if _, err := s.dag.getNode(objid, raw.Version(i+1)); err != nil {
- t.Errorf("GetNode() can not find object %d %d in DAG, err %v", objid, i, err)
- }
- }
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- if len(s.hdlInitiator.updObjects) != 1 {
- t.Errorf("Unexpected number of updated objects %d", len(s.hdlInitiator.updObjects))
- }
- st := s.hdlInitiator.updObjects[objid]
- if st.isConflict {
- t.Errorf("Detected a conflict %v", st)
- }
- if st.newHead != 3 || st.oldHead != raw.NoVersion {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
- if st.resolvVal.Mutation.PriorVersion != raw.NoVersion || st.resolvVal.Mutation.Version != 3 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
- m, err := s.hdlInitiator.storeMutation(objid, st.resolvVal)
- if err != nil {
- t.Errorf("Could not translate mutation %v", err)
- }
- if m.Version != raw.NoVersion || m.PriorVersion != raw.NoVersion {
- t.Errorf("Data mismatch in mutation translation %v", m)
- }
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 0 || s.log.head.Curorder != 1 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
-
- // Verify DAG state.
- if head, err := s.dag.getHead(objid); err != nil || head != 3 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
- node, err := s.dag.getNode(objid, raw.Version(3))
- if err != nil {
- t.Errorf("cannot find dag node for object %d v3: %s", objid, err)
- }
- if !node.Deleted {
- t.Errorf("deleted node not found for object %d v3", objid)
- }
- if !s.dag.hasDeletedDescendant(objid, raw.Version(2)) {
- t.Errorf("link to deleted node not found for object %d from v2", objid)
- }
- if !s.dag.hasDeletedDescendant(objid, raw.Version(1)) {
- t.Errorf("link to deleted node not found for object %d from v1", objid)
- }
-}
-
-// TestLogStreamDel2Objs tests that a local and a remote log stream
-// can be correctly applied when there is local and a remote delete on
-// 2 different objects. Commands are in files
-// testdata/<local-init-01.log.sync,remote-2obj-del.log.sync>.
-func TestLogStreamDel2Objs(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- if _, err = logReplayCommands(s.log, "local-init-01.log.sync"); err != nil {
- t.Error(err)
- }
-
- stream, err := createReplayStream("remote-2obj-del.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
-
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
-
- // Check minGens.
- expVec := GenVector{"VeyronPhone": 1}
- if !reflect.DeepEqual(expVec, minGens) {
- t.Errorf("Data mismatch for minGens: %v instead of %v",
- minGens, expVec)
- }
- // Check generation metadata.
- curVal, err := s.log.getGenMetadata("VeyronPhone", 1)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object in log file for VeyronPhone err %v", err)
- }
- expVal := &genMetadata{Pos: 0, Count: 4, MaxLSN: 3}
- if !reflect.DeepEqual(expVal, curVal) {
- t.Errorf("Data mismatch for generation metadata: %v instead of %v",
- curVal, expVal)
- }
-
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- if len(s.hdlInitiator.updObjects) != 2 {
- t.Errorf("Unexpected number of updated objects %d", len(s.hdlInitiator.updObjects))
- }
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
-
- objs := []string{"123", "456"}
- newHeads := []raw.Version{6, 2}
- conflicts := []bool{false, true}
- for pos, o := range objs {
- objid, err := strToObjID(o)
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
-
- st := s.hdlInitiator.updObjects[objid]
-
- if st.isConflict != conflicts[pos] {
- t.Errorf("Detected a wrong conflict %v", st)
- }
- if st.newHead != newHeads[pos] {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
- if pos == 1 {
- // Force a blessing to remote version for testing.
- st.resolvVal.Mutation.Version = st.newHead
- st.resolvVal.Mutation.PriorVersion = st.oldHead
- st.resolvVal.Delete = false
- }
- m, err := s.hdlInitiator.storeMutation(objid, st.resolvVal)
- if err != nil {
- t.Errorf("Could not translate mutation %v", err)
- }
-
- if pos == 0 {
- if st.oldHead != 3 {
- t.Errorf("Conflict detection didn't succeed for obj123 %v", st)
- }
- if m.Version != raw.NoVersion || m.PriorVersion != 3 {
- t.Errorf("Data mismatch in mutation translation for obj123 %v", m)
- }
- // Test echo back from watch for these mutations.
- if err := s.log.processWatchRecord(objid, 0, raw.Version(3), &LogValue{}, NoTxID); err != nil {
- t.Errorf("Echo processing from watch failed %v", err)
- }
- }
-
- if pos == 1 {
- if st.oldHead == raw.NoVersion {
- t.Errorf("Conflict detection didn't succeed for obj456 %v", st)
- }
- if m.Version != 2 || m.PriorVersion != raw.NoVersion {
- t.Errorf("Data mismatch in mutation translation for obj456 %v", m)
- }
- // Test echo back from watch for these mutations.
- if err := s.log.processWatchRecord(objid, raw.Version(2), 0, &LogValue{}, NoTxID); err != nil {
- t.Errorf("Echo processing from watch failed %v", err)
- }
- }
- }
-
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 6 || s.log.head.Curorder != 1 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
-}
-
-// TestLogStreamNoConflict tests that a local and a remote log stream
-// can be correctly applied (when there are no conflicts). Commands
-// are in files
-// testdata/<local-init-00.log.sync,remote-noconf-00.log.sync>.
-func TestLogStreamNoConflict(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- if _, err = logReplayCommands(s.log, "local-init-00.log.sync"); err != nil {
- t.Error(err)
- }
-
- stream, err := createReplayStream("remote-noconf-00.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
-
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
-
- // Check minGens.
- expVec := GenVector{"VeyronPhone": 1}
- if !reflect.DeepEqual(expVec, minGens) {
- t.Errorf("Data mismatch for minGens: %v instead of %v",
- minGens, expVec)
- }
-
- // Check generation metadata.
- curVal, err := s.log.getGenMetadata("VeyronPhone", 1)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object in log file for VeyronPhone err %v", err)
- }
- expVal := &genMetadata{Pos: 0, Count: 3, MaxLSN: 2}
- if !reflect.DeepEqual(expVal, curVal) {
- t.Errorf("Data mismatch for generation metadata: %v instead of %v",
- curVal, expVal)
- }
-
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- // Check all log records.
- for _, devid := range []DeviceID{"VeyronPhone", "VeyronTab"} {
- v := raw.Version(1)
- for i := LSN(0); i < 3; i++ {
- curRec, err := s.log.getLogRec(devid, GenID(1), i)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:%d in log file err %v",
- devid, i, err)
- }
- if curRec.ObjID != objid {
- t.Errorf("Data mismatch in log record %v", curRec)
- }
- // Verify DAG state.
- if _, err := s.dag.getNode(objid, v); err != nil {
- t.Errorf("GetNode() can not find object %d %d in DAG, err %v", objid, i, err)
- }
- v = v + 1
- }
- }
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- if len(s.hdlInitiator.updObjects) != 1 {
- t.Errorf("Unexpected number of updated objects %d", len(s.hdlInitiator.updObjects))
- }
- st := s.hdlInitiator.updObjects[objid]
- if st.isConflict {
- t.Errorf("Detected a conflict %v", st)
- }
- if st.newHead != 6 || st.oldHead != 3 {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
- if st.resolvVal.Mutation.PriorVersion != 3 || st.resolvVal.Mutation.Version != 6 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 3 || s.log.head.Curorder != 1 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
- // Verify DAG state.
- if head, err := s.dag.getHead(objid); err != nil || head != 6 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-}
-
-// TestLogStreamConflict tests that a local and a remote log stream
-// can be correctly applied (when there are conflicts). Commands are
-// in files testdata/<local-init-00.log.sync,remote-conf-00.log.sync>.
-func TestLogStreamConflict(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- conflictResolutionPolicy = useTime
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- if _, err = logReplayCommands(s.log, "local-init-00.log.sync"); err != nil {
- t.Error(err)
- }
-
- stream, err := createReplayStream("remote-conf-00.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
-
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
-
- // Check minGens.
- expVec := GenVector{"VeyronPhone": 1}
- if !reflect.DeepEqual(expVec, minGens) {
- t.Errorf("Data mismatch for minGens: %v instead of %v",
- minGens, expVec)
- }
-
- // Check generation metadata.
- curVal, err := s.log.getGenMetadata("VeyronPhone", 1)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object in log file for VeyronPhone err %v", err)
- }
- expVal := &genMetadata{Pos: 0, Count: 3, MaxLSN: 2}
- if !reflect.DeepEqual(expVal, curVal) {
- t.Errorf("Data mismatch for generation metadata: %v instead of %v",
- curVal, expVal)
- }
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
-
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- lcount := []LSN{3, 4}
- // Check all log records.
- for index, devid := range []DeviceID{"VeyronPhone", "VeyronTab"} {
- v := raw.Version(1)
- for i := LSN(0); i < lcount[index]; i++ {
- curRec, err := s.log.getLogRec(devid, GenID(1), i)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:%d in log file err %v",
- devid, i, err)
- }
- if curRec.ObjID != objid {
- t.Errorf("Data mismatch in log record %v", curRec)
- }
- if devid == "VeyronTab" && index == 3 && curRec.RecType != LinkRec {
- t.Errorf("Data mismatch in log record %v", curRec)
- }
- // Verify DAG state.
- if _, err := s.dag.getNode(objid, v); err != nil {
- t.Errorf("GetNode() can not find object %d %d in DAG, err %v", objid, i, err)
- }
- v = v + 1
- }
- }
- if len(s.hdlInitiator.updObjects) != 1 {
- t.Errorf("Unexpected number of updated objects %d", len(s.hdlInitiator.updObjects))
- }
- st := s.hdlInitiator.updObjects[objid]
- if !st.isConflict {
- t.Errorf("Didn't detect a conflict %v", st)
- }
- if st.newHead != 6 || st.oldHead != 3 {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
- if st.resolvVal.Mutation.PriorVersion != 3 || st.resolvVal.Mutation.Version != 6 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
- // Curlsn == 4 for the log record that resolves conflict.
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 4 || s.log.head.Curorder != 1 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
- // Verify DAG state.
- if head, err := s.dag.getHead(objid); err != nil || head != 6 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-}
-
-// TestMultipleLogStream tests that a local and 2 remote log streams
-// can be correctly applied (when there are conflicts). Commands are
-// in file testdata/<local-init-00.log.sync,remote-conf-01.log.sync>.
-func TestMultipleLogStream(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- conflictResolutionPolicy = useTime
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- if _, err = logReplayCommands(s.log, "local-init-00.log.sync"); err != nil {
- t.Error(err)
- }
-
- stream, err := createReplayStream("remote-conf-01.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
-
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
-
- // Check minGens.
- expVec := GenVector{"VeyronPhone": 1, "VeyronLaptop": 1}
- if !reflect.DeepEqual(expVec, minGens) {
- t.Errorf("Data mismatch for minGens: %v instead of %v",
- minGens, expVec)
- }
-
- // Check generation metadata.
- curVal, err := s.log.getGenMetadata("VeyronLaptop", 1)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object in log file for VeyronPhone err %v", err)
- }
- expVal := &genMetadata{Pos: 0, Count: 1, MaxLSN: 0}
- if !reflect.DeepEqual(expVal, curVal) {
- t.Errorf("Data mismatch for generation metadata: %v instead of %v",
- curVal, expVal)
- }
-
- curVal, err = s.log.getGenMetadata("VeyronPhone", 1)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object in log file for VeyronPhone err %v", err)
- }
- expVal.Pos = 1
- expVal.Count = 2
- expVal.MaxLSN = 1
- if !reflect.DeepEqual(expVal, curVal) {
- t.Errorf("Data mismatch for generation metadata: %v instead of %v",
- curVal, expVal)
- }
-
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
-
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- // Check all log records.
- lcount := []LSN{2, 4, 1}
- for index, devid := range []DeviceID{"VeyronPhone", "VeyronTab", "VeyronLaptop"} {
- v := raw.Version(1)
- for i := LSN(0); i < lcount[index]; i++ {
- curRec, err := s.log.getLogRec(devid, GenID(1), i)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:%d in log file err %v",
- devid, i, err)
- }
- if curRec.ObjID != objid {
- t.Errorf("Data mismatch in log record %v", curRec)
- }
- if devid == "VeyronTab" && index == 3 && curRec.RecType != LinkRec {
- t.Errorf("Data mismatch in log record %v", curRec)
- }
- // Verify DAG state.
- if _, err := s.dag.getNode(objid, v); err != nil {
- t.Errorf("GetNode() can not find object %d %d in DAG, err %v", objid, i, err)
- }
- v = v + 1
- }
- }
- if len(s.hdlInitiator.updObjects) != 1 {
- t.Errorf("Unexpected number of updated objects %d", len(s.hdlInitiator.updObjects))
- }
- st := s.hdlInitiator.updObjects[objid]
- if !st.isConflict {
- t.Errorf("Didn't detect a conflict %v", st)
- }
- if st.newHead != 6 || st.oldHead != 3 {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
- if st.resolvVal.Mutation.PriorVersion != 3 || st.resolvVal.Mutation.Version != 6 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
- // Curlsn == 4 for the log record that resolves conflict.
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 4 || s.log.head.Curorder != 2 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
- // Verify DAG state.
- if head, err := s.dag.getHead(objid); err != nil || head != 6 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-}
-
-// TestInitiatorBlessNoConf0 tests that a local and a remote log
-// record stream can be correctly applied, when the conflict is
-// resolved by a blessing. In this test, local head of the object is
-// unchanged at the end of replay. Commands are in files
-// testdata/<local-init-00.log.sync,remote-noconf-link-00.log.sync>.
-func TestInitiatorBlessNoConf0(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- if _, err = logReplayCommands(s.log, "local-init-00.log.sync"); err != nil {
- t.Error(err)
- }
- stream, err := createReplayStream("remote-noconf-link-00.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
-
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- // Check that there are no conflicts.
- if len(s.hdlInitiator.updObjects) != 1 {
- t.Errorf("Too many objects %v", len(s.hdlInitiator.updObjects))
- }
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- st := s.hdlInitiator.updObjects[objid]
- if st.isConflict {
- t.Errorf("Detected a conflict %v", st)
- }
- if st.newHead != 3 || st.oldHead != 3 {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
-
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
- if st.resolvVal.Mutation.Version != 3 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
- // No new log records should be added.
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 3 || s.log.head.Curorder != 1 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
- // Verify DAG state.
- if head, err := s.dag.getHead(objid); err != nil || head != 3 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-}
-
-// TestInitiatorBlessNoConf1 tests that a local and a remote log
-// record stream can be correctly applied, when the conflict is
-// resolved by a blessing. In this test, local head of the object is
-// updated at the end of the replay. Commands are in files
-// testdata/<local-init-00.log.sync,remote-noconf-link-01.log.sync>.
-func TestInitiatorBlessNoConf1(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- if _, err = logReplayCommands(s.log, "local-init-00.log.sync"); err != nil {
- t.Error(err)
- }
- stream, err := createReplayStream("remote-noconf-link-01.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
-
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- // Check that there are no conflicts.
- if len(s.hdlInitiator.updObjects) != 1 {
- t.Errorf("Too many objects %v", len(s.hdlInitiator.updObjects))
- }
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- st := s.hdlInitiator.updObjects[objid]
- if st.isConflict {
- t.Errorf("Detected a conflict %v", st)
- }
- if st.newHead != 4 || st.oldHead != 3 {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
-
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
- if st.resolvVal.Mutation.Version != 4 || st.resolvVal.Mutation.PriorVersion != 3 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
- // No new log records should be added.
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 3 || s.log.head.Curorder != 1 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
- // Verify DAG state.
- if head, err := s.dag.getHead(objid); err != nil || head != 4 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-}
-
-// TestInitiatorBlessNoConf2 tests that a local and a remote log
-// record stream can be correctly applied, when the conflict is
-// resolved by a blessing. In this test, local head of the object is
-// updated at the end of the first replay. In the second replay, a
-// conflict resolved locally is rediscovered since it was also
-// resolved remotely. Commands are in files
-// testdata/<local-init-00.log.sync,remote-noconf-link-02.log.sync,
-// remote-noconf-link-repeat.log.sync>.
-func TestInitiatorBlessNoConf2(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- if _, err = logReplayCommands(s.log, "local-init-00.log.sync"); err != nil {
- t.Error(err)
- }
- stream, err := createReplayStream("remote-noconf-link-02.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
-
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- // Check that there are no conflicts.
- if len(s.hdlInitiator.updObjects) != 1 {
- t.Errorf("Too many objects %v", len(s.hdlInitiator.updObjects))
- }
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- st := s.hdlInitiator.updObjects[objid]
- if st.isConflict {
- t.Errorf("Detected a conflict %v", st)
- }
- if st.newHead != 5 || st.oldHead != 3 {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
-
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{"VeyronTab": 0}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
- if st.resolvVal.Mutation.Version != 5 || st.resolvVal.Mutation.PriorVersion != 3 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
- // No new log records should be added.
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 3 || s.log.head.Curorder != 2 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
- // Verify DAG state.
- if head, err := s.dag.getHead(objid); err != nil || head != 5 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-
- // Test simultaneous conflict resolution.
- stream, err = createReplayStream("remote-noconf-link-repeat.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
-
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- // Check that there are no conflicts.
- if len(s.hdlInitiator.updObjects) != 1 {
- t.Errorf("Too many objects %v", len(s.hdlInitiator.updObjects))
- }
- st = s.hdlInitiator.updObjects[objid]
- if st.isConflict {
- t.Errorf("Detected a conflict %v", st)
- }
- if st.newHead != 5 || st.oldHead != 5 {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
-
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronLaptop"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
- if st.resolvVal.Mutation.Version != 5 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
- // No new log records should be added.
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 3 || s.log.head.Curorder != 3 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
- // Verify DAG state.
- if head, err := s.dag.getHead(objid); err != nil || head != 5 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-}
-
-// TestInitiatorBlessConf tests that a local and a remote log record
-// stream can be correctly applied, when the conflict is resolved by a
-// blessing. Commands are in files
-// testdata/<local-init-00.log.sync,remote-conf-link.log.sync>.
-func TestInitiatorBlessConf(t *testing.T) {
- dir, err := createTempDir()
- if err != nil {
- t.Errorf("Could not create tempdir %v", err)
- }
- // Set a large value to prevent the threads from firing.
- // Test is not thread safe.
- peerSyncInterval = 1 * time.Hour
- garbageCollectInterval = 1 * time.Hour
- s := NewSyncd("", "", "VeyronTab", dir, "", 0)
-
- defer s.Close()
- defer os.RemoveAll(dir)
-
- if _, err = logReplayCommands(s.log, "local-init-00.log.sync"); err != nil {
- t.Error(err)
- }
- stream, err := createReplayStream("remote-conf-link.log.sync")
- if err != nil {
- t.Fatalf("createReplayStream failed with err %v", err)
- }
-
- var minGens GenVector
- if minGens, err = s.hdlInitiator.processLogStream(stream); err != nil {
- t.Fatalf("processLogStream failed with err %v", err)
- }
- if err := s.hdlInitiator.detectConflicts(); err != nil {
- t.Fatalf("detectConflicts failed with err %v", err)
- }
- // Check that there are no conflicts.
- if len(s.hdlInitiator.updObjects) != 1 {
- t.Errorf("Too many objects %v", len(s.hdlInitiator.updObjects))
- }
- objid, err := strToObjID("12345")
- if err != nil {
- t.Errorf("Could not create objid %v", err)
- }
- st := s.hdlInitiator.updObjects[objid]
- if !st.isConflict {
- t.Errorf("Didn't detect a conflict %v", st)
- }
- if st.newHead != 4 || st.oldHead != 3 || st.ancestor != 2 {
- t.Errorf("Conflict detection didn't succeed %v", st)
- }
-
- if err := s.hdlInitiator.resolveConflicts(); err != nil {
- t.Fatalf("resolveConflicts failed with err %v", err)
- }
- if st.resolvVal.Mutation.Version != 4 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
-
- if err := s.hdlInitiator.updateStoreAndSync(nil, GenVector{}, minGens, GenVector{}, "VeyronPhone"); err != nil {
- t.Fatalf("updateStoreAndSync failed with err %v", err)
- }
- if st.resolvVal.Mutation.Version != 4 || st.resolvVal.Mutation.PriorVersion != 3 {
- t.Errorf("Mutation generation is not accurate %v", st)
- }
- // New log records should be added.
- if s.log.head.Curgen != 1 || s.log.head.Curlsn != 4 || s.log.head.Curorder != 1 {
- t.Errorf("Data mismatch in log header %v", s.log.head)
- }
- curRec, err := s.log.getLogRec(s.id, GenID(1), LSN(3))
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:1:3 in log file err %v",
- s.id, err)
- }
- if curRec.ObjID != objid || curRec.RecType != LinkRec {
- t.Errorf("Data mismatch in log record %v", curRec)
- }
- // Verify DAG state.
- if head, err := s.dag.getHead(objid); err != nil || head != 4 {
- t.Errorf("Invalid object %d head in DAG %s, err %v", objid, head, err)
- }
-}
diff --git a/runtimes/google/vsync/kvdb.go b/runtimes/google/vsync/kvdb.go
deleted file mode 100644
index 7efefde..0000000
--- a/runtimes/google/vsync/kvdb.go
+++ /dev/null
@@ -1,176 +0,0 @@
-package vsync
-
-// Helpful wrappers to a persistent key/value (K/V) DB used by Veyron Sync.
-// The current underlying DB is gkvlite.
-
-import (
- "bytes"
- "fmt"
- "io/ioutil"
- "os"
- "path"
-
- "github.com/steveyen/gkvlite"
-
- "veyron2/vom"
-)
-
-type kvdb struct {
- store *gkvlite.Store
- fdesc *os.File
-}
-
-type kvtable struct {
- coll *gkvlite.Collection
-}
-
-// kvdbOpen opens or creates a K/V DB for the given filename and tables names
-// within the DB. It returns the DB handler and handlers for each table.
-func kvdbOpen(filename string, tables []string) (*kvdb, []*kvtable, error) {
- // Open the file and create it if it does not exist.
- fdesc, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
- if err != nil {
- return nil, nil, err
- }
-
- // Initialize the DB (store) and its tables (collections).
- // The store takes ownership of fdesc on success.
- store, err := gkvlite.NewStore(fdesc)
- if err != nil {
- fdesc.Close()
- return nil, nil, err
- }
-
- flush := false
- tbls := make([]*kvtable, len(tables))
-
- for i, table := range tables {
- coll := store.GetCollection(table)
- if coll == nil {
- if coll = store.SetCollection(table, nil); coll == nil {
- store.Close()
- return nil, nil, fmt.Errorf("cannot create K/V DB table %s in file %s", table, filename)
- }
- flush = true
- }
- tbls[i] = &kvtable{coll: coll}
- }
-
- if flush {
- store.Flush() // Flush newly created collections.
- }
-
- db := &kvdb{store: store, fdesc: fdesc}
- return db, tbls, nil
-}
-
-// close closes the given K/V DB.
-func (db *kvdb) close() {
- db.store.Close()
- db.fdesc.Close()
-}
-
-// flush flushes the given K/V DB to disk.
-func (db *kvdb) flush() {
- db.store.Flush()
- db.fdesc.Sync()
-}
-
-// set stores (or overwrites) the given key/value pair in the DB table.
-func (t *kvtable) set(key string, value interface{}) error {
- val := new(bytes.Buffer)
- if err := vom.NewEncoder(val).Encode(value); err != nil {
- return err
- }
- return t.coll.Set([]byte(key), val.Bytes())
-}
-
-// create stores the given key/value pair in the DB table only if
-// the key does not already exist. Otherwise it returns an error.
-func (t *kvtable) create(key string, value interface{}) error {
- if t.hasKey(key) {
- return fmt.Errorf("key %s exists", key)
- }
- return t.set(key, value)
-}
-
-// update stores the given key/value pair in the DB table only if
-// the key already exists. Otherwise it returns an error.
-func (t *kvtable) update(key string, value interface{}) error {
- if !t.hasKey(key) {
- return fmt.Errorf("key %s does not exist", key)
- }
- return t.set(key, value)
-}
-
-// get retrieves the value of a key from the DB table.
-func (t *kvtable) get(key string, value interface{}) error {
- val, err := t.coll.Get([]byte(key))
- if err != nil {
- return err
- }
- if val == nil {
- return fmt.Errorf("entry %s not found in the K/V DB table", key)
- }
- return vom.NewDecoder(bytes.NewBuffer(val)).Decode(value)
-}
-
-// del deletes the entry in the DB table given its key.
-func (t *kvtable) del(key string) error {
- _, err := t.coll.Delete([]byte(key))
- return err
-}
-
-// hasKey returns true if the given key exists in the DB table.
-func (t *kvtable) hasKey(key string) bool {
- item, err := t.coll.GetItem([]byte(key), false)
- return err == nil && item != nil
-}
-
-// keyIter iterates over all keys in a DB table invoking the given callback
-// function for each one. The key iterator callback is passed the item key.
-func (t *kvtable) keyIter(keyIterCB func(key string)) error {
- return t.coll.VisitItemsAscend(nil, false, func(item *gkvlite.Item) bool {
- keyIterCB(string(item.Key))
- return true
- })
-}
-
-// compact compacts the K/V DB file on disk. It flushs the DB file, creates
-// a compact copy of the file under /tmp, then closes the DB, moves the new
-// file to replace the old one, then re-opens the new DB file.
-func (db *kvdb) compact(filename string, tables []string) (*kvdb, []*kvtable, error) {
- db.store.Flush()
-
- // Create a unique temporary filename to copy the compact store into.
- prefix := path.Base(filename)
- if prefix == "." || prefix == "/" {
- return nil, nil, fmt.Errorf("invalid DB filename %s", filename)
- }
-
- fdesc, err := ioutil.TempFile("", prefix)
- if err != nil {
- return nil, nil, err
- }
- tmpfile := fdesc.Name()
- defer os.Remove(tmpfile)
- defer fdesc.Close()
-
- // Make a compact copy of the store.
- store, err := db.store.CopyTo(fdesc, 0)
- if err == nil {
- err = store.Flush()
- }
- store.Close()
- if err != nil {
- return nil, nil, err
- }
-
- // Swap the files and re-open the new store.
- if err = os.Rename(tmpfile, filename); err != nil {
- return nil, nil, err
- }
-
- db.close() // close it, after the rename there is no turning back
- return kvdbOpen(filename, tables)
-}
diff --git a/runtimes/google/vsync/kvdb_test.go b/runtimes/google/vsync/kvdb_test.go
deleted file mode 100644
index 5b0b6e1..0000000
--- a/runtimes/google/vsync/kvdb_test.go
+++ /dev/null
@@ -1,454 +0,0 @@
-package vsync
-
-// Tests for the Veyron Sync K/V DB component.
-
-import (
- "fmt"
- "os"
- "reflect"
- "testing"
- "time"
-)
-
-// A user structure stores info in the "users" table.
-type user struct {
- Username string
- Drinks []string
-}
-
-// A drink structure stores info in the "drinks" table.
-type drink struct {
- Name string
- Alcohol bool
-}
-
-var (
- users = []user{
- {Username: "lancelot", Drinks: []string{"beer", "coffee"}},
- {Username: "arthur", Drinks: []string{"coke", "beer", "coffee"}},
- {Username: "robin", Drinks: []string{"pepsi"}},
- {Username: "galahad"},
- }
- drinks = []drink{
- {Name: "coke", Alcohol: false},
- {Name: "pepsi", Alcohol: false},
- {Name: "beer", Alcohol: true},
- {Name: "coffee", Alcohol: false},
- }
-)
-
-// createTestDB creates a K/V DB with 2 tables.
-func createTestDB(t *testing.T) (fname string, db *kvdb, usersTbl, drinksTbl *kvtable) {
- fname = fmt.Sprintf("%s/sync_kvdb_test_%d_%d", os.TempDir(), os.Getpid(), time.Now().UnixNano())
- db, tbls, err := kvdbOpen(fname, []string{"users", "drinks"})
- if err != nil {
- os.Remove(fname)
- t.Fatalf("cannot create new K/V DB file %s: %v", fname, err)
- }
-
- if _, err = os.Stat(fname); err != nil {
- os.Remove(fname)
- t.Fatalf("newly created K/V DB file %s not found: %v", fname, err)
- }
-
- usersTbl, drinksTbl = tbls[0], tbls[1]
- return
-}
-
-// initTestTables initializes the K/V tables used by the tests.
-func initTestTables(t *testing.T, usersTbl, drinksTbl *kvtable, useCreate bool) {
- userPut, drinkPut, funcName := usersTbl.set, drinksTbl.set, "set()"
- if useCreate {
- userPut, drinkPut, funcName = usersTbl.create, drinksTbl.create, "create()"
- }
-
- for _, uu := range users {
- if err := userPut(uu.Username, &uu); err != nil {
- t.Fatalf("%s failed for user %s", funcName, uu.Username)
- }
- }
-
- for _, dd := range drinks {
- if err := drinkPut(dd.Name, &dd); err != nil {
- t.Fatalf("%s failed for drink %s", funcName, dd.Name)
- }
- }
-
- return
-}
-
-// checkTestTables verifies the contents of the K/V tables.
-func checkTestTables(t *testing.T, usersTbl, drinksTbl *kvtable) {
- for _, uu := range users {
- var u2 user
- if err := usersTbl.get(uu.Username, &u2); err != nil {
- t.Fatalf("get() failed for user %s", uu.Username)
- }
- if !reflect.DeepEqual(u2, uu) {
- t.Fatalf("got wrong data for user %s: %#v instead of %#v", uu.Username, u2, uu)
- }
- if !usersTbl.hasKey(uu.Username) {
- t.Fatalf("hasKey() did not find user %s", uu.Username)
- }
- }
- for _, dd := range drinks {
- var d2 drink
- if err := drinksTbl.get(dd.Name, &d2); err != nil {
- t.Fatalf("get() failed for drink %s", dd.Name)
- }
- if !reflect.DeepEqual(d2, dd) {
- t.Fatalf("got wrong data for drink %s: %#v instead of %#v", dd.Name, d2, dd)
- }
- if !drinksTbl.hasKey(dd.Name) {
- t.Fatalf("hasKey() did not find drink %s", dd.Name)
- }
- }
-}
-
-func TestKVDBSet(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
-
- db.flush()
-}
-
-func TestKVDBCreate(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, true)
-
- db.flush()
-}
-
-func TestKVDBBadGet(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- // The DB is empty, all gets must fail.
- for _, uu := range users {
- var u2 user
- if err := usersTbl.get(uu.Username, &u2); err == nil {
- t.Fatalf("get() found non-existent user %s in file %s: %v", uu.Username, kvdbfile, u2)
- }
- }
- for _, dd := range drinks {
- var d2 drink
- if err := drinksTbl.get(dd.Name, &d2); err == nil {
- t.Fatalf("get() found non-existent drink %s in file %s: %v", dd.Name, kvdbfile, d2)
- }
- }
-}
-
-func TestKVDBBadUpdate(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- // The DB is empty, all updates must fail.
- for _, uu := range users {
- u2 := user{Username: uu.Username}
- if err := usersTbl.update(uu.Username, &u2); err == nil {
- t.Fatalf("update() worked for a non-existent user %s in file %s", uu.Username, kvdbfile)
- }
- }
- for _, dd := range drinks {
- d2 := drink{Name: dd.Name}
- if err := drinksTbl.update(dd.Name, &d2); err == nil {
- t.Fatalf("update() worked for a non-existent drink %s in file %s", dd.Name, kvdbfile)
- }
- }
-}
-
-func TestKVDBBadHasKey(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- // The DB is empty, all key-checks must fail.
- for _, uu := range users {
- if usersTbl.hasKey(uu.Username) {
- t.Fatalf("hasKey() found non-existent user %s in file %s", uu.Username, kvdbfile)
- }
- }
- for _, dd := range drinks {
- if drinksTbl.hasKey(dd.Name) {
- t.Fatalf("hasKey() found non-existent drink %s in file %s", dd.Name, kvdbfile)
- }
- }
-}
-
-func TestKVDBGet(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
- checkTestTables(t, usersTbl, drinksTbl)
-
- db.flush()
- checkTestTables(t, usersTbl, drinksTbl)
-}
-
-func TestKVDBBadCreate(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
-
- // Must not be able to re-create the same entries.
- for _, uu := range users {
- u2 := user{Username: uu.Username}
- if err := usersTbl.create(uu.Username, &u2); err == nil {
- t.Fatalf("create() worked for an existing user %s in file %s", uu.Username, kvdbfile)
- }
- }
- for _, dd := range drinks {
- d2 := drink{Name: dd.Name}
- if err := drinksTbl.create(dd.Name, &d2); err == nil {
- t.Fatalf("create() worked for an existing drink %s in file %s", dd.Name, kvdbfile)
- }
- }
-
- db.flush()
-}
-
-func TestKVDBReopen(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
-
- initTestTables(t, usersTbl, drinksTbl, true)
-
- // Close the re-open the file.
- db.flush()
- db.close()
-
- db, tbls, err := kvdbOpen(kvdbfile, []string{"users", "drinks"})
- if err != nil {
- t.Fatalf("Cannot re-open existing K/V DB file %s", kvdbfile)
- }
- defer db.close()
-
- usersTbl, drinksTbl = tbls[0], tbls[1]
- checkTestTables(t, usersTbl, drinksTbl)
-}
-
-func TestKVDBKeyIter(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
-
- // Get the list of all entry keys in each table.
- keylist := ""
- err := usersTbl.keyIter(func(key string) {
- keylist += key + ","
- })
- if err != nil || keylist != "arthur,galahad,lancelot,robin," {
- t.Fatalf("keyIter() failed in file %s: err %v, user names: %v", kvdbfile, err, keylist)
- }
- keylist = ""
- err = drinksTbl.keyIter(func(key string) {
- keylist += key + ","
- })
- if err != nil || keylist != "beer,coffee,coke,pepsi," {
- t.Fatalf("keyIter() failed in file %s: err %v, drink names: %v", kvdbfile, err, keylist)
- }
-
- db.flush()
-}
-
-func TestKVDBUpdate(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
-
- initTestTables(t, usersTbl, drinksTbl, false)
- db.flush()
- db.close()
-
- db, tbls, err := kvdbOpen(kvdbfile, []string{"users", "drinks"})
- if err != nil {
- t.Fatalf("Cannot re-open existing K/V DB file %s", kvdbfile)
- }
- defer db.close()
-
- usersTbl, drinksTbl = tbls[0], tbls[1]
-
- for _, uu := range users {
- key := uu.Username
- u2 := uu
- u2.Username += "-New"
-
- if err = usersTbl.update(key, &u2); err != nil {
- t.Fatalf("update() failed for user %s in file %s", key, kvdbfile)
- }
-
- var u3 user
- if err = usersTbl.get(key, &u3); err != nil {
- t.Fatalf("get() failed for user %s in file %s", key, kvdbfile)
- }
- if !reflect.DeepEqual(u3, u2) {
- t.Fatalf("got wrong new data for user %s in file %s: %#v instead of %#v", key, kvdbfile, u3, u2)
- }
- }
-
- for _, dd := range drinks {
- key := dd.Name
- d2 := dd
- d2.Alcohol = !d2.Alcohol
-
- if err = drinksTbl.update(key, &d2); err != nil {
- t.Fatalf("update() failed for drink %s in file %s", key, kvdbfile)
- }
-
- var d3 drink
- if err = drinksTbl.get(key, &d3); err != nil {
- t.Fatalf("get() failed for drink %s in file %s", key, kvdbfile)
- }
- if !reflect.DeepEqual(d3, d2) {
- t.Fatalf("got wrong new data for drink %s in file %s: %#v instead of %#v", key, kvdbfile, d3, d2)
- }
- }
-
- db.flush()
-}
-
-func TestKVDBSetAgain(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
-
- for _, uu := range users {
- key := uu.Username
- u2 := uu
- u2.Username += "-New"
-
- if err := usersTbl.set(key, &u2); err != nil {
- t.Fatalf("set() again failed for user %s in file %s", key, kvdbfile)
- }
-
- var u3 user
- if err := usersTbl.get(key, &u3); err != nil {
- t.Fatalf("get() failed for user %s in file %s", key, kvdbfile)
- }
- if !reflect.DeepEqual(u3, u2) {
- t.Fatalf("got wrong new data for user %s in file %s: %#v instead of %#v", key, kvdbfile, u3, u2)
- }
- }
-
- for _, dd := range drinks {
- key := dd.Name
- d2 := dd
- d2.Alcohol = !d2.Alcohol
-
- if err := drinksTbl.update(key, &d2); err != nil {
- t.Fatalf("set() again failed for drink %s in file %s", key, kvdbfile)
- }
-
- var d3 drink
- if err := drinksTbl.get(key, &d3); err != nil {
- t.Fatalf("get() failed for drink %s in file %s", key, kvdbfile)
- }
- if !reflect.DeepEqual(d3, d2) {
- t.Fatalf("got wrong new data for drink %s in file %s: %#v instead of %#v", key, kvdbfile, d3, d2)
- }
- }
-
- db.flush()
-}
-
-func TestKVDBCompact(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
-
- db.flush()
-
- // Make some NOP changes and flush the DB after each change to grow the file.
- for _, uu := range users {
- key := uu.Username
- if err := usersTbl.set(key, &uu); err != nil {
- t.Fatalf("set() NOP failed for user %s in file %s", key, kvdbfile)
- }
- db.flush()
- }
-
- for _, dd := range drinks {
- key := dd.Name
- if err := drinksTbl.update(key, &dd); err != nil {
- t.Fatalf("set() NOP failed for drink %s in file %s", key, kvdbfile)
- }
- db.flush()
- }
-
- // Compact the DB file and verify that it contains the same data and has a smaller size.
- oldSize := fileSize(kvdbfile)
-
- db, tbls, err := db.compact(kvdbfile, []string{"users", "drinks"})
- if err != nil {
- t.Fatalf("compact() failed to create a smaller version of K/V DB file %s: %v", kvdbfile, err)
- }
- defer db.close()
-
- usersTbl, drinksTbl = tbls[0], tbls[1]
-
- newSize := fileSize(kvdbfile)
- if newSize >= oldSize {
- t.Errorf("compact() failed to shrink file %s: old size %d, new size %d", kvdbfile, oldSize, newSize)
- }
-
- checkTestTables(t, usersTbl, drinksTbl)
-
- // Verify that compact() rejects a filename without a basename.
- for _, badfile := range []string{"/", ".", ".."} {
- _, _, err := db.compact(badfile, []string{"users", "drinks"})
- if err == nil {
- t.Errorf("compact() did not fail when given the invalid filename: %s", badfile)
- }
- }
-}
-
-func TestKVDBDelete(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
-
- db.flush()
-
- // Delete entries and verify that they no longer exist.
-
- for _, uu := range users {
- key := uu.Username
- if err := usersTbl.del(key); err != nil {
- t.Errorf("del() failed for user %s in file %s", key, kvdbfile)
- }
- if usersTbl.hasKey(key) {
- t.Errorf("hasKey() still finds deleted user %s in file %s", key, kvdbfile)
- }
- }
-
- for _, dd := range drinks {
- key := dd.Name
- if err := drinksTbl.del(key); err != nil {
- t.Errorf("del() failed for drink %s in file %s", key, kvdbfile)
- }
- if drinksTbl.hasKey(key) {
- t.Errorf("hasKey() still finds deleted drink %s in file %s", key, kvdbfile)
- }
- }
-
- db.flush()
-}
diff --git a/runtimes/google/vsync/replay_test.go b/runtimes/google/vsync/replay_test.go
deleted file mode 100644
index 1904f31..0000000
--- a/runtimes/google/vsync/replay_test.go
+++ /dev/null
@@ -1,239 +0,0 @@
-package vsync
-
-// Used to ease the setup of Veyron Sync test scenarios.
-// Parses a sync command file and returns a vector of commands to execute.
-//
-// Used by different test replay engines:
-// - dagReplayCommands() executes the parsed commands at the DAG API level.
-// - logReplayCommands() executes the parsed commands at the Log API level.
-
-import (
- "bufio"
- "encoding/binary"
- "fmt"
- "os"
- "strconv"
- "strings"
-
- "veyron/services/store/raw"
-
- "veyron2/storage"
-)
-
-const (
- addLocal = iota
- addRemote
- setDevTable
- linkLocal
- linkRemote
-)
-
-type syncCommand struct {
- cmd int
- objID storage.ID
- version raw.Version
- parents []raw.Version
- logrec string
- devID DeviceID
- genVec GenVector
- continued bool
- deleted bool
-}
-
-func strToObjID(objStr string) (storage.ID, error) {
- var objID storage.ID
- id, err := strconv.ParseUint(objStr, 10, 64)
- if err != nil {
- return objID, err
- }
- idbuf := make([]byte, len(objID))
- if binary.PutUvarint(idbuf, id) == 0 {
- return objID, fmt.Errorf("cannot store ID %d into a binary buffer", id)
- }
- for i := 0; i < len(objID); i++ {
- objID[i] = idbuf[i]
- }
- return objID, nil
-}
-
-func strToVersion(verStr string) (raw.Version, error) {
- ver, err := strconv.ParseUint(verStr, 10, 64)
- if err != nil {
- return 0, err
- }
- return raw.Version(ver), nil
-}
-
-func strToGenID(genIDStr string) (GenID, error) {
- id, err := strconv.ParseUint(genIDStr, 10, 64)
- if err != nil {
- return 0, err
- }
- return GenID(id), nil
-}
-
-func parseSyncCommands(file string) ([]syncCommand, error) {
- cmds := []syncCommand{}
- sf, err := os.Open("testdata/" + file)
- if err != nil {
- return nil, err
- }
- defer sf.Close()
-
- scanner := bufio.NewScanner(sf)
- lineno := 0
- for scanner.Scan() {
- lineno++
- line := strings.TrimSpace(scanner.Text())
- if line == "" || line[0] == '#' {
- continue
- }
-
- args := strings.Split(line, "|")
- nargs := len(args)
-
- switch args[0] {
- case "addl", "addr":
- expNargs := 8
- if nargs != expNargs {
- return nil, fmt.Errorf("%s:%d: need %d args instead of %d", file, lineno, expNargs, nargs)
- }
- version, err := strToVersion(args[2])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid version: %s", file, lineno, args[2])
- }
- var parents []raw.Version
- for i := 3; i <= 4; i++ {
- if args[i] != "" {
- pver, err := strToVersion(args[i])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid parent: %s", file, lineno, args[i])
- }
- parents = append(parents, pver)
- }
- }
-
- continued, err := strconv.ParseBool(args[6])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid continued bit: %s", file, lineno, args[6])
- }
- del, err := strconv.ParseBool(args[7])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid deleted bit: %s", file, lineno, args[7])
- }
- cmd := syncCommand{version: version, parents: parents, logrec: args[5], continued: continued, deleted: del}
- if args[0] == "addl" {
- cmd.cmd = addLocal
- } else {
- cmd.cmd = addRemote
- }
- if cmd.objID, err = strToObjID(args[1]); err != nil {
- return nil, fmt.Errorf("%s:%d: invalid object ID: %s", file, lineno, args[1])
- }
- cmds = append(cmds, cmd)
-
- case "setdev":
- expNargs := 3
- if nargs != expNargs {
- return nil, fmt.Errorf("%s:%d: need %d args instead of %d", file, lineno, expNargs, nargs)
- }
-
- genVec := make(GenVector)
- for _, elem := range strings.Split(args[2], ",") {
- kv := strings.Split(elem, ":")
- if len(kv) != 2 {
- return nil, fmt.Errorf("%s:%d: invalid gen vector key/val: %s", file, lineno, elem)
- }
- genID, err := strToGenID(kv[1])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid gen ID: %s", file, lineno, kv[1])
- }
- genVec[DeviceID(kv[0])] = genID
- }
-
- cmd := syncCommand{cmd: setDevTable, devID: DeviceID(args[1]), genVec: genVec}
- cmds = append(cmds, cmd)
-
- case "linkl", "linkr":
- expNargs := 6
- if nargs != expNargs {
- return nil, fmt.Errorf("%s:%d: need %d args instead of %d", file, lineno, expNargs, nargs)
- }
-
- version, err := strToVersion(args[2])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid version: %s", file, lineno, args[2])
- }
- if args[3] == "" {
- return nil, fmt.Errorf("%s:%d: parent (to-node) version not specified", file, lineno)
- }
- if args[4] != "" {
- return nil, fmt.Errorf("%s:%d: cannot specify a 2nd parent (to-node): %s", file, lineno, args[4])
- }
- parent, err := strToVersion(args[3])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid parent (to-node) version: %s", file, lineno, args[3])
- }
-
- cmd := syncCommand{version: version, parents: []raw.Version{parent}, logrec: args[5]}
- if args[0] == "linkl" {
- cmd.cmd = linkLocal
- } else {
- cmd.cmd = linkRemote
- }
- if cmd.objID, err = strToObjID(args[1]); err != nil {
- return nil, fmt.Errorf("%s:%d: invalid object ID: %s", file, lineno, args[1])
- }
- cmds = append(cmds, cmd)
-
- default:
- return nil, fmt.Errorf("%s:%d: invalid operation: %s", file, lineno, args[0])
- }
- }
-
- err = scanner.Err()
- return cmds, err
-}
-
-func dagReplayCommands(dag *dag, syncfile string) error {
- cmds, err := parseSyncCommands(syncfile)
- if err != nil {
- return err
- }
-
- for _, cmd := range cmds {
- switch cmd.cmd {
- case addLocal:
- err = dag.addNode(cmd.objID, cmd.version, false, cmd.deleted, cmd.parents, cmd.logrec, NoTxID)
- if err != nil {
- return fmt.Errorf("cannot add local node %d:%d to DAG: %v", cmd.objID, cmd.version, err)
- }
- if err := dag.moveHead(cmd.objID, cmd.version); err != nil {
- return fmt.Errorf("cannot move head to %d:%d in DAG: %v", cmd.objID, cmd.version, err)
- }
- dag.flush()
-
- case addRemote:
- err = dag.addNode(cmd.objID, cmd.version, true, cmd.deleted, cmd.parents, cmd.logrec, NoTxID)
- if err != nil {
- return fmt.Errorf("cannot add remote node %d:%d to DAG: %v", cmd.objID, cmd.version, err)
- }
- dag.flush()
-
- case linkLocal:
- if err = dag.addParent(cmd.objID, cmd.version, cmd.parents[0], false); err != nil {
- return fmt.Errorf("cannot add local parent %d to DAG node %d:%d: %v",
- cmd.parents[0], cmd.objID, cmd.version, err)
- }
- dag.flush()
-
- case linkRemote:
- if err = dag.addParent(cmd.objID, cmd.version, cmd.parents[0], true); err != nil {
- return fmt.Errorf("cannot add remote parent %d to DAG node %d:%d: %v",
- cmd.parents[0], cmd.objID, cmd.version, err)
- }
- dag.flush()
- }
- }
- return nil
-}
diff --git a/runtimes/google/vsync/sgtable.go b/runtimes/google/vsync/sgtable.go
deleted file mode 100644
index aa15b92..0000000
--- a/runtimes/google/vsync/sgtable.go
+++ /dev/null
@@ -1,495 +0,0 @@
-package vsync
-
-// The SyncGroup Table stores the group information in a K/V DB. It also
-// maintains an index to provide access by SyncGroup ID or name.
-//
-// The SyncGroup info is fetched from the SyncGroup server by the create or
-// join operations, and is regularly updated after that.
-//
-// The DB contains two tables persisted to disk (data, names) and one
-// in-memory (ephemeral) map (members):
-// * data: one entry per SyncGroup ID containing the SyncGroup data
-// * names: one entry per SyncGroup name pointing to its SyncGroup ID
-// * members: an inverted index of SyncGroup members to SyncGroup IDs
-// built from the list of SyncGroup joiners
-// * peerSGs: an inverted index of SyncGroup RootOIDs to sets of peer
-// SyncGroup IDs, i.e. SyncGroups defined on the same root
-// path in the Store (RootOID)
-
-import (
- "errors"
- "fmt"
-
- "veyron/services/syncgroup"
-
- "veyron2/storage"
-)
-
-var (
- errBadSGTable = errors.New("invalid SyncGroup Table")
-)
-
-type syncGroupTable struct {
- fname string // file pathname
- store *kvdb // underlying K/V store
- sgData *kvtable // pointer to "data" table in the kvdb
- sgNames *kvtable // pointer to "names" table in the kvdb
- members map[string]*memberInfo // in-memory tracking of SyncGroup member info
- peerSGs map[storage.ID]sgSet // in-memory tracking of peer SyncGroups per RootOID
-}
-
-type syncGroupData struct {
- SrvInfo syncgroup.SyncGroupInfo // SyncGroup info from SyncGroupServer
- LocalPath string // local path of the SyncGroup in the Store
-}
-
-type memberInfo struct {
- gids map[syncgroup.ID]*memberMetaData // map of SyncGroup IDs joined and their metadata
-}
-
-type memberMetaData struct {
- metaData syncgroup.JoinerMetaData // joiner metadata at the SyncGroup server
- identity string // joiner security identity
-}
-
-type sgSet map[syncgroup.ID]struct{} // a set of SyncGroups
-
-// openSyncGroupTable opens or creates a syncGroupTable for the given filename.
-func openSyncGroupTable(filename string) (*syncGroupTable, error) {
- // Open the file and create it if it does not exist.
- // Also initialize the store and its tables.
- db, tbls, err := kvdbOpen(filename, []string{"data", "names"})
- if err != nil {
- return nil, err
- }
-
- s := &syncGroupTable{
- fname: filename,
- store: db,
- sgData: tbls[0],
- sgNames: tbls[1],
- members: make(map[string]*memberInfo),
- peerSGs: make(map[storage.ID]sgSet),
- }
-
- // Reconstruct the in-memory tracking maps by iterating over the SyncGroups.
- // This is needed when an existing SyncGroup Table file is re-opened.
- s.sgData.keyIter(func(gidStr string) {
- // Get the SyncGroup data given the group ID in string format (as the data table key).
- gid, err := syncgroup.ParseID(gidStr)
- if err != nil {
- return
- }
-
- data, err := s.getSyncGroupByID(gid)
- if err != nil {
- return
- }
-
- // Add all SyncGroup members to the members inverted index.
- s.addAllMembers(data)
-
- // Add the SyncGroup ID to its peer SyncGroup set based on the RootOID.
- s.addPeerSyncGroup(gid, data.SrvInfo.RootOID)
- })
-
- return s, nil
-}
-
-// close closes the syncGroupTable and invalidates its structure.
-func (s *syncGroupTable) close() {
- if s.store != nil {
- s.store.close() // this also closes the tables
- }
- *s = syncGroupTable{} // zero out the structure
-}
-
-// flush flushes the syncGroupTable store to disk.
-func (s *syncGroupTable) flush() {
- if s.store != nil {
- s.store.flush()
- }
-}
-
-// compact compacts the kvdb file of the syncGroupTable.
-func (s *syncGroupTable) compact() error {
- if s.store == nil {
- return errBadSGTable
- }
- db, tbls, err := s.store.compact(s.fname, []string{"data", "names"})
- if err != nil {
- return err
- }
- s.store = db
- s.sgData = tbls[0]
- s.sgNames = tbls[1]
- return nil
-}
-
-// addGroup adds a new SyncGroup given its information.
-func (s *syncGroupTable) addSyncGroup(sgData *syncGroupData) error {
- if s.store == nil {
- return errBadSGTable
- }
- if sgData == nil {
- return errors.New("group information not specified")
- }
- gid, name := sgData.SrvInfo.SGOID, sgData.SrvInfo.Name
- if name == "" {
- return errors.New("group name not specified")
- }
- if len(sgData.SrvInfo.Joiners) == 0 {
- return errors.New("group has no joiners")
- }
-
- if s.hasSGDataEntry(gid) {
- return fmt.Errorf("group %d already exists", gid)
- }
- if s.hasSGNameEntry(name) {
- return fmt.Errorf("group name %s already exists", name)
- }
-
- // Add the group name and data entries.
- if err := s.setSGNameEntry(name, gid); err != nil {
- return err
- }
-
- if err := s.setSGDataEntry(gid, sgData); err != nil {
- s.delSGNameEntry(name)
- return err
- }
-
- s.addAllMembers(sgData)
- s.addPeerSyncGroup(gid, sgData.SrvInfo.RootOID)
- return nil
-}
-
-// getSyncGroupID retrieves the SyncGroup ID given its name.
-func (s *syncGroupTable) getSyncGroupID(name string) (syncgroup.ID, error) {
- return s.getSGNameEntry(name)
-}
-
-// getSyncGroupName retrieves the SyncGroup name given its ID.
-func (s *syncGroupTable) getSyncGroupName(gid syncgroup.ID) (string, error) {
- data, err := s.getSyncGroupByID(gid)
- if err != nil {
- return "", err
- }
-
- return data.SrvInfo.Name, nil
-}
-
-// getSyncGroupByID retrieves the SyncGroup given its ID.
-func (s *syncGroupTable) getSyncGroupByID(gid syncgroup.ID) (*syncGroupData, error) {
- return s.getSGDataEntry(gid)
-}
-
-// getSyncGroupByName retrieves the SyncGroup given its name.
-func (s *syncGroupTable) getSyncGroupByName(name string) (*syncGroupData, error) {
- gid, err := s.getSyncGroupID(name)
- if err != nil {
- return nil, err
- }
- return s.getSyncGroupByID(gid)
-}
-
-// updateSyncGroup updates the SyncGroup data.
-func (s *syncGroupTable) updateSyncGroup(data *syncGroupData) error {
- if s.store == nil {
- return errBadSGTable
- }
- if data == nil {
- return errors.New("SyncGroup data not specified")
- }
- if data.SrvInfo.Name == "" {
- return errors.New("group name not specified")
- }
- if len(data.SrvInfo.Joiners) == 0 {
- return errors.New("group has no joiners")
- }
-
- oldData, err := s.getSyncGroupByName(data.SrvInfo.Name)
- if err != nil {
- return err
- }
-
- if data.SrvInfo.SGOID != oldData.SrvInfo.SGOID {
- return fmt.Errorf("cannot change ID of SyncGroup name %s", data.SrvInfo.Name)
- }
- if data.SrvInfo.RootOID != oldData.SrvInfo.RootOID {
- return fmt.Errorf("cannot change root ID of SyncGroup name %s", data.SrvInfo.Name)
- }
-
- // Get the old set of SyncGroup joiners and diff it with the new set.
- // Add all the current members because this inserts the new members and
- // updates the metadata of the existing ones (addMember() is like a "put").
- // Delete the members that are no longer part of the SyncGroup.
- gid := oldData.SrvInfo.SGOID
- newJoiners, oldJoiners := data.SrvInfo.Joiners, oldData.SrvInfo.Joiners
-
- for member, memberData := range newJoiners {
- s.addMember(member.Name, gid, member.Identity, memberData)
- }
-
- for member := range oldJoiners {
- if _, ok := newJoiners[member]; !ok {
- s.delMember(member.Name, gid)
- }
- }
-
- return s.setSGDataEntry(gid, data)
-}
-
-// delSyncGroupByID deletes the SyncGroup given its ID.
-func (s *syncGroupTable) delSyncGroupByID(gid syncgroup.ID) error {
- data, err := s.getSyncGroupByID(gid)
- if err != nil {
- return err
- }
- if err = s.delSGNameEntry(data.SrvInfo.Name); err != nil {
- return err
- }
-
- s.delAllMembers(data)
- s.delPeerSyncGroup(gid, data.SrvInfo.RootOID)
- return s.delSGDataEntry(gid)
-}
-
-// delSyncGroupByName deletes the SyncGroup given its name.
-func (s *syncGroupTable) delSyncGroupByName(name string) error {
- gid, err := s.getSyncGroupID(name)
- if err != nil {
- return err
- }
-
- return s.delSyncGroupByID(gid)
-}
-
-// getMembers returns all SyncGroup members and the count of SyncGroups each one joined.
-func (s *syncGroupTable) getMembers() (map[string]uint32, error) {
- if s.store == nil {
- return nil, errBadSGTable
- }
-
- members := make(map[string]uint32)
- for member, info := range s.members {
- members[member] = uint32(len(info.gids))
- }
-
- return members, nil
-}
-
-// getMemberInfo returns SyncGroup information for a given member.
-func (s *syncGroupTable) getMemberInfo(member string) (*memberInfo, error) {
- if s.store == nil {
- return nil, errBadSGTable
- }
-
- info, ok := s.members[member]
- if !ok {
- return nil, fmt.Errorf("unknown member: %s", member)
- }
-
- return info, nil
-}
-
-// addMember inserts or updates a (member, group ID) entry in the in-memory
-// structure that indexes SyncGroup memberships based on member names and stores
-// in it the member's joiner metadata.
-func (s *syncGroupTable) addMember(member string, gid syncgroup.ID, identity string, metadata syncgroup.JoinerMetaData) {
- if s.store == nil {
- return
- }
-
- info, ok := s.members[member]
- if !ok {
- info = &memberInfo{gids: make(map[syncgroup.ID]*memberMetaData)}
- s.members[member] = info
- }
-
- info.gids[gid] = &memberMetaData{metaData: metadata, identity: identity}
-}
-
-// delMember removes a (member, group ID) entry from the in-memory structure
-// that indexes SyncGroup memberships based on member names.
-func (s *syncGroupTable) delMember(member string, gid syncgroup.ID) {
- if s.store == nil {
- return
- }
-
- info, ok := s.members[member]
- if !ok {
- return
- }
-
- delete(info.gids, gid)
- if len(info.gids) == 0 {
- delete(s.members, member)
- }
-}
-
-// addAllMembers inserts all members of a SyncGroup in the in-memory structure
-// that indexes SyncGroup memberships based on member names.
-func (s *syncGroupTable) addAllMembers(data *syncGroupData) {
- if s.store == nil || data == nil {
- return
- }
-
- gid := data.SrvInfo.SGOID
- for member, memberData := range data.SrvInfo.Joiners {
- s.addMember(member.Name, gid, member.Identity, memberData)
- }
-}
-
-// delAllMembers removes all members of a SyncGroup from the in-memory structure
-// that indexes SyncGroup memberships based on member names.
-func (s *syncGroupTable) delAllMembers(data *syncGroupData) {
- if s.store == nil || data == nil {
- return
- }
-
- gid := data.SrvInfo.SGOID
- for member := range data.SrvInfo.Joiners {
- s.delMember(member.Name, gid)
- }
-}
-
-// addPeerSyncGroup inserts the group ID into the in-memory set of peer SyncGroups
-// that use the same RootOID in the Store.
-func (s *syncGroupTable) addPeerSyncGroup(gid syncgroup.ID, rootOID storage.ID) {
- if s.store == nil {
- return
- }
-
- peers, ok := s.peerSGs[rootOID]
- if !ok {
- peers = make(sgSet)
- s.peerSGs[rootOID] = peers
- }
-
- peers[gid] = struct{}{}
-}
-
-// delPeerSyncGroup removes the group ID from the in-memory set of peer SyncGroups
-// that use the same RootOID in the Store.
-func (s *syncGroupTable) delPeerSyncGroup(gid syncgroup.ID, rootOID storage.ID) {
- if s.store == nil {
- return
- }
-
- peers, ok := s.peerSGs[rootOID]
- if !ok {
- return
- }
-
- delete(peers, gid)
- if len(peers) == 0 {
- delete(s.peerSGs, rootOID)
- }
-}
-
-// getPeerSyncGroups returns the set of peer SyncGroups for a given SyncGroup.
-// The given SyncGroup ID is included in that set.
-func (s *syncGroupTable) getPeerSyncGroups(gid syncgroup.ID) (sgSet, error) {
- if s.store == nil {
- return sgSet{}, errBadSGTable
- }
-
- data, err := s.getSyncGroupByID(gid)
- if err != nil {
- return sgSet{}, err
- }
-
- return s.peerSGs[data.SrvInfo.RootOID], nil
-}
-
-// Low-level functions to access the tables in the K/V DB.
-// They directly access the table entries without tracking their relationships.
-
-// sgDataKey returns the key used to access the SyncGroup data in the DB.
-func sgDataKey(gid syncgroup.ID) string {
- return gid.String()
-}
-
-// hasSGDataEntry returns true if the SyncGroup data entry exists in the DB.
-func (s *syncGroupTable) hasSGDataEntry(gid syncgroup.ID) bool {
- if s.store == nil {
- return false
- }
- key := sgDataKey(gid)
- return s.sgData.hasKey(key)
-}
-
-// setSGDataEntry stores the SyncGroup data in the DB.
-func (s *syncGroupTable) setSGDataEntry(gid syncgroup.ID, data *syncGroupData) error {
- if s.store == nil {
- return errBadSGTable
- }
- key := sgDataKey(gid)
- return s.sgData.set(key, data)
-}
-
-// getSGDataEntry retrieves from the DB the SyncGroup data for a given group ID.
-func (s *syncGroupTable) getSGDataEntry(gid syncgroup.ID) (*syncGroupData, error) {
- if s.store == nil {
- return nil, errBadSGTable
- }
- var data syncGroupData
- key := sgDataKey(gid)
- if err := s.sgData.get(key, &data); err != nil {
- return nil, err
- }
- return &data, nil
-}
-
-// delSGDataEntry deletes the SyncGroup data from the DB.
-func (s *syncGroupTable) delSGDataEntry(gid syncgroup.ID) error {
- if s.store == nil {
- return errBadSGTable
- }
- key := sgDataKey(gid)
- return s.sgData.del(key)
-}
-
-// sgNameKey returns the key used to access the SyncGroup name in the DB.
-func sgNameKey(name string) string {
- return name
-}
-
-// hasSGNameEntry returns true if the SyncGroup name entry exists in the DB.
-func (s *syncGroupTable) hasSGNameEntry(name string) bool {
- if s.store == nil {
- return false
- }
- key := sgNameKey(name)
- return s.sgNames.hasKey(key)
-}
-
-// setSGNameEntry stores the SyncGroup name to ID mapping in the DB.
-func (s *syncGroupTable) setSGNameEntry(name string, gid syncgroup.ID) error {
- if s.store == nil {
- return errBadSGTable
- }
- key := sgNameKey(name)
- return s.sgNames.set(key, gid)
-}
-
-// getSGNameEntry retrieves the SyncGroup name to ID mapping from the DB.
-func (s *syncGroupTable) getSGNameEntry(name string) (syncgroup.ID, error) {
- var gid syncgroup.ID
- if s.store == nil {
- return gid, errBadSGTable
- }
- key := sgNameKey(name)
- err := s.sgNames.get(key, &gid)
- return gid, err
-}
-
-// delSGNameEntry deletes the SyncGroup name to ID mapping from the DB.
-func (s *syncGroupTable) delSGNameEntry(name string) error {
- if s.store == nil {
- return errBadSGTable
- }
- key := sgNameKey(name)
- return s.sgNames.del(key)
-}
diff --git a/runtimes/google/vsync/sgtable_test.go b/runtimes/google/vsync/sgtable_test.go
deleted file mode 100644
index 1f914b4..0000000
--- a/runtimes/google/vsync/sgtable_test.go
+++ /dev/null
@@ -1,894 +0,0 @@
-package vsync
-
-// Tests for the Veyron SyncGroup Table.
-
-import (
- "os"
- "reflect"
- "testing"
-
- "veyron/services/syncgroup"
-)
-
-// TestSyncGroupTableOpen tests the creation of a SyncGroup Table, closing and re-opening it.
-// It also verifies that its backing file is created and that a 2nd close is safe.
-func TestSyncGroupTableOpen(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- fsize := getFileSize(sgfile)
- if fsize < 0 {
- t.Fatalf("SyncGroup Table file %s not created", sgfile)
- }
-
- sg.flush()
- oldfsize := fsize
- fsize = getFileSize(sgfile)
- if fsize <= oldfsize {
- t.Fatalf("SyncGroup Table file %s not flushed", sgfile)
- }
-
- sg.close()
-
- sg, err = openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot re-open existing SyncGroup Table file %s", sgfile)
- }
-
- oldfsize = fsize
- fsize = getFileSize(sgfile)
- if fsize != oldfsize {
- t.Fatalf("SyncGroup Table file %s size changed across re-open", sgfile)
- }
-
- sg.close()
- sg.close() // multiple closes should be a safe NOP
-
- fsize = getFileSize(sgfile)
- if fsize != oldfsize {
- t.Fatalf("SyncGroup Table file %s size changed across close", sgfile)
- }
-
- // Fail opening a SyncGroup Table in a non-existent directory.
- _, err = openSyncGroupTable("/not/really/there/junk.sg")
- if err == nil {
- t.Fatalf("openSyncGroupTable() did not fail when using a bad pathname")
- }
-}
-
-// TestInvalidSyncGroupTable tests using methods on an invalid (closed) SyncGroup Table.
-func TestInvalidSyncGroupTable(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- sg.close()
-
- sgid, err := syncgroup.ParseID("1234")
- if err != nil {
- t.Error(err)
- }
- rootid, err := strToObjID("5678")
- if err != nil {
- t.Fatal(err)
- }
-
- validateError := func(t *testing.T, err error, funcName string) {
- if err == nil || err.Error() != "invalid SyncGroup Table" {
- t.Errorf("%s() did not fail on a closed SyncGroup Table: %v", funcName, err)
- }
- }
-
- err = sg.compact()
- validateError(t, err, "compact")
-
- err = sg.addSyncGroup(&syncGroupData{})
- validateError(t, err, "addSyncGroup")
-
- _, err = sg.getSyncGroupID("foobar")
- validateError(t, err, "getSyncGroupID")
-
- _, err = sg.getSyncGroupName(sgid)
- validateError(t, err, "getSyncGroupName")
-
- _, err = sg.getSyncGroupByID(sgid)
- validateError(t, err, "getSyncGroupByID")
-
- _, err = sg.getSyncGroupByName("foobar")
- validateError(t, err, "getSyncGroupByName")
-
- err = sg.updateSyncGroup(&syncGroupData{})
- validateError(t, err, "updateSyncGroup")
-
- err = sg.delSyncGroupByID(sgid)
- validateError(t, err, "delSyncGroupByID")
-
- err = sg.delSyncGroupByName("foobar")
- validateError(t, err, "delSyncGroupByName")
-
- _, err = sg.getMembers()
- validateError(t, err, "getMembers")
-
- _, err = sg.getMemberInfo("foobar")
- validateError(t, err, "getMemberInfo")
-
- _, err = sg.getPeerSyncGroups(sgid)
- validateError(t, err, "getPeerSyncGroups")
-
- err = sg.setSGDataEntry(sgid, &syncGroupData{})
- validateError(t, err, "setSGDataEntry")
-
- _, err = sg.getSGDataEntry(sgid)
- validateError(t, err, "getSGDataEntry")
-
- err = sg.delSGDataEntry(sgid)
- validateError(t, err, "delSGDataEntry")
-
- err = sg.setSGNameEntry("foobar", sgid)
- validateError(t, err, "setSGNameEntry")
-
- _, err = sg.getSGNameEntry("foobar")
- validateError(t, err, "getSGNameEntry")
-
- err = sg.delSGNameEntry("foobar")
- validateError(t, err, "delSGNameEntry")
-
- // These calls should be harmless NOPs.
- sg.flush()
- sg.close()
- sg.addMember("foobar", sgid, "foobar identity", syncgroup.JoinerMetaData{})
- sg.delMember("foobar", sgid)
- sg.addAllMembers(&syncGroupData{})
- sg.delAllMembers(&syncGroupData{})
- sg.addPeerSyncGroup(sgid, rootid)
- sg.delPeerSyncGroup(sgid, rootid)
-
- if sg.hasSGDataEntry(sgid) {
- t.Errorf("hasSGDataEntry() found an entry on a closed SyncGroup Table")
- }
- if sg.hasSGNameEntry("foobar") {
- t.Errorf("hasSGNameEntry() found an entry on a closed SyncGroup Table")
- }
-}
-
-// TestAddSyncGroup tests adding SyncGroups.
-func TestAddSyncGroup(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- sgname := "foobar"
- sgid, err := syncgroup.ParseID("1234")
- if err != nil {
- t.Fatal(err)
- }
- rootid, err := strToObjID("5678")
- if err != nil {
- t.Fatal(err)
- }
-
- sgData := &syncGroupData{
- SrvInfo: syncgroup.SyncGroupInfo{
- SGOID: sgid,
- RootOID: rootid,
- Name: sgname,
- Joiners: map[syncgroup.NameIdentity]syncgroup.JoinerMetaData{
- syncgroup.NameIdentity{Name: "phone", Identity: "A"}: syncgroup.JoinerMetaData{SyncPriority: 10},
- syncgroup.NameIdentity{Name: "tablet", Identity: "B"}: syncgroup.JoinerMetaData{SyncPriority: 25},
- syncgroup.NameIdentity{Name: "cloud", Identity: "C"}: syncgroup.JoinerMetaData{SyncPriority: 1},
- },
- },
- }
-
- err = sg.addSyncGroup(sgData)
- if err != nil {
- t.Errorf("adding SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- // Verify SyncGroup ID, name, and data.
- if id, err := sg.getSyncGroupID(sgname); err != nil || id != sgid {
- t.Errorf("cannot get back ID of SyncGroup %s: got ID %d instead of %d; err: %v", sgname, id, sgid, err)
- }
- if name, err := sg.getSyncGroupName(sgid); err != nil || name != sgname {
- t.Errorf("cannot get back name of SyncGroup ID %d: got %s instead of %s; err: %v", sgid, name, sgname, err)
- }
-
- data, err := sg.getSyncGroupByID(sgid)
- if err != nil {
- t.Errorf("cannot get SyncGroup by ID %d: %v", sgid, err)
- }
- if !reflect.DeepEqual(data, sgData) {
- t.Errorf("invalid SyncGroup data for group ID %d: got %v instead of %v", sgid, data, sgData)
- }
-
- data, err = sg.getSyncGroupByName(sgname)
- if err != nil {
- t.Errorf("cannot get SyncGroup by Name %s: %v", sgname, err)
- }
- if !reflect.DeepEqual(data, sgData) {
- t.Errorf("invalid SyncGroup data for group name %s: got %v instead of %v", sgname, data, sgData)
- }
-
- // Verify membership data.
- members, err := sg.getMembers()
- if err != nil {
- t.Errorf("cannot get all SyncGroup members: %v", err)
- }
- expMembers := map[string]uint32{"phone": 1, "tablet": 1, "cloud": 1}
- if !reflect.DeepEqual(members, expMembers) {
- t.Errorf("invalid SyncGroup members: got %v instead of %v", members, expMembers)
- }
-
- expMetaData := map[string]*memberMetaData{
- "phone": &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 10}, identity: "A"},
- "tablet": &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 25}, identity: "B"},
- "cloud": &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 1}, identity: "C"},
- }
- for mm := range members {
- info, err := sg.getMemberInfo(mm)
- if err != nil || info == nil {
- t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
- }
- if len(info.gids) != 1 {
- t.Errorf("invalid info for SyncGroup member %s: %v", mm, info)
- }
- expJoinerMetaData := expMetaData[mm]
- joinerMetaData := info.gids[sgid]
- if !reflect.DeepEqual(joinerMetaData, expJoinerMetaData) {
- t.Errorf("invalid joiner Data for SyncGroup member %s under group ID %d: got %v instead of %v",
- mm, sgid, joinerMetaData, expJoinerMetaData)
- }
- }
-
- // Use a non-existent member.
- if info, err := sg.getMemberInfo("should-not-be-there"); err == nil {
- t.Errorf("found info for invalid SyncGroup member: %v", info)
- }
-
- // Verify peer SyncGroup info.
- peers, err := sg.getPeerSyncGroups(sgid)
- if err != nil {
- t.Errorf("cannot get peer SyncGroups for group ID %d: %v", sgid, err)
- }
- expPeers := sgSet{sgid: struct{}{}}
- if !reflect.DeepEqual(peers, expPeers) {
- t.Errorf("invalid peer SyncGroups: got %v instead of %v", peers, expPeers)
- }
-
- // Use a non-existent group ID.
- xid, err := syncgroup.ParseID("7788998877")
- if err != nil {
- t.Fatal(err)
- }
- if peers, err := sg.getPeerSyncGroups(xid); err == nil {
- t.Errorf("found peer SyncGroup for invalid SyncGroup: %v", peers)
- }
-
- // Adding a SyncGroup for a pre-existing group ID or name should fail.
- err = sg.addSyncGroup(sgData)
- if err == nil {
- t.Errorf("re-adding SyncGroup %d did not fail", sgid)
- }
-
- sgData.SrvInfo.SGOID, err = syncgroup.ParseID("5555")
- if err != nil {
- t.Fatal(err)
- }
- err = sg.addSyncGroup(sgData)
- if err == nil {
- t.Errorf("adding SyncGroup %s with a different ID did not fail", sgname)
- }
-
- sg.close()
-}
-
-// TestInvalidAddSyncGroup tests adding SyncGroups.
-func TestInvalidAddSyncGroup(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- sgname := "foobar"
- sgid, err := syncgroup.ParseID("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- err = sg.addSyncGroup(nil)
- if err == nil {
- t.Errorf("adding a nil SyncGroup did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData := &syncGroupData{}
- sgData.SrvInfo.SGOID = sgid
- sgData.SrvInfo.RootOID, err = strToObjID("5678")
- if err != nil {
- t.Fatal(err)
- }
-
- err = sg.addSyncGroup(sgData)
- if err == nil {
- t.Errorf("adding a SyncGroup with an empty name did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData.SrvInfo.Name = sgname
-
- err = sg.addSyncGroup(sgData)
- if err == nil {
- t.Errorf("adding a SyncGroup with no joiners did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sg.close()
-}
-
-// TestUpdateSyncGroup tests updating a SyncGroup.
-func TestUpdateSyncGroup(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- err = sg.updateSyncGroup(nil)
- if err == nil {
- t.Errorf("updating a nil SyncGroup did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData := &syncGroupData{}
- err = sg.updateSyncGroup(sgData)
- if err == nil {
- t.Errorf("updating a SyncGroup with an empty name did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData.SrvInfo.Name = "blabla"
- err = sg.updateSyncGroup(sgData)
- if err == nil {
- t.Errorf("updating a SyncGroup with no joiners did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData.SrvInfo.Joiners = map[syncgroup.NameIdentity]syncgroup.JoinerMetaData{
- syncgroup.NameIdentity{Name: "phone", Identity: "X"}: syncgroup.JoinerMetaData{SyncPriority: 10},
- }
- err = sg.updateSyncGroup(sgData)
- if err == nil {
- t.Errorf("updating a SyncGroup with a non-existing name did not fail in SyncGroup Table file %s", sgfile)
- }
-
- // Create the SyncGroup to update later.
- sgname := "foobar"
- sgid, err := syncgroup.ParseID("1234")
- if err != nil {
- t.Fatal(err)
- }
- rootid, err := strToObjID("5678")
- if err != nil {
- t.Fatal(err)
- }
-
- sgData = &syncGroupData{
- SrvInfo: syncgroup.SyncGroupInfo{
- SGOID: sgid,
- RootOID: rootid,
- Name: sgname,
- Joiners: map[syncgroup.NameIdentity]syncgroup.JoinerMetaData{
- syncgroup.NameIdentity{Name: "phone", Identity: "A"}: syncgroup.JoinerMetaData{SyncPriority: 10},
- syncgroup.NameIdentity{Name: "tablet", Identity: "B"}: syncgroup.JoinerMetaData{SyncPriority: 25},
- syncgroup.NameIdentity{Name: "cloud", Identity: "C"}: syncgroup.JoinerMetaData{SyncPriority: 1},
- },
- },
- }
-
- err = sg.addSyncGroup(sgData)
- if err != nil {
- t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- // Update it using different group or root IDs, which is not allowed.
- xid, err := syncgroup.ParseID("9999")
- if err != nil {
- t.Fatal(err)
- }
-
- sgData.SrvInfo.SGOID = xid
-
- err = sg.updateSyncGroup(sgData)
- if err == nil {
- t.Errorf("updating a SyncGroup with an ID mismatch did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData.SrvInfo.SGOID = sgid
- sgData.SrvInfo.RootOID, err = strToObjID("9999")
- if err != nil {
- t.Fatal(err)
- }
-
- err = sg.updateSyncGroup(sgData)
- if err == nil {
- t.Errorf("updating a SyncGroup with a root ID mismatch did not fail in SyncGroup Table file %s", sgfile)
- }
-
- // Update it using a modified set of joiners.
- sgData.SrvInfo.RootOID = rootid
- sgData.SrvInfo.Joiners[syncgroup.NameIdentity{Name: "universe", Identity: "Y"}] = syncgroup.JoinerMetaData{SyncPriority: 0}
- delete(sgData.SrvInfo.Joiners, syncgroup.NameIdentity{Name: "cloud", Identity: "C"})
-
- err = sg.updateSyncGroup(sgData)
- if err != nil {
- t.Errorf("updating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- // Do some NOP member deletions (bad member, bad group ID).
- // SyncGroup verification (below) should see the expected info asserting these were NOPs.
- sg.delMember("blablablablabla", sgid)
- sg.delMember("phone", xid)
-
- // Verify updated SyncGroup.
- if id, err := sg.getSyncGroupID(sgname); err != nil || id != sgid {
- t.Errorf("cannot get back ID of updated SyncGroup %s: got ID %d instead of %d; err: %v", sgname, id, sgid, err)
- }
- if name, err := sg.getSyncGroupName(sgid); err != nil || name != sgname {
- t.Errorf("cannot get back name of updated SyncGroup ID %d: got %s instead of %s; err: %v", sgid, name, sgname, err)
- }
-
- expData := &syncGroupData{
- SrvInfo: syncgroup.SyncGroupInfo{
- SGOID: sgid,
- RootOID: rootid,
- Name: sgname,
- Joiners: map[syncgroup.NameIdentity]syncgroup.JoinerMetaData{
- syncgroup.NameIdentity{Name: "phone", Identity: "A"}: syncgroup.JoinerMetaData{SyncPriority: 10},
- syncgroup.NameIdentity{Name: "tablet", Identity: "B"}: syncgroup.JoinerMetaData{SyncPriority: 25},
- syncgroup.NameIdentity{Name: "universe", Identity: "Y"}: syncgroup.JoinerMetaData{SyncPriority: 0},
- },
- },
- }
-
- data, err := sg.getSyncGroupByID(sgid)
- if err != nil {
- t.Errorf("cannot get updated SyncGroup by ID %d: %v", sgid, err)
- }
- if !reflect.DeepEqual(data, expData) {
- t.Errorf("invalid SyncGroup data for updated group ID %d: got %v instead of %v", sgid, data, expData)
- }
-
- data, err = sg.getSyncGroupByName(sgname)
- if err != nil {
- t.Errorf("cannot get updated SyncGroup by Name %s: %v", sgname, err)
- }
- if !reflect.DeepEqual(data, expData) {
- t.Errorf("invalid SyncGroup data for updated group name %s: got %v instead of %v", sgname, data, expData)
- }
-
- // Verify membership data.
- members, err := sg.getMembers()
- if err != nil {
- t.Errorf("cannot get all SyncGroup members after update: %v", err)
- }
- expMembers := map[string]uint32{"phone": 1, "tablet": 1, "universe": 1}
- if !reflect.DeepEqual(members, expMembers) {
- t.Errorf("invalid SyncGroup members after update: got %v instead of %v", members, expMembers)
- }
-
- expMetaData := map[string]*memberMetaData{
- "phone": &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 10}, identity: "A"},
- "tablet": &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 25}, identity: "B"},
- "universe": &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 0}, identity: "Y"},
- }
- for mm := range members {
- info, err := sg.getMemberInfo(mm)
- if err != nil || info == nil {
- t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
- }
- if len(info.gids) != 1 {
- t.Errorf("invalid info for SyncGroup member %s: %v", mm, info)
- }
- expJoinerMetaData := expMetaData[mm]
- joinerMetaData := info.gids[sgid]
- if !reflect.DeepEqual(joinerMetaData, expJoinerMetaData) {
- t.Errorf("invalid joiner Data for SyncGroup member %s under group ID %d: got %v instead of %v",
- mm, sgid, joinerMetaData, expJoinerMetaData)
- }
- }
-
- // Verify peer SyncGroup info.
- peers, err := sg.getPeerSyncGroups(sgid)
- if err != nil {
- t.Errorf("cannot get peer SyncGroups for group ID %d after an update: %v", sgid, err)
- }
- expPeers := sgSet{sgid: struct{}{}}
- if !reflect.DeepEqual(peers, expPeers) {
- t.Errorf("invalid peer SyncGroups after an update: got %v instead of %v", peers, expPeers)
- }
-
- sg.close()
-}
-
-// TestDeleteSyncGroup tests deleting a SyncGroup.
-func TestDeleteSyncGroup(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- sgname := "foobar"
- sgid, err := syncgroup.ParseID("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- // Delete non-existing SyncGroups.
- err = sg.delSyncGroupByID(sgid)
- if err == nil {
- t.Errorf("deleting a non-existing SyncGroup ID did not fail in SyncGroup Table file %s", sgfile)
- }
-
- err = sg.delSyncGroupByName(sgname)
- if err == nil {
- t.Errorf("deleting a non-existing SyncGroup name did not fail in SyncGroup Table file %s", sgfile)
- }
-
- // Create the SyncGroup to delete later.
- rootid, err := strToObjID("5678")
- if err != nil {
- t.Fatal(err)
- }
-
- sgData := &syncGroupData{
- SrvInfo: syncgroup.SyncGroupInfo{
- SGOID: sgid,
- RootOID: rootid,
- Name: sgname,
- Joiners: map[syncgroup.NameIdentity]syncgroup.JoinerMetaData{
- syncgroup.NameIdentity{Name: "phone", Identity: "A"}: syncgroup.JoinerMetaData{SyncPriority: 10},
- syncgroup.NameIdentity{Name: "tablet", Identity: "B"}: syncgroup.JoinerMetaData{SyncPriority: 25},
- syncgroup.NameIdentity{Name: "cloud", Identity: "C"}: syncgroup.JoinerMetaData{SyncPriority: 1},
- },
- },
- }
-
- err = sg.addSyncGroup(sgData)
- if err != nil {
- t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- // Delete it by ID.
- err = sg.delSyncGroupByID(sgid)
- if err != nil {
- t.Errorf("deleting SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- // Create it again then delete it by name.
- err = sg.addSyncGroup(sgData)
- if err != nil {
- t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- err = sg.delSyncGroupByName(sgname)
- if err != nil {
- t.Errorf("deleting SyncGroup name %s failed in SyncGroup Table file %s: %v", sgname, sgfile, err)
- }
-
- // Verify peer SyncGroup info.
- if peers, err := sg.getPeerSyncGroups(sgid); err == nil {
- t.Errorf("got get peer SyncGroups for deleted group ID %d: %v", sgid, peers)
- }
-
- sg.close()
-}
-
-// TestSyncGroupTableCompact tests compacting the SyncGroup Table K/V DB file.
-func TestSyncGroupTableCompact(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- sgname := "foobar"
- sgid, err := syncgroup.ParseID("1234")
- if err != nil {
- t.Fatal(err)
- }
- rootid, err := strToObjID("5678")
- if err != nil {
- t.Fatal(err)
- }
-
- // Add a SyncGroup and use flushes to increase the K/V DB file size.
- sgData := &syncGroupData{
- SrvInfo: syncgroup.SyncGroupInfo{
- SGOID: sgid,
- RootOID: rootid,
- Name: sgname,
- Joiners: map[syncgroup.NameIdentity]syncgroup.JoinerMetaData{
- syncgroup.NameIdentity{Name: "phone", Identity: "A"}: syncgroup.JoinerMetaData{SyncPriority: 10},
- syncgroup.NameIdentity{Name: "tablet", Identity: "B"}: syncgroup.JoinerMetaData{SyncPriority: 25},
- syncgroup.NameIdentity{Name: "cloud", Identity: "C"}: syncgroup.JoinerMetaData{SyncPriority: 1},
- },
- },
- }
-
- sg.flush()
-
- err = sg.addSyncGroup(sgData)
- if err != nil {
- t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- sg.flush()
-
- // Verify SyncGroup and membership info after 2 steps: close/reopen, and compact.
-
- for i := 0; i < 2; i++ {
- switch i {
- case 0:
- sg.close()
- sg, err = openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot re-open SyncGroup Table file %s", sgfile)
- }
-
- case 1:
- if err = sg.compact(); err != nil {
- t.Fatalf("cannot compact SyncGroup Table file %s", sgfile)
- }
- }
-
- // Verify SyncGroup data.
- data, err := sg.getSyncGroupByID(sgid)
- if err != nil {
- t.Errorf("cannot get SyncGroup ID %d (iter %d) in SyncGroup Table file %s: %v", sgid, i, sgfile, err)
- }
- if !reflect.DeepEqual(data, sgData) {
- t.Errorf("invalid SyncGroup data for ID %d (iter %d): got %v instead of %v", sgid, i, data, sgData)
- }
-
- data, err = sg.getSyncGroupByName(sgname)
- if err != nil {
- t.Errorf("cannot get SyncGroup name %s (iter %d) in SyncGroup Table file %s: %v", sgname, i, sgfile, err)
- }
- if !reflect.DeepEqual(data, sgData) {
- t.Errorf("invalid SyncGroup data for name %s (iter %d): got %v instead of %v", sgname, i, data, sgData)
- }
-
- // Verify membership data.
- members, err := sg.getMembers()
- if err != nil {
- t.Errorf("cannot get all SyncGroup members (iter %d): %v", i, err)
- }
- expMembers := map[string]uint32{"phone": 1, "tablet": 1, "cloud": 1}
- if !reflect.DeepEqual(members, expMembers) {
- t.Errorf("invalid SyncGroup members (iter %d): got %v instead of %v", i, members, expMembers)
- }
-
- expMetaData := map[string]*memberMetaData{
- "phone": &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 10}, identity: "A"},
- "tablet": &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 25}, identity: "B"},
- "cloud": &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 1}, identity: "C"},
- }
- for mm := range members {
- info, err := sg.getMemberInfo(mm)
- if err != nil || info == nil {
- t.Errorf("cannot get info for SyncGroup member %s (iter %d): info: %v, err: %v", mm, i, info, err)
- }
- if len(info.gids) != 1 {
- t.Errorf("invalid info for SyncGroup member %s (iter %d): %v", mm, i, info)
- }
- expJoinerMetaData := expMetaData[mm]
- joinerMetaData := info.gids[sgid]
- if !reflect.DeepEqual(joinerMetaData, expJoinerMetaData) {
- t.Errorf("invalid joiner Data for SyncGroup member %s (iter %d) in group ID %d: %v instead of %v",
- mm, i, sgid, joinerMetaData, expJoinerMetaData)
- }
- }
- }
-
- sg.close()
-}
-
-// TestPeerSyncGroups tests creating peer SyncGroup on the same root OID.
-func TestPeerSyncGroups(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- sgname1, sgname2 := "foo", "bar"
- sgid1, err := syncgroup.ParseID("1234")
- if err != nil {
- t.Fatal(err)
- }
- sgid2, err := syncgroup.ParseID("8888")
- if err != nil {
- t.Fatal(err)
- }
- rootid, err := strToObjID("5678")
- if err != nil {
- t.Fatal(err)
- }
-
- // Add both SyncGroups using the same root OID.
- sgData1 := &syncGroupData{
- SrvInfo: syncgroup.SyncGroupInfo{
- SGOID: sgid1,
- RootOID: rootid,
- Name: sgname1,
- Joiners: map[syncgroup.NameIdentity]syncgroup.JoinerMetaData{
- syncgroup.NameIdentity{Name: "phone", Identity: "A"}: syncgroup.JoinerMetaData{SyncPriority: 10},
- syncgroup.NameIdentity{Name: "tablet", Identity: "B"}: syncgroup.JoinerMetaData{SyncPriority: 25},
- syncgroup.NameIdentity{Name: "cloud", Identity: "C"}: syncgroup.JoinerMetaData{SyncPriority: 1},
- },
- },
- }
-
- sgData2 := &syncGroupData{
- SrvInfo: syncgroup.SyncGroupInfo{
- SGOID: sgid2,
- RootOID: rootid,
- Name: sgname2,
- Joiners: map[syncgroup.NameIdentity]syncgroup.JoinerMetaData{
- syncgroup.NameIdentity{Name: "tablet", Identity: "B"}: syncgroup.JoinerMetaData{SyncPriority: 111},
- syncgroup.NameIdentity{Name: "door", Identity: "X"}: syncgroup.JoinerMetaData{SyncPriority: 33},
- syncgroup.NameIdentity{Name: "lamp", Identity: "Z"}: syncgroup.JoinerMetaData{SyncPriority: 9},
- },
- },
- }
-
- err = sg.addSyncGroup(sgData1)
- if err != nil {
- t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid1, sgfile, err)
- }
- err = sg.addSyncGroup(sgData2)
- if err != nil {
- t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid2, sgfile, err)
- }
-
- // Verify peer SyncGroup info.
- expPeers := sgSet{sgid1: struct{}{}, sgid2: struct{}{}}
-
- for _, sgid := range []syncgroup.ID{sgid1, sgid2} {
- peers, err := sg.getPeerSyncGroups(sgid)
- if err != nil {
- t.Errorf("cannot get peer SyncGroups for group ID %d: %v", sgid, err)
- }
- if !reflect.DeepEqual(peers, expPeers) {
- t.Errorf("invalid peer SyncGroups got group ID %d: got %v instead of %v", sgid, peers, expPeers)
- }
- }
-
- // Verify SyncGroup membership data.
- members, err := sg.getMembers()
- if err != nil {
- t.Errorf("cannot get all SyncGroup members: %v", err)
- }
-
- expMembers := map[string]uint32{"phone": 1, "tablet": 2, "cloud": 1, "door": 1, "lamp": 1}
- if !reflect.DeepEqual(members, expMembers) {
- t.Errorf("invalid SyncGroup members: got %v instead of %v", members, expMembers)
- }
-
- expMemberInfo := map[string]*memberInfo{
- "phone": &memberInfo{
- gids: map[syncgroup.ID]*memberMetaData{
- sgid1: &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 10}, identity: "A"},
- },
- },
- "tablet": &memberInfo{
- gids: map[syncgroup.ID]*memberMetaData{
- sgid1: &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 25}, identity: "B"},
- sgid2: &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 111}, identity: "B"},
- },
- },
- "cloud": &memberInfo{
- gids: map[syncgroup.ID]*memberMetaData{
- sgid1: &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 1}, identity: "C"},
- },
- },
- "door": &memberInfo{
- gids: map[syncgroup.ID]*memberMetaData{
- sgid2: &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 33}, identity: "X"},
- },
- },
- "lamp": &memberInfo{
- gids: map[syncgroup.ID]*memberMetaData{
- sgid2: &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 9}, identity: "Z"},
- },
- },
- }
-
- for mm := range members {
- info, err := sg.getMemberInfo(mm)
- if err != nil || info == nil {
- t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
- }
- expInfo := expMemberInfo[mm]
- if !reflect.DeepEqual(info, expInfo) {
- t.Errorf("invalid info for SyncGroup member %s: got %v instead of %v", mm, info, expInfo)
- }
- }
-
- // Delete the 1st SyncGroup.
- err = sg.delSyncGroupByID(sgid1)
- if err != nil {
- t.Errorf("deleting SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid1, sgfile, err)
- }
-
- // Verify peer SyncGroup info.
- expPeers = sgSet{sgid2: struct{}{}}
- peers, err := sg.getPeerSyncGroups(sgid2)
- if err != nil {
- t.Errorf("cannot get peer SyncGroups for group ID %d: %v", sgid2, err)
- }
- if !reflect.DeepEqual(peers, expPeers) {
- t.Errorf("invalid peer SyncGroups got group ID %d: got %v instead of %v", sgid2, peers, expPeers)
- }
-
- // Verify SyncGroup membership data.
- members, err = sg.getMembers()
- if err != nil {
- t.Errorf("cannot get all SyncGroup members: %v", err)
- }
-
- expMembers = map[string]uint32{"tablet": 1, "door": 1, "lamp": 1}
- if !reflect.DeepEqual(members, expMembers) {
- t.Errorf("invalid SyncGroup members: got %v instead of %v", members, expMembers)
- }
-
- expMemberInfo = map[string]*memberInfo{
- "tablet": &memberInfo{
- gids: map[syncgroup.ID]*memberMetaData{
- sgid2: &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 111}, identity: "B"},
- },
- },
- "door": &memberInfo{
- gids: map[syncgroup.ID]*memberMetaData{
- sgid2: &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 33}, identity: "X"},
- },
- },
- "lamp": &memberInfo{
- gids: map[syncgroup.ID]*memberMetaData{
- sgid2: &memberMetaData{metaData: syncgroup.JoinerMetaData{SyncPriority: 9}, identity: "Z"},
- },
- },
- }
-
- for mm := range members {
- info, err := sg.getMemberInfo(mm)
- if err != nil || info == nil {
- t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
- }
- expInfo := expMemberInfo[mm]
- if !reflect.DeepEqual(info, expInfo) {
- t.Errorf("invalid info for SyncGroup member %s: got %v instead of %v", mm, info, expInfo)
- }
- }
-
- sg.close()
-}
diff --git a/runtimes/google/vsync/testdata/local-init-00.log.sync b/runtimes/google/vsync/testdata/local-init-00.log.sync
deleted file mode 100644
index 3eb51de..0000000
--- a/runtimes/google/vsync/testdata/local-init-00.log.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object locally and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addl|12345|1|||logrec-00|false|false
-addl|12345|2|1||logrec-01|false|false
-addl|12345|3|2||logrec-02|false|false
diff --git a/runtimes/google/vsync/testdata/local-init-00.sync b/runtimes/google/vsync/testdata/local-init-00.sync
deleted file mode 100644
index d1dace2..0000000
--- a/runtimes/google/vsync/testdata/local-init-00.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object locally and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addl|12345|0|||logrec-00|false|false
-addl|12345|1|0||logrec-01|false|false
-addl|12345|2|1||logrec-02|false|false
diff --git a/runtimes/google/vsync/testdata/local-init-01.log.sync b/runtimes/google/vsync/testdata/local-init-01.log.sync
deleted file mode 100644
index 578f795..0000000
--- a/runtimes/google/vsync/testdata/local-init-01.log.sync
+++ /dev/null
@@ -1,9 +0,0 @@
-# Create objects locally and update one and delete another.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addl|123|1|||logrec-00|false|false
-addl|123|2|1||logrec-01|false|false
-addl|123|3|2||logrec-02|false|false
-
-addl|456|1|||logrec-00|false|false
-addl|456|0|1||logrec-00|false|true
\ No newline at end of file
diff --git a/runtimes/google/vsync/testdata/local-init-01.sync b/runtimes/google/vsync/testdata/local-init-01.sync
deleted file mode 100644
index 525dd09..0000000
--- a/runtimes/google/vsync/testdata/local-init-01.sync
+++ /dev/null
@@ -1,12 +0,0 @@
-# Create an object DAG locally with branches and resolved conflicts.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addl|12345|0|||logrec-00|false|false
-addl|12345|1|0||logrec-01|false|false
-addl|12345|2|1||logrec-02|false|false
-addl|12345|3|1||logrec-03|false|false
-addl|12345|4|2|3|logrec-04|false|false
-addl|12345|5|4||logrec-05|false|false
-addl|12345|6|1||logrec-06|false|false
-addl|12345|7|5|6|logrec-07|false|false
-addl|12345|8|7||logrec-08|false|false
diff --git a/runtimes/google/vsync/testdata/local-init-02.sync b/runtimes/google/vsync/testdata/local-init-02.sync
deleted file mode 100644
index 70b1319..0000000
--- a/runtimes/google/vsync/testdata/local-init-02.sync
+++ /dev/null
@@ -1,10 +0,0 @@
-# Create DAGs for 3 objects locally.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addl|12345|1|||logrec-a-01|false|false
-addl|12345|2|1||logrec-a-02|false|false
-
-addl|67890|1|||logrec-b-01|false|false
-addl|67890|2|1||logrec-b-02|false|false
-
-addl|222|1|||logrec-c-01|false|false
diff --git a/runtimes/google/vsync/testdata/local-init-03.sync b/runtimes/google/vsync/testdata/local-init-03.sync
deleted file mode 100644
index 3b3fba3..0000000
--- a/runtimes/google/vsync/testdata/local-init-03.sync
+++ /dev/null
@@ -1,10 +0,0 @@
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addl|12345|1|||logrec-01|false|false
-addl|12345|2|1||logrec-02|false|false
-addl|12345|3|1||logrec-03|false|false
-addl|12345|4|2||logrec-04|false|false
-addl|12345|5|2||logrec-05|false|true
-addl|12345|6|4|5|logrec-06|false|false
-addl|12345|7|3|5|logrec-07|false|false
-addl|12345|8|6|7|logrec-08|false|false
diff --git a/runtimes/google/vsync/testdata/local-resolve-00.sync b/runtimes/google/vsync/testdata/local-resolve-00.sync
deleted file mode 100644
index 02e1c88..0000000
--- a/runtimes/google/vsync/testdata/local-resolve-00.sync
+++ /dev/null
@@ -1,4 +0,0 @@
-# Create an object locally and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addl|12345|6|2|5|logrec-06|false|false
diff --git a/runtimes/google/vsync/testdata/remote-2obj-del.log.sync b/runtimes/google/vsync/testdata/remote-2obj-del.log.sync
deleted file mode 100644
index 403da95..0000000
--- a/runtimes/google/vsync/testdata/remote-2obj-del.log.sync
+++ /dev/null
@@ -1,7 +0,0 @@
-# Update one object and delete another object remotely.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|123|4|3||VeyronPhone:1:0|false|false
-addr|123|5|4||VeyronPhone:1:1|false|false
-addr|123|6|5||VeyronPhone:1:2|false|true
-addr|456|2|1||VeyronPhone:1:3|false|false
\ No newline at end of file
diff --git a/runtimes/google/vsync/testdata/remote-conf-00.log.sync b/runtimes/google/vsync/testdata/remote-conf-00.log.sync
deleted file mode 100644
index a391c14..0000000
--- a/runtimes/google/vsync/testdata/remote-conf-00.log.sync
+++ /dev/null
@@ -1,8 +0,0 @@
-# Update an object remotely three times triggering one conflict after
-# it was created locally up to v3 (i.e. assume the remote sync received
-# it from the local sync at v2, then updated separately).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|4|2||VeyronPhone:1:0|false|false
-addr|12345|5|4||VeyronPhone:1:1|false|false
-addr|12345|6|5||VeyronPhone:1:2|false|false
diff --git a/runtimes/google/vsync/testdata/remote-conf-00.sync b/runtimes/google/vsync/testdata/remote-conf-00.sync
deleted file mode 100644
index 7e555a6..0000000
--- a/runtimes/google/vsync/testdata/remote-conf-00.sync
+++ /dev/null
@@ -1,8 +0,0 @@
-# Update an object remotely three times triggering one conflict after
-# it was created locally up to v2 (i.e. assume the remote sync received
-# it from the local sync at v1, then updated separately).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|3|1||logrec-03|false|false
-addr|12345|4|3||logrec-04|false|false
-addr|12345|5|4||logrec-05|false|false
diff --git a/runtimes/google/vsync/testdata/remote-conf-01.log.sync b/runtimes/google/vsync/testdata/remote-conf-01.log.sync
deleted file mode 100644
index 63117bf..0000000
--- a/runtimes/google/vsync/testdata/remote-conf-01.log.sync
+++ /dev/null
@@ -1,11 +0,0 @@
-# Update an object remotely three times triggering a conflict with
-# 2 graft points: v1 and v4. This assumes that the remote sync got
-# v1, made its own conflicting v4 that it resolved into v5 (against v2)
-# then made a v6 change. When the local sync gets back this info it
-# sees 2 graft points: v1-v4 and v2-v5.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|4|1||VeyronLaptop:1:0|false|false
-addr|12345|5|2|4|VeyronPhone:1:0|false|false
-addr|12345|6|5||VeyronPhone:1:1|false|false
-
diff --git a/runtimes/google/vsync/testdata/remote-conf-01.sync b/runtimes/google/vsync/testdata/remote-conf-01.sync
deleted file mode 100644
index 4655a1e..0000000
--- a/runtimes/google/vsync/testdata/remote-conf-01.sync
+++ /dev/null
@@ -1,10 +0,0 @@
-# Update an object remotely three times triggering a conflict with
-# 2 graft points: v0 and v2. This assumes that the remote sync got
-# v0, made its own conflicting v3 that it resolved into v4 (against v1)
-# then made a v5 change. When the local sync gets back this info it
-# sees 2 graft points: v0-v3 and v1-v4.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|3|0||logrec-03|false|false
-addr|12345|4|1|3|logrec-04|false|false
-addr|12345|5|4||logrec-05|false|false
diff --git a/runtimes/google/vsync/testdata/remote-conf-link.log.sync b/runtimes/google/vsync/testdata/remote-conf-link.log.sync
deleted file mode 100644
index e6093a2..0000000
--- a/runtimes/google/vsync/testdata/remote-conf-link.log.sync
+++ /dev/null
@@ -1,5 +0,0 @@
-# Update an object remotely, detect conflict, and bless the local version.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|4|1||VeyronPhone:1:0|false|false
-linkr|12345|4|2||VeyronPhone:1:1
diff --git a/runtimes/google/vsync/testdata/remote-init-00.log.sync b/runtimes/google/vsync/testdata/remote-init-00.log.sync
deleted file mode 100644
index 5e12809..0000000
--- a/runtimes/google/vsync/testdata/remote-init-00.log.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object remotely and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|1|||VeyronPhone:1:0|false|false
-addr|12345|2|1||VeyronPhone:1:1|false|false
-addr|12345|3|2||VeyronPhone:1:2|false|false
diff --git a/runtimes/google/vsync/testdata/remote-init-00.sync b/runtimes/google/vsync/testdata/remote-init-00.sync
deleted file mode 100644
index 7f2189a..0000000
--- a/runtimes/google/vsync/testdata/remote-init-00.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object remotely and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|0|||logrec-00|false|false
-addr|12345|1|0||logrec-01|false|false
-addr|12345|2|1||logrec-02|false|false
diff --git a/runtimes/google/vsync/testdata/remote-init-01.log.sync b/runtimes/google/vsync/testdata/remote-init-01.log.sync
deleted file mode 100644
index ad022b4..0000000
--- a/runtimes/google/vsync/testdata/remote-init-01.log.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object remotely and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|1|||VeyronPhone:5:0|false|false
-addr|12345|2|1||VeyronPhone:5:1|false|false
-addr|12345|3|2||VeyronPhone:5:2|false|false
diff --git a/runtimes/google/vsync/testdata/remote-init-02.log.sync b/runtimes/google/vsync/testdata/remote-init-02.log.sync
deleted file mode 100644
index 8885949..0000000
--- a/runtimes/google/vsync/testdata/remote-init-02.log.sync
+++ /dev/null
@@ -1,17 +0,0 @@
-# Create objects and transactions remotely.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|123|1|||VeyronPhone:1:0|false|false
-
-addr|123|2|1||VeyronPhone:1:1|true|false
-addr|456|1|||VeyronPhone:1:2|true|false
-addr|789|1|||VeyronPhone:1:3|false|false
-
-addr|789|2|1||VeyronPhone:1:4|false|false
-
-addr|789|3|1||VeyronTab:1:0|false|false
-
-addr|789|4|2|3|VeyronPhone:2:0|false|false
-
-addr|123|3|2||VeyronPhone:2:1|true|false
-addr|456|2|1||VeyronPhone:2:2|false|false
diff --git a/runtimes/google/vsync/testdata/remote-init-03.log.sync b/runtimes/google/vsync/testdata/remote-init-03.log.sync
deleted file mode 100644
index 85083c7..0000000
--- a/runtimes/google/vsync/testdata/remote-init-03.log.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object remotely and delete it.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|1|||VeyronPhone:1:0|false|false
-addr|12345|2|1||VeyronPhone:1:1|false|false
-addr|12345|3|2||VeyronPhone:1:2|false|true
diff --git a/runtimes/google/vsync/testdata/remote-noconf-00.log.sync b/runtimes/google/vsync/testdata/remote-noconf-00.log.sync
deleted file mode 100644
index c56208b..0000000
--- a/runtimes/google/vsync/testdata/remote-noconf-00.log.sync
+++ /dev/null
@@ -1,8 +0,0 @@
-# Update an object remotely three times without triggering a conflict
-# after it was created locally up to v3 (i.e. assume the remote sync
-# received it from the local sync first, then updated it).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|4|3||VeyronPhone:1:0|false|false
-addr|12345|5|4||VeyronPhone:1:1|false|false
-addr|12345|6|5||VeyronPhone:1:2|false|false
diff --git a/runtimes/google/vsync/testdata/remote-noconf-00.sync b/runtimes/google/vsync/testdata/remote-noconf-00.sync
deleted file mode 100644
index 9b95c86..0000000
--- a/runtimes/google/vsync/testdata/remote-noconf-00.sync
+++ /dev/null
@@ -1,8 +0,0 @@
-# Update an object remotely three times without triggering a conflict
-# after it was created locally up to v2 (i.e. assume the remote sync
-# received it from the local sync first, then updated it).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|3|2||logrec-03|false|false
-addr|12345|4|3||logrec-04|false|false
-addr|12345|5|4||logrec-05|false|false
diff --git a/runtimes/google/vsync/testdata/remote-noconf-link-00.log.sync b/runtimes/google/vsync/testdata/remote-noconf-link-00.log.sync
deleted file mode 100644
index 1da37e3..0000000
--- a/runtimes/google/vsync/testdata/remote-noconf-link-00.log.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Update an object remotely, detect conflict, and bless the remote version.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|4|1||VeyronPhone:1:0|false|false
-linkr|12345|2|4||VeyronPhone:1:1
-
diff --git a/runtimes/google/vsync/testdata/remote-noconf-link-01.log.sync b/runtimes/google/vsync/testdata/remote-noconf-link-01.log.sync
deleted file mode 100644
index 5530231..0000000
--- a/runtimes/google/vsync/testdata/remote-noconf-link-01.log.sync
+++ /dev/null
@@ -1,5 +0,0 @@
-# Update an object remotely, detect conflict, and bless the local version.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|4|1||VeyronPhone:1:0|false|false
-linkr|12345|4|3||VeyronPhone:1:1
diff --git a/runtimes/google/vsync/testdata/remote-noconf-link-02.log.sync b/runtimes/google/vsync/testdata/remote-noconf-link-02.log.sync
deleted file mode 100644
index c628985..0000000
--- a/runtimes/google/vsync/testdata/remote-noconf-link-02.log.sync
+++ /dev/null
@@ -1,7 +0,0 @@
-# Update an object remotely, detect conflict, and bless the remote version, and continue updating.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-addr|12345|4|1||VeyronPhone:1:0|false|false
-linkr|12345|3|4||VeyronPhone:1:1
-addr|12345|5|3||VeyronPhone:2:0|false|false
-
diff --git a/runtimes/google/vsync/testdata/remote-noconf-link-repeat.log.sync b/runtimes/google/vsync/testdata/remote-noconf-link-repeat.log.sync
deleted file mode 100644
index 1ab977d..0000000
--- a/runtimes/google/vsync/testdata/remote-noconf-link-repeat.log.sync
+++ /dev/null
@@ -1,5 +0,0 @@
-# Resolve the same conflict on two different devices.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-
-linkr|12345|3|4||VeyronLaptop:1:0
-
diff --git a/runtimes/google/vsync/testdata/test-1obj.gc.sync b/runtimes/google/vsync/testdata/test-1obj.gc.sync
deleted file mode 100644
index 93eca9e..0000000
--- a/runtimes/google/vsync/testdata/test-1obj.gc.sync
+++ /dev/null
@@ -1,13 +0,0 @@
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-# Local node is A. Remote nodes are B and C.
-addr|12345|0|||C:1:0|false|false
-addr|12345|1|0||B:1:0|false|false
-addl|12345|2|0||A:1:0|false|false
-addl|12345|3|1|2|A:2:0|false|false
-addr|12345|4|3||C:2:0|false|false
-addr|12345|5|3||B:2:0|false|false
-addr|12345|6|4|5|B:3:0|false|false
-# Devtable state
-setdev|A|A:2,B:3,C:2
-setdev|B|A:2,B:3,C:2
-setdev|C|A:2,B:1,C:2
diff --git a/runtimes/google/vsync/testdata/test-3obj.gc.sync b/runtimes/google/vsync/testdata/test-3obj.gc.sync
deleted file mode 100644
index d9a3899..0000000
--- a/runtimes/google/vsync/testdata/test-3obj.gc.sync
+++ /dev/null
@@ -1,44 +0,0 @@
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<continued>|<deleted>
-# Local node is A. Remote nodes are B and C.
-addl|123|1|||A:1:0|false|false
-
-addr|456|1|||B:1:0|false|false
-
-addr|456|2|1||B:2:0|false|false
-addr|123|2|1||B:2:1|false|false
-
-addl|456|3|2||A:2:0|false|false
-addl|123|4|2||A:2:1|false|false
-
-addr|789|1|||C:1:0|false|false
-
-addr|789|2|1||C:2:0|false|false
-
-addr|123|3|1||C:3:0|false|false
-addr|789|3|2||C:3:1|false|false
-
-addr|123|5|3|2|C:4:0|false|false
-
-addl|123|6|4|5|A:3:0|false|false
-addl|456|4|3||A:3:1|false|false
-addl|789|4|3||A:3:2|false|false
-
-addr|456|5|2||B:3:0|false|false
-
-addl|456|7|4|5|A:4:0|false|false
-
-addr|456|6|2||C:5:0|false|false
-addr|123|7|5||C:5:1|false|false
-addr|123|8|7||C:5:2|false|false
-addr|789|5|3||C:5:3|false|false
-
-addl|123|9|6|8|A:5:0|false|false
-addl|456|8|6|7|A:5:1|false|false
-addl|789|6|4|5|A:5:2|false|false
-
-addl|123|10|9||A:6:0|false|false
-
-# Devtable state
-setdev|A|A:6,B:3,C:5
-setdev|B|A:4,B:3,C:4
-setdev|C|A:4,B:3,C:4
diff --git a/runtimes/google/vsync/util_test.go b/runtimes/google/vsync/util_test.go
deleted file mode 100644
index b669ffe..0000000
--- a/runtimes/google/vsync/util_test.go
+++ /dev/null
@@ -1,259 +0,0 @@
-package vsync
-
-// Utilities for testing.
-import (
- "container/list"
- "fmt"
- "os"
- "time"
-
- "veyron/services/store/raw"
-)
-
-// getFileName generates a filename for a temporary (per unit test) kvdb file.
-func getFileName() string {
- return fmt.Sprintf("%s/sync_test_%d_%d", os.TempDir(), os.Getpid(), time.Now().UnixNano())
-}
-
-// createTempDir creates a unique temporary directory to store kvdb files.
-func createTempDir() (string, error) {
- dir := fmt.Sprintf("%s/sync_test_%d_%d/", os.TempDir(), os.Getpid(), time.Now().UnixNano())
- if err := os.MkdirAll(dir, 0700); err != nil {
- return "", err
- }
- return dir, nil
-}
-
-// getFileSize returns the size of a file.
-func getFileSize(fname string) int64 {
- finfo, err := os.Stat(fname)
- if err != nil {
- return -1
- }
- return finfo.Size()
-}
-
-// dummyStream struct emulates stream of log records received from RPC.
-type dummyStream struct {
- l *list.List
- value LogRec
-}
-
-func newStream() *dummyStream {
- ds := &dummyStream{
- l: list.New(),
- }
- return ds
-}
-
-func (ds *dummyStream) Advance() bool {
- if ds.l.Len() > 0 {
- ds.value = ds.l.Remove(ds.l.Front()).(LogRec)
- return true
- }
- return false
-}
-
-func (ds *dummyStream) Value() LogRec {
- return ds.value
-}
-
-func (ds *dummyStream) RecvStream() interface {
- Advance() bool
- Value() LogRec
- Err() error
-} {
- return ds
-}
-
-func (*dummyStream) Err() error { return nil }
-
-func (ds *dummyStream) Finish() (GenVector, error) {
- return GenVector{}, nil
-}
-
-func (ds *dummyStream) Cancel() {
-}
-
-func (ds *dummyStream) add(rec LogRec) {
- ds.l.PushBack(rec)
-}
-
-// logReplayCommands replays log records parsed from the input file.
-func logReplayCommands(log *iLog, syncfile string) (GenVector, error) {
- cmds, err := parseSyncCommands(syncfile)
- if err != nil {
- return nil, err
- }
-
- var minGens GenVector
- remote := false
- var stream *dummyStream
- for _, cmd := range cmds {
- switch cmd.cmd {
- case addLocal:
- parent := raw.NoVersion
- if cmd.parents != nil {
- parent = cmd.parents[0]
- }
- err = log.processWatchRecord(cmd.objID, cmd.version, parent, &LogValue{Mutation: raw.Mutation{Version: cmd.version}, Delete: cmd.deleted}, NoTxID)
- if err != nil {
- return nil, fmt.Errorf("cannot replay local log records %d:%s err %v",
- cmd.objID, cmd.version, err)
- }
-
- case addRemote:
- // TODO(hpucha): This code is no longer
- // used. Will be deleted when
- // processWatchRecord is moved to watcher.go
- if !remote {
- stream = newStream()
- }
- remote = true
- id, gnum, lsn, err := splitLogRecKey(cmd.logrec)
- if err != nil {
- return nil, err
- }
- rec := LogRec{
- DevID: id,
- GNum: gnum,
- LSN: lsn,
- ObjID: cmd.objID,
- CurVers: cmd.version,
- Parents: cmd.parents,
- Value: LogValue{},
- }
- stream.add(rec)
- }
- }
-
- return minGens, nil
-}
-
-// createReplayStream creates a dummy stream of log records parsed from the input file.
-func createReplayStream(syncfile string) (*dummyStream, error) {
- cmds, err := parseSyncCommands(syncfile)
- if err != nil {
- return nil, err
- }
-
- stream := newStream()
- for _, cmd := range cmds {
- id, gnum, lsn, err := splitLogRecKey(cmd.logrec)
- if err != nil {
- return nil, err
- }
- rec := LogRec{
- DevID: id,
- GNum: gnum,
- LSN: lsn,
- ObjID: cmd.objID,
- CurVers: cmd.version,
- Parents: cmd.parents,
- Value: LogValue{
- Mutation: raw.Mutation{Version: cmd.version},
- Continued: cmd.continued,
- Delete: cmd.deleted,
- },
- }
-
- switch cmd.cmd {
- case addRemote:
- rec.RecType = NodeRec
- case linkRemote:
- rec.RecType = LinkRec
- default:
- return nil, err
- }
- stream.add(rec)
- }
-
- return stream, nil
-}
-
-// populates the log and dag state as part of state initialization.
-func populateLogAndDAG(s *syncd, rec *LogRec) error {
- logKey, err := s.log.putLogRec(rec)
- if err != nil {
- return err
- }
-
- if err := s.dag.addNode(rec.ObjID, rec.CurVers, false, rec.Value.Delete, rec.Parents, logKey, NoTxID); err != nil {
- return err
- }
- if err := s.dag.moveHead(rec.ObjID, rec.CurVers); err != nil {
- return err
- }
- return nil
-}
-
-// vsyncInitState initializes log, dag and devtable state obtained from an input trace-like file.
-func vsyncInitState(s *syncd, syncfile string) error {
- cmds, err := parseSyncCommands(syncfile)
- if err != nil {
- return err
- }
-
- var curGen GenID
- genMap := make(map[string]*genMetadata)
-
- for _, cmd := range cmds {
- switch cmd.cmd {
- case addLocal, addRemote:
- id, gnum, lsn, err := splitLogRecKey(cmd.logrec)
- if err != nil {
- return err
- }
- rec := &LogRec{
- DevID: id,
- GNum: gnum,
- LSN: lsn,
- ObjID: cmd.objID,
- CurVers: cmd.version,
- Parents: cmd.parents,
- Value: LogValue{Continued: cmd.continued, Delete: cmd.deleted},
- }
- if err := populateLogAndDAG(s, rec); err != nil {
- return err
- }
- key := generationKey(id, gnum)
- if m, ok := genMap[key]; !ok {
- genMap[key] = &genMetadata{
- Pos: s.log.head.Curorder,
- Count: 1,
- MaxLSN: rec.LSN,
- }
- s.log.head.Curorder++
- } else {
- m.Count++
- if rec.LSN > m.MaxLSN {
- m.MaxLSN = rec.LSN
- }
- }
- if cmd.cmd == addLocal {
- curGen = gnum
- }
-
- case setDevTable:
- if err := s.devtab.putGenVec(cmd.devID, cmd.genVec); err != nil {
- return err
- }
- }
- }
-
- // Initializing genMetadata.
- for key, gen := range genMap {
- dev, gnum, err := splitGenerationKey(key)
- if err != nil {
- return err
- }
- if err := s.log.putGenMetadata(dev, gnum, gen); err != nil {
- return err
- }
- }
-
- // Initializing generation in log header.
- s.log.head.Curgen = curGen + 1
-
- return nil
-}
diff --git a/runtimes/google/vsync/vsync.vdl b/runtimes/google/vsync/vsync.vdl
deleted file mode 100644
index e5ac7e9..0000000
--- a/runtimes/google/vsync/vsync.vdl
+++ /dev/null
@@ -1,73 +0,0 @@
-package vsync
-
-import (
- "veyron/services/store/raw"
-
- "veyron2/storage"
-)
-
-// DeviceID is the globally unique ID of a device.
-type DeviceID string
-// GenID is the unique ID per generation per device.
-type GenID uint64
-// LSN is the log sequence number.
-type LSN uint64
-// GenVector is the generation vector.
-type GenVector map[DeviceID]GenID
-
-const (
- // NodeRec type log record adds a new node in the dag.
- NodeRec = byte(0)
- // LinkRec type log record adds a new link in the dag.
- LinkRec = byte(1)
-)
-
-// LogRec represents a single log record that is exchanged between two
-// peers.
-//
-// It contains log related metadata: DevID is the id of the
-// device that created the log record, GNum is the ID of the
-// generation that the log record is part of, LSN is the log
-// sequence number of the log record in the generation GNum,
-// and RecType is the type of log record.
-//
-// It also contains information relevant to the updates to an object
-// in the store: ObjID is the id of the object that was
-// updated. CurVers is the current version number of the
-// object. Parents can contain 0, 1 or 2 parent versions that the
-// current version is derived from, and Value is the actual value of
-// the object mutation.
-type LogRec struct {
- // Log related information.
- DevID DeviceID
- GNum GenID
- LSN LSN
- RecType byte
- // Object related information.
- ObjID storage.ID
- CurVers raw.Version
- Parents []raw.Version
- Value LogValue
-}
-
-// LogValue represents an object mutation within a transaction.
-type LogValue struct {
- // Mutation is the store mutation representing the change in the object.
- Mutation raw.Mutation
- // SyncTime is the timestamp of the mutation when it arrives at the Sync server.
- SyncTime int64
- // Delete indicates whether the mutation resulted in the object being
- // deleted from the store.
- Delete bool
- // Continued tracks the transaction boundaries in a range of mutations.
- // It is set to true in all transaction mutations except the last one
- // in which it is set to false to mark the end of the transaction.
- Continued bool
-}
-
-// Sync allows a device to GetDeltas from another device.
-type Sync interface {
- // GetDeltas returns a device's current generation vector and all the missing log records
- // when compared to the incoming generation vector.
- GetDeltas(In GenVector, ClientID DeviceID) stream<_, LogRec> (Out GenVector, Err error)
-}
diff --git a/runtimes/google/vsync/vsync.vdl.go b/runtimes/google/vsync/vsync.vdl.go
deleted file mode 100644
index 057838e..0000000
--- a/runtimes/google/vsync/vsync.vdl.go
+++ /dev/null
@@ -1,416 +0,0 @@
-// This file was auto-generated by the veyron vdl tool.
-// Source: vsync.vdl
-
-package vsync
-
-import (
- "veyron/services/store/raw"
-
- "veyron2/storage"
-
- // The non-user imports are prefixed with "_gen_" to prevent collisions.
- _gen_io "io"
- _gen_veyron2 "veyron2"
- _gen_context "veyron2/context"
- _gen_ipc "veyron2/ipc"
- _gen_naming "veyron2/naming"
- _gen_vdlutil "veyron2/vdl/vdlutil"
- _gen_wiretype "veyron2/wiretype"
-)
-
-// DeviceID is the globally unique ID of a device.
-type DeviceID string
-
-// GenID is the unique ID per generation per device.
-type GenID uint64
-
-// LSN is the log sequence number.
-type LSN uint64
-
-// GenVector is the generation vector.
-type GenVector map[DeviceID]GenID
-
-// LogRec represents a single log record that is exchanged between two
-// peers.
-//
-// It contains log related metadata: DevID is the id of the
-// device that created the log record, GNum is the ID of the
-// generation that the log record is part of, LSN is the log
-// sequence number of the log record in the generation GNum,
-// and RecType is the type of log record.
-//
-// It also contains information relevant to the updates to an object
-// in the store: ObjID is the id of the object that was
-// updated. CurVers is the current version number of the
-// object. Parents can contain 0, 1 or 2 parent versions that the
-// current version is derived from, and Value is the actual value of
-// the object mutation.
-type LogRec struct {
- // Log related information.
- DevID DeviceID
- GNum GenID
- LSN LSN
- RecType byte
- // Object related information.
- ObjID storage.ID
- CurVers raw.Version
- Parents []raw.Version
- Value LogValue
-}
-
-// LogValue represents an object mutation within a transaction.
-type LogValue struct {
- // Mutation is the store mutation representing the change in the object.
- Mutation raw.Mutation
- // SyncTime is the timestamp of the mutation when it arrives at the Sync server.
- SyncTime int64
- // Delete indicates whether the mutation resulted in the object being
- // deleted from the store.
- Delete bool
- // Continued tracks the transaction boundaries in a range of mutations.
- // It is set to true in all transaction mutations except the last one
- // in which it is set to false to mark the end of the transaction.
- Continued bool
-}
-
-// NodeRec type log record adds a new node in the dag.
-const NodeRec = byte(0)
-
-// LinkRec type log record adds a new link in the dag.
-const LinkRec = byte(1)
-
-// TODO(bprosnitz) Remove this line once signatures are updated to use typevals.
-// It corrects a bug where _gen_wiretype is unused in VDL pacakges where only bootstrap types are used on interfaces.
-const _ = _gen_wiretype.TypeIDInvalid
-
-// Sync allows a device to GetDeltas from another device.
-// Sync is the interface the client binds and uses.
-// Sync_ExcludingUniversal is the interface without internal framework-added methods
-// to enable embedding without method collisions. Not to be used directly by clients.
-type Sync_ExcludingUniversal interface {
- // GetDeltas returns a device's current generation vector and all the missing log records
- // when compared to the incoming generation vector.
- GetDeltas(ctx _gen_context.T, In GenVector, ClientID DeviceID, opts ..._gen_ipc.CallOpt) (reply SyncGetDeltasCall, err error)
-}
-type Sync interface {
- _gen_ipc.UniversalServiceMethods
- Sync_ExcludingUniversal
-}
-
-// SyncService is the interface the server implements.
-type SyncService interface {
-
- // GetDeltas returns a device's current generation vector and all the missing log records
- // when compared to the incoming generation vector.
- GetDeltas(context _gen_ipc.ServerContext, In GenVector, ClientID DeviceID, stream SyncServiceGetDeltasStream) (reply GenVector, err error)
-}
-
-// SyncGetDeltasCall is the interface for call object of the method
-// GetDeltas in the service interface Sync.
-type SyncGetDeltasCall interface {
- // RecvStream returns the recv portion of the stream
- RecvStream() interface {
- // Advance stages an element so the client can retrieve it
- // with Value. Advance returns true iff there is an
- // element to retrieve. The client must call Advance before
- // calling Value. Advance may block if an element is not
- // immediately available.
- Advance() bool
-
- // Value returns the element that was staged by Advance.
- // Value may panic if Advance returned false or was not
- // called at all. Value does not block.
- Value() LogRec
-
- // Err returns a non-nil error iff the stream encountered
- // any errors. Err does not block.
- Err() error
- }
-
- // Finish blocks until the server is done and returns the positional
- // return values for call.
- //
- // If Cancel has been called, Finish will return immediately; the output of
- // Finish could either be an error signalling cancelation, or the correct
- // positional return values from the server depending on the timing of the
- // call.
- //
- // Calling Finish is mandatory for releasing stream resources, unless Cancel
- // has been called or any of the other methods return an error.
- // Finish should be called at most once.
- Finish() (reply GenVector, err error)
-
- // Cancel cancels the RPC, notifying the server to stop processing. It
- // is safe to call Cancel concurrently with any of the other stream methods.
- // Calling Cancel after Finish has returned is a no-op.
- Cancel()
-}
-
-type implSyncGetDeltasStreamIterator struct {
- clientCall _gen_ipc.Call
- val LogRec
- err error
-}
-
-func (c *implSyncGetDeltasStreamIterator) Advance() bool {
- c.val = LogRec{}
- c.err = c.clientCall.Recv(&c.val)
- return c.err == nil
-}
-
-func (c *implSyncGetDeltasStreamIterator) Value() LogRec {
- return c.val
-}
-
-func (c *implSyncGetDeltasStreamIterator) Err() error {
- if c.err == _gen_io.EOF {
- return nil
- }
- return c.err
-}
-
-// Implementation of the SyncGetDeltasCall interface that is not exported.
-type implSyncGetDeltasCall struct {
- clientCall _gen_ipc.Call
- readStream implSyncGetDeltasStreamIterator
-}
-
-func (c *implSyncGetDeltasCall) RecvStream() interface {
- Advance() bool
- Value() LogRec
- Err() error
-} {
- return &c.readStream
-}
-
-func (c *implSyncGetDeltasCall) Finish() (reply GenVector, err error) {
- if ierr := c.clientCall.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (c *implSyncGetDeltasCall) Cancel() {
- c.clientCall.Cancel()
-}
-
-type implSyncServiceGetDeltasStreamSender struct {
- serverCall _gen_ipc.ServerCall
-}
-
-func (s *implSyncServiceGetDeltasStreamSender) Send(item LogRec) error {
- return s.serverCall.Send(item)
-}
-
-// SyncServiceGetDeltasStream is the interface for streaming responses of the method
-// GetDeltas in the service interface Sync.
-type SyncServiceGetDeltasStream interface {
- // SendStream returns the send portion of the stream.
- SendStream() interface {
- // Send places the item onto the output stream, blocking if there is no buffer
- // space available. If the client has canceled, an error is returned.
- Send(item LogRec) error
- }
-}
-
-// Implementation of the SyncServiceGetDeltasStream interface that is not exported.
-type implSyncServiceGetDeltasStream struct {
- writer implSyncServiceGetDeltasStreamSender
-}
-
-func (s *implSyncServiceGetDeltasStream) SendStream() interface {
- // Send places the item onto the output stream, blocking if there is no buffer
- // space available. If the client has canceled, an error is returned.
- Send(item LogRec) error
-} {
- return &s.writer
-}
-
-// BindSync returns the client stub implementing the Sync
-// interface.
-//
-// If no _gen_ipc.Client is specified, the default _gen_ipc.Client in the
-// global Runtime is used.
-func BindSync(name string, opts ..._gen_ipc.BindOpt) (Sync, error) {
- var client _gen_ipc.Client
- switch len(opts) {
- case 0:
- // Do nothing.
- case 1:
- if clientOpt, ok := opts[0].(_gen_ipc.Client); opts[0] == nil || ok {
- client = clientOpt
- } else {
- return nil, _gen_vdlutil.ErrUnrecognizedOption
- }
- default:
- return nil, _gen_vdlutil.ErrTooManyOptionsToBind
- }
- stub := &clientStubSync{defaultClient: client, name: name}
-
- return stub, nil
-}
-
-// NewServerSync creates a new server stub.
-//
-// It takes a regular server implementing the SyncService
-// interface, and returns a new server stub.
-func NewServerSync(server SyncService) interface{} {
- return &ServerStubSync{
- service: server,
- }
-}
-
-// clientStubSync implements Sync.
-type clientStubSync struct {
- defaultClient _gen_ipc.Client
- name string
-}
-
-func (__gen_c *clientStubSync) client(ctx _gen_context.T) _gen_ipc.Client {
- if __gen_c.defaultClient != nil {
- return __gen_c.defaultClient
- }
- return _gen_veyron2.RuntimeFromContext(ctx).Client()
-}
-
-func (__gen_c *clientStubSync) GetDeltas(ctx _gen_context.T, In GenVector, ClientID DeviceID, opts ..._gen_ipc.CallOpt) (reply SyncGetDeltasCall, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "GetDeltas", []interface{}{In, ClientID}, opts...); err != nil {
- return
- }
- reply = &implSyncGetDeltasCall{clientCall: call, readStream: implSyncGetDeltasStreamIterator{clientCall: call}}
- return
-}
-
-func (__gen_c *clientStubSync) UnresolveStep(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply []string, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "UnresolveStep", nil, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubSync) Signature(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply _gen_ipc.ServiceSignature, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "Signature", nil, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubSync) GetMethodTags(ctx _gen_context.T, method string, opts ..._gen_ipc.CallOpt) (reply []interface{}, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "GetMethodTags", []interface{}{method}, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-// ServerStubSync wraps a server that implements
-// SyncService and provides an object that satisfies
-// the requirements of veyron2/ipc.ReflectInvoker.
-type ServerStubSync struct {
- service SyncService
-}
-
-func (__gen_s *ServerStubSync) GetMethodTags(call _gen_ipc.ServerCall, method string) ([]interface{}, error) {
- // TODO(bprosnitz) GetMethodTags() will be replaces with Signature().
- // Note: This exhibits some weird behavior like returning a nil error if the method isn't found.
- // This will change when it is replaced with Signature().
- switch method {
- case "GetDeltas":
- return []interface{}{}, nil
- default:
- return nil, nil
- }
-}
-
-func (__gen_s *ServerStubSync) Signature(call _gen_ipc.ServerCall) (_gen_ipc.ServiceSignature, error) {
- result := _gen_ipc.ServiceSignature{Methods: make(map[string]_gen_ipc.MethodSignature)}
- result.Methods["GetDeltas"] = _gen_ipc.MethodSignature{
- InArgs: []_gen_ipc.MethodArgument{
- {Name: "In", Type: 67},
- {Name: "ClientID", Type: 65},
- },
- OutArgs: []_gen_ipc.MethodArgument{
- {Name: "Out", Type: 67},
- {Name: "Err", Type: 68},
- },
-
- OutStream: 79,
- }
-
- result.TypeDefs = []_gen_vdlutil.Any{
- _gen_wiretype.NamedPrimitiveType{Type: 0x3, Name: "veyron/runtimes/google/vsync.DeviceID", Tags: []string(nil)}, _gen_wiretype.NamedPrimitiveType{Type: 0x35, Name: "veyron/runtimes/google/vsync.GenID", Tags: []string(nil)}, _gen_wiretype.MapType{Key: 0x41, Elem: 0x42, Name: "veyron/runtimes/google/vsync.GenVector", Tags: []string(nil)}, _gen_wiretype.NamedPrimitiveType{Type: 0x1, Name: "error", Tags: []string(nil)}, _gen_wiretype.NamedPrimitiveType{Type: 0x35, Name: "veyron/runtimes/google/vsync.LSN", Tags: []string(nil)}, _gen_wiretype.NamedPrimitiveType{Type: 0x32, Name: "byte", Tags: []string(nil)}, _gen_wiretype.ArrayType{Elem: 0x46, Len: 0x10, Name: "veyron2/storage.ID", Tags: []string(nil)}, _gen_wiretype.NamedPrimitiveType{Type: 0x35, Name: "veyron/services/store/raw.Version", Tags: []string(nil)}, _gen_wiretype.SliceType{Elem: 0x48, Name: "", Tags: []string(nil)}, _gen_wiretype.NamedPrimitiveType{Type: 0x1, Name: "anydata", Tags: []string(nil)}, _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x3, Name: "Name"},
- _gen_wiretype.FieldType{Type: 0x47, Name: "ID"},
- },
- "veyron/services/store/raw.DEntry", []string(nil)},
- _gen_wiretype.SliceType{Elem: 0x4b, Name: "", Tags: []string(nil)}, _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x47, Name: "ID"},
- _gen_wiretype.FieldType{Type: 0x48, Name: "PriorVersion"},
- _gen_wiretype.FieldType{Type: 0x48, Name: "Version"},
- _gen_wiretype.FieldType{Type: 0x2, Name: "IsRoot"},
- _gen_wiretype.FieldType{Type: 0x4a, Name: "Value"},
- _gen_wiretype.FieldType{Type: 0x4c, Name: "Dir"},
- },
- "veyron/services/store/raw.Mutation", []string(nil)},
- _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x4d, Name: "Mutation"},
- _gen_wiretype.FieldType{Type: 0x25, Name: "SyncTime"},
- _gen_wiretype.FieldType{Type: 0x2, Name: "Delete"},
- _gen_wiretype.FieldType{Type: 0x2, Name: "Continued"},
- },
- "veyron/runtimes/google/vsync.LogValue", []string(nil)},
- _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x41, Name: "DevID"},
- _gen_wiretype.FieldType{Type: 0x42, Name: "GNum"},
- _gen_wiretype.FieldType{Type: 0x45, Name: "LSN"},
- _gen_wiretype.FieldType{Type: 0x46, Name: "RecType"},
- _gen_wiretype.FieldType{Type: 0x47, Name: "ObjID"},
- _gen_wiretype.FieldType{Type: 0x48, Name: "CurVers"},
- _gen_wiretype.FieldType{Type: 0x49, Name: "Parents"},
- _gen_wiretype.FieldType{Type: 0x4e, Name: "Value"},
- },
- "veyron/runtimes/google/vsync.LogRec", []string(nil)},
- }
-
- return result, nil
-}
-
-func (__gen_s *ServerStubSync) UnresolveStep(call _gen_ipc.ServerCall) (reply []string, err error) {
- if unresolver, ok := __gen_s.service.(_gen_ipc.Unresolver); ok {
- return unresolver.UnresolveStep(call)
- }
- if call.Server() == nil {
- return
- }
- var published []string
- if published, err = call.Server().Published(); err != nil || published == nil {
- return
- }
- reply = make([]string, len(published))
- for i, p := range published {
- reply[i] = _gen_naming.Join(p, call.Name())
- }
- return
-}
-
-func (__gen_s *ServerStubSync) GetDeltas(call _gen_ipc.ServerCall, In GenVector, ClientID DeviceID) (reply GenVector, err error) {
- stream := &implSyncServiceGetDeltasStream{writer: implSyncServiceGetDeltasStreamSender{serverCall: call}}
- reply, err = __gen_s.service.GetDeltas(call, In, ClientID, stream)
- return
-}
diff --git a/runtimes/google/vsync/vsyncd.go b/runtimes/google/vsync/vsyncd.go
deleted file mode 100644
index ceb0785..0000000
--- a/runtimes/google/vsync/vsyncd.go
+++ /dev/null
@@ -1,285 +0,0 @@
-package vsync
-
-// Package vsync provides veyron sync daemon utility functions. Sync
-// daemon serves incoming GetDeltas requests and contacts other peers
-// to get deltas from them. When it receives a GetDeltas request, the
-// incoming generation vector is diffed with the local generation
-// vector, and missing generations are sent back. When it receives
-// log records in response to a GetDeltas request, it replays those
-// log records to get in sync with the sender.
-import (
- "fmt"
- "strings"
- "sync"
- "time"
-
- "veyron/services/store/raw"
-
- "veyron2/ipc"
- "veyron2/naming"
- "veyron2/security"
- "veyron2/vlog"
- "veyron2/vom"
-
- _ "veyron/services/store/typeregistryhack"
-)
-
-// syncd contains the metadata for the sync daemon.
-type syncd struct {
- // Pointers to metadata structures.
- log *iLog
- devtab *devTable
- dag *dag
-
- // Local device id.
- id DeviceID
-
- // RWlock to concurrently access log and device table data structures.
- lock sync.RWMutex
- // State to coordinate shutting down all spawned goroutines.
- pending sync.WaitGroup
- closed chan struct{}
-
- // Local Veyron store.
- storeEndpoint string
- store raw.Store
-
- // Handlers for goroutine procedures.
- hdlGC *syncGC
- hdlWatcher *syncWatcher
- hdlInitiator *syncInitiator
-}
-
-type syncDispatcher struct {
- server ipc.Invoker
- auth security.Authorizer
-}
-
-// NewSyncDispatcher returns an object dispatcher.
-func NewSyncDispatcher(s interface{}, auth security.Authorizer) ipc.Dispatcher {
- return &syncDispatcher{ipc.ReflectInvoker(s), auth}
-}
-
-func (d *syncDispatcher) Lookup(suffix, method string) (ipc.Invoker, security.Authorizer, error) {
- if strings.HasSuffix(suffix, "sync") {
- return d.server, d.auth, nil
- }
- return nil, nil, fmt.Errorf("Lookup:: failed on suffix: %s", suffix)
-}
-
-// NewSyncd creates a new syncd instance.
-//
-// Syncd concurrency: syncd initializes three goroutines at
-// startup. The "watcher" thread is responsible for watching the store
-// for changes to its objects. The "initiator" thread is responsible
-// for periodically checking the neighborhood and contacting a peer to
-// obtain changes from that peer. The "gc" thread is responsible for
-// periodically checking if any log records and dag state can be
-// pruned. All these 3 threads perform write operations to the data
-// structures, and synchronize by acquiring a write lock on s.lock. In
-// addition, when syncd receives an incoming RPC, it responds to the
-// request by acquiring a read lock on s.lock. Thus, at any instant in
-// time, either one of the watcher, initiator or gc threads is active,
-// or any number of responders can be active, serving incoming
-// requests. Fairness between these threads follows from
-// sync.RWMutex. The spec says that the writers cannot be starved by
-// the readers but it does not guarantee FIFO. We may have to revisit
-// this in the future.
-func NewSyncd(peerEndpoints, peerDeviceIDs, devid, storePath, storeEndpoint string, syncTick time.Duration) *syncd {
- // Connect to the local Veyron store.
- // At present this is optional to allow testing (from the command-line) w/o Veyron store running.
- // TODO: connecting to Veyron store should be mandatory.
- var store raw.Store
- if storeEndpoint != "" {
- var err error
- store, err = raw.BindStore(naming.JoinAddressName(storeEndpoint, raw.RawStoreSuffix))
- if err != nil {
- vlog.Fatalf("NewSyncd: cannot connect to Veyron store endpoint (%s): %s", storeEndpoint, err)
- }
- }
-
- return newSyncdCore(peerEndpoints, peerDeviceIDs, devid, storePath, storeEndpoint, store, syncTick)
-}
-
-// newSyncdCore is the internal function that creates the Syncd
-// structure and initilizes its thread (goroutines). It takes a
-// Veyron Store parameter to separate the core of Syncd setup from the
-// external dependency on Veyron Store.
-func newSyncdCore(peerEndpoints, peerDeviceIDs, devid, storePath, storeEndpoint string,
- store raw.Store, syncTick time.Duration) *syncd {
- s := &syncd{}
-
- // Bootstrap my own DeviceID.
- s.id = DeviceID(devid)
-
- var err error
- // Log init.
- if s.log, err = openILog(storePath+"/ilog", s); err != nil {
- vlog.Fatalf("newSyncd: ILogInit failed: err %v", err)
- }
-
- // DevTable init.
- if s.devtab, err = openDevTable(storePath+"/dtab", s); err != nil {
- vlog.Fatalf("newSyncd: DevTableInit failed: err %v", err)
- }
-
- // Dag Init.
- if s.dag, err = openDAG(storePath + "/dag"); err != nil {
- vlog.Fatalf("newSyncd: OpenDag failed: err %v", err)
- }
-
- // Veyron Store.
- s.storeEndpoint = storeEndpoint
- s.store = store
- vlog.VI(1).Infof("newSyncd: Local Veyron store: %s", s.storeEndpoint)
-
- // Register these Watch data types with VOM.
- // TODO(tilaks): why aren't they auto-retrieved from the IDL?
- vom.Register(&raw.Mutation{})
- vom.Register(&raw.DEntry{})
-
- // Channel to propagate close event to all threads.
- s.closed = make(chan struct{})
-
- s.pending.Add(3)
-
- // Get deltas every peerSyncInterval.
- s.hdlInitiator = newInitiator(s, peerEndpoints, peerDeviceIDs, syncTick)
- go s.hdlInitiator.contactPeers()
-
- // Garbage collect every garbageCollectInterval.
- s.hdlGC = newGC(s)
- go s.hdlGC.garbageCollect()
-
- // Start a watcher thread that will get updates from local store.
- s.hdlWatcher = newWatcher(s)
- go s.hdlWatcher.watchStore()
-
- return s
-}
-
-// Close cleans up syncd state.
-func (s *syncd) Close() {
- close(s.closed)
- s.pending.Wait()
-
- // TODO(hpucha): close without flushing.
-}
-
-// isSyncClosing returns true if Close() was called i.e. the "closed" channel is closed.
-func (s *syncd) isSyncClosing() bool {
- select {
- case <-s.closed:
- return true
- default:
- return false
- }
-}
-
-// GetDeltas responds to the incoming request from a client by sending missing generations to the client.
-func (s *syncd) GetDeltas(_ ipc.ServerContext, In GenVector, ClientID DeviceID, Stream SyncServiceGetDeltasStream) (GenVector, error) {
- vlog.VI(1).Infof("GetDeltas:: Received vector %v from client %s", In, ClientID)
-
- // Handle misconfiguration: the client cannot have the same ID as me.
- if ClientID == s.id {
- vlog.VI(1).Infof("GetDeltas:: impostor alert: client ID %s is the same as mine %s", ClientID, s.id)
- return GenVector{}, fmt.Errorf("impostor: you cannot be %s, for I am %s", ClientID, s.id)
- }
-
- if err := s.updateDeviceInfo(ClientID, In); err != nil {
- vlog.Fatalf("GetDeltas:: updateDeviceInfo failed with err %v", err)
- }
-
- out, gens, gensInfo, err := s.prepareGensToReply(In)
- if err != nil {
- vlog.Fatalf("GetDeltas:: prepareGensToReply failed with err %v", err)
- }
-
- for pos, v := range gens {
- gen := gensInfo[pos]
- var count uint64
- for i := LSN(0); i <= gen.MaxLSN; i++ {
- count++
- rec, err := s.getLogRec(v.devID, v.genID, i)
- if err != nil {
- vlog.Fatalf("GetDeltas:: Couldn't get log record %s %d %d, err %v",
- v.devID, v.genID, i, err)
- }
- vlog.VI(1).Infof("Sending log record %v", rec)
- if err := Stream.SendStream().Send(*rec); err != nil {
- vlog.Errorf("GetDeltas:: Couldn't send stream err: %v", err)
- return GenVector{}, err
- }
- }
- if count != gen.Count {
- vlog.Fatalf("GetDeltas:: GenMetadata has incorrect log records for generation %s %d %v",
- v.devID, v.genID, gen)
- }
- }
- return out, nil
-}
-
-// updateDeviceInfo updates the remote device's information based on
-// the incoming GetDeltas request.
-func (s *syncd) updateDeviceInfo(ClientID DeviceID, In GenVector) error {
- s.lock.Lock()
- defer s.lock.Unlock()
-
- // Note that the incoming client generation vector cannot be
- // used for garbage collection. We can only garbage collect
- // based on the generations we receive from other
- // devices. Receiving a set of generations assures that all
- // updates branching from those generations are also received
- // and hence generations present on all devices can be
- // GC'ed. This function sends generations to other devices and
- // hence does not use the generation vector for GC.
- //
- // TODO(hpucha): Cache the client's incoming generation vector
- // to assist in tracking missing generations and hence next
- // peer to contact.
- if !s.devtab.hasDevInfo(ClientID) {
- if err := s.devtab.addDevice(ClientID); err != nil {
- return err
- }
- }
- return nil
-}
-
-// prepareGensToReply processes the incoming generation vector and
-// returns the metadata of all the missing generations between the
-// incoming and the local generation vector.
-func (s *syncd) prepareGensToReply(In GenVector) (GenVector, []*genOrder, []*genMetadata, error) {
- s.lock.RLock()
- defer s.lock.RUnlock()
-
- // Get local generation vector.
- out, err := s.devtab.getGenVec(s.id)
- if err != nil {
- return GenVector{}, nil, nil, err
- }
-
- // Diff the two generation vectors.
- gens, err := s.devtab.diffGenVectors(out, In)
- if err != nil {
- return GenVector{}, nil, nil, err
- }
-
- // Get the metadata for all the generations in the reply.
- gensInfo := make([]*genMetadata, len(gens))
- for pos, v := range gens {
- gen, err := s.log.getGenMetadata(v.devID, v.genID)
- if err != nil || gen.Count <= 0 {
- return GenVector{}, nil, nil, err
- }
- gensInfo[pos] = gen
- }
-
- return out, gens, gensInfo, nil
-}
-
-// getLogRec gets the log record for a given generation and lsn.
-func (s *syncd) getLogRec(dev DeviceID, gen GenID, lsn LSN) (*LogRec, error) {
- s.lock.RLock()
- defer s.lock.RUnlock()
- return s.log.getLogRec(dev, gen, lsn)
-}
diff --git a/runtimes/google/vsync/vsyncd/main.go b/runtimes/google/vsync/vsyncd/main.go
deleted file mode 100644
index a915550..0000000
--- a/runtimes/google/vsync/vsyncd/main.go
+++ /dev/null
@@ -1,73 +0,0 @@
-// Binary vsyncd is the sync daemon.
-package main
-
-import (
- "flag"
- "os"
-
- "veyron/runtimes/google/vsync"
- vflag "veyron/security/flag"
-
- "veyron2/rt"
- "veyron2/vlog"
-)
-
-var (
- // TODO(rthellend): Remove the protocol and address flags when the config
- // manager is working.
- protocol = flag.String("protocol", "tcp", "protocol to listen on")
- address = flag.String("address", ":0", "address to listen on")
-
- peerEndpoints = flag.String("peers", "",
- "comma separated list of endpoints of the vsync peer")
- peerDeviceIDs = flag.String("peerids", "",
- "comma separated list of deviceids of the vsync peer")
- devid = flag.String("devid", "", "Device ID")
- storePath = flag.String("store", os.TempDir(), "path to store files")
- vstoreEndpoint = flag.String("vstore", "", "endpoint of the local Veyron store")
- syncTick = flag.Duration("synctick", 0, "clock tick duration for sync with a peer (e.g. 10s)")
-)
-
-func main() {
- flag.Parse()
- if *devid == "" {
- vlog.Fatalf("syncd:: --devid needs to be specified")
- }
-
- // Create the runtime.
- r := rt.Init()
- defer r.Cleanup()
-
- // Create a new server instance.
- s, err := r.NewServer()
- if err != nil {
- vlog.Fatalf("syncd:: failure creating server: err %v", err)
- }
-
- // Create a new SyncService.
- syncd := vsync.NewSyncd(*peerEndpoints, *peerDeviceIDs, *devid, *storePath, *vstoreEndpoint, *syncTick)
- syncService := vsync.NewServerSync(syncd)
-
- // Create the authorizer.
- auth := vflag.NewAuthorizerOrDie()
-
- // Register the service.
- syncDisp := vsync.NewSyncDispatcher(syncService, auth)
-
- // Create an endpoint and begin listening.
- if endpoint, err := s.Listen(*protocol, *address); err == nil {
- vlog.VI(0).Infof("syncd:: Listening now at %v", endpoint)
- } else {
- vlog.Fatalf("syncd:: error listening to service: err %v", err)
- }
-
- // Publish the vsync service.
- name := "global/vsync/" + *devid
- if err := s.Serve(name, syncDisp); err != nil {
- vlog.Fatalf("syncd: error publishing service: err %v", err)
- }
-
- // Wait forever.
- done := make(chan struct{})
- <-done
-}
diff --git a/runtimes/google/vsync/watcher.go b/runtimes/google/vsync/watcher.go
deleted file mode 100644
index 8d3f461..0000000
--- a/runtimes/google/vsync/watcher.go
+++ /dev/null
@@ -1,202 +0,0 @@
-package vsync
-
-// Veyron Sync hook to the Store Watch API. When applications update objects
-// in the local Veyron Store, Sync learns about these via the Watch API stream
-// of object mutations. In turn, this Sync watcher thread updates the DAG and
-// ILog records to track the object change histories.
-
-import (
- "fmt"
- "io"
- "time"
-
- "veyron/services/store/raw"
-
- "veyron2/context"
- "veyron2/rt"
- "veyron2/services/watch"
- "veyron2/services/watch/types"
- "veyron2/vlog"
-)
-
-var (
- // watchRetryDelay is how long the watcher waits before calling the Watch() RPC again
- // after the previous call fails.
- watchRetryDelay = 2 * time.Second
- // streamRetryDelay is how long the watcher waits before creating a new Watch stream
- // after the previous stream ends.
- streamRetryDelay = 1 * time.Second
-)
-
-// syncWatcher contains the metadata for the Sync Watcher thread.
-type syncWatcher struct {
- // syncd is a pointer to the Syncd instance owning this Watcher.
- syncd *syncd
-
- // curTx is the transaction ID of the latest transaction being processed by the Watcher.
- curTx TxID
-
- // curTxSyncTime is the timestamp of the latest transaction being processed by the Watcher.
- curTxSyncTime int64
-}
-
-// newWatcher creates a new Sync Watcher instance attached to the given Syncd instance.
-func newWatcher(syncd *syncd) *syncWatcher {
- return &syncWatcher{syncd: syncd, curTx: NoTxID}
-}
-
-// watchStreamCanceler is a helper goroutine that cancels the watcher RPC when Syncd notifies
-// its goroutines to exit by closing its internal channel. This in turn unblocks the watcher
-// enabling it to exit. If the RPC fails, the watcher notifies the canceler to exit by
-// closing a private "done" channel between them.
-func (w *syncWatcher) watchStreamCanceler(stream watch.GlobWatcherWatchGlobCall, done chan struct{}) {
- select {
- case <-w.syncd.closed:
- vlog.VI(1).Info("watchStreamCanceler: Syncd channel closed, cancel stream and exit")
- stream.Cancel()
- case <-done:
- vlog.VI(1).Info("watchStreamCanceler: done, exit")
- }
-}
-
-// watchStore consumes change records obtained by watching the store
-// for updates and applies them to the sync DBs.
-func (w *syncWatcher) watchStore() {
- defer w.syncd.pending.Done()
-
- // If no Veyron store is configured, there is nothing to watch.
- if w.syncd.store == nil {
- vlog.VI(1).Info("watchStore: Veyron store not configured; skipping the watcher")
- return
- }
-
- // Get a Watch stream, process it, repeat till end-of-life.
- ctx := rt.R().NewContext()
- for {
- stream := w.getWatchStream(ctx)
- if stream == nil {
- return // Syncd is exiting.
- }
-
- // Spawn a goroutine to detect the Syncd "closed" channel and cancel the RPC stream
- // to unblock the watcher. The "done" channel lets the watcher terminate the goroutine.
- done := make(chan struct{})
- go w.watchStreamCanceler(stream, done)
-
- // Process the stream of Watch updates until it closes (similar to "tail -f").
- w.processWatchStream(stream)
-
- if w.syncd.isSyncClosing() {
- return // Syncd is exiting.
- }
-
- stream.Finish()
- close(done)
-
- // TODO(rdaoud): we need a rate-limiter here in case the stream closes too quickly.
- // If the stream stays up long enough, no need to sleep before getting a new one.
- time.Sleep(streamRetryDelay)
- }
-}
-
-// getWatchStream() returns a Watch API stream and handles retries if the Watch() call fails.
-// If the stream is nil, it means Syncd is exiting cleanly and the caller should terminate.
-func (w *syncWatcher) getWatchStream(ctx context.T) watch.GlobWatcherWatchGlobCall {
- for {
- req := raw.Request{}
- if resmark := w.syncd.devtab.head.Resmark; resmark != nil {
- req.ResumeMarker = resmark
- }
-
- stream, err := w.syncd.store.Watch(ctx, req)
- if err == nil {
- return stream
- }
-
- if w.syncd.isSyncClosing() {
- vlog.VI(1).Info("getWatchStream: exiting, Syncd closed its channel: ", err)
- return nil
- }
-
- vlog.VI(1).Info("getWatchStream: call to Watch() failed, retrying a bit later: ", err)
- time.Sleep(watchRetryDelay)
- }
-}
-
-// processWatchStream reads the stream of Watch updates and applies the object mutations.
-// Ideally this call does not return as the stream should be un-ending (like "tail -f").
-// If the stream is closed, distinguish between the cases of end-of-stream vs Syncd canceling
-// the stream to trigger a clean exit.
-func (w *syncWatcher) processWatchStream(call watch.GlobWatcherWatchGlobCall) {
- w.curTx = NoTxID
- stream := call.RecvStream()
- for stream.Advance() {
- change := stream.Value()
-
- // Timestamp of the change arriving at the Sync server.
- syncTime := time.Now().UnixNano()
-
- if err := w.processChange(change, syncTime); err != nil {
- // TODO(rdaoud): don't crash, instead add retry policies to attempt some degree of
- // self-healing from a data corruption where feasible, otherwise quarantine this device
- // from the cluster and stop Syncd to avoid propagating data corruptions.
- vlog.Fatal("processWatchStream:", err)
- }
- }
-
- err := stream.Err()
- if err == nil {
- err = io.EOF
- }
- if w.syncd.isSyncClosing() {
- vlog.VI(1).Info("processWatchStream: exiting, Syncd closed its channel: ", err)
- } else {
- vlog.VI(1).Info("processWatchStream: RPC stream error, re-issue Watch(): ", err)
- }
-}
-
-// processChange applies a change (object mutation) received from the Watch API.
-// The function grabs the write-lock to access the Log and DAG DBs.
-func (w *syncWatcher) processChange(ch types.Change, syncTime int64) error {
- w.syncd.lock.Lock()
- defer w.syncd.lock.Unlock()
-
- vlog.VI(1).Infof("processChange:: ready to process change")
-
- mu, ok := ch.Value.(*raw.Mutation)
- if !ok {
- return fmt.Errorf("invalid change value, not a mutation: %#v", ch)
- }
-
- // Begin a new transaction if needed.
- if w.curTx == NoTxID && ch.Continued {
- w.curTx = w.syncd.dag.addNodeTxStart()
- w.curTxSyncTime = syncTime
- }
-
- time := syncTime
- if w.curTx != NoTxID {
- // All LogValues belonging to the same transaction get the same timestamp.
- time = w.curTxSyncTime
- }
- val := &LogValue{Mutation: *mu, SyncTime: time, Delete: ch.State == types.DoesNotExist, Continued: ch.Continued}
- vlog.VI(2).Infof("processChanges:: processing record %v, Tx %v", val, w.curTx)
- if err := w.syncd.log.processWatchRecord(mu.ID, mu.Version, mu.PriorVersion, val, w.curTx); err != nil {
- return fmt.Errorf("cannot process mutation: %#v: %s", ch, err)
- }
-
- // End the previous transaction if any.
- if w.curTx != NoTxID && !ch.Continued {
- if err := w.syncd.dag.addNodeTxEnd(w.curTx); err != nil {
- return err
- }
- w.curTx = NoTxID
- }
-
- // If the resume marker changed, update the device table.
- if !ch.Continued {
- w.syncd.devtab.head.Resmark = ch.ResumeMarker
- }
-
- return nil
-}
diff --git a/runtimes/google/vsync/watcher_test.go b/runtimes/google/vsync/watcher_test.go
deleted file mode 100644
index 899d58c..0000000
--- a/runtimes/google/vsync/watcher_test.go
+++ /dev/null
@@ -1,505 +0,0 @@
-package vsync
-
-// Tests for the Veyron Sync watcher.
-
-import (
- "bytes"
- "fmt"
- "os"
- "reflect"
- "testing"
- "time"
-
- "veyron/services/store/raw"
-
- "veyron2/context"
- "veyron2/ipc"
- "veyron2/rt"
- "veyron2/services/watch/types"
- "veyron2/storage"
-)
-
-var (
- info testInfo
- recvBlocked chan struct{}
-)
-
-// testInfo controls the flow through the fake store and fake reply stream used
-// to simulate the Watch API.
-type testInfo struct {
- failWatch bool
- failWatchCount int
- failRecv bool
- failRecvCount int
- eofRecv bool
- blockRecv bool
- watchResmark []byte
-}
-
-// fakeVStore is used to simulate the Watch() API of the store and stubs the other store APIs.
-type fakeVStore struct {
-}
-
-func (*fakeVStore) GetMethodTags(_ context.T, _ string, _ ...ipc.CallOpt) ([]interface{}, error) {
- panic("not implemented")
-}
-
-func (*fakeVStore) UnresolveStep(_ context.T, _ ...ipc.CallOpt) ([]string, error) {
- panic("not implemented")
-}
-
-func (*fakeVStore) Signature(_ context.T, _ ...ipc.CallOpt) (ipc.ServiceSignature, error) {
- panic("not implemented")
-}
-
-func (v *fakeVStore) Watch(_ context.T, req raw.Request, _ ...ipc.CallOpt) (raw.StoreWatchCall, error) {
- // If "failWatch" is set, simulate a failed RPC call.
- if info.failWatch {
- info.failWatchCount++
- return nil, fmt.Errorf("fakeWatch forced error: %d", info.failWatchCount)
- }
-
- // Save the resmark from the Watch request.
- info.watchResmark = req.ResumeMarker
-
- // Return a fake stream to access the batch of changes (store mutations).
- return newFakeStream(), nil
-}
-
-func (*fakeVStore) PutMutations(_ context.T, _ ...ipc.CallOpt) (raw.StorePutMutationsCall, error) {
- panic("not implemented")
-}
-
-// fakeStream is used to simulate the reply stream of the Watch() API.
-type fakeStream struct {
- changes []types.Change
- i int
- resmark byte
- canceled chan struct{}
- err error
-}
-
-func newFakeStream() *fakeStream {
- s := &fakeStream{}
- if info.watchResmark != nil {
- s.resmark = info.watchResmark[0]
- }
- s.changes = getChanges()
- s.i = -1
- s.canceled = make(chan struct{})
- return s
-}
-
-func (s *fakeStream) RecvStream() interface {
- Advance() bool
- Value() types.Change
- Err() error
-} {
- return s
-}
-
-func (s *fakeStream) Advance() bool {
- // If "failRecv" is set, simulate a failed call.
- if info.failRecv {
- info.failRecvCount++
- s.err = fmt.Errorf("fake recv error on fake stream: %d", info.failRecvCount)
- return false
- }
-
- if s.i+1 == len(s.changes) {
- // Make sure the next Recv() call returns EOF on the stream.
- info.eofRecv = true
- }
-
- // If "eofRecv" is set, simulate a closed stream and make sure the next Recv() call blocks.
- if info.eofRecv {
- info.eofRecv, info.blockRecv = false, true
- s.err = nil
- return false
- }
-
- // If "blockRecv" is set, simulate blocking the call until the stream is canceled.
- if info.blockRecv {
- close(recvBlocked)
- <-s.canceled
- s.err = nil
- return false
- }
-
- // Otherwise return a change.
- // Adjust the resume marker of the change records to follow the one given to the Watch request.
- s.i++
- return true
-}
-
-func (s *fakeStream) Value() types.Change {
- ch := s.changes[s.i]
-
- if !ch.Continued {
- s.resmark++
- resmark := []byte{s.resmark, 0, 0, 0, 0, 0, 0, 0}
- ch.ResumeMarker = resmark
- }
-
- return ch
-}
-
-func (s *fakeStream) Err() error {
- return s.err
-}
-func (s *fakeStream) Finish() error {
- return nil
-}
-
-func (s *fakeStream) Cancel() {
- close(s.canceled)
-}
-
-// getChanges returns a batch of store mutations used to simulate the Watch API.
-// The batch contains two transactions to verify both new-object creation and the
-// mutation of an existing object.
-func getChanges() []types.Change {
- return []types.Change{
- // 1st transaction: create "/" and "/a" and "/a/b" as 3 new objects (prior versions are 0).
- types.Change{
- Name: "",
- State: 0,
- Value: &raw.Mutation{
- ID: storage.ID{0x4c, 0x6d, 0xb5, 0x1a, 0xa7, 0x40, 0xd8, 0xc6,
- 0x2b, 0x90, 0xdf, 0x87, 0x45, 0x3, 0xe2, 0x85},
- PriorVersion: 0x0,
- Version: 0x4d65822107fcfd52,
- Value: "value-root",
- Dir: []raw.DEntry{
- raw.DEntry{
- Name: "a",
- ID: storage.ID{0x8, 0x2b, 0xc4, 0x2e, 0x15, 0xaf, 0x4f, 0xcf,
- 0x61, 0x1d, 0x7f, 0x19, 0xa8, 0xd7, 0x83, 0x1f},
- },
- },
- },
- ResumeMarker: nil,
- Continued: true,
- },
- types.Change{
- Name: "",
- State: 0,
- Value: &raw.Mutation{
- ID: storage.ID{0x8, 0x2b, 0xc4, 0x2e, 0x15, 0xaf, 0x4f, 0xcf,
- 0x61, 0x1d, 0x7f, 0x19, 0xa8, 0xd7, 0x83, 0x1f},
- PriorVersion: 0x0,
- Version: 0x57e9d1860d1d68d8,
- Value: "value-a",
- Dir: []raw.DEntry{
- raw.DEntry{
- Name: "b",
- ID: storage.ID{0x6e, 0x4a, 0x32, 0x7c, 0x29, 0x7d, 0x76, 0xfb,
- 0x51, 0x42, 0xb1, 0xb1, 0xd9, 0x5b, 0x2d, 0x7},
- },
- },
- },
- ResumeMarker: nil,
- Continued: true,
- },
- types.Change{
- Name: "",
- State: 0,
- Value: &raw.Mutation{
- ID: storage.ID{0x6e, 0x4a, 0x32, 0x7c, 0x29, 0x7d, 0x76, 0xfb,
- 0x51, 0x42, 0xb1, 0xb1, 0xd9, 0x5b, 0x2d, 0x7},
- PriorVersion: 0x0,
- Version: 0x55104dc76695721d,
- Value: "value-b",
- Dir: nil,
- },
- ResumeMarker: nil,
- Continued: false,
- },
-
- // 2nd transaction: create "/a/c" as a new object, which also updates "a" (its "Dir" field).
- types.Change{
- Name: "",
- State: 0,
- Value: &raw.Mutation{
- ID: storage.ID{0x8, 0x2b, 0xc4, 0x2e, 0x15, 0xaf, 0x4f, 0xcf,
- 0x61, 0x1d, 0x7f, 0x19, 0xa8, 0xd7, 0x83, 0x1f},
- PriorVersion: 0x57e9d1860d1d68d8,
- Version: 0x365a858149c6e2d1,
- Value: "value-a",
- Dir: []raw.DEntry{
- raw.DEntry{
- Name: "b",
- ID: storage.ID{0x6e, 0x4a, 0x32, 0x7c, 0x29, 0x7d, 0x76, 0xfb,
- 0x51, 0x42, 0xb1, 0xb1, 0xd9, 0x5b, 0x2d, 0x7},
- },
- raw.DEntry{
- Name: "c",
- ID: storage.ID{0x70, 0xff, 0x65, 0xec, 0xf, 0x82, 0x5f, 0x44,
- 0xb6, 0x9f, 0x89, 0x5e, 0xea, 0x75, 0x9d, 0x71},
- },
- },
- },
- ResumeMarker: nil,
- Continued: true,
- },
- types.Change{
- Name: "",
- State: 0,
- Value: &raw.Mutation{
- ID: storage.ID{0x70, 0xff, 0x65, 0xec, 0xf, 0x82, 0x5f, 0x44,
- 0xb6, 0x9f, 0x89, 0x5e, 0xea, 0x75, 0x9d, 0x71},
- PriorVersion: 0x0,
- Version: 0x380704bb7b4d7c03,
- Value: "value-c",
- Dir: nil,
- },
- ResumeMarker: nil,
- Continued: false,
- },
-
- // 3rd transaction: remove "/a/b" which updates "a" (its "Dir" field) and deletes "b".
- types.Change{
- Name: "",
- State: 0,
- Value: &raw.Mutation{
- ID: storage.ID{0x8, 0x2b, 0xc4, 0x2e, 0x15, 0xaf, 0x4f, 0xcf,
- 0x61, 0x1d, 0x7f, 0x19, 0xa8, 0xd7, 0x83, 0x1f},
- PriorVersion: 0x365a858149c6e2d1,
- Version: 0xa858149c6e2d1000,
- Value: "value-a",
- Dir: []raw.DEntry{
- raw.DEntry{
- Name: "c",
- ID: storage.ID{0x70, 0xff, 0x65, 0xec, 0xf, 0x82, 0x5f, 0x44,
- 0xb6, 0x9f, 0x89, 0x5e, 0xea, 0x75, 0x9d, 0x71},
- },
- },
- },
- ResumeMarker: nil,
- Continued: true,
- },
- types.Change{
- Name: "",
- State: types.DoesNotExist,
- Value: &raw.Mutation{
- ID: storage.ID{0x6e, 0x4a, 0x32, 0x7c, 0x29, 0x7d, 0x76, 0xfb,
- 0x51, 0x42, 0xb1, 0xb1, 0xd9, 0x5b, 0x2d, 0x7},
- PriorVersion: 0x55104dc76695721d,
- Version: 0x0,
- Value: "",
- Dir: nil,
- },
- ResumeMarker: nil,
- Continued: false,
- },
- }
-}
-
-// initTestDir creates a per-test directory to store the Sync DB files and returns it.
-// It also initializes (resets) the test control metadata.
-func initTestDir(t *testing.T) string {
- info = testInfo{}
- recvBlocked = make(chan struct{})
- watchRetryDelay = 10 * time.Millisecond
- streamRetryDelay = 5 * time.Millisecond
-
- path := fmt.Sprintf("%s/sync_test_%d_%d/", os.TempDir(), os.Getpid(), time.Now().UnixNano())
- if err := os.Mkdir(path, 0775); err != nil {
- t.Fatalf("makeTestDir: cannot create directory %s: %s", path, err)
- }
- return path
-}
-
-// fakeSyncd creates a Syncd server structure with enough metadata to be used
-// in watcher unit tests. If "withStore" is true, create a fake store entry.
-// Otherwise simulate a no-store Sync server.
-func fakeSyncd(t *testing.T, storeDir string, withStore bool) *syncd {
- var s *syncd
- if withStore {
- s = newSyncdCore("", "", "fake-dev", storeDir, "", &fakeVStore{}, 0)
- } else {
- s = newSyncdCore("", "", "fake-dev", storeDir, "", nil, 0)
- }
- if s == nil {
- t.Fatal("cannot create a Sync server")
- }
- return s
-}
-
-// TestWatcherNoStore tests the watcher without a connection to a local store.
-// It verifies that the watcher exits without side-effects.
-func TestWatcherNoStore(t *testing.T) {
- dir := initTestDir(t)
- defer os.RemoveAll(dir)
-
- s := fakeSyncd(t, dir, false)
- s.Close()
-}
-
-// TestWatcherRPCError tests the watcher reacting to an error from the Watch() RPC.
-// It verifies that the watcher retries the RPC after a delay.
-func TestWatcherRPCError(t *testing.T) {
- rt.Init()
- dir := initTestDir(t)
- defer os.RemoveAll(dir)
-
- info.failWatch = true
- s := fakeSyncd(t, dir, true)
-
- n := 4
- time.Sleep(time.Duration(n) * watchRetryDelay)
-
- s.Close()
-
- if info.failWatchCount == 0 {
- t.Fatal("Watch() RPC retry count is zero")
- }
-}
-
-// TestWatcherRecvError tests the watcher reacting to an error from the stream receive.
-// It verifies that the watcher retries the Watch() RPC after a delay.
-func TestWatcherRecvError(t *testing.T) {
- rt.Init()
- dir := initTestDir(t)
- defer os.RemoveAll(dir)
-
- info.failRecv = true
- s := fakeSyncd(t, dir, true)
-
- n := 2
- time.Sleep(time.Duration(n) * streamRetryDelay)
-
- s.Close()
-
- if info.failRecvCount == 0 {
- t.Fatal("Recv() retry count is zero")
- }
-}
-
-// TestWatcherChanges tests the watcher applying changes received from store.
-func TestWatcherChanges(t *testing.T) {
- rt.Init()
- dir := initTestDir(t)
- defer os.RemoveAll(dir)
-
- s := fakeSyncd(t, dir, true)
-
- // Wait for the watcher to block on the Recv(), i.e. it finished processing the updates.
- <-recvBlocked
-
- // Verify the state of the Sync DAG and Device Table before terminating it.
- oidRoot := storage.ID{0x4c, 0x6d, 0xb5, 0x1a, 0xa7, 0x40, 0xd8, 0xc6, 0x2b, 0x90, 0xdf, 0x87, 0x45, 0x3, 0xe2, 0x85}
- oidA := storage.ID{0x8, 0x2b, 0xc4, 0x2e, 0x15, 0xaf, 0x4f, 0xcf, 0x61, 0x1d, 0x7f, 0x19, 0xa8, 0xd7, 0x83, 0x1f}
- oidB := storage.ID{0x6e, 0x4a, 0x32, 0x7c, 0x29, 0x7d, 0x76, 0xfb, 0x51, 0x42, 0xb1, 0xb1, 0xd9, 0x5b, 0x2d, 0x7}
- oidC := storage.ID{0x70, 0xff, 0x65, 0xec, 0xf, 0x82, 0x5f, 0x44, 0xb6, 0x9f, 0x89, 0x5e, 0xea, 0x75, 0x9d, 0x71}
-
- oids := []storage.ID{oidRoot, oidA, oidC}
- heads := []raw.Version{0x4d65822107fcfd52, 0xa858149c6e2d1000, 0x380704bb7b4d7c03}
-
- for i, oid := range oids {
- expHead := heads[i]
- head, err := s.dag.getHead(oid)
- if err != nil {
- t.Errorf("cannot find head node for object %d: %s", oid, err)
- } else if head != expHead {
- t.Errorf("wrong head for object %d: %d instead of %d", oid, head, expHead)
- }
- }
-
- // Verify oidB.
- headB, err := s.dag.getHead(oidB)
- if err != nil {
- t.Errorf("cannot find head node for object %d: %s", oidB, err)
- }
- if headB == raw.NoVersion || headB == raw.Version(0x55104dc76695721d) {
- t.Errorf("wrong head for object B %d: %d ", oidB, headB)
- }
-
- // Verify transaction state for the first transaction.
- node, err := s.dag.getNode(oidRoot, heads[0])
- if err != nil {
- t.Errorf("cannot find dag node for object %d %v: %s", oidRoot, heads[0], err)
- }
- if node.TxID == NoTxID {
- t.Errorf("expecting non nil txid for object %d:%v", oidRoot, heads[0])
- }
- txMap, err := s.dag.getTransaction(node.TxID)
- if err != nil {
- t.Errorf("cannot find transaction for id %v: %s", node.TxID, err)
- }
- expTxMap := dagTxMap{
- oidRoot: heads[0],
- oidA: raw.Version(0x57e9d1860d1d68d8),
- oidB: raw.Version(0x55104dc76695721d),
- }
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Data mismatch for txid %v txmap %v instead of %v",
- node.TxID, txMap, expTxMap)
- }
-
- // Verify transaction state for the second transaction.
- node, err = s.dag.getNode(oidC, heads[2])
- if err != nil {
- t.Errorf("cannot find dag node for object %d %v: %s", oidC, heads[2], err)
- }
- if node.TxID == NoTxID {
- t.Errorf("expecting non nil txid for object %d:%v", oidC, heads[2])
- }
- txMap, err = s.dag.getTransaction(node.TxID)
- if err != nil {
- t.Errorf("cannot find transaction for id %v: %s", node.TxID, err)
- }
- expTxMap = dagTxMap{
- oidA: raw.Version(0x365a858149c6e2d1),
- oidC: heads[2],
- }
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Data mismatch for txid %v txmap %v instead of %v",
- node.TxID, txMap, expTxMap)
- }
-
- // Verify transaction state for the third transaction.
- node, err = s.dag.getNode(oidA, heads[1])
- if err != nil {
- t.Errorf("cannot find dag node for object %d %v: %s", oidA, heads[1], err)
- }
- if node.TxID == NoTxID {
- t.Errorf("expecting non nil txid for object %d:%v", oidA, heads[1])
- }
- txMap, err = s.dag.getTransaction(node.TxID)
- if err != nil {
- t.Errorf("cannot find transaction for id %v: %s", node.TxID, err)
- }
- expTxMap = dagTxMap{
- oidA: heads[1],
- oidB: headB,
- }
- if !reflect.DeepEqual(txMap, expTxMap) {
- t.Errorf("Data mismatch for txid %v txmap %v instead of %v",
- node.TxID, txMap, expTxMap)
- }
-
- // Verify deletion tracking.
- node, err = s.dag.getNode(oidB, headB)
- if err != nil {
- t.Errorf("cannot find dag node for object %d %v: %s", oidB, headB, err)
- }
- if !node.Deleted {
- t.Errorf("deleted node not found for object %d %v: %s", oidB, headB, err)
- }
- if !s.dag.hasDeletedDescendant(oidB, raw.Version(0x55104dc76695721d)) {
- t.Errorf("link to deleted node not found for object %d %v: %s", oidB, headB, err)
- }
-
- expResmark := []byte{3, 0, 0, 0, 0, 0, 0, 0}
-
- if bytes.Compare(s.devtab.head.Resmark, expResmark) != 0 {
- t.Errorf("error in watch device table resume marker: %v instead of %v", s.devtab.head.Resmark, expResmark)
- }
-
- if bytes.Compare(info.watchResmark, expResmark) != 0 {
- t.Errorf("error in watch call final resume marker: %v instead of %v", info.watchResmark, expResmark)
- }
-
- s.Close()
-}
diff --git a/services/mgmt/application/applicationd/test.sh b/services/mgmt/application/applicationd/test.sh
old mode 100755
new mode 100644
index af10d2a..e73fcd6
--- a/services/mgmt/application/applicationd/test.sh
+++ b/services/mgmt/application/applicationd/test.sh
@@ -11,7 +11,7 @@
build() {
local -r GO="${REPO_ROOT}/scripts/build/go"
"${GO}" build veyron/services/mgmt/application/applicationd || shell_test::fail "line ${LINENO}: failed to build 'applicationd'"
- "${GO}" build veyron/services/store/stored || shell_test::fail "line ${LINENO}: failed to build 'stored'"
+ "${GO}" build veyron.io/store/veyron/services/store/stored || shell_test::fail "line ${LINENO}: failed to build 'stored'"
"${GO}" build veyron/tools/application || shell_test::fail "line ${LINENO}: failed to build 'application'"
}
diff --git a/services/mgmt/application/impl/impl_test.go b/services/mgmt/application/impl/impl_test.go
index de8d13b..eb228eb 100644
--- a/services/mgmt/application/impl/impl_test.go
+++ b/services/mgmt/application/impl/impl_test.go
@@ -4,8 +4,8 @@
"reflect"
"testing"
+ "veyron.io/store/veyron/services/store/testutil"
"veyron/services/mgmt/repository"
- "veyron/services/store/testutil"
"veyron2/naming"
"veyron2/rt"
diff --git a/services/mgmt/application/impl/invoker.go b/services/mgmt/application/impl/invoker.go
index a0f3d2f..b79b136 100644
--- a/services/mgmt/application/impl/invoker.go
+++ b/services/mgmt/application/impl/invoker.go
@@ -4,8 +4,8 @@
"errors"
"strings"
+ _ "veyron.io/store/veyron/services/store/typeregistryhack"
"veyron/services/mgmt/lib/fs"
- _ "veyron/services/store/typeregistryhack"
"veyron2/ipc"
"veyron2/naming"
"veyron2/services/mgmt/application"
diff --git a/services/mgmt/profile/impl/impl_test.go b/services/mgmt/profile/impl/impl_test.go
index 8a2fb7f..44e17e4 100644
--- a/services/mgmt/profile/impl/impl_test.go
+++ b/services/mgmt/profile/impl/impl_test.go
@@ -4,9 +4,9 @@
"reflect"
"testing"
+ "veyron.io/store/veyron/services/store/testutil"
"veyron/services/mgmt/profile"
"veyron/services/mgmt/repository"
- "veyron/services/store/testutil"
"veyron2/naming"
"veyron2/rt"
diff --git a/services/mgmt/profile/impl/invoker.go b/services/mgmt/profile/impl/invoker.go
index 3e51159..cf79341 100644
--- a/services/mgmt/profile/impl/invoker.go
+++ b/services/mgmt/profile/impl/invoker.go
@@ -5,7 +5,7 @@
"veyron/services/mgmt/lib/fs"
"veyron/services/mgmt/profile"
- // _ "veyron/services/store/typeregistryhack"
+ // _ "veyron.io/store/veyron/services/store/typeregistryhack"
"veyron2/ipc"
"veyron2/naming"
diff --git a/services/mgmt/profile/profiled/test.sh b/services/mgmt/profile/profiled/test.sh
old mode 100755
new mode 100644
index 72bccce..31fd7e5
--- a/services/mgmt/profile/profiled/test.sh
+++ b/services/mgmt/profile/profiled/test.sh
@@ -12,7 +12,7 @@
build() {
local -r GO="${REPO_ROOT}/scripts/build/go"
"${GO}" build veyron/services/mgmt/profile/profiled || shell_test::fail "line ${LINENO}: failed to build 'profiled'"
- "${GO}" build veyron/services/store/stored || shell_test::fail "line ${LINENO}: failed to build 'stored'"
+ "${GO}" build veyron.io/store/veyron/services/store/stored || shell_test::fail "line ${LINENO}: failed to build 'stored'"
"${GO}" build veyron/tools/profile || shell_test::fail "line ${LINENO}: failed to build 'profile'"
}
diff --git a/services/store/memstore/blackbox/graph_test.go b/services/store/memstore/blackbox/graph_test.go
deleted file mode 100644
index 6afbb4b..0000000
--- a/services/store/memstore/blackbox/graph_test.go
+++ /dev/null
@@ -1,157 +0,0 @@
-package blackbox
-
-import (
- "testing"
-
- _ "veyron/lib/testutil"
- "veyron/services/store/memstore"
-
- "veyron2/storage"
-)
-
-// A Node has a Label and some Children.
-type Node struct {
- Label string
- Children map[string]storage.ID
-}
-
-// Create a linear graph and truncate it.
-func TestLinear(t *testing.T) {
- const linearNodeCount = 10
-
- st, err := memstore.New(rootPublicID, "")
- if err != nil {
- t.Fatalf("memstore.New() failed: %v", err)
- }
- if v, err := st.Bind("/").Get(rootPublicID, nil); v != nil || err == nil {
- t.Errorf("Unexpected root")
- }
-
- if _, err := st.Bind("/").Put(rootPublicID, nil, &Node{}); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
-
- // Create a linked list.
- path := ""
- var nodes [linearNodeCount]*Node
- var ids [linearNodeCount]storage.ID
- tr := memstore.NewTransaction()
- for i := 0; i != linearNodeCount; i++ {
- path = path + "/Children/a"
- node := &Node{Label: path}
- nodes[i] = node
- stat, err := st.Bind(path).Put(rootPublicID, tr, node)
- if err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- ids[i] = stat.ID
- if _, err := st.Bind(path).Get(rootPublicID, tr); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- }
- tr.Commit()
-
- // Verify that all the nodes still exist.
- st.GC()
- for i, node := range nodes {
- path := node.Label
- id := ids[i]
- ExpectExists(t, st, path, id)
- }
-
- // Truncate the graph to length 3.
- {
- node := nodes[2]
- node.Children = nil
- if _, err := st.Bind("/Children/a/Children/a/Children/a").Put(rootPublicID, nil, node); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- }
-
- st.GC()
- for i, node := range nodes {
- path := node.Label
- id := ids[i]
- if i < 3 {
- ExpectExists(t, st, path, id)
- } else {
- ExpectNotExists(t, st, path, id)
- }
- }
-}
-
-// Create a lollipop graph and remove part of the cycle.
-func TestLollipop(t *testing.T) {
- const linearNodeCount = 10
- const loopNodeIndex = 5
- const cutNodeIndex = 7
-
- st, err := memstore.New(rootPublicID, "")
- if err != nil {
- t.Fatalf("memstore.New() failed: %v", err)
- }
- if v, err := st.Bind("/").Get(rootPublicID, nil); v != nil || err == nil {
- t.Errorf("Unexpected root")
- }
-
- stat, err := st.Bind("/").Put(rootPublicID, nil, &Node{})
- if err != nil || stat == nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- id := stat.ID
-
- // Create a linked list.
- path := ""
- var nodes [linearNodeCount]*Node
- var ids [linearNodeCount]storage.ID
- tr := memstore.NewTransaction()
- for i := 0; i != linearNodeCount; i++ {
- path = path + "/Children/a"
- node := &Node{Label: path}
- nodes[i] = node
- stat, err := st.Bind(path).Put(rootPublicID, tr, node)
- if err != nil || stat == nil {
- t.Errorf("Unexpected error: %s: %s", path, err)
- }
- id = stat.ID
- ids[i] = id
- }
-
- // Add a back-loop.
- {
- node := nodes[linearNodeCount-1]
- node.Children = map[string]storage.ID{"a": ids[loopNodeIndex]}
- if _, err := st.Bind(node.Label).Put(rootPublicID, tr, node); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- }
- tr.Commit()
-
- // Verify that all the nodes still exist.
- st.GC()
- for i, node := range nodes {
- path := node.Label
- id := ids[i]
- ExpectExists(t, st, path, id)
- }
-
- // Truncate part of the loop.
- {
- node := nodes[cutNodeIndex]
- node.Children = nil
- if _, err := st.Bind(node.Label).Put(rootPublicID, nil, node); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- }
-
- st.GC()
- for i, node := range nodes {
- path := node.Label
- id := ids[i]
- if i <= cutNodeIndex {
- ExpectExists(t, st, path, id)
- } else {
- ExpectNotExists(t, st, path, id)
- }
- }
-}
diff --git a/services/store/memstore/blackbox/many_to_many/doc.go b/services/store/memstore/blackbox/many_to_many/doc.go
deleted file mode 100644
index b1a48ba..0000000
--- a/services/store/memstore/blackbox/many_to_many/doc.go
+++ /dev/null
@@ -1,4 +0,0 @@
-// Package blackbox is placed under its own directory many_to_many to separate
-// it from memstore/blackbox, since they'd otherwise clash on the names of the
-// types and methods they each define for their specific tests.
-package blackbox
diff --git a/services/store/memstore/blackbox/many_to_many/many_to_many_test.go b/services/store/memstore/blackbox/many_to_many/many_to_many_test.go
deleted file mode 100644
index a925681..0000000
--- a/services/store/memstore/blackbox/many_to_many/many_to_many_test.go
+++ /dev/null
@@ -1,295 +0,0 @@
-package blackbox
-
-import (
- "runtime"
- "testing"
-
- "veyron/services/store/memstore"
- "veyron/services/store/memstore/blackbox"
-
- "veyron2/security"
- "veyron2/storage"
- "veyron2/vom"
-)
-
-var (
- rootPublicID security.PublicID = security.FakePublicID("root")
-)
-
-func init() {
- vom.Register(&Person{})
- vom.Register(&Player{})
- vom.Register(&Team{})
- vom.Register(&Role{})
- vom.Register(&DirectPlayer{})
- vom.Register(&DirectTeam{})
-}
-
-// This schema uses a Role relation to indicate who plays for what team.
-// There are also indexes; each Player and each Team have a list of Roles/
-//
-// Person : a person.
-// Player : belongs to many teams.
-// Team : contains many players.
-// Role : (Player, Team) pair.
-//
-// / : Dir
-// /People : Dir
-// /People/John : Person
-// /Players : Dir
-// /Players/John : Player
-// /Teams : Dir
-// /Teams/Rockets : Team
-
-// Dir is a type of directories.
-type Dir struct{}
-
-// Person is a person.
-type Person struct {
- FullName string
- SSN int
-}
-
-// Player is a person who has a Role.
-type Player struct {
- Person storage.ID
- Roles []storage.ID // Role
-}
-
-// Team has a set of Roles/
-type Team struct {
- Roles []storage.ID // Role
-}
-
-// Role associates a Player with a Team.
-type Role struct {
- Position string
- Player storage.ID // Player
- Team storage.ID
-}
-
-func newDir() *Dir {
- return &Dir{}
-}
-
-func newPerson(name string, ssn int) *Person {
- return &Person{FullName: name, SSN: ssn}
-}
-
-func newPlayer(personID storage.ID) *Player {
- return &Player{Person: personID}
-}
-
-func newTeam() *Team {
- return &Team{}
-}
-
-func newRole(pos string, playerID, teamID storage.ID) *Role {
- return &Role{Position: pos, Player: playerID, Team: teamID}
-}
-
-func getPerson(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) (storage.ID, *Person) {
- _, file, line, _ := runtime.Caller(1)
- e := blackbox.Get(t, st, tr, path)
- v := e.Value
- p, ok := v.(*Person)
- if !ok {
- t.Fatalf("%s(%d): %s: not a Person: %v", file, line, path, v)
- }
- return e.Stat.ID, p
-}
-
-func getPlayer(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) (storage.ID, *Player) {
- _, file, line, _ := runtime.Caller(1)
- e := blackbox.Get(t, st, tr, path)
- v := e.Value
- p, ok := v.(*Player)
- if !ok {
- t.Fatalf("%s(%d): %s: not a Player: %v", file, line, path, v)
- }
- return e.Stat.ID, p
-}
-
-func getTeam(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) (storage.ID, *Team) {
- _, file, line, _ := runtime.Caller(1)
- e := blackbox.Get(t, st, tr, path)
- v := e.Value
- p, ok := v.(*Team)
- if !ok {
- t.Fatalf("%s(%d): %s: not a Team: %v", file, line, path, v)
- }
- return e.Stat.ID, p
-}
-
-func getRole(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) (storage.ID, *Role) {
- _, file, line, _ := runtime.Caller(1)
- e := blackbox.Get(t, st, tr, path)
- v := e.Value
- p, ok := v.(*Role)
- if !ok {
- t.Fatalf("%s(%d): %s: not a Role: %v", file, line, path, v)
- }
- return e.Stat.ID, p
-}
-
-func TestManyToManyWithRole(t *testing.T) {
- st, err := memstore.New(rootPublicID, "")
- if err != nil {
- t.Fatalf("memstore.New() failed: %v", err)
- }
-
- // Create a player John who plays for the Rockets team.
- {
- tr := memstore.NewTransaction()
- blackbox.Put(t, st, tr, "/", newDir())
- blackbox.Put(t, st, tr, "/People", newDir())
- blackbox.Put(t, st, tr, "/Players", newDir())
- blackbox.Put(t, st, tr, "/Teams", newDir())
-
- person := newPerson("John", 1234567809)
- personID := blackbox.Put(t, st, tr, "/People/John", person)
- player := newPlayer(personID)
- playerID := blackbox.Put(t, st, tr, "/Players/John", player)
- team := newTeam()
- teamID := blackbox.Put(t, st, tr, "/Teams/Rockets", team)
-
- // XXX(jyh): we have to update the team/player to add the cyclic
- // links. Consider whether individual operations in a transaction
- // should be exempt from the dangling-reference check.
- //
- // Note: the @ means to append the role to the Roles array.
- role := newRole("center", playerID, teamID)
- roleID := blackbox.Put(t, st, tr, "/Players/John/Roles/@", role)
- blackbox.Put(t, st, tr, "/Teams/Rockets/Roles/@", roleID)
-
- blackbox.Commit(t, tr)
- }
-
- // Verify the state.
- {
- tr := memstore.NewTransaction()
- pID, p := getPerson(t, st, tr, "/People/John")
- if p.FullName != "John" {
- t.Errorf("Expected %q, got %q", "John", p.FullName)
- }
-
- plID, pl := getPlayer(t, st, tr, "/Players/John")
- if pl.Person != pID {
- t.Errorf("Expected %s, got %s", pID, pl.Person)
- }
-
- teamID, team := getTeam(t, st, tr, "/Teams/Rockets")
- if len(team.Roles) != 1 || len(pl.Roles) != 1 || team.Roles[0] != pl.Roles[0] {
- t.Errorf("Expected one role: %v, %v", team, pl)
- }
-
- role1ID, role1 := getRole(t, st, tr, "/Players/John/Roles/0")
- role2ID, _ := getRole(t, st, tr, "/Teams/Rockets/Roles/0")
- if role1ID != role2ID {
- t.Errorf("Expected %s, got %s", role1ID, role2ID)
- }
- if role1.Player != plID {
- t.Errorf("Expected %s, got %s", plID, role1.Player)
- }
- if role1.Team != teamID {
- t.Errorf("Expected %s, got %s", teamID, role1.Team)
- }
- }
-}
-
-////////////////////////////////////////////////////////////////////////
-// This schema removes the separate Role object. Instead the Player refers
-// directly to the Teams, and vice versa.
-
-// DirectPlayer is a person who plays on a team.
-type DirectPlayer struct {
- Person storage.ID
- Teams []storage.ID
-}
-
-// DirectTeam has a set of players.
-type DirectTeam struct {
- Players []storage.ID
-}
-
-func newDirectPlayer(personID storage.ID) *DirectPlayer {
- return &DirectPlayer{Person: personID}
-}
-
-func newDirectTeam() *DirectTeam {
- return &DirectTeam{}
-}
-
-func getDirectPlayer(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) (storage.ID, *DirectPlayer) {
- _, file, line, _ := runtime.Caller(1)
- e := blackbox.Get(t, st, tr, path)
- v := e.Value
- p, ok := v.(*DirectPlayer)
- if !ok {
- t.Fatalf("%s(%d): %s: not a DirectPlayer: %v", file, line, path, v)
- }
- return e.Stat.ID, p
-}
-
-func getDirectTeam(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) (storage.ID, *DirectTeam) {
- _, file, line, _ := runtime.Caller(1)
- e := blackbox.Get(t, st, tr, path)
- v := e.Value
- p, ok := v.(*DirectTeam)
- if !ok {
- t.Fatalf("%s(%d): %s: not a DirectTeam: %v", file, line, path, v)
- }
- return e.Stat.ID, p
-}
-
-func TestManyToManyDirect(t *testing.T) {
- st, err := memstore.New(rootPublicID, "")
- if err != nil {
- t.Fatalf("memstore.New() failed: %v", err)
- }
-
- // Create a player John who plays for the Rockets team.
- {
- tr := memstore.NewTransaction()
- blackbox.Put(t, st, tr, "/", newDir())
- blackbox.Put(t, st, tr, "/People", newDir())
- blackbox.Put(t, st, tr, "/Players", newDir())
- blackbox.Put(t, st, tr, "/Teams", newDir())
-
- person := newPerson("John", 1234567809)
- personID := blackbox.Put(t, st, tr, "/People/John", person)
- player := newDirectPlayer(personID)
- playerID := blackbox.Put(t, st, tr, "/Players/John", player)
- team := newDirectTeam()
- teamID := blackbox.Put(t, st, tr, "/Teams/Rockets", team)
-
- // XXX(jyh): we have to update the team/player to add the cyclic
- // links. Consider whether individual operations in a transaction
- // should be exempt from the dangling-reference check.
- blackbox.Put(t, st, tr, "/Players/John/Teams/@", teamID)
- blackbox.Put(t, st, tr, "/Teams/Rockets/Players/@", playerID)
-
- blackbox.Commit(t, tr)
- }
-
- // Verify the state.
- {
- tr := memstore.NewTransaction()
- pID, p := getPerson(t, st, tr, "/People/John")
- plID, pl := getDirectPlayer(t, st, tr, "/Players/John")
- teamID, team := getDirectTeam(t, st, tr, "/Teams/Rockets")
-
- if p.FullName != "John" {
- t.Errorf("Expected %q, got %q", "John", p.FullName)
- }
- if pl.Person != pID {
- t.Errorf("Expected %s, got %s", pID, pl.Person)
- }
- if len(pl.Teams) != 1 || pl.Teams[0] != teamID {
- t.Errorf("Expected one team: %v, %v", pl.Teams, team.Players)
- }
- if len(team.Players) != 1 || team.Players[0] != plID {
- t.Errorf("Expected one player: %v, %v", team, pl)
- }
- }
-}
diff --git a/services/store/memstore/blackbox/photoalbum_test.go b/services/store/memstore/blackbox/photoalbum_test.go
deleted file mode 100644
index b1deb61..0000000
--- a/services/store/memstore/blackbox/photoalbum_test.go
+++ /dev/null
@@ -1,154 +0,0 @@
-package blackbox
-
-import (
- "runtime"
- "testing"
-
- "veyron/services/store/memstore"
-
- "veyron2/storage"
- "veyron2/vom"
-)
-
-func init() {
- vom.Register(&Dir{})
- vom.Register(&User{})
- vom.Register(&Photo{})
- vom.Register(&Edit{})
- vom.Register(&Album{})
-}
-
-// Dir is a "directory" containg a dictionaries of entries.
-type Dir struct{}
-
-// User represents a "user", with a username and a "home" directory.
-// The name of the user is part of the path to the object.
-type User struct {
- Dir
- SSN int
-}
-
-// Photo represents an image. It contains the Object name for the data,
-// stored elsewhere on some content server.
-type Photo struct {
- Dir
- Comment string
- Content string // Object name
- Edits []Edit
-}
-
-// Edit is an edit to a Photo.
-type Edit struct {
- // ...
-}
-
-// Album is a photoalbum.
-type Album struct {
- Title string
- Photos map[string]storage.ID
-}
-
-func newDir() *Dir {
- return &Dir{}
-}
-
-func newUser(ssn int) *User {
- return &User{SSN: ssn}
-}
-
-func newAlbum(title string) *Album {
- return &Album{Title: title}
-}
-
-func newPhoto(content, comment string, edits ...Edit) *Photo {
- return &Photo{Content: content, Comment: comment}
-}
-
-func getPhoto(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) *Photo {
- _, file, line, _ := runtime.Caller(1)
- v := Get(t, st, tr, path).Value
- p, ok := v.(*Photo)
- if !ok {
- t.Fatalf("%s(%d): %s: not a Photo: %#v", file, line, path, v)
- }
- return p
-}
-
-func TestPhotoAlbum(t *testing.T) {
- st, err := memstore.New(rootPublicID, "")
- if err != nil {
- t.Fatalf("memstore.New() failed: %v", err)
- }
-
- // Create directories.
- {
- tr := memstore.NewTransaction()
- Put(t, st, tr, "/", newDir())
- Put(t, st, tr, "/Users", newDir())
- Put(t, st, tr, "/Users/jyh", newUser(1234567890))
- Put(t, st, tr, "/Users/jyh/ByDate", newDir())
- Put(t, st, tr, "/Users/jyh/ByDate/2014_01_01", newDir())
- Put(t, st, tr, "/Users/jyh/Albums", newDir())
- Commit(t, tr)
- }
-
- // Add some photos by date.
- {
- p1 := newPhoto("/global/contentd/DSC1000.jpg", "Half Dome")
- p2 := newPhoto("/global/contentd/DSC1001.jpg", "I don't want to hike")
- p3 := newPhoto("/global/contentd/DSC1002.jpg", "Crying kids")
- p4 := newPhoto("/global/contentd/DSC1003.jpg", "Ice cream")
- p5 := newPhoto("/global/contentd/DSC1004.jpg", "Let's go home")
-
- tr := memstore.NewTransaction()
- Put(t, st, tr, "/Users/jyh/ByDate/2014_01_01/09:00", p1)
- Put(t, st, tr, "/Users/jyh/ByDate/2014_01_01/09:15", p2)
- Put(t, st, tr, "/Users/jyh/ByDate/2014_01_01/09:16", p3)
- Put(t, st, tr, "/Users/jyh/ByDate/2014_01_01/10:00", p4)
- Put(t, st, tr, "/Users/jyh/ByDate/2014_01_01/10:05", p5)
- Commit(t, tr)
- }
-
- // Add an Album with some of the photos.
- {
- tr := memstore.NewTransaction()
- Put(t, st, tr, "/Users/jyh/Albums/Yosemite", newAlbum("Yosemite selected photos"))
- e5 := Get(t, st, tr, "/Users/jyh/ByDate/2014_01_01/10:05")
- Put(t, st, tr, "/Users/jyh/Albums/Yosemite/Photos/1", e5.Stat.ID)
- e3 := Get(t, st, tr, "/Users/jyh/ByDate/2014_01_01/09:16")
- Put(t, st, tr, "/Users/jyh/Albums/Yosemite/Photos/2", e3.Stat.ID)
- Commit(t, tr)
- }
-
- // Verify some of the photos.
- {
- p1 := getPhoto(t, st, nil, "/Users/jyh/ByDate/2014_01_01/09:00")
- if p1.Comment != "Half Dome" {
- t.Errorf("Expected %q, got %q", "Half Dome", p1.Comment)
- }
- }
-
- {
- p3 := getPhoto(t, st, nil, "/Users/jyh/Albums/Yosemite/Photos/2")
- if p3.Comment != "Crying kids" {
- t.Errorf("Expected %q, got %q", "Crying kids", p3.Comment)
- }
- }
-
- // Update p3.Comment = "Happy"
- {
- tr := memstore.NewTransaction()
- p3 := getPhoto(t, st, tr, "/Users/jyh/ByDate/2014_01_01/09:16")
- p3.Comment = "Happy"
- Put(t, st, tr, "/Users/jyh/ByDate/2014_01_01/09:16", p3)
- Commit(t, tr)
- }
-
- // Verify that the photo in the album has also changed.
- {
- p3 := getPhoto(t, st, nil, "/Users/jyh/Albums/Yosemite/Photos/2")
- if p3.Comment != "Happy" {
- t.Errorf("Expected %q, got %q", "Happy", p3.Comment)
- }
- }
-}
diff --git a/services/store/memstore/blackbox/sync_integration_test.go b/services/store/memstore/blackbox/sync_integration_test.go
deleted file mode 100644
index 98b23e9..0000000
--- a/services/store/memstore/blackbox/sync_integration_test.go
+++ /dev/null
@@ -1,122 +0,0 @@
-package blackbox
-
-import (
- "testing"
-
- "veyron/services/store/memstore"
- watchtesting "veyron/services/store/memstore/testing"
- "veyron/services/store/raw"
-
- "veyron2/rt"
- "veyron2/services/watch/types"
-)
-
-func recv(t *testing.T, call raw.StoreWatchCall, n int) []types.Change {
- rStream := call.RecvStream()
- changes := make([]types.Change, n)
- for i := 0; i < n; i++ {
- if !rStream.Advance() {
- t.Error("Advance() failed: %v", rStream.Err())
- }
- changes[i] = rStream.Value()
- }
- return changes
-}
-
-func TestSyncState(t *testing.T) {
- rt.Init()
-
- // Create a new store.
- dbName, st, cleanup := CreateStore(t, "vstore_source")
- defer cleanup()
-
- // Put /, /a, /a/b
- tr := memstore.NewTransaction()
- id1 := Put(t, st, tr, "/", "val1")
- id2 := Put(t, st, tr, "/a", "val2")
- id3 := Put(t, st, tr, "/a/b", "val3")
- Commit(t, tr)
-
- // Remove /a/b
- tr = memstore.NewTransaction()
- Remove(t, st, tr, "/a/b")
- Commit(t, tr)
- GC(t, st)
-
- if err := st.Close(); err != nil {
- t.Fatalf("Close() failed: %v", err)
- }
-
- // Create a target store for integration testing.
- _, target, cleanup := CreateStore(t, "vstore_target")
- defer cleanup()
-
- // Re-create a new store. This should compress the log, creating an initial
- // state containing / and /a.
- st, cleanup = OpenStore(t, dbName)
- defer cleanup()
-
- // Create the watcher
- w, cleanup := OpenWatch(t, dbName)
- defer cleanup()
- // Create a sync request
- stream := watchtesting.WatchRaw(rootPublicID, w.WatchRaw, raw.Request{})
-
- // Update target
- changes := recv(t, stream, 2)
- PutMutations(t, target, Mutations(changes))
- GC(t, target)
-
- // Expect that the target contains id1 and id2 but not id3
- ExpectExists(t, target, "/", id1)
- ExpectExists(t, target, "/a", id2)
- ExpectNotExists(t, target, "/a/b", id3)
-}
-
-func TestSyncTransaction(t *testing.T) {
- rt.Init()
-
- _, target, cleanup := CreateStore(t, "vstore_target")
- defer cleanup()
-
- dbName, st, cleanup := CreateStore(t, "vstore_source")
- defer cleanup()
-
- // Create the watcher
- w, cleanup := OpenWatch(t, dbName)
- defer cleanup()
- // Create a sync request
- stream := watchtesting.WatchRaw(rootPublicID, w.WatchRaw, raw.Request{})
-
- // First transaction, put /, /a, /a/b
- tr := memstore.NewTransaction()
- id1 := Put(t, st, tr, "/", "val1")
- id2 := Put(t, st, tr, "/a", "val2")
- id3 := Put(t, st, tr, "/a/b", "val3")
- Commit(t, tr)
-
- // Update target
- changes := recv(t, stream, 3)
- PutMutations(t, target, Mutations(changes))
- GC(t, target)
-
- // Expect that the target contains id1, id2, id3
- ExpectExists(t, target, "/", id1)
- ExpectExists(t, target, "/a", id2)
- ExpectExists(t, target, "/a/b", id3)
-
- // Next transaction, remove /a/b
- tr = memstore.NewTransaction()
- Remove(t, st, tr, "/a/b")
- Commit(t, tr)
-
- // Update target
- changes = recv(t, stream, 1)
- PutMutations(t, target, Mutations(changes))
- GC(t, target)
-
- // Expect that the target contains id1, id2, but not id3
- ExpectExists(t, target, "/", id1)
- ExpectExists(t, target, "/a", id2)
- ExpectNotExists(t, target, "/a/b", id3)
-}
diff --git a/services/store/memstore/blackbox/team_player_test.go b/services/store/memstore/blackbox/team_player_test.go
deleted file mode 100644
index a086129..0000000
--- a/services/store/memstore/blackbox/team_player_test.go
+++ /dev/null
@@ -1,225 +0,0 @@
-package blackbox
-
-import (
- "runtime"
- "testing"
-
- "veyron/services/store/memstore"
- "veyron/services/store/memstore/state"
-
- "veyron2/storage"
- "veyron2/vom"
-)
-
-func init() {
- vom.Register(&Player{})
- vom.Register(&Team{})
-}
-
-// This schema uses the Team/Player schema in the query doc.
-//
-// Player : belongs to many teams.
-// Team : contains many players.
-
-// Player is a person who has a Role.
-type Player struct {
- FullName string
-}
-
-// Team has a set of Roles/
-type Team struct {
- FullName string
- Players []storage.ID
-}
-
-func newPlayer(name string) *Player {
- return &Player{FullName: name}
-}
-
-func newTeam(name string) *Team {
- return &Team{FullName: name}
-}
-
-func getPlayer(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) (storage.ID, *Player) {
- _, file, line, _ := runtime.Caller(1)
- e := Get(t, st, tr, path)
- p, ok := e.Value.(*Player)
- if !ok {
- t.Fatalf("%s(%d): %s: not a Player: %v", file, line, path, e.Value)
- }
- return e.Stat.ID, p
-}
-
-func getTeam(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) (storage.ID, *Team) {
- _, file, line, _ := runtime.Caller(1)
- e := Get(t, st, tr, path)
- p, ok := e.Value.(*Team)
- if !ok {
- t.Fatalf("%s(%d): %s: not a Team: %v", file, line, path, e.Value)
- }
- return e.Stat.ID, p
-}
-
-func TestManyToManyWithRole(t *testing.T) {
- st, err := memstore.New(rootPublicID, "")
- if err != nil {
- t.Fatalf("memstore.New() failed: %v", err)
- }
-
- john := newPlayer("John")
- jane := newPlayer("Jane")
- joan := newPlayer("Joan")
-
- rockets := newTeam("Rockets")
- hornets := newTeam("Hornets")
-
- // Create the state.
- {
- tr := memstore.NewTransaction()
- Put(t, st, tr, "/", newDir())
- Put(t, st, tr, "/teamsapp", newDir())
- Put(t, st, tr, "/teamsapp/teams", newDir())
- Put(t, st, tr, "/teamsapp/teams/rockets", rockets)
- Put(t, st, tr, "/teamsapp/teams/hornets", hornets)
- Put(t, st, tr, "/teamsapp/players", newDir())
-
- johnID := Put(t, st, tr, "/teamsapp/teams/rockets/Players/@", john)
- janeID := Put(t, st, tr, "/teamsapp/teams/rockets/Players/@", jane)
- Put(t, st, tr, "/teamsapp/teams/hornets/Players/@", janeID)
- joanID := Put(t, st, tr, "/teamsapp/teams/hornets/Players/@", joan)
-
- Put(t, st, tr, "/teamsapp/players/John", johnID)
- Put(t, st, tr, "/teamsapp/players/Jane", janeID)
- Put(t, st, tr, "/teamsapp/players/Joan", joanID)
-
- Commit(t, tr)
- }
-
- rocketsID := Get(t, st, nil, "/teamsapp/teams/rockets").Stat.ID
- hornetsID := Get(t, st, nil, "/teamsapp/teams/hornets").Stat.ID
- johnID := Get(t, st, nil, "/teamsapp/players/John").Stat.ID
- janeID := Get(t, st, nil, "/teamsapp/players/Jane").Stat.ID
- joanID := Get(t, st, nil, "/teamsapp/players/Joan").Stat.ID
-
- // Verify some of the state.
- {
- tr := memstore.NewTransaction()
- _, john := getPlayer(t, st, tr, "/teamsapp/players/John")
- _, rockets := getTeam(t, st, tr, "/teamsapp/teams/rockets")
-
- if john.FullName != "John" {
- t.Errorf("Expected %q, got %q", "John", john.FullName)
- }
- if len(rockets.Players) != 2 {
- t.Fatalf("Expected two players: got %v", rockets.Players)
- }
- if rockets.Players[0] != johnID {
- t.Errorf("Expected %s, got %s", johnID, rockets.Players[0])
- }
- }
-
- // Iterate over the rockets.
- players := make(map[storage.ID]*Player)
- name := storage.ParsePath("/teamsapp/players")
- for it := st.Snapshot().NewIterator(rootPublicID, name,
- state.ListPaths, nil); it.IsValid(); it.Next() {
-
- e := it.Get()
- if p, ok := e.Value.(*Player); ok {
- if _, ok := players[e.Stat.ID]; ok {
- t.Errorf("Player already exists: %v", p)
- continue
- }
- players[e.Stat.ID] = p
- }
- }
- if len(players) != 3 {
- t.Errorf("Should have 3 players: have %v", players)
- }
- if p, ok := players[johnID]; !ok || p.FullName != "John" {
- t.Errorf("Should have John, have %v", p)
- }
- if p, ok := players[janeID]; !ok || p.FullName != "Jane" {
- t.Errorf("Should have Jane, have %v", p)
- }
- if p, ok := players[joanID]; !ok || p.FullName != "Joan" {
- t.Errorf("Should have Joan, have %v", p)
- }
-
- // Iterate over all teams, nonrecursively.
- teams := make(map[storage.ID]*Team)
- name = storage.ParsePath("/teamsapp/teams")
- for it := st.Snapshot().NewIterator(rootPublicID, name,
- state.ListPaths, state.ImmediateFilter); it.IsValid(); it.Next() {
-
- e := it.Get()
- v := e.Value
- if _, ok := v.(*Player); ok {
- t.Errorf("Nonrecursive iteration should not show players")
- }
- if team, ok := v.(*Team); ok {
- if _, ok := teams[e.Stat.ID]; ok {
- t.Errorf("Team already exists: %v", team)
- continue
- }
- teams[e.Stat.ID] = team
- }
- }
- if len(teams) != 2 {
- t.Errorf("Should have 2 teams: have %v", teams)
- }
- if team, ok := teams[rocketsID]; !ok || team.FullName != "Rockets" {
- t.Errorf("Should have Rockets, have %v", team)
- }
- if team, ok := teams[hornetsID]; !ok || team.FullName != "Hornets" {
- t.Errorf("Should have Hornets, have %v", team)
- }
-
- // Iterate over all teams, recursively.
- contractCount := 0
- teamCount := 0
- players = make(map[storage.ID]*Player)
- teams = make(map[storage.ID]*Team)
- name = storage.ParsePath("/teamsapp/teams")
- for it := st.Snapshot().NewIterator(rootPublicID, name,
- state.ListPaths, nil); it.IsValid(); it.Next() {
-
- e := it.Get()
- v := e.Value
- if p, ok := v.(*Player); ok {
- players[e.Stat.ID] = p
- contractCount++
- }
- if team, ok := v.(*Team); ok {
- teams[e.Stat.ID] = team
- teamCount++
- }
- }
- if teamCount != 2 {
- t.Errorf("Should have 2 teams: have %d", teamCount)
- }
- if len(teams) != 2 {
- t.Errorf("Should have 2 teams: have %v", teams)
- }
- if team, ok := teams[rocketsID]; !ok || team.FullName != "Rockets" {
- t.Errorf("Should have Rockets, have %v", team)
- }
- if team, ok := teams[hornetsID]; !ok || team.FullName != "Hornets" {
- t.Errorf("Should have Hornets, have %v", team)
- }
- if contractCount != 4 {
- t.Errorf("Should have 4 contracts: have %d", contractCount)
- }
- if len(players) != 3 {
- t.Errorf("Should have 3 players: have %v", players)
- }
- if p, ok := players[johnID]; !ok || p.FullName != "John" {
- t.Errorf("Should have John, have %v", p)
- }
- if p, ok := players[janeID]; !ok || p.FullName != "Jane" {
- t.Errorf("Should have Jane, have %v", p)
- }
- if p, ok := players[joanID]; !ok || p.FullName != "Joan" {
- t.Errorf("Should have Joan, have %v", p)
- }
-}
diff --git a/services/store/memstore/blackbox/util.go b/services/store/memstore/blackbox/util.go
deleted file mode 100644
index c494310..0000000
--- a/services/store/memstore/blackbox/util.go
+++ /dev/null
@@ -1,177 +0,0 @@
-package blackbox
-
-import (
- "io/ioutil"
- "os"
- "runtime"
- "testing"
-
- "veyron/services/store/memstore"
- mtest "veyron/services/store/memstore/testing"
- memwatch "veyron/services/store/memstore/watch"
- "veyron/services/store/raw"
-
- "veyron2/security"
- "veyron2/services/watch/types"
- "veyron2/storage"
-)
-
-var (
- rootPublicID security.PublicID = security.FakePublicID("root")
- nullMutation = raw.Mutation{}
-)
-
-func Get(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) *storage.Entry {
- _, file, line, _ := runtime.Caller(1)
- e, err := st.Bind(path).Get(rootPublicID, tr)
- if err != nil {
- t.Fatalf("%s(%d): can't get %s: %s", file, line, path, err)
- }
- return e
-}
-
-func Put(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string, v interface{}) storage.ID {
- _, file, line, _ := runtime.Caller(1)
- stat, err := st.Bind(path).Put(rootPublicID, tr, v)
- if err != nil {
- t.Fatalf("%s(%d): can't put %s: %s", file, line, path, err)
- }
- if stat != nil {
- return stat.ID
- }
- if id, ok := v.(storage.ID); ok {
- return id
- }
- t.Errorf("%s(%d): can't find id", file, line)
- return storage.ID{}
-}
-
-func Remove(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) {
- _, file, line, _ := runtime.Caller(1)
- if err := st.Bind(path).Remove(rootPublicID, tr); err != nil {
- t.Fatalf("%s(%d): can't remove %s: %s", file, line, path, err)
- }
-}
-
-func Commit(t *testing.T, tr *memstore.Transaction) {
- if err := tr.Commit(); err != nil {
- t.Fatalf("Transaction aborted: %s", err)
- }
-}
-
-func ExpectExists(t *testing.T, st *memstore.Store, path string, id storage.ID) {
- _, file, line, _ := runtime.Caller(1)
- e, err := st.Bind(path).Get(rootPublicID, nil)
- if err != nil {
- t.Errorf("%s(%d): Expected value for ID: %x", file, line, id)
- }
- if e.Stat.ID != id {
- t.Errorf("%s(%d): expected id to be %v, but was %v", file, line, id, e.Stat.ID)
- }
-}
-
-func ExpectNotExists(t *testing.T, st *memstore.Store, path string, id storage.ID) {
- _, file, line, _ := runtime.Caller(1)
- e, err := st.Bind(path).Get(rootPublicID, nil)
- if err == nil {
- t.Errorf("%s(%d): Unexpected value: %v", file, line, e)
- }
-}
-
-func GC(t *testing.T, st *memstore.Store) {
- if err := st.GC(); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): can't gc: %s", file, line, err)
- }
-}
-
-func CreateStore(t *testing.T, dbSuffix string) (string, *memstore.Store, func()) {
- dbName, err := ioutil.TempDir(os.TempDir(), dbSuffix)
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
- cleanup := func() {
- os.RemoveAll(dbName)
- }
-
- st, err := memstore.New(rootPublicID, dbName)
- if err != nil {
- cleanup()
- t.Fatalf("New() failed: %v", err)
- }
-
- return dbName, st, cleanup
-}
-
-func OpenStore(t *testing.T, dbName string) (*memstore.Store, func()) {
- st, err := memstore.New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
-
- return st, func() {
- os.RemoveAll(dbName)
- }
-}
-
-func OpenWatch(t *testing.T, dbName string) (*memwatch.Watcher, func()) {
- w, err := memwatch.New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
-
- return w, func() {
- w.Close()
- }
-}
-
-// putMutationsStream implements raw.StoreServicePutMutationsStream.
-type putMutationsStream struct {
- mus []raw.Mutation
- index int
-}
-
-func newPutMutationsStream(mus []raw.Mutation) raw.StoreServicePutMutationsStream {
- return &putMutationsStream{
- mus: mus,
- index: -1,
- }
-}
-
-func (s *putMutationsStream) RecvStream() interface {
- Advance() bool
- Value() raw.Mutation
- Err() error
-} {
- return s
-}
-
-func (s *putMutationsStream) Advance() bool {
- s.index++
- return s.index < len(s.mus)
-}
-
-func (s *putMutationsStream) Value() raw.Mutation {
- return s.mus[s.index]
-}
-
-func (*putMutationsStream) Err() error {
- return nil
-}
-
-func PutMutations(t *testing.T, st *memstore.Store, mus []raw.Mutation) {
- stream := newPutMutationsStream(mus)
- rootCtx := mtest.NewFakeServerContext(rootPublicID)
- if err := st.PutMutations(rootCtx, stream); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): can't put mutations %s: %s", file, line, mus, err)
- }
-}
-
-func Mutations(changes []types.Change) []raw.Mutation {
- mutations := make([]raw.Mutation, len(changes))
- for i, change := range changes {
- mutations[i] = *(change.Value.(*raw.Mutation))
- }
- return mutations
-}
diff --git a/services/store/memstore/doc.go b/services/store/memstore/doc.go
deleted file mode 100644
index fd50ad8..0000000
--- a/services/store/memstore/doc.go
+++ /dev/null
@@ -1,7 +0,0 @@
-// Package memstore implements an in-memory version of the veyron2/storage API.
-// The store is logged to a file, and it is persistent, but the entire state is
-// also kept in memory at all times.
-//
-// The purpose of this fake is to explore the model. It is for testing; it
-// isn't intended to be deployed.
-package memstore
diff --git a/services/store/memstore/field/reflect.go b/services/store/memstore/field/reflect.go
deleted file mode 100644
index 891cb31..0000000
--- a/services/store/memstore/field/reflect.go
+++ /dev/null
@@ -1,271 +0,0 @@
-package field
-
-// reflect.go uses reflection to access the fields in a value.
-//
-// Get(v, path)
-// Set(v, path, x)
-// Remove(v, path)
-//
-// The path is JSON-style, using field names. For example, consider the
-// following value.
-//
-// type MyValue struct {
-// A int
-// B []int
-// C map[string]int
-// D map[string]struct{E int}
-// }
-//
-// Here are some possible paths:
-//
-// var x MyValue = ...
-// Get(x, storage.PathName{"A"}) == x.A
-// Get(x, storage.PathName{"B", "7"}) == x.B[7]
-// Get(x, storage.PathName{"C", "a"}) == x.C["a"]
-// Get(x, storage.PathName{"D", "a", "E"}) == x.D["a"].E
-//
-// Set(x, storage.PathName{"A"}, 17)
-// Get(x, storage.PathName{"A"}) == 17
-//
-// Set(x, storage.PathName{"D", "a"}, struct{E: 12})
-// Get(x, storage.PathName{"D", "a", "E"}) == 12
-//
-// Remove(x, storage.PathName{"D", "a"}
-// Get(x, storage.PathName{"D", "a", "E"}) fails
-
-import (
- "reflect"
- "strconv"
-
- "veyron2/storage"
-)
-
-type SetResult uint32
-
-const (
- SetFailedNotFound SetResult = iota
- SetFailedWrongType
- SetAsValue
- SetAsID
-)
-
-const (
- SliceAppendSuffix = "@"
-)
-
-var (
- nullID storage.ID
- nullValue reflect.Value
-
- tyID = reflect.TypeOf(nullID)
-)
-
-// Get returns the value associated with a path, stopping at values that
-// can't be resolved. It returns the value at the maximum prefix of the path
-// that can be resolved, and any suffix that remains.
-func Get(val interface{}, path storage.PathName) (reflect.Value, storage.PathName) {
- v, suffix := findField(reflect.ValueOf(val), path)
- return v, path[suffix:]
-}
-
-// findField returns the field specified by the path, using reflection to
-// traverse the value. Returns the field, and how many components of the path
-// were resolved.
-func findField(v reflect.Value, path storage.PathName) (reflect.Value, int) {
- for i, field := range path {
- v1 := followPointers(v)
- if !v1.IsValid() {
- return v, i
- }
- v2 := findNextField(v1, field)
- if !v2.IsValid() {
- return v1, i
- }
- v = v2
- }
- return v, len(path)
-}
-
-func followPointers(v reflect.Value) reflect.Value {
- if !v.IsValid() {
- return v
- }
- kind := v.Type().Kind()
- for kind == reflect.Ptr || kind == reflect.Interface {
- v = v.Elem()
- if !v.IsValid() {
- return v
- }
- kind = v.Type().Kind()
- }
- return v
-}
-
-func findNextField(v reflect.Value, field string) reflect.Value {
- switch v.Type().Kind() {
- case reflect.Array, reflect.Slice:
- return findSliceField(v, field)
- case reflect.Map:
- return findMapField(v, field)
- case reflect.Struct:
- return v.FieldByName(field)
- default:
- return reflect.Value{}
- }
-}
-
-func findSliceField(v reflect.Value, field string) reflect.Value {
- l := v.Len()
- i, err := strconv.Atoi(field)
- if err != nil || i < 0 || i >= l {
- return reflect.Value{}
- }
- return v.Index(i)
-}
-
-func findMapField(v reflect.Value, field string) reflect.Value {
- tyKey := v.Type().Key()
- if v.IsNil() || tyKey.Kind() != reflect.String {
- return reflect.Value{}
- }
- return v.MapIndex(reflect.ValueOf(field).Convert(tyKey))
-}
-
-// Set assigns the value associated with a subfield of an object.
-// If the field has type storage.ID, then id is stored instead of xval.
-//
-// Here are the possible cases:
-//
-// 1. SetFailedNotFound if the operation failed because the path doesn't exist.
-//
-// 2. SetFailedWrongType if the operation failed because the value has the wrong type.
-//
-// 3. SetAsValue if the operation was successful, and the value xval was
-// stored. The returned storage.ID is null.
-//
-// 4. SetAsId if the operation was successful, but the type of the field is
-// storage.ID and xval does not have type storage.ID. In this case, the value
-// xval is not stored; the storage.ID is returned instead. If the field does
-// not already exist, a new storage.ID is created (and returned).
-//
-// The setAsID case means that the value xval is to be stored as a separate
-// value in the store, not as a subfield of the current value.
-//
-// As a special case, if the field type is storage.ID, and xval has type storage.ID,
-// then it is case #2, setAsValue. The returned storage.ID is zero.
-func Set(v reflect.Value, name string, xval interface{}) (SetResult, storage.ID) {
- v = followPointers(v)
- if !v.IsValid() {
- return SetFailedNotFound, nullID
- }
- switch v.Type().Kind() {
- case reflect.Map:
- return setMapField(v, name, xval)
- case reflect.Array, reflect.Slice:
- return setSliceField(v, name, xval)
- case reflect.Struct:
- return setStructField(v, name, xval)
- default:
- return SetFailedNotFound, nullID
- }
-}
-
-func setMapField(v reflect.Value, name string, xval interface{}) (SetResult, storage.ID) {
- tyV := v.Type()
- tyKey := tyV.Key()
- if tyKey.Kind() != reflect.String {
- return SetFailedNotFound, nullID
- }
- key := reflect.ValueOf(name).Convert(tyKey)
- r, x, id := coerceValue(tyV.Elem(), v.MapIndex(key), xval)
- if r == SetFailedWrongType {
- return SetFailedWrongType, nullID
- }
- v.SetMapIndex(key, x)
- return r, id
-}
-
-func setSliceField(v reflect.Value, field string, xval interface{}) (SetResult, storage.ID) {
- if field == SliceAppendSuffix {
- r, x, id := coerceValue(v.Type().Elem(), nullValue, xval)
- if r == SetFailedWrongType {
- return SetFailedWrongType, nullID
- }
- // This can panic if v is not settable. It is a requirement that users of this method
- // ensure that it is settable.
- v.Set(reflect.Append(v, x))
- return r, id
- }
- l := v.Len()
- i, err := strconv.Atoi(field)
- if err != nil || i < 0 || i >= l {
- return SetFailedNotFound, nullID
- }
- r, x, id := coerceValue(v.Type().Elem(), v.Index(i), xval)
- if r == SetFailedWrongType {
- return SetFailedWrongType, nullID
- }
- v.Index(i).Set(x)
- return r, id
-}
-
-func setStructField(v reflect.Value, name string, xval interface{}) (SetResult, storage.ID) {
- field, found := v.Type().FieldByName(name)
- if !found {
- return SetFailedNotFound, nullID
- }
- fieldVal := v.FieldByName(name)
- r, x, id := coerceValue(field.Type, fieldVal, xval)
- if r == SetFailedWrongType {
- return SetFailedWrongType, nullID
- }
- fieldVal.Set(x)
- return r, id
-}
-
-func coerceValue(ty reflect.Type, prev reflect.Value, xval interface{}) (SetResult, reflect.Value, storage.ID) {
- x := reflect.ValueOf(xval)
- switch {
- case ty == tyID:
- if x.Type() == tyID {
- return SetAsValue, x, xval.(storage.ID)
- }
- var id storage.ID
- if prev.IsValid() {
- var ok bool
- if id, ok = prev.Interface().(storage.ID); !ok {
- return SetFailedWrongType, nullValue, nullID
- }
- } else {
- id = storage.NewID()
- }
- return SetAsID, reflect.ValueOf(id), id
- case x.Type().AssignableTo(ty):
- return SetAsValue, x, nullID
- case x.Type().ConvertibleTo(ty):
- return SetAsValue, x.Convert(ty), nullID
- default:
- return SetFailedWrongType, nullValue, nullID
- }
-}
-
-// Remove removes a field associated with a path.
-// Return the old value and true iff the update succeeded.
-func Remove(v reflect.Value, name string) bool {
- v = followPointers(v)
- if !v.IsValid() || v.Type().Kind() != reflect.Map || v.IsNil() {
- return false
- }
- return removeMapField(v, name)
-}
-
-func removeMapField(v reflect.Value, name string) bool {
- // TODO(jyh): Also handle cases where field is a primitive scalar type like
- // int or bool.
- tyKey := v.Type().Key()
- if tyKey.Kind() != reflect.String {
- return false
- }
- v.SetMapIndex(reflect.ValueOf(name).Convert(tyKey), reflect.Value{})
- return true
-}
diff --git a/services/store/memstore/field/reflect_test.go b/services/store/memstore/field/reflect_test.go
deleted file mode 100644
index 5c85253..0000000
--- a/services/store/memstore/field/reflect_test.go
+++ /dev/null
@@ -1,382 +0,0 @@
-package field_test
-
-import (
- "reflect"
- "testing"
-
- _ "veyron/lib/testutil"
- "veyron/services/store/memstore/field"
-
- "veyron2/storage"
-)
-
-type A struct {
- B int
- C []int
- D map[string]int
-
- E storage.ID
- F []storage.ID
- G map[string]storage.ID
-}
-
-type V struct {
- UID storage.ID
-}
-
-func pathEq(p1, p2 storage.PathName) bool {
- if len(p1) != len(p2) {
- return false
- }
- for i, x := range p1 {
- if p2[i] != x {
- return false
- }
- }
- return true
-}
-
-func TestGetField(t *testing.T) {
- v := &A{
- B: 5,
- C: []int{6, 7},
- D: map[string]int{"a": 8, "b": 9},
- E: storage.NewID(),
- F: []storage.ID{storage.NewID()},
- G: map[string]storage.ID{"a": storage.NewID()},
- }
-
- // Identity.
- x, s := field.Get(v, storage.PathName{})
- if x.Interface() != v || !pathEq(storage.PathName{}, s) {
- t.Errorf("Identity failed: %v", s)
- }
-
- // B field.
- x, s = field.Get(v, storage.PathName{"B"})
- if x.Interface() != 5 || !pathEq(storage.PathName{}, s) {
- t.Errorf("Expected 5, got %v, suffix=%v", x.Interface(), s)
- }
-
- // C field.
- x, s = field.Get(v, storage.PathName{"C"})
- if !pathEq(storage.PathName{}, s) {
- t.Errorf("Failed to get C: %v")
- }
- {
- y, ok := x.Interface().([]int)
- if !ok || len(y) != 2 || y[0] != 6 || y[1] != 7 {
- t.Errorf("C has the wrong value: %v", x)
- }
- }
- x, s = field.Get(v, storage.PathName{"C", "0"})
- if x.Interface() != 6 || !pathEq(storage.PathName{}, s) {
- t.Errorf("Expected 6, got %v, %v", x, s)
- }
- x, s = field.Get(v, storage.PathName{"C", "1"})
- if x.Interface() != 7 || !pathEq(storage.PathName{}, s) {
- t.Errorf("Expected 7, got %v, %v", x, s)
- }
- x, s = field.Get(v, storage.PathName{"C", "2"})
- if !pathEq(storage.PathName{"2"}, s) {
- t.Errorf("Expected %v, got %v", storage.PathName{"2"}, s)
- }
- {
- y, ok := x.Interface().([]int)
- if !ok || len(y) != 2 || y[0] != 6 || y[1] != 7 {
- t.Errorf("C has the wrong value: %v", x)
- }
- }
-
- // D field.
- x, s = field.Get(v, storage.PathName{"D"})
- if !pathEq(storage.PathName{}, s) {
- t.Errorf("Failed to get D")
- }
- {
- y, ok := x.Interface().(map[string]int)
- if !ok || len(y) != 2 || y["a"] != 8 || y["b"] != 9 {
- t.Errorf("Bad value: %v", y)
- }
- }
- x, s = field.Get(v, storage.PathName{"D", "a"})
- if x.Interface() != 8 || !pathEq(storage.PathName{}, s) {
- t.Errorf("Expected 8, got %v", x)
- }
- x, s = field.Get(v, storage.PathName{"D", "b"})
- if x.Interface() != 9 || !pathEq(storage.PathName{}, s) {
- t.Errorf("Expected 9, got %v", x)
- }
- x, s = field.Get(v, storage.PathName{"D", "c"})
- if !pathEq(storage.PathName{"c"}, s) {
- t.Errorf("Expected %v, got %v", storage.PathName{"c"}, s)
- }
- {
- y, ok := x.Interface().(map[string]int)
- if !ok || len(y) != 2 || y["a"] != 8 || y["b"] != 9 {
- t.Errorf("Bad value: %v", y)
- }
- }
-
- // E field.
- x, s = field.Get(v, storage.PathName{"E"})
- if x.Interface() != v.E || !pathEq(storage.PathName{}, s) {
- t.Errorf("Failed to get E: %v", x.Interface())
- }
- x, s = field.Get(v, storage.PathName{"E", "a", "b", "c"})
- if x.Interface() != v.E || !pathEq(storage.PathName{"a", "b", "c"}, s) {
- t.Errorf("Failed to get E: %v, %v", x.Interface(), s)
- }
-}
-
-func TestSetSliceField(t *testing.T) {
- v := &[]string{"a", "b", "c"}
- rv := reflect.ValueOf(v)
-
- // Test simple get and set.
- b, _ := field.Get(v, storage.PathName{"1"})
- if "b" != b.Interface() {
- t.Errorf(`Expected "b", got %v`, b.Interface())
- }
- if ok, _ := field.Set(rv, "1", "other"); ok != field.SetAsValue {
- t.Errorf("field.Set failed on slice: %v", ok)
- }
- b, _ = field.Get(v, storage.PathName{"1"})
- if "other" != b.Interface() {
- t.Errorf(`Expected "a", got %v`, b.Interface())
- }
-
- // Test get on a non-existant field.
- ne, _ := field.Get(v, storage.PathName{"4"})
- if ne.Kind() != reflect.Slice {
- t.Errorf("Expected to get a top level slice, got: %v", ne.Interface())
- }
- ne, _ = field.Get(v, storage.PathName{"-1"})
- if ne.Kind() != reflect.Slice {
- t.Errorf("Expected to get a top level slice, got: %v", ne.Interface())
- }
- nepath := storage.PathName{"X"}
- ne, s := field.Get(v, nepath)
- if ne.Kind() != reflect.Slice {
- t.Errorf("Expected to get a top level slice, got: %v", ne.Interface())
- }
- if !reflect.DeepEqual(s, nepath) {
- t.Errorf("Expected path %v, got %v.", nepath, s)
- }
-
- // Test adding a value.
- if ok, _ := field.Set(rv, "@", "AppendedVal"); ok != field.SetAsValue {
- t.Errorf("Expected to succeed in appending value: %v", ok)
- }
- appended, _ := field.Get(v, storage.PathName{"3"})
- if "AppendedVal" != appended.Interface() {
- t.Errorf(`Expected "AppendedVal", got %v`, appended.Interface())
- }
-
- // Test set of an incompatible value fails.
- if ok, _ := field.Set(rv, "1", true); ok == field.SetAsValue {
- t.Errorf("Expected field.Set to fail when an incompatible value is being set.")
- }
-}
-
-func TestSetEmptySliceField(t *testing.T) {
- v := &[]string{}
- rv := reflect.ValueOf(v)
-
- ne, _ := field.Get(v, storage.PathName{"0"})
- if ne.Kind() != reflect.Slice {
- t.Errorf("Expected to get a top level slice, got: %v", ne.Interface())
- }
- if ok, _ := field.Set(rv, "0", "a"); ok == field.SetAsValue {
- t.Errorf("Expected field.Set to fail")
- }
-}
-
-func TestSetMapField(t *testing.T) {
- v := &map[string]string{
- "A": "a",
- "B": "b",
- }
- rv := reflect.ValueOf(v)
-
- // Test simple get and set.
- a, _ := field.Get(v, storage.PathName{"A"})
- if "a" != a.Interface() {
- t.Errorf(`Expected "a", got %v`, a.Interface())
- }
- if ok, _ := field.Set(rv, "A", "other"); ok != field.SetAsValue {
- t.Errorf("field.Set failed on map: %v", ok)
- }
- a, _ = field.Get(v, storage.PathName{"A"})
- if "other" != a.Interface() {
- t.Errorf(`Expected "a", got %v`, a.Interface())
- }
-
- // Test get on a non-existant field.
- nepath := storage.PathName{"NonExistant"}
- ne, s := field.Get(v, nepath)
- if !reflect.DeepEqual(s, nepath) {
- t.Errorf("Expected path %v, got %v.", nepath, s)
- }
- if ne.Kind() != reflect.Map {
- t.Errorf("Expected to get a top level map, got: %v", ne.Interface())
- }
-
- // Test that set on a non-existant field adds the field.
- if ok, _ := field.Set(rv, "C", "c"); ok != field.SetAsValue {
- t.Errorf("Expected field.Set to succeed: %v", ok)
- }
- c, _ := field.Get(v, storage.PathName{"C"})
- if "c" != c.Interface() {
- t.Errorf(`Expected "c", got %v`, c.Interface())
- }
-
- // Test set of an incompatible value fails.
- if ok, _ := field.Set(rv, "A", true); ok == field.SetAsValue {
- t.Errorf("Expected field.Set to fail when an incompatible value is being set.")
- }
-}
-
-func TestSetEmptyMapField(t *testing.T) {
- v := &map[string]interface{}{}
- rv := reflect.ValueOf(v)
-
- ne, _ := field.Get(v, storage.PathName{"A"})
- if ne.Kind() != reflect.Map {
- t.Errorf("Expected to get a top level map, got: %v", ne.Interface())
- }
- if ok, _ := field.Set(rv, "A", "a"); ok != field.SetAsValue {
- t.Errorf("Expected field.Set to succeed: %v", ok)
- }
- a, _ := field.Get(v, storage.PathName{"A"})
- if "a" != a.Interface() {
- t.Errorf(`Expected "a", got %v`, a.Interface())
- }
-}
-
-func TestSetStructField(t *testing.T) {
- a := &A{
- B: 5,
- C: []int{6, 7},
- D: map[string]int{"a": 8, "b": 9},
- E: storage.NewID(),
- F: []storage.ID{storage.NewID()},
- G: map[string]storage.ID{"a": storage.NewID()},
- }
- v := reflect.ValueOf(a)
-
- // B field.
- x, _ := field.Get(a, storage.PathName{"B"})
- if x.Interface() != 5 {
- t.Errorf("Expected 5, got %v", x)
- }
- if ok, _ := field.Set(v, "B", 15); ok != field.SetAsValue {
- t.Errorf("field.Set failed: %v", ok)
- }
- x, _ = field.Get(a, storage.PathName{"B"})
- if x.Interface() != 15 {
- t.Errorf("Expected 15, got %v", x)
- }
-
- // C field.
- if ok, _ := field.Set(v, "C", []int{7}); ok != field.SetAsValue {
- t.Errorf("Failed to set C: %v", ok)
- }
- x, _ = field.Get(a, storage.PathName{"C", "0"})
- if x.Interface() != 7 {
- t.Errorf("Expected 6, got %v", x)
- }
-
- p, _ := field.Get(a, storage.PathName{"C"})
- if ok, _ := field.Set(p, "0", 8); ok != field.SetAsValue {
- t.Errorf("Failed to set C: %v", ok)
- }
- x, _ = field.Get(a, storage.PathName{"C", "0"})
- if x.Interface() != 8 {
- t.Errorf("Expected 8, got %v", x)
- }
-
- p, _ = field.Get(a, storage.PathName{"C"})
- if ok, _ := field.Set(p, "@", 9); ok != field.SetAsValue {
- t.Errorf("Failed to set C")
- }
- x, _ = field.Get(a, storage.PathName{"C", "1"})
- if x.Interface() != 9 {
- t.Errorf("Expected 9, got %v", x)
- }
-
- // D field.
- if ok, _ := field.Set(v, "D", map[string]int{"a": 1}); ok != field.SetAsValue {
- t.Errorf("Failed to set D")
- }
- x, _ = field.Get(a, storage.PathName{"D", "a"})
- if x.Interface() != 1 {
- t.Errorf("Expected 1, got %v", x)
- }
-
- p, _ = field.Get(a, storage.PathName{"D"})
- if ok, _ := field.Set(p, "a", 2); ok != field.SetAsValue {
- t.Errorf("Failed to set D")
- }
- x, _ = field.Get(a, storage.PathName{"D", "a"})
- if x.Interface() != 2 {
- t.Errorf("Expected 2, got %v", x)
- }
-
- // E field.
- id := storage.NewID()
- ok, id2 := field.Set(v, "E", id)
- if ok != field.SetAsValue || id2 != id {
- t.Errorf("Failed to set E: %b, %s/%s", ok, id, id2)
- }
-
- // F field.
- p, _ = field.Get(a, storage.PathName{"F"})
- ok, fid := field.Set(p, "0", "fail")
- if ok != field.SetAsID {
- t.Errorf("Failed to set F: %v", x)
- }
- x, _ = field.Get(a, storage.PathName{"F", "0"})
- if x.Interface() != fid {
- t.Errorf("Expected %v, got %v", id, x.Interface())
- }
-
- // G field.
- p, _ = field.Get(a, storage.PathName{"G"})
- ok, fid = field.Set(p, "key", "fail")
- if ok != field.SetAsID {
- t.Errorf("Failed to set G")
- }
- x, _ = field.Get(a, storage.PathName{"G", "key"})
- if x.Interface() != fid {
- t.Errorf("Expected %v, got %v", id, x)
- }
-}
-
-func TestRemoveField(t *testing.T) {
- a := &A{
- B: 5,
- C: []int{6, 7},
- D: map[string]int{"a": 8, "b": 9},
- E: storage.NewID(),
- F: []storage.ID{storage.NewID()},
- G: map[string]storage.ID{"a": storage.NewID()},
- }
- v := reflect.ValueOf(a)
-
- if field.Remove(v, "B") {
- t.Errorf("Unexpected success")
- }
- p, _ := field.Get(a, storage.PathName{"C"})
- if field.Remove(p, "0") {
- t.Errorf("Unexpected success")
- }
- p, _ = field.Get(a, storage.PathName{"D"})
- if !field.Remove(p, "a") {
- t.Errorf("Unexpected failure")
- }
- x, s := field.Get(a, storage.PathName{"D", "a"})
- if !pathEq(storage.PathName{"a"}, s) {
- t.Errorf("Unexpected value: %v", x)
- }
-}
diff --git a/services/store/memstore/log.go b/services/store/memstore/log.go
deleted file mode 100644
index 3b69326..0000000
--- a/services/store/memstore/log.go
+++ /dev/null
@@ -1,225 +0,0 @@
-package memstore
-
-// A log consist of:
-// 1. A header.
-// 2. A state snapshot.
-// 3. A sequence of transactions.
-//
-// The header includes information about the log version, the size of the state
-// snapshot, the root storage.ID, etc. The snapshot is a sequence of entries for
-// each of the values in the state. A transaction is a *mutations object.
-//
-// There are separate interfaces for reading writing; *wlog is used for writing,
-// and *rlog is used for reading.
-import (
- "io"
- "os"
- "path"
- "sync"
- "time"
-
- "veyron/runtimes/google/lib/follow"
- "veyron/services/store/memstore/state"
-
- "veyron2/security"
- "veyron2/verror"
- "veyron2/vom"
-)
-
-const (
- logfileName = "storage.log"
- // TODO(tilaks): determine correct permissions for the logs.
- dirPerm = os.FileMode(0700)
- filePerm = os.FileMode(0600)
-)
-
-var (
- errLogIsClosed = verror.Abortedf("log is closed")
-)
-
-// wlog is the type of log writers.
-type wlog struct {
- file *os.File
- enc *vom.Encoder
-}
-
-// RLog is the type of log readers.
-type RLog struct {
- mu sync.Mutex
- closed bool // GUARDED_BY(mu)
- reader io.ReadCloser
- dec *vom.Decoder
-}
-
-// createLog creates a log writer. dbName is the path of the database directory,
-// to which transaction logs are written.
-func createLog(dbName string) (*wlog, error) {
- // Create the log file at the default path in the database directory.
- filePath := path.Join(dbName, logfileName)
- if err := os.MkdirAll(dbName, dirPerm); err != nil {
- return nil, err
- }
- file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, filePerm)
- if err != nil {
- return nil, err
- }
- return &wlog{
- file: file,
- enc: vom.NewEncoder(file),
- }, nil
-}
-
-// close closes the log file.
-func (l *wlog) close() {
- l.file.Close()
- l.file = nil
- l.enc = nil
-}
-
-// writeState writes the initial state.
-func (l *wlog) writeState(st *Store) error {
- // If writeState returns a nil error, the caller should assume that
- // future reads from the log file will discover the new state.
- // Therefore we commit the log file's new content to stable storage.
- // Note: l.enc does not buffer output, and doesn't need to be flushed.
- if l.file == nil {
- return errLogIsClosed
- }
- return st.State.Write(l.enc)
-}
-
-// appendTransaction adds a transaction to the end of the log.
-func (l *wlog) appendTransaction(m *state.Mutations) error {
- // If appendTransaction returns a nil error, the caller should assume that
- // future reads from the log file will discover the new transaction.
- // Therefore we commit the log file's new content to stable storage.
- // Note: l.enc does not buffer output, and doesn't need to be flushed.
- if l.file == nil {
- return errLogIsClosed
- }
- return l.enc.Encode(m)
-}
-
-// OpenLog opens a log for reading. dbName is the path of the database directory.
-// If followLog is true, reads block until records can be read. Otherwise,
-// reads return EOF when no record can be read.
-func OpenLog(dbName string, followLog bool) (*RLog, error) {
- // Open the log file at the default path in the database directory.
- filePath := path.Join(dbName, logfileName)
- var reader io.ReadCloser
- var err error
- if followLog {
- reader, err = follow.NewReader(filePath)
- } else {
- reader, err = os.Open(filePath)
- }
- if err != nil {
- return nil, err
- }
- return &RLog{
- reader: reader,
- dec: vom.NewDecoder(reader),
- }, nil
-}
-
-// Close closes the log. If Close is called concurrently with ReadState or
-// ReadTransaction, ongoing reads will terminate. Close is idempotent.
-func (l *RLog) Close() {
- l.mu.Lock()
- defer l.mu.Unlock()
- if l.closed {
- return
- }
- l.closed = true
- l.reader.Close()
-}
-
-func (l *RLog) isClosed() bool {
- l.mu.Lock()
- defer l.mu.Unlock()
- return l.closed
-}
-
-// ReadState reads the initial state state. ReadState returns an error if the
-// log is closed before or during the read. ReadState should not be invoked
-// concurrently with other reads.
-func (l *RLog) ReadState(adminID security.PublicID) (*Store, error) {
- if l.isClosed() {
- return nil, errLogIsClosed
- }
-
- // Create the state struct.
- st, err := New(adminID, "")
- if err != nil {
- return nil, err
- }
-
- // Create the state without refcounts.
- if err := st.State.Read(l.dec); err != nil {
- return nil, err
- }
-
- return st, nil
-}
-
-// ReadTransaction reads a transaction entry from the log. ReadTransaction
-// returns an error if the log is closed before or during the read.
-// ReadTransaction should not be invoked concurrently with other reads.
-func (l *RLog) ReadTransaction() (*state.Mutations, error) {
- if l.isClosed() {
- return nil, errLogIsClosed
- }
-
- var ms state.Mutations
- if err := l.dec.Decode(&ms); err != nil {
- return nil, err
- }
- for _, m := range ms.Delta {
- m.UpdateRefs()
- }
- return &ms, nil
-}
-
-// backup the log file.
-func backupLog(dbName string) error {
- srcPath := path.Join(dbName, logfileName)
- dstPath := srcPath + "." + time.Now().Format(time.RFC3339)
- return os.Rename(srcPath, dstPath)
-}
-
-// openDB opens the log file if it exists. dbName is the path of the database
-// directory. If followLog is true, reads block until records can be read.
-// Otherwise, reads return EOF when no record can be read.
-func openDB(dbName string, followLog bool) (*RLog, error) {
- if dbName == "" {
- return nil, nil
- }
- rlog, err := OpenLog(dbName, followLog)
- if err != nil && os.IsNotExist(err) {
- // It is not an error for the log not to exist.
- err = nil
- }
- return rlog, err
-}
-
-// readAndCloseDB reads the state from the log file.
-func readAndCloseDB(admin security.PublicID, rlog *RLog) (*Store, error) {
- defer rlog.Close()
- st, err := rlog.ReadState(admin)
- if err != nil {
- return nil, err
- }
- for {
- mu, err := rlog.ReadTransaction()
- if err == io.EOF {
- break
- }
- if err != nil {
- return nil, err
- }
- if err := st.ApplyMutations(mu); err != nil {
- return nil, err
- }
- }
- return st, nil
-}
diff --git a/services/store/memstore/log_test.go b/services/store/memstore/log_test.go
deleted file mode 100644
index 923e27a..0000000
--- a/services/store/memstore/log_test.go
+++ /dev/null
@@ -1,291 +0,0 @@
-package memstore
-
-import (
- "fmt"
- "io/ioutil"
- "os"
- "runtime"
- "testing"
-
- "veyron/services/store/memstore/refs"
- "veyron/services/store/memstore/state"
-
- "veyron2/security"
- "veyron2/storage"
- "veyron2/vom"
-)
-
-var (
- rootPublicID security.PublicID = security.FakePublicID("root")
-)
-
-type Dir struct {
- Entries map[string]storage.ID
-}
-
-type Data struct {
- Comment string
-}
-
-func init() {
- vom.Register(&Data{})
-}
-
-func newData(c string) *Data {
- return &Data{Comment: c}
-}
-
-func expectEqDir(t *testing.T, file string, line int, d1, d2 *Dir) {
- for name, id1 := range d1.Entries {
- id2, ok := d2.Entries[name]
- if !ok {
- t.Errorf("%s(%d): does not exist: %s", file, line, name)
- } else if id2 != id1 {
- t.Errorf("%s(%d): expected ID %s, got %s", file, line, id1, id2)
- }
- }
- for name, _ := range d2.Entries {
- _, ok := d1.Entries[name]
- if !ok {
- t.Errorf("%s(%d): should not exist: %s", file, line, name)
- }
- }
-}
-
-func expectEqData(t *testing.T, file string, line int, d1, d2 *Data) {
- if d1.Comment != d2.Comment {
- t.Errorf("%s(%d): expected %q, got %q", d1.Comment, d2.Comment)
- }
-}
-
-// expectEqValues compares two items. They are equal only if they have the same
-// type, and their contents are equal.
-func expectEqValues(t *testing.T, file string, line int, v1, v2 interface{}) {
- switch x1 := v1.(type) {
- case *Dir:
- x2, ok := v2.(*Dir)
- if !ok {
- t.Errorf("%s(%d): not a Dir: %v", file, line, v2)
- } else {
- expectEqDir(t, file, line, x1, x2)
- }
- case *Data:
- x2, ok := v2.(*Data)
- if !ok {
- t.Errorf("%s(%d): not a Data: %v", file, line, v2)
- } else {
- expectEqData(t, file, line, x1, x2)
- }
- default:
- t.Errorf("Unknown type: %T, %v", v1, v1)
- }
-}
-
-// expectEqImplicitDir compares two directories.
-func expectEqImplicitDir(t *testing.T, file string, line int, d1, d2 refs.Dir) {
- l1 := refs.FlattenDir(d1)
- l2 := refs.FlattenDir(d2)
- i1 := 0
- i2 := 0
- for i1 < len(l1) && i2 < len(l2) {
- e1 := l1[i1]
- e2 := l2[i2]
- if e1.Name == e2.Name {
- if e1.ID != e2.ID {
- t.Errorf("%s(%d): expected id %s, got %s", file, line, e1.ID, e2.ID)
- }
- i1++
- i2++
- } else if e1.Name < e2.Name {
- t.Errorf("%s(%d): missing directory %s", file, line, e1.Name)
- i1++
- } else {
- t.Errorf("%s(%d): unexpected directory %s", file, line, e2.Name)
- i2++
- }
- }
- for _, e1 := range l1[i1:] {
- t.Errorf("%s(%d): missing directory %s", file, line, e1.Name)
- }
- for _, e2 := range l2[i2:] {
- t.Errorf("%s(%d): unexpected directory %s", file, line, e2.Name)
- }
-}
-
-func readTransaction(r *RLog) (*state.Mutations, error) {
- type result struct {
- mu *state.Mutations
- err error
- }
- results := make(chan result)
- go func() {
- defer close(results)
- mu, err := r.ReadTransaction()
- results <- result{mu: mu, err: err}
- }()
- res := <-results
- return res.mu, res.err
-}
-
-// readTransactions reads and applies the transactions. Returns the last transaction read.
-func readTransactions(t *testing.T, r *RLog, st *Store, n int) *state.Mutations {
- _, file, line, _ := runtime.Caller(1)
- var m *state.Mutations
- for i := 0; i < n; i++ {
- var err error
- m, err = readTransaction(r)
- if err != nil {
- t.Errorf("%s(%d): error in readTransaction(): %s", file, line, err)
- return nil
- }
- if err := st.ApplyMutations(m); err != nil {
- t.Errorf("%s(%d): error in ApplyMutations(): %s", file, line, err)
- return nil
- }
- }
- return m
-}
-
-func TestTransaction(t *testing.T) {
- dbName, err := ioutil.TempDir(os.TempDir(), "vstore")
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
- defer os.RemoveAll(dbName)
-
- w, err := createLog(dbName)
- if err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
-
- // Create an initial state.
- st1, err := New(rootPublicID, "")
- if err != nil {
- t.Fatalf("newState() failed: %v", err)
- }
- tr := &Transaction{}
- mkdir(t, st1, tr, "/")
- mkdir(t, st1, tr, "/a")
- mkdir(t, st1, tr, "/a/b")
- mkdir(t, st1, tr, "/a/b/c")
- commit(t, tr)
-
- // Write the initial state.
- defer w.close()
- w.writeState(st1)
-
- // Write some Transactions.
- var data [5]*Data
- var paths [5]string
- for i := 0; i != 5; i++ {
- name := fmt.Sprintf("data%d", i)
- tr := &Transaction{}
- data[i] = newData(name)
- paths[i] = "/a/b/c/" + name
- put(t, st1, tr, paths[i], data[i])
- commit(t, tr)
- w.appendTransaction(tr.snapshot.Mutations())
- }
-
- r, err := OpenLog(dbName, true)
- if err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- defer r.Close()
- st2, err := r.ReadState(rootPublicID)
- if err != nil {
- t.Fatalf("Can't read state: %s", err)
- }
- readTransactions(t, r, st2, 5)
-
- // Remove data3.
- {
- tr := &Transaction{}
- remove(t, st1, tr, "/a/b/c/data3")
- commit(t, tr)
- w.appendTransaction(tr.snapshot.Mutations())
- }
- readTransactions(t, r, st2, 1)
- {
- tr := &Transaction{}
- expectExists(t, st1, tr, paths[0])
- expectNotExists(t, st1, tr, paths[3])
- }
-
- // Remove all entries.
- {
- tr := &Transaction{}
- remove(t, st1, tr, "/a/b/c")
- commit(t, tr)
- w.appendTransaction(tr.snapshot.Mutations())
- }
- readTransactions(t, r, st2, 1)
- {
- tr := &Transaction{}
- for i := 0; i != 5; i++ {
- expectNotExists(t, st1, tr, paths[i])
- }
- }
-}
-
-func TestDeletions(t *testing.T) {
- dbName, err := ioutil.TempDir(os.TempDir(), "vstore")
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
- defer os.RemoveAll(dbName)
-
- // Create an initial state.
- st1, err := New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("newState() failed: %v", err)
- }
- tr := &Transaction{}
- mkdir(t, st1, tr, "/")
- mkdir(t, st1, tr, "/a")
- ids := make(map[string]storage.ID)
- ids["/a/b"], _ = mkdir(t, st1, tr, "/a/b")
- ids["/a/b/c"], _ = mkdir(t, st1, tr, "/a/b/c")
- ids["/a/b/d"], _ = mkdir(t, st1, tr, "/a/b/d")
- commit(t, tr)
-
- // Reconstruct the state from the log.
- r, err := OpenLog(dbName, true)
- if err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- defer r.Close()
- st2, err := r.ReadState(rootPublicID)
- if err != nil {
- t.Fatalf("Can't read state: %s", err)
- }
- readTransactions(t, r, st2, 1)
-
- // Remove b.
- {
- tr := &Transaction{}
- remove(t, st1, tr, "/a/b")
- commit(t, tr)
- }
- readTransactions(t, r, st2, 1)
- {
- tr := &Transaction{}
- expectExists(t, st1, tr, "/a")
- expectNotExists(t, st1, tr, "/a/b")
- }
-
- // Perform a GC.
- if err := st1.GC(); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- // The transaction should include deletions.
- mu := readTransactions(t, r, st2, 1)
- for name, id := range ids {
- if _, ok := mu.Deletions[id]; !ok {
- t.Errorf("Expected deletion for path %s", name)
- }
- if len(mu.Deletions) != len(ids) {
- t.Errorf("Unexpected deletion: %v", mu.Deletions)
- }
- }
-}
diff --git a/services/store/memstore/object.go b/services/store/memstore/object.go
deleted file mode 100644
index 9fb0540..0000000
--- a/services/store/memstore/object.go
+++ /dev/null
@@ -1,93 +0,0 @@
-package memstore
-
-import (
- iquery "veyron/services/store/memstore/query"
-
- "veyron2/query"
- "veyron2/security"
- "veyron2/storage"
- "veyron2/verror"
-)
-
-// Object is a binding to a store value. This is currently represented as a
-// path. This means that in different transactions, the object may refer to
-// different values with different store.IDs.
-type Object struct {
- path storage.PathName
- store *Store
-}
-
-// Exists returns true iff the object has a value in the current transaction.
-func (o *Object) Exists(pid security.PublicID, trans *Transaction) (bool, error) {
- tr, _, err := o.store.getTransaction(trans)
- if err != nil {
- return false, err
- }
- v, err := tr.snapshot.Get(pid, o.path)
- ok := v != nil && err == nil
- return ok, nil
-}
-
-// Get returns the value for an object.
-func (o *Object) Get(pid security.PublicID, trans *Transaction) (*storage.Entry, error) {
- tr, _, err := o.store.getTransaction(trans)
- if err != nil {
- return nil, err
- }
- return tr.snapshot.Get(pid, o.path)
-}
-
-// Put updates the value for an object.
-func (o *Object) Put(pid security.PublicID, trans *Transaction, v interface{}) (*storage.Stat, error) {
- tr, commit, err := o.store.getTransaction(trans)
- if err != nil {
- return nil, err
- }
- st, err := tr.snapshot.Put(pid, o.path, v)
- if err != nil {
- return nil, err
- }
- if commit {
- err = tr.Commit()
- }
- return st, err
-}
-
-// Remove removes the value for an object.
-func (o *Object) Remove(pid security.PublicID, trans *Transaction) error {
- tr, commit, err := o.store.getTransaction(trans)
- if err != nil {
- return err
- }
- if err := tr.snapshot.Remove(pid, o.path); err != nil {
- return err
- }
- if commit {
- return tr.Commit()
- }
- return nil
-}
-
-// Stat returns entry info.
-func (o *Object) Stat(pid security.PublicID, trans *Transaction) (*storage.Stat, error) {
- return nil, verror.Internalf("Stat not yet implemented")
-}
-
-// Query returns entries matching the given query.
-func (o *Object) Query(pid security.PublicID, trans *Transaction, q query.Query) (iquery.QueryStream, error) {
- tr, _, err := o.store.getTransaction(trans)
- if err != nil {
- return nil, err
- }
- stream := iquery.Eval(tr.snapshot.GetSnapshot(), pid, o.path, q)
- return stream, nil
-}
-
-// Glob returns names that match the given pattern.
-func (o *Object) Glob(pid security.PublicID, trans *Transaction, pattern string) (iquery.GlobStream, error) {
- tr, _, err := o.store.getTransaction(trans)
- if err != nil {
- return nil, err
- }
- return iquery.Glob(tr.snapshot.GetSnapshot(), pid, o.path, pattern)
-}
diff --git a/services/store/memstore/object_test.go b/services/store/memstore/object_test.go
deleted file mode 100644
index c5232b9..0000000
--- a/services/store/memstore/object_test.go
+++ /dev/null
@@ -1,346 +0,0 @@
-package memstore
-
-import (
- "runtime"
- "testing"
-
- "veyron2/query"
-)
-
-func newValue() interface{} {
- return &Dir{}
-}
-
-func TestPutGetRemoveRoot(t *testing.T) {
- st, err := New(rootPublicID, "")
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
- o := st.Bind("/")
- testGetPutRemove(t, st, o)
-}
-
-func TestPutGetRemoveChild(t *testing.T) {
- st, err := New(rootPublicID, "")
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
- tr := NewTransaction()
- mkdir(t, st, tr, "/")
- commit(t, tr)
- o := st.Bind("/Entries/a")
- testGetPutRemove(t, st, o)
-}
-
-func testGetPutRemove(t *testing.T, st *Store, o *Object) {
- value := newValue()
-
- {
- // Check that the root object does not exist.
- tr := &Transaction{}
- if ok, err := o.Exists(rootPublicID, tr); ok && err == nil {
- t.Errorf("Should not exist")
- }
- if v, err := o.Get(rootPublicID, tr); v != nil && err == nil {
- t.Errorf("Should not exist: %v, %s", v, err)
- }
- }
-
- {
- // Add the root object.
- tr1 := &Transaction{}
- s, err := o.Put(rootPublicID, tr1, value)
- if err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- if ok, err := o.Exists(rootPublicID, tr1); !ok || err != nil {
- t.Errorf("Should exist: %s", err)
- }
- e, err := o.Get(rootPublicID, tr1)
- if err != nil {
- t.Errorf("Object should exist: %s", err)
- }
- if e.Stat.ID != s.ID {
- t.Errorf("Expected %s, got %s", s.ID, e.Stat.ID)
- }
-
- // Transactions are isolated.
- tr2 := &Transaction{}
- if ok, err := o.Exists(rootPublicID, tr2); ok && err != nil {
- t.Errorf("Should not exist")
- }
- if v, err := o.Get(rootPublicID, tr2); v != nil && err == nil {
- t.Errorf("Should not exist: %v, %s", v, err)
- }
-
- // Apply tr1.
- if err := tr1.Commit(); err != nil {
- t.Errorf("Unexpected error")
- }
- if ok, err := o.Exists(rootPublicID, tr1); !ok || err != nil {
- t.Errorf("Should exist: %s", err)
- }
- if _, err := o.Get(rootPublicID, tr1); err != nil {
- t.Errorf("Object should exist: %s", err)
- }
-
- // tr2 is still isolated.
- if ok, err := o.Exists(rootPublicID, tr2); ok && err == nil {
- t.Errorf("Should not exist")
- }
- if v, err := o.Get(rootPublicID, tr2); v != nil && err == nil {
- t.Errorf("Should not exist: %v, %s", v, err)
- }
-
- // tr3 observes the commit.
- tr3 := &Transaction{}
- if ok, err := o.Exists(rootPublicID, tr3); !ok || err != nil {
- t.Errorf("Should exist")
- }
- if _, err := o.Get(rootPublicID, tr3); err != nil {
- t.Errorf("Object should exist: %s", err)
- }
- }
-
- {
- // Remove the root object.
- tr1 := &Transaction{}
- if err := o.Remove(rootPublicID, tr1); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- if ok, err := o.Exists(rootPublicID, tr1); ok && err != nil {
- t.Errorf("Should not exist")
- }
- if v, err := o.Get(rootPublicID, tr1); v != nil || err == nil {
- t.Errorf("Object should exist: %v", v)
- }
-
- // The removal is isolated.
- tr2 := &Transaction{}
- if ok, err := o.Exists(rootPublicID, tr2); !ok || err != nil {
- t.Errorf("Should exist")
- }
- if _, err := o.Get(rootPublicID, tr2); err != nil {
- t.Errorf("Object should exist: %s", err)
- }
-
- // Apply tr1.
- if err := tr1.Commit(); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- if ok, err := o.Exists(rootPublicID, tr1); ok && err == nil {
- t.Errorf("Should not exist")
- }
- if v, err := o.Get(rootPublicID, tr1); v != nil || err == nil {
- t.Errorf("Object should exist: %v", v)
- }
-
- // The removal is isolated.
- if ok, err := o.Exists(rootPublicID, tr2); !ok || err != nil {
- t.Errorf("Should exist: %s", err)
- }
- if _, err := o.Get(rootPublicID, tr2); err != nil {
- t.Errorf("Object should exist: %s", err)
- }
- }
-
- {
- // Check that the root object does not exist.
- tr1 := &Transaction{}
- if ok, err := o.Exists(rootPublicID, tr1); ok && err == nil {
- t.Errorf("Should not exist")
- }
- if v, err := o.Get(rootPublicID, tr1); v != nil && err == nil {
- t.Errorf("Should not exist: %v, %s", v, err)
- }
- }
-}
-
-func TestConcurrentOK(t *testing.T) {
- st, err := New(rootPublicID, "")
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
- {
- tr := NewTransaction()
- mkdir(t, st, tr, "/")
- mkdir(t, st, tr, "/Entries/a")
- mkdir(t, st, tr, "/Entries/b")
- commit(t, tr)
- }
-
- o1 := st.Bind("/Entries/a/Entries/c")
- tr1 := &Transaction{}
- v1 := newValue()
- s1, err := o1.Put(rootPublicID, tr1, v1)
- if err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
-
- o2 := st.Bind("/Entries/b/Entries/d")
- tr2 := &Transaction{}
- v2 := newValue()
- s2, err := o2.Put(rootPublicID, tr2, v2)
- if err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
-
- if err := tr1.Commit(); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- if err := tr2.Commit(); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
-
- tr3 := &Transaction{}
- if x, err := o1.Get(rootPublicID, tr3); err != nil || x.Stat.ID != s1.ID {
- t.Errorf("Value should exist: %v, %s", x, err)
- }
- if x, err := o2.Get(rootPublicID, tr3); err != nil || x.Stat.ID != s2.ID {
- t.Errorf("Value should exist: %v, %s", x, err)
- }
-}
-
-func TestQuery(t *testing.T) {
- st, err := New(rootPublicID, "")
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
- {
- tr := NewTransaction()
- mkdir(t, st, tr, "/")
- mkdir(t, st, tr, "/Entries/a")
- mkdir(t, st, tr, "/Entries/b")
- commit(t, tr)
- }
-
- o1 := st.Bind("/Entries")
- tr1 := &Transaction{}
-
- stream, err := o1.Query(rootPublicID, tr1, query.Query{"*"})
- results := map[string]bool{}
- expected := []string{"", "a", "b"}
- for stream.Next() {
- results[stream.Get().Name] = true
- }
- if nresults, nexpected := len(results), len(expected); nresults != nexpected {
- t.Errorf("Unexpected number of query results. Want %d, got %d. %v",
- nexpected, nresults, results)
- }
- for _, expect := range expected {
- if !results[expect] {
- t.Errorf("Missing query result %s in %v.", expect, results)
- }
- }
-}
-
-func TestConcurrentConflict(t *testing.T) {
- st, err := New(rootPublicID, "")
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
- {
- tr := NewTransaction()
- mkdir(t, st, tr, "/")
- mkdir(t, st, tr, "/Entries/a")
- commit(t, tr)
- }
-
- o := st.Bind("/Entries/a/Entries/c")
- tr1 := &Transaction{}
- v1 := newValue()
- s1, err := o.Put(rootPublicID, tr1, v1)
- if err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
-
- tr2 := &Transaction{}
- v2 := newValue()
- if _, err = o.Put(rootPublicID, tr2, v2); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
-
- if err := tr1.Commit(); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- if err := tr2.Commit(); err == nil || err.Error() != "precondition failed" {
- t.Errorf("Expected precondition failed, got %q", err)
- }
-
- tr3 := &Transaction{}
- if x, err := o.Get(rootPublicID, tr3); err != nil || x.Stat.ID != s1.ID {
- t.Errorf("Value should exist: %v, %s", x, err)
- }
-}
-
-type Foo struct{}
-
-func newFoo() *Foo {
- return &Foo{}
-}
-
-func getFoo(t *testing.T, st *Store, tr *Transaction, path string) *Foo {
- _, file, line, _ := runtime.Caller(1)
- v := get(t, st, tr, path)
- res, ok := v.(*Foo)
- if !ok {
- t.Fatalf("%s(%d): %s: not a Foo: %v", file, line, path, v)
- }
- return res
-}
-
-func TestSimpleMove(t *testing.T) {
- st, err := New(rootPublicID, "")
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
-
- // Create / and /x.
- {
- tr := &Transaction{}
- put(t, st, tr, "/", newFoo())
- put(t, st, tr, "/x", newFoo())
- commit(t, tr)
- }
-
- // Move /x to /y.
- {
- tr := &Transaction{}
- x := getFoo(t, st, tr, "/x")
- remove(t, st, tr, "/x")
- put(t, st, tr, "/y", x)
- commit(t, tr)
- }
-}
-
-// Test a path conflict where some directory along the path to a value has been
-// mutated concurrently.
-func TestPathConflict(t *testing.T) {
- st, err := New(rootPublicID, "")
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
-
- {
- tr := &Transaction{}
- put(t, st, tr, "/", newFoo())
- put(t, st, tr, "/a", newFoo())
- put(t, st, tr, "/a/b", newFoo())
- put(t, st, tr, "/a/b/c", newFoo())
- commit(t, tr)
- }
-
- // Add a new value.
- tr1 := &Transaction{}
- put(t, st, tr1, "/a/b/c/d", newFoo())
-
- // Change a directory along the path.
- tr2 := &Transaction{}
- put(t, st, tr2, "/a/b", newFoo())
- commit(t, tr2)
-
- // First Transaction should abort.
- if err := tr1.Commit(); err == nil {
- t.Errorf("Expected transaction to abort")
- }
-}
diff --git a/services/store/memstore/pathregex/doc.go b/services/store/memstore/pathregex/doc.go
deleted file mode 100644
index 3098f38..0000000
--- a/services/store/memstore/pathregex/doc.go
+++ /dev/null
@@ -1,40 +0,0 @@
-// Package pathregex implements path regular expressions. These are like glob
-// patterns, with the following kinds of expressions.
-//
-// Each component of a path is specified using a glob expression, where:
-// * matches any sequence of characters.
-// ? matches a single character.
-// [...] matches any single character in a range, where the ranges include:
-// c: matches the single character c
-// c1-c2: matches all characters in the range { c1,...,c2 }
-// ]: can be specified as the first character in the range
-// ^: a leading ^ inverts the range
-// {r1,r2,...,rN} matches r1 OR r1 OR ... OR rN.
-//
-// A path regular expression composes a path.
-//
-// R ::= r // a single component glob expression
-// | ... // matches any path
-// | R1 / R2 // sequential composition
-// | { R1, R2, ..., RN } // alternation, R1 OR R2 OR ... OR RN
-//
-// Examples:
-//
-// x.jpg - matches a path with one component x.jpg.
-// a/b - matches a two component path.
-// .../a/b/... - matches a path containing an a/b somewhere within it.
-// .../[abc]/... - matches a path containing an "a" or "b" or "c" component.
-// .../{a,b,c}/... - same as above.
-// {.../a/...,.../b/...,.../c/...} - same as above.
-//
-// The usage is to compile a path expression to a finite automaton, then execute
-// the automaton.
-//
-// ss, err := pathregex.Compile(".../{a,b/c,d/e/f}/...")
-// ss = ss.Step("x") // Match against x/d/e/f/y.
-// ss = ss.Step("d")
-// ss = ss.Step("e")
-// ss = ss.Step("f")
-// ss = ss.Step("y")
-// if ss.IsFinal() { log.Printf("Matched") }
-package pathregex
diff --git a/services/store/memstore/pathregex/nfa.go b/services/store/memstore/pathregex/nfa.go
deleted file mode 100644
index 80b0f3d..0000000
--- a/services/store/memstore/pathregex/nfa.go
+++ /dev/null
@@ -1,226 +0,0 @@
-package pathregex
-
-import (
- "regexp"
- "sort"
- "unsafe"
-)
-
-// state is a state of the NFA.
-type state struct {
- // isFinal is true iff the state is a final state.
- isFinal bool
-
- // isClosed is true iff the epsilon transitions are transitively closed.
- isClosed bool
-
- // trans is a non-epsilon transitions, or nil.
- trans *transition
-
- // epsilon are epsilon transitions, meaning the automation can take a
- // state transition without reading any input.
- epsilon hashStateSet
-}
-
-// transition represents a non-empty transition. If the current input symbol matches the
-// regular expression, the NFA can take a transition to the next states.
-type transition struct {
- re *regexp.Regexp // nil means accept anything.
- next hashStateSet
-}
-
-// hashStateSet represents a set of states using a map.
-type hashStateSet map[*state]struct{}
-
-// StateSet represents a set of states of the NFA as a sorted slice of state pointers.
-type StateSet []*state
-
-// byPointer is used to sort the pointers in StateSet.
-type byPointer StateSet
-
-// tt accepts anything.
-func (r tt) compile(final *state) *state {
- t := &transition{next: hashStateSet{final: struct{}{}}}
- return &state{trans: t, epsilon: make(hashStateSet)}
-}
-
-// ff has no transititons.
-func (r *ff) compile(final *state) *state {
- return &state{epsilon: make(hashStateSet)}
-}
-
-// epsilon doesn't require a transition.
-func (r *epsilon) compile(final *state) *state {
- return final
-}
-
-// single has a single transition from initial to final state.
-func (r *single) compile(final *state) *state {
- t := &transition{re: r.r, next: hashStateSet{final: struct{}{}}}
- return &state{trans: t, epsilon: make(hashStateSet)}
-}
-
-// sequence composes the automata.
-func (r *sequence) compile(final *state) *state {
- return r.r1.compile(r.r2.compile(final))
-}
-
-// alt constructs the separate automata, then defines a new start state that
-// includes epsilon transitions to the two separate automata.
-func (r *alt) compile(final *state) *state {
- s1 := r.r1.compile(final)
- s2 := r.r2.compile(final)
- return &state{epsilon: hashStateSet{s1: struct{}{}, s2: struct{}{}}}
-}
-
-// star contains a loop to accepts 0-or-more occurrences of r.re. There is an
-// epsilon transition from the start state s1 to the final state (for 0
-// occurrences), and a back epsilon-transition for 1-or-more occurrences.
-func (r *star) compile(final *state) *state {
- s2 := &state{epsilon: make(hashStateSet)}
- s1 := r.re.compile(s2)
- s2.epsilon = hashStateSet{s1: struct{}{}, final: struct{}{}}
- s1.epsilon[s2] = struct{}{}
- return s1
-}
-
-// close takes the transitive closure of the epsilon transitions.
-func (s *state) close() {
- if !s.isClosed {
- s.isClosed = true
- s.epsilon[s] = struct{}{}
- s.epsilon.closeEpsilon()
- if s.trans != nil {
- s.trans.next.closeEpsilon()
- }
- }
-}
-
-// isUseless returns true iff the state has only epsilon transitions and it is
-// not a final state.
-func (s *state) isUseless() bool {
- return s.trans == nil && !s.isFinal
-}
-
-// addStates folds the src states into the dst.
-func (dst hashStateSet) addStates(src hashStateSet) {
- for s, _ := range src {
- dst[s] = struct{}{}
- }
-}
-
-// stateSet converts the hashStateSet to a StateSet.
-func (set hashStateSet) stateSet() StateSet {
- states := StateSet{}
- for s, _ := range set {
- if !s.isUseless() {
- states = append(states, s)
- }
- }
- sort.Sort(byPointer(states))
- return states
-}
-
-// closeEpsilon closes the state set under epsilon transitions.
-func (states hashStateSet) closeEpsilon() {
- for changed := true; changed; {
- size := len(states)
- for s, _ := range states {
- s.close()
- states.addStates(s.epsilon)
- }
- changed = len(states) != size
- }
-
- // Remove useless states.
- for s, _ := range states {
- if s.isUseless() {
- delete(states, s)
- }
- }
-}
-
-// Step takes a transition for input name.
-func (ss StateSet) Step(name string) StateSet {
- states := make(hashStateSet)
- for _, s := range ss {
- if s.trans != nil && (s.trans.re == nil || s.trans.re.MatchString(name)) {
- s.close()
- states.addStates(s.trans.next)
- }
- }
- return states.stateSet()
-}
-
-// IsFinal returns true iff the StateSet contains a final state.
-func (ss StateSet) IsFinal() bool {
- for _, s := range ss {
- if s.isFinal {
- return true
- }
- }
- return false
-}
-
-// IsReject returns true iff the StateSet is empty.
-func (ss StateSet) IsReject() bool {
- return len(ss) == 0
-}
-
-// Union combines the state sets, returning a new StateSet.
-func (s1 StateSet) Union(s2 StateSet) StateSet {
- // As a space optimization, detect the case where the two states sets are
- // equal. If so, return s1 unchanged.
- if s1.Equals(s2) {
- return s1
- }
-
- i1 := 0
- i2 := 0
- var result StateSet
- for i1 < len(s1) && i2 < len(s2) {
- p1 := uintptr(unsafe.Pointer(s1[i1]))
- p2 := uintptr(unsafe.Pointer(s2[i2]))
- switch {
- case p1 == p2:
- i2++
- fallthrough
- case p1 < p2:
- result = append(result, s1[i1])
- i1++
- case p2 < p1:
- result = append(result, s2[i2])
- i2++
- }
- }
- result = append(result, s1[i1:]...)
- result = append(result, s2[i2:]...)
- return result
-}
-
-// Equals returns true iff the state sets are equal.
-func (s1 StateSet) Equals(s2 StateSet) bool {
- if len(s1) != len(s2) {
- return false
- }
- for i, s := range s1 {
- if s2[i] != s {
- return false
- }
- }
- return true
-}
-
-// sorting methods.
-func (a byPointer) Len() int { return len(a) }
-func (a byPointer) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
-func (a byPointer) Less(i, j int) bool {
- return uintptr(unsafe.Pointer(a[i])) < uintptr(unsafe.Pointer(a[j]))
-}
-
-// Compile compiles the Regex to a NFA.
-func compileNFA(r regex) StateSet {
- s := r.compile(&state{isFinal: true, epsilon: make(hashStateSet)})
- s.close()
- return s.epsilon.stateSet()
-}
diff --git a/services/store/memstore/pathregex/parser.go b/services/store/memstore/pathregex/parser.go
deleted file mode 100644
index 101cce1..0000000
--- a/services/store/memstore/pathregex/parser.go
+++ /dev/null
@@ -1,278 +0,0 @@
-package pathregex
-
-import (
- "bytes"
- "io"
- "regexp"
- "strings"
-
- "veyron2/verror"
-)
-
-// parser is a recursive-descent parser for path regular expressions.
-type parser struct {
- reader *strings.Reader
-
- // commas are treated specially within braces, but are otherwise normal
- // characters.
- commaIsSpecial bool
-
- // isErrored is true iff an error has been encountered.
- isErrored bool
-}
-
-// Compile compiles a string to a finite automaton. Returns a StateSet of the
-// initial states of the automaton.
-func Compile(s string) (StateSet, error) {
- re, err := compileRegex(s)
- if err != nil {
- return nil, err
- }
- return compileNFA(re), nil
-}
-
-// CompileReverse compiles a path regular expression to a finite automaton,
-// reversing the regular expression. Returns a StateSet of the initial states
-// of the automaton.
-func CompileReverse(s string) (StateSet, error) {
- re, err := compileRegex(s)
- if err != nil {
- return nil, err
- }
- re.reverse()
- return compileNFA(re), nil
-}
-
-func compileRegex(s string) (regex, error) {
- p := &parser{reader: strings.NewReader(s)}
- re := p.parsePath()
- if p.isErrored || p.reader.Len() != 0 {
- pos, _ := p.reader.Seek(0, 1)
- err := verror.BadArgf("Syntax error at char %d: %q", pos, s)
- return nil, err
- }
- return re, nil
-}
-
-// parsePath reads a path regular expression. Reads as much of the input as
-// possible, stopping at special characters, like '}' or ',' (if
-// commaIsSpecial).
-func (p *parser) parsePath() regex {
- var path []regex
- for !p.isErrored {
- // If the next rune is '{', parse as an alternation; otherwise, parse
- // the next component.
- c, ok := p.readRune()
- if !ok {
- break
- }
- var re regex
- if c == '{' {
- re = p.parsePathAlt()
- } else {
- p.unreadRune()
- re = p.parseComponent()
- }
- if re != nil {
- path = append(path, re)
- }
-
- c, ok = p.readRune()
- if !ok {
- break
- }
- if c != '/' {
- p.unreadRune()
- break
- }
- }
- return newSequence(path)
-}
-
-// parsePathAlt reads an alternation {p1,p2,...,pn}. Assumes the opening brace
-// has already been read; consumes the closing brace.
-func (p *parser) parsePathAlt() regex {
- s := p.commaIsSpecial
- defer func() { p.commaIsSpecial = s }()
- p.commaIsSpecial = true
-
- var choices []regex
-parseLoop:
- for !p.isErrored {
- if re := p.parsePath(); re != nil {
- choices = append(choices, re)
- }
- c, ok := p.readRune()
- if !ok {
- break parseLoop
- }
- switch c {
- case ',':
- // skip
- case '}':
- return newAlt(choices)
- default:
- break parseLoop
- }
- }
- p.setError()
- return nil
-}
-
-// parseComponent parses a single component of a path. This is a glob
-// expression.
-func (p *parser) parseComponent() regex {
- // p.reader.Seek(0, 1) just returns the current position.
- startPos, _ := p.reader.Seek(0, 1)
- var buf bytes.Buffer
- p.parseComponentPiece(&buf)
- if buf.Len() == 0 {
- return nil
- }
- endPos, _ := p.reader.Seek(0, 1)
-
- // The ... component name is special.
- if endPos-startPos == 3 {
- var literal [3]byte
- p.reader.ReadAt(literal[:], startPos)
- if string(literal[:]) == "..." {
- return dotDotDotRegex
- }
- }
-
- // Everything else is a regular expression.
- re, err := regexp.Compile("^" + buf.String() + "$")
- if err != nil {
- p.setError()
- return nil
- }
- return newSingle(re)
-}
-
-// parseComponentPiece reads a glob expression and converts it to a regular
-// expression.
-func (p *parser) parseComponentPiece(buf *bytes.Buffer) {
- for !p.isErrored {
- c, ok := p.readRune()
- if !ok {
- return
- }
- switch c {
- case '*':
- buf.WriteString(".*")
- case '?':
- buf.WriteString(".")
- case '.', '(', ')':
- buf.WriteRune('\\')
- buf.WriteRune(c)
- case '\\':
- p.parseEscapedRune(buf)
- case '[':
- buf.WriteRune('[')
- p.parseCharRange(buf)
- buf.WriteRune(']')
- case '{':
- buf.WriteRune('(')
- p.parseComponentAlt(buf)
- buf.WriteRune(')')
- case '}', ']', '/':
- p.unreadRune()
- return
- case ',':
- if p.commaIsSpecial {
- p.unreadRune()
- return
- } else {
- buf.WriteRune(c)
- }
- default:
- buf.WriteRune(c)
- }
- }
-}
-
-// parseCharRange copies the input range literally.
-//
-// TODO(jyh): Translate the glob range.
-func (p *parser) parseCharRange(buf *bytes.Buffer) {
- // Initial ] does not close the range.
- c, ok := p.readRune()
- if !ok {
- p.setError()
- return
- }
- buf.WriteRune(c)
- for {
- c, ok := p.readRune()
- if !ok {
- p.setError()
- return
- }
- if c == ']' {
- break
- }
- buf.WriteRune(c)
- }
-}
-
-// parseComponentAlt parses an alternation.
-func (p *parser) parseComponentAlt(buf *bytes.Buffer) {
- s := p.commaIsSpecial
- p.commaIsSpecial = true
- defer func() { p.commaIsSpecial = s }()
- for {
- p.parseComponentPiece(buf)
- c, ok := p.readRune()
- if !ok {
- p.setError()
- return
- }
- switch c {
- case ',':
- buf.WriteRune('|')
- case '}':
- return
- default:
- p.setError()
- return
- }
- }
-}
-
-// parseEscapedRune parses a rune immediately after a backslash.
-func (p *parser) parseEscapedRune(buf *bytes.Buffer) {
- c, ok := p.readRune()
- if !ok {
- return
- }
- // TODO(jyh): Are there any special escape sequences?
- buf.WriteRune('\\')
- buf.WriteRune(c)
-}
-
-// readRune reads the next rune from the input reader. If there is an error,
-// returns false and sets the isErrored flag if the error is not io.EOF.
-func (p *parser) readRune() (rune, bool) {
- if p.isErrored {
- return 0, false
- }
- c, _, err := p.reader.ReadRune()
- switch {
- case err == io.EOF:
- return c, false
- case err != nil:
- p.setError()
- return c, false
- }
- return c, true
-}
-
-// unreadRune pushes back the last rune read.
-func (p *parser) unreadRune() {
- p.reader.UnreadRune()
-}
-
-// setError sets the error flag.
-func (p *parser) setError() {
- p.isErrored = true
-}
diff --git a/services/store/memstore/pathregex/parser_test.go b/services/store/memstore/pathregex/parser_test.go
deleted file mode 100644
index 616f195..0000000
--- a/services/store/memstore/pathregex/parser_test.go
+++ /dev/null
@@ -1,128 +0,0 @@
-package pathregex
-
-import (
- "runtime"
- "strings"
- "testing"
-)
-
-type testCase struct {
- // path expression.
- path string
-
- // expected regular expression afer parsing.
- expected string
-
- // expected reversed regular expression.
- reversed string
-}
-
-type errorCase struct {
- // path expression.
- path string
-
- // expected error position.
- pos int64
-}
-
-var (
- testCases = []testCase{
- testCase{"a", "^a$", "^a$"},
- testCase{"*", "^.*$", "^.*$"},
- testCase{"?", "^.$", "^.$"},
- testCase{".", "^\\.$", "^\\.$"},
- testCase{"[a]", "^[a]$", "^[a]$"},
- testCase{"[]]", "^[]]$", "^[]]$"},
- testCase{"[a-z]", "^[a-z]$", "^[a-z]$"},
- testCase{"[^a-z]", "^[^a-z]$", "^[^a-z]$"},
- testCase{
- "DSC[0-9][0-9][0-9][0-9].jpg",
- "^DSC[0-9][0-9][0-9][0-9]\\.jpg$",
- "^DSC[0-9][0-9][0-9][0-9]\\.jpg$",
- },
- testCase{"abc/def", "^abc$/^def$", "^def$/^abc$"},
- testCase{"abc{foo,bar}def", "^abc(foo|bar)def$", "^abc(foo|bar)def$"},
- testCase{"{abc,def}", "(^abc$|^def$)", "(^abc$|^def$)"},
- testCase{
- "abc/{bbb,ccc,ddd}/def",
- "^abc$/(^bbb$|(^ccc$|^ddd$))/^def$",
- "^def$/(^bbb$|(^ccc$|^ddd$))/^abc$",
- },
- testCase{
- ".../abc/def/...",
- "(<true>)*/^abc$/^def$/(<true>)*",
- "(<true>)*/^def$/^abc$/(<true>)*",
- },
- testCase{
- "abc/{bbb,ccc}/ddd,eee",
- "^abc$/(^bbb$|^ccc$)/^ddd,eee$",
- "^ddd,eee$/(^bbb$|^ccc$)/^abc$",
- },
- testCase{
- "aaa/bbb{ccc,ddd}eee,fff/ggg",
- "^aaa$/^bbb(ccc|ddd)eee,fff$/^ggg$",
- "^ggg$/^bbb(ccc|ddd)eee,fff$/^aaa$",
- },
- testCase{
- "aaa/{bbb/ccc,ddd{eee,fff}ggg,ggg/{hhh,iii}/jjj}/kkk",
- "^aaa$/(^bbb$/^ccc$|(^ddd(eee|fff)ggg$|^ggg$/(^hhh$|^iii$)/^jjj$))/^kkk$",
- "^kkk$/(^ccc$/^bbb$|(^ddd(eee|fff)ggg$|^jjj$/(^hhh$|^iii$)/^ggg$))/^aaa$",
- },
- }
-
- errorCases = []errorCase{
- errorCase{"[", 1},
- errorCase{"[]", 2},
- errorCase{"{", 1},
- errorCase{"{,", 2},
- errorCase{"aaa{bbb/ccc}ddd", 8},
- errorCase{"aaa/bbb/ccc}", 11},
- }
-)
-
-func parsePath(t *testing.T, c *testCase) {
- p := &parser{reader: strings.NewReader(c.path)}
- re := p.parsePath()
- pos, _ := p.reader.Seek(0, 1)
- if p.isErrored || p.reader.Len() != 0 {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): syntax error at char %d: %q", file, line, pos, c.path)
- }
- s := re.String()
- if s != c.expected {
- t.Errorf("Expected %q, got %q", c.expected, s)
- }
-
- re.reverse()
- s = re.String()
- if s != c.reversed {
- t.Errorf("Expected %q, got %q", c.reversed, s)
- }
-}
-
-func parsePathError(t *testing.T, e *errorCase) {
- _, file, line, _ := runtime.Caller(1)
- p := &parser{reader: strings.NewReader(e.path)}
- p.parsePath()
- if p.reader.Len() != 0 {
- p.setError()
- }
- if !p.isErrored {
- t.Errorf("%s(%d): expected error: %q", file, line, e.path)
- }
- if pos, _ := p.reader.Seek(0, 1); pos != e.pos {
- t.Errorf("%s(%d): %q: expected error at pos %d, got %d", file, line, e.path, e.pos, pos)
- }
-}
-
-func TestParser(t *testing.T) {
- for _, c := range testCases {
- parsePath(t, &c)
- }
-}
-
-func TestParserErrors(t *testing.T) {
- for _, e := range errorCases {
- parsePathError(t, &e)
- }
-}
diff --git a/services/store/memstore/pathregex/pathregex_test.go b/services/store/memstore/pathregex/pathregex_test.go
deleted file mode 100644
index c43a0ef..0000000
--- a/services/store/memstore/pathregex/pathregex_test.go
+++ /dev/null
@@ -1,128 +0,0 @@
-package pathregex_test
-
-import (
- "runtime"
- "strings"
- "testing"
-
- "veyron/services/store/memstore/pathregex"
-)
-
-func pathMatch(ss pathregex.StateSet, s string) bool {
- var path []string
- if s != "" {
- path = strings.Split(s, "/")
- }
- for _, s := range path {
- ss = ss.Step(s)
- }
- return ss.IsFinal()
-}
-
-func expectMatch(t *testing.T, ss pathregex.StateSet, s string) {
- if !pathMatch(ss, s) {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): expected match: %q", file, line, s)
- }
-}
-
-func expectNoMatch(t *testing.T, ss pathregex.StateSet, s string) {
- if pathMatch(ss, s) {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): unexpected match: %q", file, line, s)
- }
-}
-
-func TestSingle(t *testing.T) {
- ss, err := pathregex.Compile("a*b*c")
- if err != nil {
- t.Fatalf("Bad regex: %s", err)
- }
- expectNoMatch(t, ss, "")
- expectNoMatch(t, ss, "b")
- expectMatch(t, ss, "aabccc")
- expectNoMatch(t, ss, "aabccc/b")
-}
-
-func TestSequence(t *testing.T) {
- ss, err := pathregex.Compile("abc/def")
- if err != nil {
- t.Fatalf("Bad regex: %s", err)
- }
- expectNoMatch(t, ss, "")
- expectNoMatch(t, ss, "abcdef")
- expectMatch(t, ss, "abc/def")
- expectNoMatch(t, ss, "abc/def/ghi")
-}
-
-func TestAlt(t *testing.T) {
- ss, err := pathregex.Compile("{abc,def}")
- if err != nil {
- t.Fatalf("Bad regex: %s", err)
- }
- expectNoMatch(t, ss, "")
- expectNoMatch(t, ss, "abcdef")
- expectMatch(t, ss, "abc")
- expectMatch(t, ss, "def")
- expectNoMatch(t, ss, "abc/def")
-}
-
-func TestAltPath(t *testing.T) {
- ss, err := pathregex.Compile("{abc/def,def/abc}")
- if err != nil {
- t.Fatalf("Bad regex: %s", err)
- }
- expectNoMatch(t, ss, "")
- expectNoMatch(t, ss, "abcdef")
- expectNoMatch(t, ss, "abc")
- expectNoMatch(t, ss, "def")
- expectMatch(t, ss, "abc/def")
- expectMatch(t, ss, "def/abc")
-}
-
-func TestUnanchored(t *testing.T) {
- ss, err := pathregex.Compile(".../abc/...")
- if err != nil {
- t.Fatalf("Bad regex: %s", err)
- }
- expectNoMatch(t, ss, "")
- expectMatch(t, ss, "abc")
- expectNoMatch(t, ss, "def")
- expectMatch(t, ss, "x/x/abc/x/x")
-}
-
-func TestStar(t *testing.T) {
- ss, err := pathregex.Compile("{,a,*/aa,*/*/aaa,*/*/*/aaaa}")
- if err != nil {
- t.Fatalf("Bad regex: %s", err)
- }
- expectMatch(t, ss, "")
- expectMatch(t, ss, "a")
- expectNoMatch(t, ss, "b/a")
- expectMatch(t, ss, "b/aa")
- expectMatch(t, ss, "c/b/aaa")
- expectMatch(t, ss, "d/c/b/aaaa")
- expectNoMatch(t, ss, "d/c/b/aaa")
-}
-
-func TestSequenceReverse(t *testing.T) {
- ss, err := pathregex.CompileReverse("abc/def")
- if err != nil {
- t.Fatalf("Bad regex: %s", err)
- }
- expectNoMatch(t, ss, "")
- expectNoMatch(t, ss, "abcdef")
- expectMatch(t, ss, "def/abc")
- expectNoMatch(t, ss, "def/abc/ghi")
-}
-
-func TestAltReverse(t *testing.T) {
- ss, err := pathregex.CompileReverse(".../123/{abc/def,g/*/i}/456/...")
- if err != nil {
- t.Fatalf("Bad regex: %s", err)
- }
- expectNoMatch(t, ss, "")
- expectNoMatch(t, ss, "123/abc/def/456")
- expectMatch(t, ss, "456/def/abc/123")
- expectMatch(t, ss, "x/y/456/i/am/g/123/z/w")
-}
diff --git a/services/store/memstore/pathregex/regex.go b/services/store/memstore/pathregex/regex.go
deleted file mode 100644
index 3a50219..0000000
--- a/services/store/memstore/pathregex/regex.go
+++ /dev/null
@@ -1,148 +0,0 @@
-package pathregex
-
-import (
- "regexp"
-)
-
-// regex is the type of regular expressions.
-type regex interface {
- // compile constructs an NFA. The argument is the final state. Returns the
- // initial state.
- compile(final *state) *state
-
- // reverse reverses the regular expression in place.
- reverse()
-
- // String returns a string representation of the regular expression.
- String() string
-}
-
-// regexCommon is a common base type of regular expressions.
-type regexCommon struct{}
-
-// tt accepts everything.
-type tt struct {
- regexCommon
-}
-
-// ff rejects everything.
-type ff struct {
- regexCommon
-}
-
-// epsilon matches the empty path.
-type epsilon struct {
- regexCommon
-}
-
-// single matches a single component of a path, using a regexp.Regexp for the
-// match.
-type single struct {
- regexCommon
- r *regexp.Regexp
-}
-
-// sequence matches a sequence of components.
-type sequence struct {
- regexCommon
- r1, r2 regex
-}
-
-// alt matches one of two regular expressions.
-type alt struct {
- regexCommon
- r1, r2 regex
-}
-
-// star is the Kleene closure.
-type star struct {
- regexCommon
- re regex
-}
-
-var (
- trueRegex = &tt{}
- falseRegex = &ff{}
- epsilonRegex = &epsilon{}
- dotDotDotRegex = &star{re: trueRegex}
-)
-
-// newSingle returns a Regex that matches one component of a path.
-func newSingle(r *regexp.Regexp) regex {
- return &single{r: r}
-}
-
-// newSequence returns a Regex that matches a sequence of path components.
-func newSequence(rl []regex) regex {
- if len(rl) == 0 {
- return epsilonRegex
- }
- r := rl[len(rl)-1]
- for i := len(rl) - 2; i >= 0; i-- {
- r = &sequence{r1: rl[i], r2: r}
- }
- return r
-}
-
-// newAlt returns a Regex that matches one of a set of regular expressions.
-func newAlt(rl []regex) regex {
- if len(rl) == 0 {
- return falseRegex
- }
- r := rl[len(rl)-1]
- for i := len(rl) - 2; i >= 0; i-- {
- r = &alt{r1: rl[i], r2: r}
- }
- return r
-}
-
-// newStar returns the Kleene closure of a regular expression.
-func newStar(re regex) regex {
- return &star{re: re}
-}
-
-func (re *tt) String() string {
- return "<true>"
-}
-
-func (re *ff) String() string {
- return "<false>"
-}
-
-func (re *epsilon) String() string {
- return "<epsilon>"
-}
-
-func (re *single) String() string {
- return re.r.String()
-}
-
-func (re *sequence) String() string {
- return re.r1.String() + "/" + re.r2.String()
-}
-
-func (re *alt) String() string {
- return "(" + re.r1.String() + "|" + re.r2.String() + ")"
-}
-
-func (re *star) String() string {
- return "(" + re.re.String() + ")*"
-}
-
-func (re *regexCommon) reverse() {
-}
-
-func (re *sequence) reverse() {
- re.r1, re.r2 = re.r2, re.r1
- re.r1.reverse()
- re.r2.reverse()
-}
-
-func (re *alt) reverse() {
- re.r1.reverse()
- re.r2.reverse()
-}
-
-func (re *star) reverse() {
- re.re.reverse()
-}
diff --git a/services/store/memstore/pathregex/regex_test.go b/services/store/memstore/pathregex/regex_test.go
deleted file mode 100644
index d96e5bf..0000000
--- a/services/store/memstore/pathregex/regex_test.go
+++ /dev/null
@@ -1,92 +0,0 @@
-package pathregex
-
-import (
- "regexp"
- "runtime"
- "strings"
- "testing"
-)
-
-func pathMatch(re regex, s string) bool {
- var path []string
- if s != "" {
- path = strings.Split(s, "/")
- }
- ss := compileNFA(re)
- for _, s := range path {
- ss = ss.Step(s)
- }
- return ss.IsFinal()
-}
-
-func expectMatch(t *testing.T, re regex, s string) {
- if !pathMatch(re, s) {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): expected match: %s, %q", file, line, re, s)
- }
-}
-
-func expectNoMatch(t *testing.T, re regex, s string) {
- if pathMatch(re, s) {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): unexpected match: %s, %q", file, line, re, s)
- }
-}
-
-func TestSingle(t *testing.T) {
- p, err := regexp.Compile("a*bc*")
- if err != nil {
- t.Fatalf("Bad regular expression: %s", err)
- }
- re := newSingle(p)
- expectNoMatch(t, re, "")
- expectMatch(t, re, "b")
- expectMatch(t, re, "aabccc")
- expectNoMatch(t, re, "aabccc/b")
-}
-
-func TestSequence(t *testing.T) {
- re := newSequence([]regex{
- newSingle(regexp.MustCompile("abc")),
- newSingle(regexp.MustCompile("def")),
- })
- expectNoMatch(t, re, "")
- expectNoMatch(t, re, "abcdef")
- expectMatch(t, re, "abc/def")
- expectNoMatch(t, re, "abc/def/ghi")
-}
-
-func TestAlt(t *testing.T) {
- re := newAlt([]regex{
- newSingle(regexp.MustCompile("abc")),
- newSingle(regexp.MustCompile("def")),
- })
- expectNoMatch(t, re, "")
- expectMatch(t, re, "abc")
- expectMatch(t, re, "def")
- expectNoMatch(t, re, "x/abc")
-}
-
-func TestStar(t *testing.T) {
- re := newStar(newSingle(regexp.MustCompile("abc")))
- expectMatch(t, re, "")
- expectMatch(t, re, "abc")
- expectMatch(t, re, "abc/abc")
- expectNoMatch(t, re, "abc/abc/a/abc")
-}
-
-func TestComplex(t *testing.T) {
- // Case-insensitive match on (j/a/s/o/n)*.
- var rl [5]regex
- for i, s := range []string{"j", "a", "s", "o", "n"} {
- rl[i] = newAlt([]regex{
- newSingle(regexp.MustCompile(s)),
- newSingle(regexp.MustCompile(strings.ToUpper(s))),
- })
- }
- re := newStar(newSequence(rl[:]))
- expectMatch(t, re, "")
- expectMatch(t, re, "j/A/s/O/n")
- expectNoMatch(t, re, "j/A/O/s/n")
- expectMatch(t, re, "j/A/s/O/n/j/a/S/o/N")
-}
diff --git a/services/store/memstore/query/eval.go b/services/store/memstore/query/eval.go
deleted file mode 100644
index e059390..0000000
--- a/services/store/memstore/query/eval.go
+++ /dev/null
@@ -1,1223 +0,0 @@
-package query
-
-import (
- "crypto/rand"
- "fmt"
- "math/big"
- "reflect"
- "runtime"
- "sort"
- "strings"
- "sync"
-
- vsync "veyron/runtimes/google/lib/sync"
- "veyron/services/store/memstore/state"
-
- "veyron2/naming"
- "veyron2/query"
- "veyron2/query/parse"
- "veyron2/security"
- "veyron2/services/store"
- "veyron2/storage"
- "veyron2/vdl/vdlutil"
- "veyron2/vlog"
-)
-
-// maxChannelSize is the maximum size of the channels used for concurrent
-// query evaluation.
-const maxChannelSize = 100
-
-// QueryStream yields the results of a query. Usage:
-// qs, _ := obj.Query
-// for qs.Next() {
-// result := qs.Get()
-// ...
-// if enough_results {
-// qs.Abort()
-// }
-// }
-// if err := qs.Err(); err != nil {
-// ...
-// }
-// Iterator is thread-safe.
-type QueryStream interface {
- // Next advances the iterator. It must be called before calling Get.
- // Returns true if there is a value to be retrieved with Get. Returns
- // false when iteration is finished.
- Next() bool
-
- // Get retrieves a query result. It is idempotent.
- Get() *store.QueryResult
-
- // Err returns the first error encountered during query evaluation. It is
- // idempotent.
- Err() error
-
- // Abort stops query evaluation early. The client must call Abort unless
- // iteration goes to completion (i.e. Next returns false). It is
- // idempotent and can be called from any thread.
- Abort()
-}
-
-// evalIterator implements QueryStream.
-type evalIterator struct {
- // mu guards 'result', 'err', and the closing of 'abort'.
- mu sync.Mutex
- // result is what Get will return. It will be nil if there are no more
- // query results. Guarded by mu.
- result *store.QueryResult
- // err is the first error encountered during query evaluation.
- // Guarded by mu.
- err error
- // abort is used as the signal to query evaluation to terminate early.
- // evaluator implementations will test for abort closing. The close()
- // call is guarded by mu.
- abort chan struct{}
-
- // results represents a stack of result channels. evalIterator is
- // essentially an iterator of iterators. Each subquery adds a nestedChannel
- // to this stack. The top of the stack is the end of the slice.
- results []nestedChannel
- // cleanup is used for testing to ensure that no goroutines are leaked.
- cleanup vsync.WaitGroup
- // maxNesting is the largest value used for nestedChannel.nesting. It
- // is the maximum nesting over the duration of the query while len(results)
- // is just the instantaneous nesting.
- maxNesting int
-}
-
-// nestedChannel contains the results of a subquery.
-type nestedChannel struct {
- // nesting is the value to use for the NestedResult field of all
- // objects that come out of the results channel.
- nesting int
- // results is the stream of results for this subquery.
- results <-chan *store.QueryResult
-}
-
-// hiddenResult wraps a value so evalIterator can elide it from
-// storage.QueryResult.Fields that are sent to the client.
-type hiddenResult struct {
- value vdlutil.Any
-}
-
-// Next implements the QueryStream method.
-func (it *evalIterator) Next() bool {
- it.mu.Lock()
- if it.err != nil {
- it.mu.Unlock()
- return false
- }
- it.mu.Unlock()
-
- depth := len(it.results) - 1
- select {
- case result, ok := <-it.results[depth].results:
- if !ok {
- it.results = it.results[:depth]
- if len(it.results) > 0 {
- return it.Next()
- }
- return false
- }
-
- // Set the correct value for NestedResult.
- result.NestedResult = store.NestedResult(it.results[depth].nesting)
-
- it.enqueueNestedChannels(result)
-
- it.mu.Lock()
- defer it.mu.Unlock()
- it.result = result
- return true
- case <-it.abort:
- return false
- }
-}
-
-// enqueueNestedChannels looks through result.Fields for nested result
-// channels. If there are any, they are appended to it.results. We use
-// the result.Fields key to append them in reverse alphabetical order.
-// We use reverse alphabetical order because it.results is a stack--
-// we want to process them in alphabetical order.
-//
-// enqueueNestedChannels also removes any result.Fields that are of the
-// type hiddenResult.
-func (it *evalIterator) enqueueNestedChannels(result *store.QueryResult) {
- if result.Fields == nil {
- return
- }
- var nestingKeys []string
- for key, val := range result.Fields {
- switch val.(type) {
- case chan *store.QueryResult:
- nestingKeys = append(nestingKeys, key)
- case hiddenResult:
- // If a field is "hidden", the value will be wrapped in the type
- // hiddenResult to make it possible for evalIterator to elide it
- // from the results sent to the client.
- delete(result.Fields, key)
- }
- }
- // Figure out the store.NestedResult values based on alphabetical order of
- // the keys.
- sort.Sort(sort.StringSlice(nestingKeys))
- var nested []nestedChannel
- for _, key := range nestingKeys {
- it.maxNesting++
- nested = append(nested,
- nestedChannel{it.maxNesting, result.Fields[key].(chan *store.QueryResult)})
- result.Fields[key] = store.NestedResult(it.maxNesting)
- }
- // Add the nested result channels in reverse alphabetical order.
- for i := len(nested) - 1; i >= 0; i-- {
- it.results = append(it.results, nested[i])
- }
-}
-
-// Get implements the QueryStream method.
-func (it *evalIterator) Get() *store.QueryResult {
- it.mu.Lock()
- defer it.mu.Unlock()
- return it.result
-}
-
-// Abort implements the QueryStream method.
-func (it *evalIterator) Abort() {
- it.mu.Lock()
- defer it.mu.Unlock()
- select {
- case <-it.abort:
- // Already closed.
- default:
- close(it.abort)
- }
- it.result = nil
-}
-
-// Err implements the QueryStream method.
-func (it *evalIterator) Err() error {
- it.mu.Lock()
- defer it.mu.Unlock()
- return it.err
-}
-
-func (it *evalIterator) setErrorf(format string, args ...interface{}) {
- it.setError(fmt.Errorf(format, args...))
-}
-
-func (it *evalIterator) setError(err error) {
- it.mu.Lock()
- if it.err == nil {
- it.err = err
- }
- it.mu.Unlock()
- it.Abort()
-}
-
-// wait blocks until all children goroutines are finished. This is useful in
-// tests to ensure that an abort cleans up correctly.
-func (it *evalIterator) wait() {
- it.cleanup.Wait()
-}
-
-func newErrorIterator(err error) QueryStream {
- return &evalIterator{
- err: err,
- abort: make(chan struct{}),
- }
-}
-
-// Eval evaluates a query and returns a QueryStream for the results. If there is
-// an error parsing the query, it will show up as an error in the QueryStream.
-// Query evaluation is concurrent, so it is important to call QueryStream.Abort
-// if the client does not consume all of the results.
-func Eval(sn state.Snapshot, clientID security.PublicID, name storage.PathName, q query.Query) QueryStream {
- ast, err := parse.Parse(q)
- if err != nil {
- return newErrorIterator(err)
- }
- evaluator, err := convert(ast)
- if err != nil {
- return newErrorIterator(err)
- }
-
- // Seed the input with the root entity.
- in := make(chan *store.QueryResult, 1)
- in <- &store.QueryResult{Name: ""}
- close(in)
-
- out := make(chan *store.QueryResult, maxChannelSize)
- it := &evalIterator{
- results: []nestedChannel{nestedChannel{0, out}},
- abort: make(chan struct{}),
- }
- if !it.cleanup.TryAdd() {
- // The query has been aborted by a call to Cancel.
- return it
- }
- go evaluator.eval(&context{
- sn: sn,
- suffix: name.String(),
- clientID: clientID,
- in: in,
- out: out,
- evalIt: it,
- abort: it.abort,
- cleanup: &it.cleanup,
- })
-
- return it
-}
-
-// context is a wrapper of all the variables that need to be passed around
-// during evaluation.
-type context struct {
- // sn is the snapshot of the store's state to use to find query results.
- sn state.Snapshot
- // suffix is the suffix we're evaluating relative to.
- suffix string
- // clientID is the identity of the client that issued the query.
- clientID security.PublicID
- // in produces the intermediate results from the previous stage of the
- // query. It will be closed when the evaluator should stop processing
- // results. It is not necessary to select on 'in' and 'errc'.
- in <-chan *store.QueryResult
- // out is where the evaluator should write the intermediate results.
- // evaluators should use context.emit instead of writing directly
- // to out.
- out chan<- *store.QueryResult
- // evalIt is the iterator that interfaces with the client. It is here
- // to allow the evaluation code to propagate errors via setError().
- evalIt *evalIterator
- // abort will be closed if query evaluation should terminate early.
- // evaluator implementations should regularly test if it is still open.
- abort chan struct{}
- // cleanup is used for testing to ensure that no goroutines are leaked.
- // evaluator.eval implementations should call Done when finished processing.
- cleanup *vsync.WaitGroup
-}
-
-// emit sends result on c.out. It is careful to watch for aborts. result can be
-// nil. Returns true if the caller should continue iterating, returns
-// false if it is time to abort.
-func (c *context) emit(result *store.QueryResult) bool {
- vlog.VI(2).Info("emit", result)
- if result == nil {
- // Check for an abort before continuing iteration.
- select {
- case <-c.abort:
- return false
- default:
- return true
- }
- } else {
- // If c.out is full, we don't want to block on it forever and ignore
- // aborts.
- select {
- case <-c.abort:
- return false
- case c.out <- result:
- return true
- }
- }
-}
-
-// evaluator is a node in the query evaluation flow. It takes intermediate
-// results produced by the previous node and produces a new set of results.
-type evaluator interface {
- // eval does the work or processing intermediate results to produce a new
- // set of results. It is expected that the client run eval in its own
- // goroutine (i.e. "go eval(ctxt)").
- eval(c *context)
-
- // singleResult returns true if this evaluator returns a single result
- // (e.g. an aggregate or a specific field). This is useful in selection
- // because we want to unbox those results. For example,
- // "teams/* | { players/* | count as numplayers }" should return
- // { Name: "teams/cardinals", Fields: {"numplayers": 5}}
- // and not
- // { Name: "teams/cardinals", Fields: {"numplayers": [{Name: "numplayers", Value: 5}]}}
- singleResult() bool
-
- // name returns a relative Object name that is appropriate for the query
- // results produced by this evaluator.
- name() string
-}
-
-// convert transforms the AST produced by parse.Parse into an AST that supports
-// evaluation specific to memstore. This transformation should not produce
-// any errors since we know all of the types that parse.Parse can produce.
-// Just in case one was overlooked, we use the panic/recover idiom to handle
-// unexpected errors. The conversion functions in the remainder of this file
-// do not return errors. Instead, they are allowed to panic, and this function
-// will recover.
-func convert(q parse.Pipeline) (ev evaluator, err error) {
- defer func() {
- if r := recover(); r != nil {
- if _, ok := r.(runtime.Error); ok {
- panic(r)
- }
- ev = nil
- err = r.(error)
- }
- }()
- return convertPipeline(q), nil
-}
-
-// convertPipeline transforms a parse.Pipeline into an evaluator.
-func convertPipeline(q parse.Pipeline) evaluator {
- switch q := q.(type) {
- case *parse.PipelineName:
- return &nameEvaluator{q.WildcardName, q.Pos}
- case *parse.PipelineType:
- return &typeEvaluator{convertPipeline(q.Src), q.Type, q.Pos}
- case *parse.PipelineFilter:
- return &filterEvaluator{convertPipeline(q.Src), convertPredicate(q.Pred), q.Pos}
- case *parse.PipelineSelection:
- return convertSelection(q)
- case *parse.PipelineFunc:
- return convertPipelineFunc(q)
- default:
- panic(fmt.Errorf("unexpected type %T", q))
- }
-}
-
-// nameEvaluator is the evaluator version of parse.PipelineName.
-type nameEvaluator struct {
- wildcardName *parse.WildcardName
-
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// eval implements the evaluator method.
-func (e *nameEvaluator) eval(c *context) {
- defer c.cleanup.Done()
- defer close(c.out)
-
- for result := range c.in {
- basepath := naming.Join(result.Name, e.wildcardName.VName)
- path := storage.ParsePath(naming.Join(c.suffix, basepath))
- vlog.VI(2).Infof("nameEvaluator suffix: %s, result.Name: %s, VName: %s",
- c.suffix, result.Name, e.wildcardName.VName)
- for it := c.sn.NewIterator(c.clientID, path,
- state.ListObjects, state.ImmediateFilter); it.IsValid(); it.Next() {
-
- entry := it.Get()
- result := &store.QueryResult{
- Name: naming.Join(basepath, it.Name()),
- Value: entry.Value,
- }
- if !c.emit(result) {
- return
- }
- if e.singleResult() {
- return
- }
- }
- }
-}
-
-// singleResult implements the evaluator method.
-func (e *nameEvaluator) singleResult() bool {
- return e.wildcardName.Exp == parse.Self
-}
-
-// name implements the evaluator method.
-func (e *nameEvaluator) name() string {
- return e.wildcardName.VName
-}
-
-// startSource creates a goroutine for src.eval(). It returns the
-// output channel for src.
-func startSource(c *context, src evaluator) chan *store.QueryResult {
- srcOut := make(chan *store.QueryResult, maxChannelSize)
- srcContext := context{
- sn: c.sn,
- suffix: c.suffix,
- clientID: c.clientID,
- in: c.in,
- out: srcOut,
- evalIt: c.evalIt,
- abort: c.abort,
- cleanup: c.cleanup,
- }
- if !c.cleanup.TryAdd() {
- // The query has been aborted by a call to Cancel.
- close(srcOut)
- return srcOut
- }
- go src.eval(&srcContext)
- return srcOut
-}
-
-// typeEvaluator is the evaluator version of parse.PipelineType.
-type typeEvaluator struct {
- // src produces the results to be filtered by type.
- src evaluator
- // ty restricts the results to a specific type of object.
- ty string
- // Pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// eval implements the evaluator method.
-func (e *typeEvaluator) eval(c *context) {
- defer c.cleanup.Done()
- defer close(c.out)
-
- for result := range startSource(c, e.src) {
- vlog.VI(2).Info("typeEvaluator", result)
- if val := reflect.ValueOf(result.Value); e.ty != val.Type().Name() {
- continue
- }
- if !c.emit(result) {
- return
- }
- }
-}
-
-// singleResult implements the evaluator method.
-func (e *typeEvaluator) singleResult() bool {
- return false
-}
-
-// name implements the evaluator method.
-func (e *typeEvaluator) name() string {
- return e.src.name()
-}
-
-// filterEvaluator is the evaluator version of parse.PipelineFilter.
-type filterEvaluator struct {
- // src produces intermediate results that will be filtered by pred.
- src evaluator
- // pred determines whether an intermediate result produced by src should be
- // filtered out.
- pred predicate
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// eval implements the evaluator method.
-func (e *filterEvaluator) eval(c *context) {
- defer c.cleanup.Done()
- defer close(c.out)
-
- for result := range startSource(c, e.src) {
- vlog.VI(2).Info("filterEvaluator", result)
- if e.pred.match(c, result) {
- if !c.emit(result) {
- return
- }
- }
- }
-}
-
-// singleResult implements the evaluator method.
-func (e *filterEvaluator) singleResult() bool {
- return false
-}
-
-// name implements the evaluator method.
-func (e *filterEvaluator) name() string {
- return e.src.name()
-}
-
-// convertSelection transforms a parse.PipelineSelection into a
-// selectionEvaluator.
-func convertSelection(p *parse.PipelineSelection) evaluator {
- e := &selectionEvaluator{
- src: convertPipeline(p.Src),
- subpipelines: make([]alias, len(p.SubPipelines), len(p.SubPipelines)),
- pos: p.Pos,
- }
- for i, a := range p.SubPipelines {
- e.subpipelines[i] = alias{convertPipeline(a.Pipeline), a.Alias, a.Hidden}
- }
- return e
-}
-
-// alias is the evaluator version of parse.Alias. It represents a pipeline
-// that has an alternate name inside of a selection using the 'as' keyword.
-type alias struct {
- // evaluator is the evaluator to be aliased.
- evaluator evaluator
- // alias is the new name for the output of evaluator.
- alias string
- // hidden is true if this field in the selection should not be included
- // in the results sent to the client.
- hidden bool
-}
-
-// selectionEvaluator is the evaluator version of parse.PipelineSelection.
-type selectionEvaluator struct {
- // src produces intermediate results on which to select.
- src evaluator
- // subpipelines is the list of pipelines to run for each result produced
- // by src.
- subpipelines []alias
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// eval implements the evaluator method.
-func (e *selectionEvaluator) eval(c *context) {
- defer c.cleanup.Done()
- defer close(c.out)
-
- for result := range startSource(c, e.src) {
- if !e.processSubpipelines(c, result) {
- return
- }
- }
-}
-
-func (e *selectionEvaluator) processSubpipelines(c *context, result *store.QueryResult) bool {
- vlog.VI(2).Info("selection: ", result.Name)
- sel := &store.QueryResult{
- Name: result.Name,
- Fields: make(map[string]vdlutil.Any),
- }
- for _, a := range e.subpipelines {
- // We create a new channel for each intermediate result, so there's no need to
- // create a large buffer.
- in := make(chan *store.QueryResult, 1)
- in <- result
- close(in)
- out := make(chan *store.QueryResult, maxChannelSize)
- ctxt := &context{
- sn: c.sn,
- suffix: c.suffix,
- clientID: c.clientID,
- in: in,
- out: out,
- evalIt: c.evalIt,
- abort: c.abort,
- cleanup: c.cleanup,
- }
- if !c.cleanup.TryAdd() {
- // The query has been aborted by a call to Cancel.
- return false
- }
- go a.evaluator.eval(ctxt)
-
- // If the subpipeline would produce a single result, use that single result
- // as the field value. Otherwise, put the channel as the field value and let
- // evalIterator do the right thing with the sub-results.
- var value interface{}
- if a.evaluator.singleResult() {
- select {
- case <-c.abort:
- return false
- case sub, ok := <-out:
- if ok {
- value = sub.Value
- }
- }
- } else {
- value = out
- }
-
- if a.alias != "" {
- if a.hidden {
- // If a field is "hidden", the value will be wrapped in the type
- // hiddenResult to make it possible for evalIterator to elide it
- // from the results sent to the client.
- value = hiddenResult{value}
- }
- sel.Fields[a.alias] = value
- } else {
- sel.Fields[a.evaluator.name()] = value
- }
- }
- return c.emit(sel)
-}
-
-// singleResult implements the evaluator method.
-func (e *selectionEvaluator) singleResult() bool {
- return false
-}
-
-// name implements the evaluator method.
-func (e *selectionEvaluator) name() string {
- return e.src.name()
-}
-
-// convertPipelineFunc transforms a parse.PipelineFunc into a funcEvaluator.
-func convertPipelineFunc(p *parse.PipelineFunc) evaluator {
- args := make([]expr, len(p.Args), len(p.Args))
- for i, a := range p.Args {
- args[i] = convertExpr(a)
- }
- src := convertPipeline(p.Src)
- switch p.FuncName {
- case "sort":
- if src.singleResult() {
- panic(fmt.Errorf("%v: sort expects multiple inputs not a single input", p.Pos))
- }
- return &funcSortEvaluator{
- src: convertPipeline(p.Src),
- args: args,
- pos: p.Pos,
- }
- case "sample":
- return convertSampleFunc(src, args, p.Pos)
- default:
- panic(fmt.Errorf("unknown function %s at Pos %v", p.FuncName, p.Pos))
- }
-}
-
-type funcSortEvaluator struct {
- // src produces intermediate results that will be sorted.
- src evaluator
- // args is the list of arguments passed to sort().
- args []expr
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-func (e *funcSortEvaluator) eval(c *context) {
- defer c.cleanup.Done()
- defer close(c.out)
- srcOut := startSource(c, e.src)
-
- sorter := argsSorter{e, c, nil}
- for result := range srcOut {
- sorter.results = append(sorter.results, result)
- }
- sort.Sort(sorter)
- for _, result := range sorter.results {
- if !c.emit(result) {
- return
- }
- }
-}
-
-// singleResult implements the evaluator method.
-func (e *funcSortEvaluator) singleResult() bool {
- // During construction, we tested that e.src is not singleResult.
- return false
-}
-
-// name implements the evaluator method.
-func (e *funcSortEvaluator) name() string {
- // A sorted resultset is still the same as the original resultset, so it
- // should have the same name.
- return e.src.name()
-}
-
-// argsSorter implements sort.Interface to sort results by e.args.
-type argsSorter struct {
- e *funcSortEvaluator
- c *context
- results []*store.QueryResult
-}
-
-func (a argsSorter) Len() int { return len(a.results) }
-func (a argsSorter) Swap(i, j int) { a.results[i], a.results[j] = a.results[j], a.results[i] }
-func (a argsSorter) Less(i, j int) bool {
- for n, arg := range a.e.args {
- // Normally, exprUnary only supports numeric operands. As part of a sort
- // expression however, it is possible to negate a string operand to
- // cause a descending sort.
- ascending := true
- unaryArg, ok := arg.(*exprUnary)
- if ok {
- // Remove the +/- operator.
- arg = unaryArg.operand
- ascending = unaryArg.op == parse.OpPos
- }
- ival := arg.value(a.c, a.results[i])
- jval := arg.value(a.c, a.results[j])
- res, err := compare(a.c, ival, jval)
- if err != nil {
- a.c.evalIt.setErrorf("error while sorting (Pos %v Arg: %d) left: %s, right: %s; %v",
- a.e.pos, n, a.results[i].Name, a.results[j].Name, err)
- return false
- }
- if res == 0 {
- continue
- }
- if ascending {
- return res < 0
- } else {
- return res > 0
- }
- }
- // Break ties based on name to get a deterministic order.
- return a.results[i].Name < a.results[j].Name
-}
-
-// funcSampleEvaluator is an evaluator that uses reservior sampling to
-// filter results to a desired number.
-type funcSampleEvaluator struct {
- // src produces intermediate results that will be transformed by func.
- src evaluator
- // numSamples is the number of samples to send to the output.
- numSamples int64
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-func convertSampleFunc(src evaluator, args []expr, pos parse.Pos) evaluator {
- if src.singleResult() {
- panic(fmt.Errorf("%v: sample expects multiple inputs not a single input", pos))
- }
- if len(args) != 1 {
- panic(fmt.Errorf("%v: sample expects exactly one integer argument specifying the number of results to include in the sample", pos))
- }
- n, ok := args[0].(*exprInt)
- if !ok {
- panic(fmt.Errorf("%v: sample expects exactly one integer argument specifying the number of results to include in the sample", pos))
- }
- return &funcSampleEvaluator{src, n.i.Int64(), pos}
-}
-
-// eval implements the evaluator method.
-func (e *funcSampleEvaluator) eval(c *context) {
- defer c.cleanup.Done()
- defer close(c.out)
- srcOut := startSource(c, e.src)
-
- reservoir := make([]*store.QueryResult, e.numSamples)
- i := int64(0)
- for result := range srcOut {
- if i < e.numSamples {
- // Fill the reservoir.
- reservoir[i] = result
- } else {
- // Sample with decreasing probability.
- bigJ, err := rand.Int(rand.Reader, big.NewInt(i+1))
- if err != nil {
- c.evalIt.setErrorf("error while sampling: %v", err)
- return
- }
- j := bigJ.Int64()
- if j < e.numSamples {
- reservoir[j] = result
- }
- }
- i++
- }
- for _, result := range reservoir {
- if !c.emit(result) {
- return
- }
- }
-}
-
-// singleResult implements the evaluator method.
-func (e *funcSampleEvaluator) singleResult() bool {
- // During construction, we tested that e.src is not singleResult.
- return false
-}
-
-// name implements the evaluator method.
-func (e *funcSampleEvaluator) name() string {
- // A sampled resultset is still the same as the original resultset, so it
- // should have the same name.
- return e.src.name()
-}
-
-// predicate determines whether an intermediate query result should be
-// filtered out.
-type predicate interface {
- match(c *context, e *store.QueryResult) bool
-}
-
-// convertPredicate transforms a parse.Predicate into a predicate.
-func convertPredicate(p parse.Predicate) predicate {
- switch p := p.(type) {
- case *parse.PredicateBool:
- return &predicateBool{p.Bool, p.Pos}
- case *parse.PredicateCompare:
- switch p.Comp {
- case parse.CompEQ, parse.CompNE, parse.CompLT, parse.CompGT, parse.CompLE, parse.CompGE:
- default:
- panic(fmt.Errorf("unknown comparator %d at %v", p.Comp, p.Pos))
- }
- return &predicateCompare{convertExpr(p.LHS), convertExpr(p.RHS), p.Comp, p.Pos}
- case *parse.PredicateAnd:
- return &predicateAnd{convertPredicate(p.LHS), convertPredicate(p.RHS), p.Pos}
- case *parse.PredicateOr:
- return &predicateOr{convertPredicate(p.LHS), convertPredicate(p.RHS), p.Pos}
- case *parse.PredicateNot:
- return &predicateNot{convertPredicate(p.Pred), p.Pos}
- // TODO(kash): Support parse.PredicateFunc.
- default:
- panic(fmt.Errorf("unexpected type %T", p))
- }
-}
-
-// predicateBool represents a boolean literal.
-type predicateBool struct {
- b bool
- pos parse.Pos
-}
-
-// match implements the predicate method.
-func (p *predicateBool) match(c *context, e *store.QueryResult) bool {
- return p.b
-}
-
-// predicateCompare handles the comparison on two expressions.
-type predicateCompare struct {
- // lhs is the left-hand-side of the comparison.
- lhs expr
- // rhs is the right-hand-side of the comparison.
- rhs expr
- // comp specifies the operator to use in the comparison.
- comp parse.Comparator
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// match implements the predicate method.
-func (p *predicateCompare) match(c *context, result *store.QueryResult) bool {
- lval := p.lhs.value(c, result)
- rval := p.rhs.value(c, result)
-
- res, err := compare(c, lval, rval)
- if err != nil {
- c.evalIt.setErrorf("error while evaluating predicate (Pos %v) for name '%s': %v",
- p.pos, result.Name, err)
- return false
- }
- switch p.comp {
- case parse.CompEQ:
- return res == 0
- case parse.CompNE:
- return res != 0
- case parse.CompLT:
- return res < 0
- case parse.CompGT:
- return res > 0
- case parse.CompLE:
- return res <= 0
- case parse.CompGE:
- return res >= 0
- default:
- c.evalIt.setErrorf("unknown comparator %d at Pos %v", p.comp, p.pos)
- return false
- }
-}
-
-// compare returns a negative number if lval is less than rval, 0 if they are
-// equal, and a positive number if lval is greater than rval.
-func compare(c *context, lval, rval interface{}) (int, error) {
- switch lval := lval.(type) {
- case string:
- rval, ok := rval.(string)
- if !ok {
- return 0, fmt.Errorf("type mismatch; left: %T, right: %T", lval, rval)
- }
- if lval < rval {
- return -1, nil
- } else if lval > rval {
- return 1, nil
- } else {
- return 0, nil
- }
- case *big.Rat:
- switch rval := rval.(type) {
- case *big.Rat:
- return lval.Cmp(rval), nil
- case *big.Int:
- return lval.Cmp(new(big.Rat).SetInt(rval)), nil
- case int, int8, int16, int32, int64:
- return lval.Cmp(new(big.Rat).SetInt64(toInt64(rval))), nil
- case uint, uint8, uint16, uint32, uint64:
- // It is not possible to convert to a big.Rat from an unsigned. Need to
- // go through big.Int first.
- return lval.Cmp(new(big.Rat).SetInt(new(big.Int).SetUint64(toUint64(rval)))), nil
- }
- case *big.Int:
- switch rval := rval.(type) {
- case *big.Rat:
- return new(big.Rat).SetInt(lval).Cmp(rval), nil
- case *big.Int:
- return lval.Cmp(rval), nil
- case int, int8, int16, int32, int64:
- return lval.Cmp(big.NewInt(toInt64(rval))), nil
- case uint, uint8, uint16, uint32, uint64:
- return lval.Cmp(new(big.Int).SetUint64(toUint64(rval))), nil
- }
- case int, int8, int16, int32, int64:
- switch rval := rval.(type) {
- case *big.Rat:
- return new(big.Rat).SetInt64(toInt64(lval)).Cmp(rval), nil
- case *big.Int:
- return new(big.Int).SetInt64(toInt64(lval)).Cmp(rval), nil
- case int, int8, int16, int32, int64:
- lint, rint := toInt64(lval), toInt64(rval)
- if lint < rint {
- return -1, nil
- } else if lint > rint {
- return 1, nil
- } else {
- return 0, nil
- }
- case uint, uint8, uint16, uint32, uint64:
- lint, rint := toUint64(lval), toUint64(rval)
- if lint < rint {
- return -1, nil
- } else if lint > rint {
- return 1, nil
- } else {
- return 0, nil
- }
- }
- }
- return 0, fmt.Errorf("unexpected type %T", lval)
-}
-
-func toInt64(i interface{}) int64 {
- switch i := i.(type) {
- case int:
- return int64(i)
- case int8:
- return int64(i)
- case int16:
- return int64(i)
- case int32:
- return int64(i)
- case int64:
- return int64(i)
- default:
- panic(fmt.Errorf("unexpected type %T", i))
- }
-}
-
-func toUint64(i interface{}) uint64 {
- switch i := i.(type) {
- case uint:
- return uint64(i)
- case uint8:
- return uint64(i)
- case uint16:
- return uint64(i)
- case uint32:
- return uint64(i)
- case uint64:
- return uint64(i)
- default:
- panic(fmt.Errorf("unexpected type %T", i))
- }
-}
-
-// predicateAnd is a predicate that is the logical conjunction of two
-// predicates.
-type predicateAnd struct {
- // lhs is the left-hand-side of the conjunction.
- lhs predicate
- // rhs is the right-hand-side of the conjunction.
- rhs predicate
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// match implements the predicate method.
-func (p *predicateAnd) match(c *context, result *store.QueryResult) bool {
- // Short circuit to avoid extra processing.
- if !p.lhs.match(c, result) {
- return false
- }
- return p.rhs.match(c, result)
-}
-
-// predicateAnd is a predicate that is the logical disjunction of two
-// predicates.
-type predicateOr struct {
- // lhs is the left-hand-side of the disjunction.
- lhs predicate
- // rhs is the right-hand-side of the disjunction.
- rhs predicate
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// match implements the predicate method.
-func (p *predicateOr) match(c *context, result *store.QueryResult) bool {
- // Short circuit to avoid extra processing.
- if p.lhs.match(c, result) {
- return true
- }
- return p.rhs.match(c, result)
-}
-
-// predicateAnd is a predicate that is the logical negation of another
-// predicate.
-type predicateNot struct {
- // pred is the predicate to be negated.
- pred predicate
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// match implements the predicate method.
-func (p *predicateNot) match(c *context, result *store.QueryResult) bool {
- return !p.pred.match(c, result)
-}
-
-// expr produces a value in the context of a store.QueryResult.
-type expr interface {
- // value returns a value in the context of result.
- value(c *context, result *store.QueryResult) interface{}
-}
-
-// convertExpr transforms a parse.Expr into an expr.
-func convertExpr(e parse.Expr) expr {
- switch e := e.(type) {
- case *parse.ExprString:
- return &exprString{e.Str, e.Pos}
- case *parse.ExprRat:
- return &exprRat{e.Rat, e.Pos}
- case *parse.ExprInt:
- return &exprInt{e.Int, e.Pos}
- case *parse.ExprName:
- return &exprName{e.Name, e.Pos}
- case *parse.ExprUnary:
- return &exprUnary{convertExpr(e.Operand), e.Op, e.Pos}
- // TODO(kash): Support the other types of expressions.
- default:
- panic(fmt.Errorf("unexpected type %T", e))
- }
-}
-
-// exprString is an expr that represents a string constant.
-type exprString struct {
- // str is the string constant specified in the query.
- str string
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// value implements the expr method.
-func (e *exprString) value(c *context, result *store.QueryResult) interface{} {
- return e.str
-}
-
-// exprRat is an expr that represents a rational number constant.
-type exprRat struct {
- rat *big.Rat
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// value implements the expr method.
-func (e *exprRat) value(c *context, result *store.QueryResult) interface{} {
- return e.rat
-}
-
-// exprInt is an expr that represents an integer constant.
-type exprInt struct {
- i *big.Int
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// value implements the expr method.
-func (e *exprInt) value(c *context, result *store.QueryResult) interface{} {
- return e.i
-}
-
-// exprName is an expr for an Object name literal.
-type exprName struct {
- // name is the Object name used in the query.
- name string
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// value implements the expr method.
-func (e *exprName) value(c *context, result *store.QueryResult) interface{} {
- if result.Fields != nil {
- // TODO(kash): Handle multipart names. This currently only works if
- // e.name has no slashes.
- val, ok := result.Fields[e.name]
- if !ok {
- c.evalIt.setErrorf("name '%s' was not selected from '%s', found: [%s]",
- e.name, result.Name, mapKeys(result.Fields))
- return nil
- }
- // If a field is "hidden", the value will be wrapped in the type
- // hiddenResult to make it possible for evalIterator to elide it
- // from the results sent to the client.
- if v, ok := val.(hiddenResult); ok {
- return v.value
- }
- return val
- }
- fullpath := naming.Join(result.Name, e.name)
- entry, err := c.sn.Get(c.clientID, storage.ParsePath(fullpath))
- if err != nil {
- c.evalIt.setErrorf("could not look up name '%s' relative to '%s': %v",
- e.name, result.Name, err)
- return nil
- }
- return entry.Value
-}
-
-func mapKeys(m map[string]vdlutil.Any) string {
- s := make([]string, 0, len(m))
- for key, _ := range m {
- s = append(s, key)
- }
- sort.Strings(s)
- return strings.Join(s, ", ")
-}
-
-// exprUnary is an expr preceded by a '+' or '-'.
-type exprUnary struct {
- // operand is the expression to be modified by Op.
- operand expr
- // op is the operator that modifies operand.
- op parse.Operator
- // pos specifies where in the query string this component started.
- pos parse.Pos
-}
-
-// value implements the expr method.
-func (e *exprUnary) value(c *context, result *store.QueryResult) interface{} {
- v := e.operand.value(c, result)
- switch e.op {
- case parse.OpNeg:
- switch v := v.(type) {
- case *big.Int:
- // Need to create a temporary big.Int since Neg mutates the Int.
- return new(big.Int).Set(v).Neg(v)
- case *big.Rat:
- // Need to create a temporary big.Rat since Neg mutates the Rat.
- return new(big.Rat).Set(v).Neg(v)
- case int:
- return -v
- case int8:
- return -v
- case int16:
- return -v
- case int32:
- return -v
- case int64:
- return -v
- case uint:
- return -v
- case uint8:
- return -v
- case uint16:
- return -v
- case uint32:
- return -v
- case uint64:
- return -v
- default:
- c.evalIt.setErrorf("cannot negate value of type %T for %s", v, result.Name)
- return nil
- }
- case parse.OpPos:
- return v
- default:
- c.evalIt.setErrorf("unknown operator %d at Pos %v", e.op, e.pos)
- return nil
- }
-}
diff --git a/services/store/memstore/query/eval_test.go b/services/store/memstore/query/eval_test.go
deleted file mode 100644
index 62fee97..0000000
--- a/services/store/memstore/query/eval_test.go
+++ /dev/null
@@ -1,686 +0,0 @@
-package query
-
-import (
- "fmt"
- "reflect"
- "testing"
- "time"
-
- _ "veyron/lib/testutil"
- "veyron/services/store/memstore/state"
-
- "veyron2/query"
- "veyron2/security"
- "veyron2/services/store"
- "veyron2/storage"
- "veyron2/vdl/vdlutil"
- "veyron2/vlog"
-)
-
-type team struct {
- Name string
- Location string
-}
-
-type player struct {
- Name string
- Age int
-}
-
-func populate(t *testing.T) *state.State {
- st := state.New(rootPublicID)
- sn := st.MutableSnapshot()
-
- // Add some objects.
- put(t, sn, "/", "")
-
- put(t, sn, "/players", "")
- alfredID := put(t, sn, "/players/alfred", player{"alfred", 17})
- aliceID := put(t, sn, "/players/alice", player{"alice", 16})
- bettyID := put(t, sn, "/players/betty", player{"betty", 23})
- bobID := put(t, sn, "/players/bob", player{"bob", 21})
-
- put(t, sn, "/players/betty/bio", "")
- put(t, sn, "/players/betty/bio/hometown", "Tampa")
-
- put(t, sn, "/teams", "")
- put(t, sn, "/teams/cardinals", team{"cardinals", "CA"})
- put(t, sn, "/teams/sharks", team{"sharks", "NY"})
- put(t, sn, "/teams/bears", team{"bears", "CO"})
-
- put(t, sn, "/teams/cardinals/players", "")
- put(t, sn, "/teams/sharks/players", "")
- put(t, sn, "/teams/bears/players", "")
-
- put(t, sn, "/teams/cardinals/players/alfred", alfredID)
- put(t, sn, "/teams/sharks/players/alice", aliceID)
- put(t, sn, "/teams/sharks/players/betty", bettyID)
- // Call him something different to make sure we are handling
- // paths correctly in subqueries. We don't want the subquery
- // "teams/sharks | type team | { players/*}" to work with
- // "/players/bob".
- put(t, sn, "/teams/sharks/players/robert", bobID)
-
- commit(t, st, sn)
- return st
-}
-
-func TestEval(t *testing.T) {
- st := populate(t)
-
- type testCase struct {
- suffix string
- query string
- expectedNames []string
- }
-
- tests := []testCase{
- // nameEvaluator:
- {"", "teams", []string{"teams"}},
- {"", "teams/.", []string{"teams"}},
- {"", "teams/*", []string{"teams", "teams/cardinals", "teams/sharks", "teams/bears"}},
-
- // With a non empty prefix:
- {"teams", ".", []string{""}},
- {"teams", "*", []string{"", "cardinals", "sharks", "bears"}},
-
- // typeEvaluator:
- {"", "teams | type team", []string{}},
- {"", "teams/. | type team", []string{}},
- {"", "teams/* | type team", []string{"teams/cardinals", "teams/sharks", "teams/bears"}},
-
- // filterEvaluator/predicateBool:
- {"", "teams | ?true", []string{"teams"}},
- {"", "teams | ?false", []string{}},
-
- // predicateCompare:
- // String constants:
- {"", "teams | ?'foo' > 'bar'", []string{"teams"}},
- {"", "teams | ?'foo' < 'bar'", []string{}},
- {"", "teams | ?'foo' == 'bar'", []string{}},
- {"", "teams | ?'foo' != 'bar'", []string{"teams"}},
- {"", "teams | ?'foo' <= 'bar'", []string{}},
- {"", "teams | ?'foo' >= 'bar'", []string{"teams"}},
- // Rational number constants:
- {"", "teams | ?2.3 > 1.0", []string{"teams"}},
- {"", "teams | ?2.3 < 1.0", []string{}},
- {"", "teams | ?2.3 == 1.0", []string{}},
- {"", "teams | ?2.3 != 1.0", []string{"teams"}},
- {"", "teams | ?2.3 <= 1.0", []string{}},
- {"", "teams | ?2.3 >= 1.0", []string{"teams"}},
- {"", "teams | ?-2.3 >= 1.0", []string{}},
- {"", "teams | ?2.3 <= -1.0", []string{}},
- // Integer constants:
- {"", "teams | ?2 > 1", []string{"teams"}},
- {"", "teams | ?2 < 1", []string{}},
- {"", "teams | ?2 == 1", []string{}},
- {"", "teams | ?2 != 1", []string{"teams"}},
- {"", "teams | ?2 <= 1", []string{}},
- {"", "teams | ?2 >= 1", []string{"teams"}},
- // Compare an integer with a rational number:
- {"", "teams | ?2 > 1.7", []string{"teams"}},
- {"", "teams | ?2.3 > 1", []string{"teams"}},
- {"", "teams | ?-2 > 1.7", []string{}},
- // Object names:
- {"", "teams/* | type team | ?Name > 'bar'", []string{"teams/cardinals", "teams/sharks", "teams/bears"}},
- {"", "teams/* | type team | ?Name > 'foo'", []string{"teams/sharks"}},
- {"", "teams/* | type team | ?Name != 'bears'", []string{"teams/cardinals", "teams/sharks"}},
- {"", "players/* | type player | ?Age > 20", []string{"players/betty", "players/bob"}},
- {"", "players/* | type player | ?-Age < -20", []string{"players/betty", "players/bob"}},
-
- // predicateAnd:
- {"", "teams | ?true && true", []string{"teams"}},
- {"", "teams | ?true && false", []string{}},
-
- // predicateOr:
- {"", "teams | ?true || true", []string{"teams"}},
- {"", "teams | ?true || false", []string{"teams"}},
- {"", "teams | ?false || false", []string{}},
-
- // predicateNot:
- {"", "teams | ?!true", []string{}},
- {"", "teams | ?!false", []string{"teams"}},
- {"", "teams | ?!(false && false)", []string{"teams"}},
- {"", "teams | ?!(true || false)", []string{}},
- }
- for _, test := range tests {
- it := Eval(st.Snapshot(), rootPublicID, storage.ParsePath(test.suffix), query.Query{test.query})
- names := map[string]bool{}
- for it.Next() {
- result := it.Get()
- if _, ok := names[result.Name]; ok {
- t.Errorf("query: %s, duplicate results for %s", test.query, result.Name)
- }
- names[result.Name] = true
- }
- if it.Err() != nil {
- t.Errorf("query: %s, Error during eval: %v", test.query, it.Err())
- continue
- }
- if len(names) != len(test.expectedNames) {
- t.Errorf("query: %s, Wrong number of names. got %v, wanted %v", test.query, names, test.expectedNames)
- continue
- }
- for _, name := range test.expectedNames {
- if !names[name] {
- t.Errorf("Did not find '%s' in %v", name, names)
- }
- }
- // Ensure that all the goroutines are cleaned up.
- it.(*evalIterator).wait()
- }
-}
-
-func TestSample(t *testing.T) {
- st := populate(t)
-
- type testCase struct {
- query string
- expectedNumNames int
- }
-
- tests := []testCase{
- {"teams/* | type team | sample(1)", 1},
- {"teams/* | type team | sample(2)", 2},
- {"teams/* | type team | sample(3)", 3},
- {"teams/* | type team | sample(4)", 3}, // Can't sample more values than exist.
- }
-
- for _, test := range tests {
- it := Eval(st.Snapshot(), rootPublicID, storage.ParsePath(""), query.Query{test.query})
- names := make(map[string]struct{})
- for it.Next() {
- result := it.Get()
- if _, ok := names[result.Name]; ok {
- t.Errorf("query: %s, duplicate results for %s", test.query, result.Name)
- }
- names[result.Name] = struct{}{}
- }
- if it.Err() != nil {
- t.Errorf("query: %s, Error during eval: %v", test.query, it.Err())
- continue
- }
- if len(names) != test.expectedNumNames {
- t.Errorf("query: %s, Wrong number of names. got %v, wanted %v", test.query, names, test.expectedNumNames)
- continue
- }
- possibleNames := map[string]struct{}{
- "teams/cardinals": struct{}{},
- "teams/sharks": struct{}{},
- "teams/bears": struct{}{},
- }
- for name, _ := range names {
- if _, ok := possibleNames[name]; !ok {
- t.Errorf("Did not find '%s' in %v", name, possibleNames)
- }
- }
- // Ensure that all the goroutines are cleaned up.
- it.(*evalIterator).wait()
- }
-}
-
-func TestSorting(t *testing.T) {
- st := populate(t)
- sn := st.MutableSnapshot()
- put(t, sn, "/teams/beavers", team{"beavers", "CO"})
- commit(t, st, sn)
-
- type testCase struct {
- query string
- expectedResults []*store.QueryResult
- }
-
- tests := []testCase{
- {
- "'teams/*' | type team | sort()",
- []*store.QueryResult{
- &store.QueryResult{0, "teams/bears", nil, team{"bears", "CO"}},
- &store.QueryResult{0, "teams/beavers", nil, team{"beavers", "CO"}},
- &store.QueryResult{0, "teams/cardinals", nil, team{"cardinals", "CA"}},
- &store.QueryResult{0, "teams/sharks", nil, team{"sharks", "NY"}},
- },
- },
- {
- "'teams/*' | type team | sort(Name)",
- []*store.QueryResult{
- &store.QueryResult{0, "teams/bears", nil, team{"bears", "CO"}},
- &store.QueryResult{0, "teams/beavers", nil, team{"beavers", "CO"}},
- &store.QueryResult{0, "teams/cardinals", nil, team{"cardinals", "CA"}},
- &store.QueryResult{0, "teams/sharks", nil, team{"sharks", "NY"}},
- },
- },
- {
- "'teams/*' | type team | sort(Location, Name)",
- []*store.QueryResult{
- &store.QueryResult{0, "teams/cardinals", nil, team{"cardinals", "CA"}},
- &store.QueryResult{0, "teams/bears", nil, team{"bears", "CO"}},
- &store.QueryResult{0, "teams/beavers", nil, team{"beavers", "CO"}},
- &store.QueryResult{0, "teams/sharks", nil, team{"sharks", "NY"}},
- },
- },
- {
- "'teams/*' | type team | sort(Location)",
- []*store.QueryResult{
- &store.QueryResult{0, "teams/cardinals", nil, team{"cardinals", "CA"}},
- &store.QueryResult{0, "teams/bears", nil, team{"bears", "CO"}},
- &store.QueryResult{0, "teams/beavers", nil, team{"beavers", "CO"}},
- &store.QueryResult{0, "teams/sharks", nil, team{"sharks", "NY"}},
- },
- },
- {
- "'teams/*' | type team | sort(+Location)",
- []*store.QueryResult{
- &store.QueryResult{0, "teams/cardinals", nil, team{"cardinals", "CA"}},
- &store.QueryResult{0, "teams/bears", nil, team{"bears", "CO"}},
- &store.QueryResult{0, "teams/beavers", nil, team{"beavers", "CO"}},
- &store.QueryResult{0, "teams/sharks", nil, team{"sharks", "NY"}},
- },
- },
- {
- "'teams/*' | type team | sort(-Location)",
- []*store.QueryResult{
- &store.QueryResult{0, "teams/sharks", nil, team{"sharks", "NY"}},
- &store.QueryResult{0, "teams/bears", nil, team{"bears", "CO"}},
- &store.QueryResult{0, "teams/beavers", nil, team{"beavers", "CO"}},
- &store.QueryResult{0, "teams/cardinals", nil, team{"cardinals", "CA"}},
- },
- },
- {
- "'teams/*' | type team | sort(-Location, Name)",
- []*store.QueryResult{
- &store.QueryResult{0, "teams/sharks", nil, team{"sharks", "NY"}},
- &store.QueryResult{0, "teams/bears", nil, team{"bears", "CO"}},
- &store.QueryResult{0, "teams/beavers", nil, team{"beavers", "CO"}},
- &store.QueryResult{0, "teams/cardinals", nil, team{"cardinals", "CA"}},
- },
- },
- {
- "'teams/*' | type team | sort(-Location, -Name)",
- []*store.QueryResult{
- &store.QueryResult{0, "teams/sharks", nil, team{"sharks", "NY"}},
- &store.QueryResult{0, "teams/beavers", nil, team{"beavers", "CO"}},
- &store.QueryResult{0, "teams/bears", nil, team{"bears", "CO"}},
- &store.QueryResult{0, "teams/cardinals", nil, team{"cardinals", "CA"}},
- },
- },
- {
- "'players/*' | type player | sort(Age)",
- []*store.QueryResult{
- &store.QueryResult{0, "players/alice", nil, player{"alice", 16}},
- &store.QueryResult{0, "players/alfred", nil, player{"alfred", 17}},
- &store.QueryResult{0, "players/bob", nil, player{"bob", 21}},
- &store.QueryResult{0, "players/betty", nil, player{"betty", 23}},
- },
- },
- {
- "'players/*' | type player | sort(-Age)",
- []*store.QueryResult{
- &store.QueryResult{0, "players/betty", nil, player{"betty", 23}},
- &store.QueryResult{0, "players/bob", nil, player{"bob", 21}},
- &store.QueryResult{0, "players/alfred", nil, player{"alfred", 17}},
- &store.QueryResult{0, "players/alice", nil, player{"alice", 16}},
- },
- },
- }
- for _, test := range tests {
- it := Eval(st.Snapshot(), rootPublicID, storage.ParsePath(""), query.Query{test.query})
- i := 0
- for it.Next() {
- result := it.Get()
- if i >= len(test.expectedResults) {
- t.Errorf("query: %s; not enough expected results (%d); found %v", test.query, len(test.expectedResults), result)
- break
- }
- if got, want := result, test.expectedResults[i]; !reflect.DeepEqual(got, want) {
- t.Errorf("query: %s;\nGOT %v\nWANT %v", test.query, got, want)
- }
- i++
- }
- if it.Err() != nil {
- t.Errorf("query: %s, Error during eval: %v", test.query, it.Err())
- continue
- }
- if i != len(test.expectedResults) {
- t.Errorf("query: %s, Got %d results, expected %d", test.query, i, len(test.expectedResults))
- continue
- }
- // Ensure that all the goroutines are cleaned up.
- it.(*evalIterator).wait()
- }
-}
-
-func TestSelection(t *testing.T) {
- st := populate(t)
-
- type testCase struct {
- suffix string
- query string
- expectedResults []*store.QueryResult
- }
-
- tests := []testCase{
- {
- "", "'teams/cardinals' | {Name: Name}",
- []*store.QueryResult{
- &store.QueryResult{0, "teams/cardinals", map[string]vdlutil.Any{"Name": "cardinals"}, nil},
- },
- },
- {
- "teams", "'cardinals' | {Name: Name}",
- []*store.QueryResult{
- &store.QueryResult{0, "cardinals", map[string]vdlutil.Any{"Name": "cardinals"}, nil},
- },
- },
- {
- "teams/cardinals", ". | {Name: Name}",
- []*store.QueryResult{
- &store.QueryResult{0, "", map[string]vdlutil.Any{"Name": "cardinals"}, nil},
- },
- },
- {
- "", "'teams/cardinals' | {Name: Name}",
- []*store.QueryResult{
- &store.QueryResult{0, "teams/cardinals", map[string]vdlutil.Any{"Name": "cardinals"}, nil},
- },
- },
- {
- "", "'teams/cardinals' | {myname: Name, myloc: Location}",
- []*store.QueryResult{
- &store.QueryResult{
- 0,
- "teams/cardinals",
- map[string]vdlutil.Any{
- "myname": "cardinals",
- "myloc": "CA",
- },
- nil,
- },
- },
- },
- {
- "", "'teams/cardinals' | {myname: Name, myloc: Location} | ? myname == 'cardinals'",
- []*store.QueryResult{
- &store.QueryResult{
- 0,
- "teams/cardinals",
- map[string]vdlutil.Any{
- "myname": "cardinals",
- "myloc": "CA",
- },
- nil,
- },
- },
- },
- {
- "", "'teams/cardinals' | {myname hidden: Name, myloc: Location} | ? myname == 'cardinals'",
- []*store.QueryResult{
- &store.QueryResult{
- 0,
- "teams/cardinals",
- map[string]vdlutil.Any{
- "myloc": "CA",
- },
- nil,
- },
- },
- },
- {
- "", "'teams/cardinals' | {self: ., myname: Name} | ? myname == 'cardinals'",
- []*store.QueryResult{
- &store.QueryResult{
- 0,
- "teams/cardinals",
- map[string]vdlutil.Any{
- "self": team{"cardinals", "CA"},
- "myname": "cardinals",
- },
- nil,
- },
- },
- },
- {
- "",
- "'teams/*' | type team | {" +
- " myname: Name," +
- " drinkers: players/* | type player | ?Age >=21 | sort()," +
- " nondrinkers: players/* | type player | ?Age < 21 | sort()" +
- "} | sort(myname)",
- []*store.QueryResult{
- &store.QueryResult{
- 0,
- "teams/bears",
- map[string]vdlutil.Any{
- "myname": "bears",
- "drinkers": store.NestedResult(1),
- "nondrinkers": store.NestedResult(2),
- },
- nil,
- },
- &store.QueryResult{
- 0,
- "teams/cardinals",
- map[string]vdlutil.Any{
- "myname": "cardinals",
- "drinkers": store.NestedResult(3),
- "nondrinkers": store.NestedResult(4),
- },
- nil,
- },
- &store.QueryResult{
- 4,
- "teams/cardinals/players/alfred",
- nil,
- player{"alfred", 17},
- },
- &store.QueryResult{
- 0,
- "teams/sharks",
- map[string]vdlutil.Any{
- "myname": "sharks",
- "drinkers": store.NestedResult(5),
- "nondrinkers": store.NestedResult(6),
- },
- nil,
- },
- &store.QueryResult{
- 5,
- "teams/sharks/players/betty",
- nil,
- player{"betty", 23},
- },
- &store.QueryResult{
- 5,
- "teams/sharks/players/robert",
- nil,
- player{"bob", 21},
- },
- &store.QueryResult{
- 6,
- "teams/sharks/players/alice",
- nil,
- player{"alice", 16},
- },
- },
- },
- // Test for selection of a nested name ('bio/hometown'). Only betty has this
- // nested name, so other players should get a nil value.
- {
- "", "'players/*' | type player | {Name: Name, hometown: 'bio/hometown'} | ? Name == 'alfred' || Name == 'betty' | sort()",
- []*store.QueryResult{
- &store.QueryResult{
- 0,
- "players/alfred",
- map[string]vdlutil.Any{
- "Name": "alfred",
- "hometown": nil,
- },
- nil,
- },
- &store.QueryResult{
- 0,
- "players/betty",
- map[string]vdlutil.Any{
- "Name": "betty",
- "hometown": "Tampa",
- },
- nil,
- },
- },
- },
- }
- for _, test := range tests {
- vlog.VI(1).Infof("Testing %s\n", test.query)
- it := Eval(st.Snapshot(), rootPublicID, storage.ParsePath(test.suffix), query.Query{test.query})
- i := 0
- for it.Next() {
- result := it.Get()
- if i >= len(test.expectedResults) {
- t.Errorf("query: %s, not enough expected results, need at least %d", i)
- it.Abort()
- break
- }
- if got, want := result, test.expectedResults[i]; !reflect.DeepEqual(got, want) {
- t.Errorf("query: %s;\nGOT %v\nWANT %v", test.query, got, want)
- }
- i++
- }
- if it.Err() != nil {
- t.Errorf("query: %s, Error during eval: %v", test.query, it.Err())
- continue
- }
- if i != len(test.expectedResults) {
- t.Errorf("query: %s, Got %d results, expected %d", test.query, i, len(test.expectedResults))
- continue
- }
- // Ensure that all the goroutines are cleaned up.
- it.(*evalIterator).wait()
- }
-}
-
-func TestError(t *testing.T) {
- st := populate(t)
-
- type testCase struct {
- query string
- expectedError string
- }
-
- tests := []testCase{
- {"teams!foo", "1:6: syntax error at token '!'"},
- {"teams | ?Name > 'foo'", "could not look up name 'Name' relative to 'teams': not found"},
- // This query results in an error because not all of the intermediate
- // results produced by "teams/*" are of type 'team'.
- // TODO(kash): We probably want an error message that says that you must
- // use a type filter.
- {"teams/* | ?Name > 'foo'", "could not look up name 'Name' relative to 'teams': not found"},
- {"'teams/cardinals' | {myname: Name, myloc: Location} | ? Name == 'foo'", "name 'Name' was not selected from 'teams/cardinals', found: [myloc, myname]"},
- {"teams/* | type team | sort(Name) | ?-Name > 'foo'", "cannot negate value of type string for teams/bears"},
- {"teams/* | type team | sample(2, 3)", "1:21: sample expects exactly one integer argument specifying the number of results to include in the sample"},
- {"teams/* | type team | sample(2.0)", "1:21: sample expects exactly one integer argument specifying the number of results to include in the sample"},
- {"teams/* | type team | sample(-1)", "1:21: sample expects exactly one integer argument specifying the number of results to include in the sample"},
-
- // TODO(kash): Selection with conflicting names.
- // TODO(kash): Trying to sort an aggregate. "... | avg | sort()"
- }
- for _, test := range tests {
- it := Eval(st.Snapshot(), rootPublicID, storage.PathName{}, query.Query{test.query})
- for it.Next() {
- }
- if it.Err() == nil {
- t.Errorf("query %s, No error, expected %s", test.query, test.expectedError)
- continue
- }
- if it.Err().Error() != test.expectedError {
- t.Errorf("query %s, got error \"%s\", expected \"%s\"", test.query, it.Err(), test.expectedError)
- continue
- }
- }
-}
-
-type mockSnapshot struct {
- it state.Iterator
-}
-
-func (m *mockSnapshot) NewIterator(pid security.PublicID, path storage.PathName, pathFilter state.PathFilter, filter state.IterFilter) state.Iterator {
- return m.it
-}
-
-func (m *mockSnapshot) Find(id storage.ID) *state.Cell {
- return nil
-}
-
-func (m *mockSnapshot) Get(pid security.PublicID, path storage.PathName) (*storage.Entry, error) {
- return nil, nil
-}
-
-type repeatForeverIterator struct {
- entry *storage.Entry
- snapshot state.Snapshot
-}
-
-func (it *repeatForeverIterator) IsValid() bool {
- return true
-}
-
-func (it *repeatForeverIterator) Get() *storage.Entry {
- return it.entry
-}
-
-func (it *repeatForeverIterator) Name() string {
- return fmt.Sprintf("teams/%v", it.entry.Stat.MTimeNS)
-}
-
-func (it *repeatForeverIterator) Next() {
- it.entry = &storage.Entry{
- storage.Stat{storage.ObjectKind, storage.NewID(), time.Now().UnixNano(), nil},
- it.entry.Value,
- }
-}
-
-func (it *repeatForeverIterator) Snapshot() state.Snapshot {
- return it.snapshot
-}
-
-func TestEvalAbort(t *testing.T) {
- type testCase struct {
- query string
- }
-
- tests := []testCase{
- testCase{"teams/*"},
- testCase{"teams/* | type team"},
- testCase{"teams/* | ?true"},
- }
-
- dummyTeam := team{"cardinals", "CA"}
- sn := &mockSnapshot{
- &repeatForeverIterator{
- entry: &storage.Entry{
- storage.Stat{storage.ObjectKind, storage.NewID(), time.Now().UnixNano(), nil},
- dummyTeam,
- },
- },
- }
- sn.it.(*repeatForeverIterator).snapshot = sn
-
- for _, test := range tests {
- // Test calling Abort immediately vs waiting until the channels are full.
- for i := 0; i < 2; i++ {
- it := Eval(sn, rootPublicID, storage.PathName{}, query.Query{test.query})
- if i == 0 {
- // Give the evaluators time to fill up the channels. Ensure that they
- // don't block forever on a full channel.
- for len(it.(*evalIterator).results[0].results) < maxChannelSize {
- time.Sleep(time.Millisecond)
- }
- }
- it.Abort()
- if it.Err() != nil {
- t.Errorf("query:%q Got non-nil error: %v", test.query, it.Err())
- }
- it.(*evalIterator).wait()
- }
- }
-}
-
-// TODO(kash): Add a test for access control.
diff --git a/services/store/memstore/query/glob.go b/services/store/memstore/query/glob.go
deleted file mode 100644
index 363580a..0000000
--- a/services/store/memstore/query/glob.go
+++ /dev/null
@@ -1,57 +0,0 @@
-package query
-
-import (
- "veyron/lib/glob"
- "veyron/services/store/memstore/refs"
- "veyron/services/store/memstore/state"
-
- "veyron2/security"
- "veyron2/storage"
-)
-
-type globIterator struct {
- state.Iterator
- pathLen int
- glob *glob.Glob
-}
-
-// GlobStream represents a sequence of results from a glob call.
-type GlobStream interface {
- // IsValid returns true iff the iterator refers to an element.
- IsValid() bool
-
- // Return one possible name for this entry.
- Name() string
-
- // Next advances to the next element.
- Next()
-}
-
-// Glob returns an iterator that emits all values that match the given pattern.
-func Glob(sn state.Snapshot, clientID security.PublicID, path storage.PathName, pattern string) (GlobStream, error) {
- return GlobIterator(sn, clientID, path, pattern)
-}
-
-// GlobIterator returns an iterator that emits all values that match the given pattern.
-func GlobIterator(sn state.Snapshot, clientID security.PublicID, path storage.PathName, pattern string) (state.Iterator, error) {
- parsed, err := glob.Parse(pattern)
- if err != nil {
- return nil, err
- }
-
- g := &globIterator{
- pathLen: len(path),
- glob: parsed,
- }
- g.Iterator = sn.NewIterator(clientID, path, state.ListPaths, state.IterFilter(g.filter))
-
- return g, nil
-}
-
-func (g *globIterator) filter(parent *refs.FullPath, path *refs.Path) (bool, bool) {
- // We never get to a path unless we've first approved its parent.
- // We can therefore only check a suffix of the glob pattern.
- prefixLen := parent.Len() - g.pathLen
- matched, suffix := g.glob.PartialMatch(prefixLen, []string(path.Name()))
- return matched && suffix.Len() == 0, matched
-}
diff --git a/services/store/memstore/query/glob_test.go b/services/store/memstore/query/glob_test.go
deleted file mode 100644
index 6690d98..0000000
--- a/services/store/memstore/query/glob_test.go
+++ /dev/null
@@ -1,124 +0,0 @@
-package query
-
-import (
- "path/filepath"
- "testing"
-
- "veyron/services/store/memstore/state"
-
- "veyron2/storage"
-)
-
-type globTest struct {
- path string
- pattern string
- expected []string
-}
-
-var globTests = []globTest{
- {"", "...", []string{
- "",
- "mvps",
- "mvps/Links/0",
- "mvps/Links/1",
- "players",
- "players/alfred",
- "players/alice",
- "players/betty",
- "players/bob",
- "teams",
- "teams/bears",
- "teams/cardinals",
- "teams/sharks",
- }},
- {"", "mvps/...", []string{
- "mvps",
- "mvps/Links/0",
- "mvps/Links/1",
- }},
- {"", "players/...", []string{
- "players",
- "players/alfred",
- "players/alice",
- "players/betty",
- "players/bob",
- }},
- // Note(mattr): This test case shows that Glob does not return
- // subfield nodes.
- {"", "mvps/*", []string{}},
- {"", "mvps/Links/*", []string{
- "mvps/Links/0",
- "mvps/Links/1",
- }},
- {"", "players/alfred", []string{
- "players/alfred",
- }},
- {"", "mvps/Links/0", []string{
- "mvps/Links/0",
- }},
- // An empty pattern returns the element referred to by the path.
- {"/mvps/Links/0", "", []string{
- "",
- }},
- {"mvps", "Links/*", []string{
- "Links/0",
- "Links/1",
- }},
- {"mvps/Links", "*", []string{
- "0",
- "1",
- }},
-}
-
-// Test that an iterator doesen't get stuck in cycles.
-func TestGlob(t *testing.T) {
- st := state.New(rootPublicID)
- sn := st.MutableSnapshot()
-
- type dir struct {
- Links []storage.ID
- }
-
- // Add some objects
- put(t, sn, "/", "")
- put(t, sn, "/teams", "")
- put(t, sn, "/teams/cardinals", "")
- put(t, sn, "/teams/sharks", "")
- put(t, sn, "/teams/bears", "")
- put(t, sn, "/players", "")
- alfredID := put(t, sn, "/players/alfred", "")
- put(t, sn, "/players/alice", "")
- bettyID := put(t, sn, "/players/betty", "")
- put(t, sn, "/players/bob", "")
-
- put(t, sn, "/mvps", &dir{[]storage.ID{alfredID, bettyID}})
-
- commit(t, st, sn)
-
- // Test that patterns starting with '/' are errors.
- _, err := Glob(st.Snapshot(), rootPublicID, storage.PathName{}, "/*")
- if err != filepath.ErrBadPattern {
- t.Errorf("Expected bad pattern error, got %v", err)
- }
-
- for _, gt := range globTests {
- path := storage.ParsePath(gt.path)
- it, err := Glob(st.Snapshot(), rootPublicID, path, gt.pattern)
- if err != nil {
- t.Errorf("Unexpected error on Glob: %s", err)
- }
- names := map[string]bool{}
- for ; it.IsValid(); it.Next() {
- names[it.Name()] = true
- }
- if len(names) != len(gt.expected) {
- t.Errorf("Wrong number of names for %s. got %v, wanted %v",
- gt.pattern, names, gt.expected)
- }
- for _, name := range gt.expected {
- if !names[name] {
- t.Errorf("Expected to find %v in %v", name, names)
- }
- }
- }
-}
diff --git a/services/store/memstore/query/util_test.go b/services/store/memstore/query/util_test.go
deleted file mode 100644
index e11b4db..0000000
--- a/services/store/memstore/query/util_test.go
+++ /dev/null
@@ -1,85 +0,0 @@
-package query
-
-import (
- "runtime"
- "testing"
-
- "veyron/services/store/memstore/state"
-
- "veyron2/security"
- "veyron2/storage"
-)
-
-var (
- rootPublicID security.PublicID = security.FakePublicID("root")
-)
-
-type Dir struct{}
-
-func mkdir(t *testing.T, sn *state.MutableSnapshot, name string) (storage.ID, interface{}) {
- _, file, line, _ := runtime.Caller(1)
- dir := &Dir{}
- path := storage.ParsePath(name)
- stat, err := sn.Put(rootPublicID, path, dir)
- if err != nil || stat == nil {
- t.Errorf("%s(%d): mkdir %s: %s", file, line, name, err)
- }
- return stat.ID, dir
-}
-
-func get(t *testing.T, sn *state.MutableSnapshot, name string) interface{} {
- _, file, line, _ := runtime.Caller(1)
- path := storage.ParsePath(name)
- e, err := sn.Get(rootPublicID, path)
- if err != nil {
- t.Fatalf("%s(%d): can't get %s: %s", file, line, name, err)
- }
- return e.Value
-}
-
-func put(t *testing.T, sn *state.MutableSnapshot, name string, v interface{}) storage.ID {
- _, file, line, _ := runtime.Caller(1)
- path := storage.ParsePath(name)
- stat, err := sn.Put(rootPublicID, path, v)
- if err != nil {
- t.Errorf("%s(%d): can't put %s: %s", file, line, name, err)
- }
- if _, err := sn.Get(rootPublicID, path); err != nil {
- t.Errorf("%s(%d): can't get %s: %s", file, line, name, err)
- }
- if stat != nil {
- return stat.ID
- }
- return storage.ID{}
-}
-
-func remove(t *testing.T, sn *state.MutableSnapshot, name string) {
- path := storage.ParsePath(name)
- if err := sn.Remove(rootPublicID, path); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): can't remove %s: %s", file, line, name, err)
- }
-}
-
-func commit(t *testing.T, st *state.State, sn *state.MutableSnapshot) {
- if err := st.ApplyMutations(sn.Mutations()); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): commit failed: %s", file, line, err)
- }
-}
-
-func expectExists(t *testing.T, sn *state.MutableSnapshot, name string) {
- _, file, line, _ := runtime.Caller(1)
- path := storage.ParsePath(name)
- if _, err := sn.Get(rootPublicID, path); err != nil {
- t.Errorf("%s(%d): does not exist: %s", file, line, path)
- }
-}
-
-func expectNotExists(t *testing.T, sn *state.MutableSnapshot, name string) {
- path := storage.ParsePath(name)
- if _, err := sn.Get(rootPublicID, path); err == nil {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): should not exist: %s", file, line, name)
- }
-}
diff --git a/services/store/memstore/refs/builder.go b/services/store/memstore/refs/builder.go
deleted file mode 100644
index b7d591f..0000000
--- a/services/store/memstore/refs/builder.go
+++ /dev/null
@@ -1,105 +0,0 @@
-package refs
-
-import (
- "reflect"
- "strconv"
-
- "veyron/runtimes/google/lib/functional"
- "veyron/services/store/raw"
-
- "veyron2/storage"
-)
-
-// Builder is used to collect all the references in a value, using
-// reflection for traversal.
-type Builder struct {
- refs Set
-}
-
-var (
- tyID = reflect.TypeOf(storage.ID{})
- tyString = reflect.TypeOf("")
-)
-
-// NewBuilder constructs a new refs builder.
-func NewBuilder() *Builder {
- return &Builder{refs: Empty}
-}
-
-// Get returns the references.
-func (b *Builder) Get() Set {
- return b.refs
-}
-
-// AddDir adds the references contained in the directory.
-func (b *Builder) AddDir(d functional.Set) {
- if d == nil {
- return
- }
- d.Iter(func(it interface{}) bool {
- b.refs = b.refs.Put(it.(*Ref))
- return true
- })
-}
-
-// AddDEntries adds the references contained in the DEntry list.
-func (b *Builder) AddDEntries(d []*raw.DEntry) {
- for _, de := range d {
- b.refs = b.refs.Put(&Ref{ID: de.ID, Path: NewSingletonPath(de.Name)})
- }
-}
-
-// AddValue adds the references contained in the value, using reflection
-// to traverse it.
-func (b *Builder) AddValue(v interface{}) {
- if v == nil {
- return
- }
- b.addRefs(nil, reflect.ValueOf(v))
-}
-
-func (b *Builder) addRefs(path *Path, v reflect.Value) {
- if !v.IsValid() {
- return
- }
- ty := v.Type()
- if ty == tyID {
- b.refs = b.refs.Put(&Ref{ID: v.Interface().(storage.ID), Path: path})
- return
- }
- switch ty.Kind() {
- case reflect.Map:
- b.addMapRefs(path, v)
- case reflect.Array, reflect.Slice:
- b.addSliceRefs(path, v)
- case reflect.Interface, reflect.Ptr:
- b.addRefs(path, v.Elem())
- case reflect.Struct:
- b.addStructRefs(path, v)
- }
-}
-
-func (b *Builder) addMapRefs(path *Path, v reflect.Value) {
- for _, key := range v.MapKeys() {
- if key.Type().Kind() == reflect.String {
- name := key.Convert(tyString).Interface().(string)
- b.addRefs(path.Append(name), v.MapIndex(key))
- }
- }
-}
-
-func (b *Builder) addSliceRefs(path *Path, v reflect.Value) {
- l := v.Len()
- for i := 0; i < l; i++ {
- b.addRefs(path.Append(strconv.Itoa(i)), v.Index(i))
- }
-}
-
-func (b *Builder) addStructRefs(path *Path, v reflect.Value) {
- l := v.NumField()
- ty := v.Type()
- for i := 0; i < l; i++ {
- name := ty.Field(i).Name
- b.addRefs(path.Append(name), v.Field(i))
- }
-}
diff --git a/services/store/memstore/refs/builder_test.go b/services/store/memstore/refs/builder_test.go
deleted file mode 100644
index 465ba93..0000000
--- a/services/store/memstore/refs/builder_test.go
+++ /dev/null
@@ -1,103 +0,0 @@
-package refs_test
-
-import (
- "strconv"
- "testing"
-
- "veyron/services/store/memstore/refs"
-
- "veyron2/storage"
-)
-
-type A struct {
- B int
- C []int
- D map[string]int
-
- E storage.ID
- F []storage.ID
- G map[string]storage.ID
-}
-
-func TestID(t *testing.T) {
- id := storage.NewID()
-
- b := refs.NewBuilder()
- b.AddValue(id)
- r := b.Get()
- if r.Len() != 1 {
- t.Errorf("Expected 1 element, got %d", r.Len())
- }
- r.Iter(func(it interface{}) bool {
- r := it.(*refs.Ref)
- if r.ID != id {
- t.Errorf("Expected %s, got %s", id, r.ID)
- }
- if r.Path != nil {
- t.Errorf("Expected nil, got %v", r.Path)
- }
- return true
- })
-}
-
-func TestArray(t *testing.T) {
- a := []storage.ID{storage.NewID(), storage.NewID()}
-
- b := refs.NewBuilder()
- b.AddValue(a)
- r := b.Get()
- if r.Len() != 2 {
- t.Errorf("Expected 2 elements, got %d", r.Len())
- }
- r.Iter(func(it interface{}) bool {
- r := it.(*refs.Ref)
- found := false
- for i, id := range a {
- if r.ID == id {
- p := refs.NewSingletonPath(strconv.Itoa(i))
- if r.Path != p {
- t.Errorf("Expected %s, got %s", p, r.Path)
- }
- found = true
- break
- }
- }
- if !found {
- t.Errorf("Unexpected reference: %v", r)
- }
- return true
- })
-}
-
-func TestStruct(t *testing.T) {
- v := &A{
- B: 5,
- C: []int{6, 7},
- D: map[string]int{"a": 8, "b": 9},
- E: storage.NewID(),
- F: []storage.ID{storage.NewID()},
- G: map[string]storage.ID{"a": storage.NewID()},
- }
- paths := make(map[storage.ID]*refs.Path)
- paths[v.E] = refs.NewSingletonPath("E")
- paths[v.F[0]] = refs.NewSingletonPath("F").Append("0")
- paths[v.G["a"]] = refs.NewSingletonPath("G").Append("a")
-
- b := refs.NewBuilder()
- b.AddValue(v)
- r := b.Get()
- if r.Len() != 3 {
- t.Errorf("Expected 3 elements, got %d", r.Len())
- }
- r.Iter(func(it interface{}) bool {
- r := it.(*refs.Ref)
- p, ok := paths[r.ID]
- if !ok {
- t.Errorf("Unexpected id %s", r.ID)
- }
- if r.Path != p {
- t.Errorf("Expected %s, got %s", p, r.Path)
- }
- return true
- })
-}
diff --git a/services/store/memstore/refs/path.go b/services/store/memstore/refs/path.go
deleted file mode 100644
index 4a151ee..0000000
--- a/services/store/memstore/refs/path.go
+++ /dev/null
@@ -1,167 +0,0 @@
-package refs
-
-import (
- "sync"
- "unsafe"
-
- "veyron2/storage"
-)
-
-// Path represents a path name, interconvertible with storage.PathName, but
-// hash-consed. To maximize sharing, the path is represented in reverse order.
-// Because of the hash-consing, this is most appropriate for common path fragments.
-// For full names, consider FullPath.
-type Path struct {
- hd string // head.
- tl *Path // tail.
-}
-
-type pathHashConsTable struct {
- sync.Mutex
- table map[Path]*Path
-}
-
-var (
- pathTable = &pathHashConsTable{table: make(map[Path]*Path)}
- nilPath *Path
-)
-
-// ComparePaths defines a total order over *Path values, based on pointer
-// comparison. Since paths are hash-consed, this is stable (but arbitrary) with
-// a process, but it is not persistent across processes.
-func ComparePaths(p1, p2 *Path) int {
- i1 := uintptr(unsafe.Pointer(p1))
- i2 := uintptr(unsafe.Pointer(p2))
- switch {
- case i1 < i2:
- return -1
- case i1 > i2:
- return 1
- default:
- return 0
- }
-}
-
-// NewSingletonPath creates a path that has just one component.
-func NewSingletonPath(name string) *Path {
- return nilPath.Append(name)
-}
-
-// String returns a printable representation of a path.
-func (p *Path) String() string {
- if p == nil {
- return ""
- }
- s := p.hd
- for p = p.tl; p != nil; p = p.tl {
- s = p.hd + "/" + s
- }
- return s
-}
-
-// Suffix prints the name corresponding to the last n elements
-// of the path.
-func (p *Path) Suffix(n int) string {
- if p == nil || n == 0 {
- return ""
- }
- s := p.hd
- for i, p := 1, p.tl; i < n && p != nil; i, p = i+1, p.tl {
- s = p.hd + "/" + s
- }
- return s
-}
-
-// Append adds a new string component to the end of a path.
-func (p *Path) Append(name string) *Path {
- pathTable.Lock()
- defer pathTable.Unlock()
- p1 := Path{hd: name, tl: p}
- if p2, ok := pathTable.table[p1]; ok {
- return p2
- }
- pathTable.table[p1] = &p1
- return &p1
-}
-
-// Name returns a storage.PathName corresponding to the path.
-func (p *Path) Name() storage.PathName {
- i := p.Len()
- pl := make(storage.PathName, i)
- for ; p != nil; p = p.tl {
- i--
- pl[i] = p.hd
- }
- return pl
-}
-
-// Len returns the number of components in the path.
-func (p *Path) Len() int {
- i := 0
- for ; p != nil; p = p.tl {
- i++
- }
- return i
-}
-
-// Split splits a path into a directory and file part.
-func (p *Path) Split() (*Path, string) {
- return p.tl, p.hd
-}
-
-// FullPath represents a path name, interconvertible with storage.PathName.
-// To maximize sharing, the path is represented in reverse order.
-type FullPath Path
-
-// NewSingletonFullPath creates a path that has just one component.
-func NewSingletonFullPath(name string) *FullPath {
- return &FullPath{hd: name}
-}
-
-// NewFullPathFromName creates a FullPath that represents the same name as path.
-func NewFullPathFromName(path storage.PathName) *FullPath {
- var fp *FullPath
- for _, el := range path {
- fp = fp.Append(el)
- }
- return fp
-}
-
-// String returns a printable representation of a path.
-func (fp *FullPath) String() string {
- return (*Path)(fp).String()
-}
-
-// Suffix prints the name corresponding to the last n elements
-// of the full path.
-func (fp *FullPath) Suffix(n int) string {
- return (*Path)(fp).Suffix(n)
-}
-
-// Append adds a new string component to the end of a path.
-func (fp *FullPath) Append(name string) *FullPath {
- return &FullPath{hd: name, tl: (*Path)(fp)}
-}
-
-// Name returns a storage.PathName corresponding to the path.
-func (fp *FullPath) Name() storage.PathName {
- return (*Path)(fp).Name()
-}
-
-// Len returns the number of components in the path.
-func (fp *FullPath) Len() int {
- return (*Path)(fp).Len()
-}
-
-// Split splits a path into a directory and file part.
-func (fp *FullPath) Split() (*FullPath, string) {
- return (*FullPath)(fp.tl), fp.hd
-}
-
-// AppendPath returns a FullPath that represents the concatenation of fp and p.
-func (fp *FullPath) AppendPath(p *Path) *FullPath {
- if p == nil {
- return fp
- }
- return fp.AppendPath(p.tl).Append(p.hd)
-}
diff --git a/services/store/memstore/refs/path_test.go b/services/store/memstore/refs/path_test.go
deleted file mode 100644
index 2a50c7f..0000000
--- a/services/store/memstore/refs/path_test.go
+++ /dev/null
@@ -1,152 +0,0 @@
-package refs
-
-import (
- "testing"
-
- "veyron2/storage"
-)
-
-func storePathEqual(p1, p2 storage.PathName) bool {
- if len(p1) != len(p2) {
- return false
- }
- for i, s := range p1 {
- if p2[i] != s {
- return false
- }
- }
- return true
-}
-
-func TestPath(t *testing.T) {
- p1 := NewSingletonPath("a")
- p2 := NewSingletonPath("a")
- if p1 != p2 {
- t.Errorf("Paths should be identical")
- }
- if !storePathEqual(p2.Name(), storage.PathName{"a"}) {
- t.Errorf("Got %s, expected 'a'", p2.Name())
- }
- if p1.Len() != 1 {
- t.Errorf("Got length %d, expected 1", p1.Len())
- }
-
- p1 = p1.Append("b")
- p2 = p2.Append("b")
- if p1 != p2 {
- t.Errorf("Paths should be identical")
- }
- if !storePathEqual(p2.Name(), storage.PathName{"a", "b"}) {
- t.Errorf("Got %s, expected 'a/b'", p2.Name())
- }
-
- p1 = p1.Append("c")
- p2 = p2.Append("c")
- if p1 != p2 {
- t.Errorf("Paths should be identical")
- }
- if !storePathEqual(p2.Name(), storage.PathName{"a", "b", "c"}) {
- t.Errorf("Got %s, expected 'a/b/c'", p2.Name())
- }
-
- p1 = p1.Append("d")
- p2 = p2.Append("e")
- if p1 == p2 {
- t.Errorf("Paths should not be identical, %s, %s", p1, p2)
- }
- if !storePathEqual(p1.Name(), storage.PathName{"a", "b", "c", "d"}) {
- t.Errorf("Got %s, expected 'a/b/c.d'", p1.Name())
- }
- if !storePathEqual(p2.Name(), storage.PathName{"a", "b", "c", "e"}) {
- t.Errorf("Got %s, expected 'a/b/c/e'", p2.Name())
- }
-}
-
-func TestPathSuffix(t *testing.T) {
- p1 := NewSingletonPath("a")
- if s := p1.Suffix(0); s != "" {
- t.Errorf("Got '%s' expected ''", s)
- }
- if s := p1.Suffix(1); s != "a" {
- t.Errorf("Got '%s' expected 'a'", s)
- }
- if s := p1.Suffix(100); s != "a" {
- t.Errorf("Got '%s' expected 'a'", s)
- }
-
- p2 := p1.Append("b").Append("c").Append("d")
- if s := p2.Suffix(0); s != "" {
- t.Errorf("Got '%s' expected ''", s)
- }
- if s := p2.Suffix(1); s != "d" {
- t.Errorf("Got '%s' expected 'd'", s)
- }
- if s := p2.Suffix(2); s != "c/d" {
- t.Errorf("Got '%s' expected 'c/d'", s)
- }
- if s := p2.Suffix(3); s != "b/c/d" {
- t.Errorf("Got '%s' expected 'b/c/d'", s)
- }
- if s := p2.Suffix(4); s != "a/b/c/d" {
- t.Errorf("Got '%s' expected 'a/b/c/d'", s)
- }
- if s := p2.Suffix(100); s != "a/b/c/d" {
- t.Errorf("Got '%s' expected 'a/b/c/d'", s)
- }
-}
-
-func TestFullPath(t *testing.T) {
- p := NewSingletonFullPath("a")
- if s := p.String(); s != "a" {
- t.Errorf("Got %s expected 'a'", s)
- }
-
- p = p.Append("b").Append("c")
- if s := p.String(); s != "a/b/c" {
- t.Errorf("Got %s expected 'a/b/c'", s)
- }
-
- suffix := NewSingletonPath("d").Append("e")
- p = p.AppendPath(suffix)
- if s := p.String(); s != "a/b/c/d/e" {
- t.Errorf("Got %s expected 'a/b/c/d/e'", s)
- }
-
- p = NewFullPathFromName(storage.PathName{"a", "b", "c"})
- if s := p.String(); s != "a/b/c" {
- t.Errorf("Got %s expected 'a/b/c'", s)
- }
-}
-
-func TestFullPathSuffix(t *testing.T) {
- p1 := NewSingletonFullPath("a")
- if s := p1.Suffix(0); s != "" {
- t.Errorf("Got '%s' expected ''", s)
- }
- if s := p1.Suffix(1); s != "a" {
- t.Errorf("Got '%s' expected 'a'", s)
- }
- if s := p1.Suffix(100); s != "a" {
- t.Errorf("Got '%s' expected 'a'", s)
- }
-
- p2 := p1.Append("b").Append("c").Append("d")
- if s := p2.Suffix(0); s != "" {
- t.Errorf("Got '%s' expected ''", s)
- }
- if s := p2.Suffix(1); s != "d" {
- t.Errorf("Got '%s' expected 'd'", s)
- }
- if s := p2.Suffix(2); s != "c/d" {
- t.Errorf("Got '%s' expected 'c/d'", s)
- }
- if s := p2.Suffix(3); s != "b/c/d" {
- t.Errorf("Got '%s' expected 'b/c/d'", s)
- }
- if s := p2.Suffix(4); s != "a/b/c/d" {
- t.Errorf("Got '%s' expected 'a/b/c/d'", s)
- }
- if s := p2.Suffix(100); s != "a/b/c/d" {
- t.Errorf("Got '%s' expected 'a/b/c/d'", s)
- }
-}
diff --git a/services/store/memstore/refs/refs.go b/services/store/memstore/refs/refs.go
deleted file mode 100644
index 9823379..0000000
--- a/services/store/memstore/refs/refs.go
+++ /dev/null
@@ -1,63 +0,0 @@
-// Package refs represents references from one value to another.
-package refs
-
-import (
- "veyron/runtimes/google/lib/functional"
- "veyron/runtimes/google/lib/functional/rb"
- "veyron/services/store/raw"
-
- "veyron2/storage"
-)
-
-// Set is a set of *Ref values.
-type Set functional.Set // *Ref
-
-// Ref represents a single reference in a store value. It includes the
-// storage.ID, and the path to the reference.
-type Ref struct {
- ID storage.ID
- Path *Path
-}
-
-// Dir represents a directory, which is a set of *Ref, sorted by path.
-type Dir functional.Set // *Ref
-
-var (
- Empty Set = rb.NewSet(compareRefs)
- EmptyDir Dir = rb.NewSet(compareRefsByPath)
-)
-
-// *ref values are sorted lexicoigraphically by (id, path).
-func compareRefs(it1, it2 interface{}) bool {
- r1 := it1.(*Ref)
- r2 := it2.(*Ref)
- cmp := storage.CompareIDs(r1.ID, r2.ID)
- return cmp < 0 || (cmp == 0 && ComparePaths(r1.Path, r2.Path) < 0)
-}
-
-// compareRefsByPath compares refs using their Path.
-func compareRefsByPath(a, b interface{}) bool {
- return ComparePaths(a.(*Ref).Path, b.(*Ref).Path) < 0
-}
-
-// FlattenDir flattens the directory map into an association list.
-func FlattenDir(d Dir) []*raw.DEntry {
- l := make([]*raw.DEntry, d.Len())
- i := 0
- d.Iter(func(v interface{}) bool {
- r := v.(*Ref)
- l[i] = &raw.DEntry{ID: r.ID, Name: r.Path.hd}
- i++
- return true
- })
- return l
-}
-
-// BuildDir builds a Dir from the association list.
-func BuildDir(l []*raw.DEntry) Dir {
- d := EmptyDir
- for _, de := range l {
- d = d.Put(&Ref{ID: de.ID, Path: NewSingletonPath(de.Name)})
- }
- return d
-}
diff --git a/services/store/memstore/state/cell.go b/services/store/memstore/state/cell.go
deleted file mode 100644
index 0422713..0000000
--- a/services/store/memstore/state/cell.go
+++ /dev/null
@@ -1,92 +0,0 @@
-package state
-
-import (
- "bytes"
-
- "veyron/services/store/memstore/refs"
- "veyron/services/store/raw"
-
- // TODO(cnicolaou): mv lib/functional into veyron somewhere.
- "veyron/runtimes/google/lib/functional"
- "veyron/runtimes/google/lib/functional/rb"
- "veyron2/storage"
-)
-
-// cell represents an entry in the store. It is reference-counted, and it
-// contains the actual value.
-type Cell struct {
- // Unique ID for the cell.
- ID storage.ID
-
- // Value
- Value interface{}
-
- // Implicit directory.
- Dir refs.Set
-
- // refs includes the references in the value and the dir.
- //
- // TODO(jyh): This is simple, but it would be more space efficient to
- // include only the refs in the value, or drop this field entirely.
- refs refs.Set
-
- // inRefs contains all the incoming references -- that is, references
- // from other values to this one.
- inRefs refs.Set
-
- // TODO(jyh): The following fields can be packed into a single word.
- refcount uint
- color color
- buffered bool
-
- // version is the version number.
- Version raw.Version
-
- // TODO(jyh): Add stat info and attributes.
-}
-
-// cellSet is a functional map from storage.ID to *cell.
-type cellSet functional.Set
-
-var (
- emptyIDTable cellSet = rb.NewSet(compareCellsByID)
-)
-
-// compareCellsByID compares the two cells' IDs.
-func compareCellsByID(a, b interface{}) bool {
- return bytes.Compare(a.(*Cell).ID[:], b.(*Cell).ID[:]) < 0
-}
-
-// newSubfieldEntry returns a storage.Entry value, ignoring the Stat info.
-func newSubfieldEntry(v interface{}) *storage.Entry {
- return &storage.Entry{Value: deepcopy(v)}
-}
-
-// setRefs sets the cell's refs field.
-func (c *Cell) setRefs() {
- r := refs.NewBuilder()
- r.AddValue(c.Value)
- r.AddDir(c.Dir)
- c.refs = r.Get()
-}
-
-// get the *storage.Entry for a cell.
-func (c *Cell) GetEntry() *storage.Entry {
- entry := newSubfieldEntry(c.Value)
- c.fillStat(&entry.Stat)
- return entry
-}
-
-// get the *storage.Stat for a cell.
-func (c *Cell) getStat() *storage.Stat {
- var stat storage.Stat
- c.fillStat(&stat)
- return &stat
-}
-
-// fillStat fills the storage.Stat struct with the the cell's metadata. Assumes
-// stat is not nil.
-func (c *Cell) fillStat(stat *storage.Stat) {
- stat.ID = c.ID
- // TODO(jyh): Fill in the missing fields
-}
diff --git a/services/store/memstore/state/doc.go b/services/store/memstore/state/doc.go
deleted file mode 100644
index 85d89bf..0000000
--- a/services/store/memstore/state/doc.go
+++ /dev/null
@@ -1,40 +0,0 @@
-// Package state implements an in-memory version of the store state.
-// There are three main types here.
-//
-// Snapshot represents an isolated read-only copy of the state.
-// It supports only iteration.
-//
-// NewIterator() : returns an iterator to traverse the state.
-//
-// MutableSnapshot is an isolated read-write copy of the state. It supports the
-// standard dictionary operations.
-//
-// Read(path) : fetches a value from the snapshot.
-// Put(path, value) : stores a value in the snapshot.
-// Remove(path) : remove a value from the snapshot.
-// Mutations() : returns the set of mutations to the snapshot.
-//
-// State is the actual shared state, with the following methods.
-//
-// Snapshot() : returns a read-only snapshot of the state.
-// MutableSnapshot() : returns a read-write snapshot of the state.
-// ApplyMutations(mutations) : applies the mutations to the state.
-//
-// ApplyMutations can fail due to concurrency where the state was modified
-// (through another call to ApplyMutations) between the time that a
-// MutableSnapshot created to the time that it is applied.
-//
-// The sequence for performing an atomic change is to copy the snapshot,
-// make the changes on the copy, the apply the changes to the original state
-// atomically.
-//
-// // Swap /a/b/c and /d/e atomically.
-// st := New(...)
-// ...
-// sn := st.MutableSnapshot()
-// x, err := sn.Get("/a/b/c")
-// y, err := sn.Get("/d/e")
-// err = sn.Put("/d/e", x)
-// err = sn.Put("/a/b/c", y)
-// err = st.ApplyMutations(sn.Mutations())
-package state
diff --git a/services/store/memstore/state/gc.go b/services/store/memstore/state/gc.go
deleted file mode 100644
index 2642431..0000000
--- a/services/store/memstore/state/gc.go
+++ /dev/null
@@ -1,241 +0,0 @@
-package state
-
-// The store values include references based on pub.ID. We use a garbage
-// collection mechanism to remove values when they are no longer referenced.
-// There are two general techniques for garbage collection: 1) infrequent
-// tracing collection (like mark/sweep, etc.), and 2) reference counting.
-// Reference counting is more responsive but it is harder to deal with cycles.
-//
-// We provide an "instant GC" semantics, meaning that once a value has become
-// garbage, it is no longer returned as the result of any subsequent query.
-//
-// To satisfy the responsiveness requirement, we use a garbage collection scheme
-// based on reference counting. Cells are collected as soon as they become
-// garbage. This happens whenever the reference count becomes zero, but it can
-// also happen when the refcount is nonzero, but the cell is part of a garbage
-// cycle.
-//
-// The scheme here is based on the following.
-//
-// Concurrent Cycle Collection in Reference Counted Systems
-// David F. Bacon and V.T. Rajan
-// IBM T.J. Watson Research Center
-// Proceedings of the European Conference on Object-Oriented Programming
-// June 2001
-// http://researcher.watson.ibm.com/researcher/files/us-bacon/Bacon01Concurrent.pdf
-//
-// This implementation is based on the "synchronous" algorithm, not the
-// concurrent one.
-//
-// The central idea is that garbage collection is invoked when a refcount is
-// decremented. If the refcount becomes zero, then the cell is certainly
-// garbage, and it can be reclaimed. If the refcount is nonzero, then the cell
-// might still be garbage. The cell is added to a "gcRoots" table for later
-// analysis.
-//
-// When GC is forced (due to a new query, for example), the graph reachable from
-// the gcRoots is traversed to see if any of the cells are involved in garbage
-// cycles. The traversal decrements refcounts based on internal references
-// (references reachable through gcRoots). If any the the refcounts now become
-// zero, that means all references are internal and the cell is actually
-// garbage.
-//
-// That means the worst case GC cost is still linear in the number of elements
-// in the store. However, it is observed that in a large number of cases in
-// practice, refcounts are either 0 or 1, so the gcRoots mechanism is not
-// invoked.
-//
-// In our case, some directories will have multiple links, coming from
-// replication group directories for example. Whenver one of the replication
-// group links is removed, we'll have to invoke the garbage collector to
-// traverse the subgraph that was replicated. This is similar to the cost that
-// we would pay if the reachable subgraph became garbage, where we have to
-// delete all of the cells in the subgraph.
-//
-// To improve performance, we delay the traversal, using the gcRoots set, until
-// the GC is forced.
-import (
- "veyron/services/store/memstore/refs"
-
- "veyron2/storage"
-)
-
-// color is the garbage collection color of a cell.
-type color uint
-
-const (
- black color = iota // In use or free.
- gray // Possible member of cycle.
- white // Member of a garbage cycle.
- purple // Possible root of cycle.
-)
-
-// ref increments the reference count. The cell can't be garbage, so color it
-// black.
-func (sn *MutableSnapshot) ref(id storage.ID) {
- c := sn.deref(id)
- c.color = black
- c.refcount++
-}
-
-// unref decrements the reference count. If the count reaches 0, the cell is
-// garbage. Otherwise the cell might be part of a cycle, so add it to the
-// gcRoots.
-func (sn *MutableSnapshot) unref(id storage.ID) {
- c := sn.deref(id)
- if c.refcount == 0 {
- panic("unref(): refcount is already zero")
- }
- c.refcount--
- if c.refcount == 0 {
- sn.release(c)
- } else {
- sn.possibleRoot(c)
- }
-}
-
-// release deletes a cell once the reference count has reached zero. Delete the
-// cell and all of its references.
-func (sn *MutableSnapshot) release(c *Cell) {
- c.color = black
- c.refs.Iter(func(it interface{}) bool {
- sn.unref(it.(*refs.Ref).ID)
- return true
- })
- if !c.buffered {
- sn.delete(c)
- }
-}
-
-// possibleRoot marks the cell as the possible root of a cycle. The cell is
-// colored purple and added to the gcRoots.
-func (sn *MutableSnapshot) possibleRoot(c *Cell) {
- if c.color != purple {
- c.color = purple
- if !c.buffered {
- c.buffered = true
- sn.gcRoots[c.ID] = struct{}{}
- }
- }
-}
-
-// GC is called to perform a garbage collection.
-//
-// - markRoots traverses all of the cells reachable from the roots,
-// decrementing reference counts due to cycles.
-//
-// - scanRoots traverses the cells reachable from the roots, coloring white if
-// the refcount is zero, or coloring black otherwise. Note that a white
-// cell might be restored to black later if is reachable through some other
-// live path.
-//
-// - collectRoots removes all remaining white nodes, which are cyclic garbage.
-func (sn *MutableSnapshot) gc() {
- sn.markRoots()
- sn.scanRoots()
- sn.collectRoots()
-}
-
-// markRoots traverses the cells reachable from the roots, marking them gray.
-// If another root is encountered during a traversal, it is removed from the
-// gcRoots.
-func (sn *MutableSnapshot) markRoots() {
- for id, _ := range sn.gcRoots {
- c := sn.deref(id)
- if c.color == purple {
- sn.markGray(c)
- } else {
- c.buffered = false
- delete(sn.gcRoots, id)
- if c.color == black && c.refcount == 0 {
- sn.delete(c)
- }
- }
- }
-}
-
-// markGray colors a cell gray, the decrements the refcounts of the children,
-// and marks them gray recursively. The result is that counts for "internal"
-// references (references reachable from the roots) are removed. Then, if the
-// reference count of a root reaches zero, it must be reachable only from
-// internal references, so it is part of a garbage cycle, and it can be deleted.
-func (sn *MutableSnapshot) markGray(c *Cell) {
- if c.color == gray {
- return
- }
- c.color = gray
- c.refs.Iter(func(it interface{}) bool {
- id := it.(*refs.Ref).ID
- nc := sn.deref(id)
- nc.refcount--
- sn.markGray(nc)
- return true
- })
-}
-
-// scanRoots traverses the graph from gcRoots. If a cell has a non-zero
-// refcount, that means it is reachable from an external reference, so it is
-// live. In that case, the cell and reachable children are live. Otherwise,
-// the refcount is zero, and the cell is colored white to indicate that it may
-// be garbage.
-func (sn *MutableSnapshot) scanRoots() {
- for id, _ := range sn.gcRoots {
- sn.scan(sn.deref(id))
- }
-}
-
-// scan gray nodes, coloring them black if the refcount is nonzero, or white
-// otherwise.
-func (sn *MutableSnapshot) scan(c *Cell) {
- if c.color != gray {
- return
- }
- if c.refcount > 0 {
- sn.scanBlack(c)
- } else {
- c.color = white
- c.refs.Iter(func(it interface{}) bool {
- id := it.(*refs.Ref).ID
- sn.scan(sn.deref(id))
- return true
- })
- }
-}
-
-// scanBlack colors a cell and its subgraph black, restoring refcounts.
-func (sn *MutableSnapshot) scanBlack(c *Cell) {
- c.color = black
- c.refs.Iter(func(it interface{}) bool {
- id := it.(*refs.Ref).ID
- nc := sn.deref(id)
- nc.refcount++
- if nc.color != black {
- sn.scanBlack(nc)
- }
- return true
- })
-}
-
-// collectRoots frees all the cells that are colored white.
-func (sn *MutableSnapshot) collectRoots() {
- for id, _ := range sn.gcRoots {
- c := sn.deref(id)
- c.buffered = false
- sn.collectWhite(c)
- }
- sn.gcRoots = make(map[storage.ID]struct{})
-}
-
-// collectWhite frees cells that are colored white.
-func (sn *MutableSnapshot) collectWhite(c *Cell) {
- if c.color != white || c.buffered {
- return
- }
- c.color = black
- c.refs.Iter(func(it interface{}) bool {
- id := it.(*refs.Ref).ID
- sn.collectWhite(sn.deref(id))
- return true
- })
- sn.delete(c)
-}
diff --git a/services/store/memstore/state/iterator.go b/services/store/memstore/state/iterator.go
deleted file mode 100644
index e7c5275..0000000
--- a/services/store/memstore/state/iterator.go
+++ /dev/null
@@ -1,270 +0,0 @@
-package state
-
-import (
- "fmt"
-
- "veyron/services/store/memstore/refs"
-
- "veyron2/security"
- "veyron2/storage"
-)
-
-// Iterator is used to iterate through the descendents of a value. The order of
-// iteration is breadth-first. Each descendent is visited at most once.
-type Iterator interface {
- // IsValid returns true iff the iterator refers to an element.
- IsValid() bool
-
- // Get returns the current value.
- Get() *storage.Entry
-
- // Return one possible name for this entry.
- Name() string
-
- // Next advances to the next element.
- Next()
-
- // Snapshot returns the iterator's snapshot.
- Snapshot() Snapshot
-}
-
-// iterator implements the Iterator interface.
-//
-// TODO(jyh): This is a simple, naive implementation. We need to implement
-// security, perform type-directed iteration, and we may need pruning (similar
-// to the -prune option to the "find" command).
-type iterator struct {
- snapshot Snapshot
-
- // Set of IDs already visited on this path.
- visited map[storage.ID]struct{}
-
- // Stack of actions to consider next. Actions are one of:
- // - visit a node accessible from the current path (the node may already
- // have been visited on the current path).
- // - unvisit a node (backtrack the current path).
- next []next
-
- // Depth of starting path.
- initialDepth int
-
- // Current value.
- entry *storage.Entry
- path *refs.FullPath
-
- pathFilter PathFilter
-
- filter IterFilter
-}
-
-type next struct {
- parent *refs.FullPath
- path *refs.Path
- id storage.ID
- action action
-}
-
-type action int
-
-const (
- visit = action(iota)
- unvisit
-)
-
-var (
- _ Iterator = (*iterator)(nil)
- _ Iterator = (*errorIterator)(nil)
-)
-
-// A PathFilter automatically limits the traversal of certain paths,
-type PathFilter int
-
-const (
- // ListPaths permits any path that does not visit the same object twice.
- ListPaths = PathFilter(iota)
- // ListObjects permits any path that does not revisit any object on a
- // previously traversed path 'Q', even if Q did not satisfy it.filter.
- ListObjects
-)
-
-// An IterFilter examines entries as they are considered by the
-// iterator and allows it to give two boolean inputs to the process:
-// ret: True if the iterator should return this value in its iteration.
-// expand: True if the iterator should consider children of this value.
-type IterFilter func(*refs.FullPath, *refs.Path) (ret bool, expand bool)
-
-// Recursive filter is an IterFilter that causes all decendents to be
-// returned during iteration. This is the default behavior.
-func RecursiveFilter(*refs.FullPath, *refs.Path) (bool, bool) {
- return true, true
-}
-
-// ImmediateFilter is a filter that causes only the specified path and
-// immediate decendents to be returned from the iterator.
-func ImmediateFilter(_ *refs.FullPath, path *refs.Path) (bool, bool) {
- return true, path == nil
-}
-
-// NewIterator returns an Iterator that starts with the value at <path>.
-// pathFilter is used to automatically limit traversal of certain paths.
-// If filter is given, it is used to limit traversal beneath certain paths and
-// limit the results of the iteration. If filter is nil, all decendents of the
-// specified path are returned.
-func (sn *snapshot) NewIterator(pid security.PublicID, path storage.PathName,
- pathFilter PathFilter, filter IterFilter) Iterator {
-
- cell, suffix, v := sn.resolveCell(path, nil)
- if cell == nil {
- return &errorIterator{snapshot: sn}
- }
-
- if filter == nil {
- filter = RecursiveFilter
- }
-
- it := &iterator{
- snapshot: sn,
- visited: make(map[storage.ID]struct{}),
- initialDepth: len(path),
- path: refs.NewFullPathFromName(path),
- pathFilter: pathFilter,
- filter: filter,
- }
-
- ret, expand := it.filter(it.path, nil)
-
- var set refs.Set
- if len(suffix) != 0 {
- // We're started from a field within cell. Calculate the refs reachable
- // from the value. Allow self references to cell.id.
- it.entry = newSubfieldEntry(v)
- r := refs.NewBuilder()
- r.AddValue(v)
- set = r.Get()
- } else {
- it.entry = cell.GetEntry()
- it.visited[cell.ID] = struct{}{}
- it.pushUnvisit(nil, cell.ID)
- set = cell.refs
- }
-
- if expand {
- it.pushVisitAll(it.path, set)
- }
- if !ret {
- it.Next()
- }
-
- return it
-}
-
-func (it *iterator) pushUnvisit(path *refs.Path, id storage.ID) {
- switch it.pathFilter {
- case ListPaths:
- it.next = append(it.next, next{nil, path, id, unvisit})
- case ListObjects:
- // Do not unvisit the object, as it is on a path already seen by
- // it.filter.
- default:
- panic("unknown PathFilter")
- }
-}
-
-func (it *iterator) pushVisitAll(parentPath *refs.FullPath, set refs.Set) {
-
- set.Iter(func(x interface{}) bool {
- ref := x.(*refs.Ref)
- it.next = append(it.next, next{parentPath, ref.Path, ref.ID, visit})
- return true
- })
-}
-
-// IsValid returns true iff the iterator refers to an element.
-func (it *iterator) IsValid() bool {
- return it.entry != nil
-}
-
-// Name returns a name for the curent value relative to the initial name.
-func (it *iterator) Name() string {
- return it.path.Suffix(it.path.Len() - it.initialDepth)
-}
-
-// Get returns the current value.
-func (it *iterator) Get() *storage.Entry {
- return it.entry
-}
-
-// Next advances to the next element.
-func (it *iterator) Next() {
- var n next
- var fullPath *refs.FullPath
- var c *Cell
- for {
- topIndex := len(it.next) - 1
- if topIndex < 0 {
- it.entry, it.path = nil, nil
- return
- }
- n, it.next = it.next[topIndex], it.next[:topIndex]
-
- if n.action == unvisit {
- delete(it.visited, n.id)
- continue
- }
-
- if _, ok := it.visited[n.id]; ok {
- continue
- }
-
- // Mark as visited.
- it.visited[n.id] = struct{}{}
- it.pushUnvisit(n.path, n.id)
-
- // Fetch the cell.
- c = it.snapshot.Find(n.id)
- if c == nil {
- // The table is inconsistent.
- panic(fmt.Sprintf("Dangling reference: %s", n.id))
- }
-
- // Check the filter
- ret, expand := it.filter(n.parent, n.path)
- fullPath = n.parent.AppendPath(n.path)
- if expand {
- it.pushVisitAll(fullPath, c.refs)
- }
- if ret {
- // Found a value.
- break
- }
- }
-
- it.entry, it.path = c.GetEntry(), fullPath
-}
-
-func (it *iterator) Snapshot() Snapshot {
- return it.snapshot
-}
-
-// errorIterator is the iterator that does nothing, has no values.
-type errorIterator struct {
- snapshot Snapshot
-}
-
-func (it *errorIterator) IsValid() bool {
- return false
-}
-
-func (it *errorIterator) Get() *storage.Entry {
- return nil
-}
-
-func (it *errorIterator) Name() string {
- return ""
-}
-
-func (it *errorIterator) Next() {}
-
-func (it *errorIterator) Snapshot() Snapshot {
- return it.snapshot
-}
diff --git a/services/store/memstore/state/iterator_test.go b/services/store/memstore/state/iterator_test.go
deleted file mode 100644
index ffc4bfc..0000000
--- a/services/store/memstore/state/iterator_test.go
+++ /dev/null
@@ -1,165 +0,0 @@
-package state_test
-
-import (
- "runtime"
- "testing"
-
- "veyron/services/store/memstore/refs"
- "veyron/services/store/memstore/state"
- "veyron2/security"
- "veyron2/storage"
-)
-
-// check that the iterator produces a set of names.
-func checkAcyclicIterator(t *testing.T, sn *state.MutableSnapshot, id security.PublicID, filter state.IterFilter, names []string) {
- _, file, line, _ := runtime.Caller(1)
-
- // Construct an index of names.
- index := map[string]bool{}
- for _, name := range names {
- index[name] = false
- }
-
- // Compute the found names.
- for it := sn.NewIterator(id, storage.ParsePath("/"), state.ListPaths, filter); it.IsValid(); it.Next() {
- name := it.Name()
- if found, ok := index[name]; ok {
- if found {
- t.Errorf("%s(%d): duplicate name %q", file, line, name)
- }
- index[name] = true
- } else {
- t.Errorf("%s(%d): unexpected name %q", file, line, name)
- }
- }
-
- // Print the not found names.
- for name, found := range index {
- if !found {
- t.Errorf("%s(%d): expected: %v", file, line, name)
- }
- }
-}
-
-// check that the iterator produces a set of names. Since entries in the store
-// can have multiple names, the names are provided using a set of equivalence
-// classes. The requirement is that the iterator produces exactly one name from
-// each equivalence class. Order doesn't matter.
-func checkUniqueObjectsIterator(t *testing.T, sn *state.MutableSnapshot, id security.PublicID, filter state.IterFilter, names [][]string) {
- _, file, line, _ := runtime.Caller(1)
-
- // Construct an index of name to equivalence class.
- index := map[string]int{}
- for i, equiv := range names {
- for _, name := range equiv {
- index[name] = i
- }
- }
-
- // Compute the found set of equivalence classes.
- found := map[int]bool{}
- for it := sn.NewIterator(id, storage.ParsePath("/"), state.ListObjects, filter); it.IsValid(); it.Next() {
- name := it.Name()
- if i, ok := index[name]; ok {
- if _, ok := found[i]; ok {
- t.Errorf("%s(%d): duplicate name %q", file, line, name)
- }
- found[i] = true
- } else {
- t.Errorf("%s(%d): unexpected name %q", file, line, name)
- }
- }
-
- // Print the not found equivalence classes.
- for i, equiv := range names {
- if !found[i] {
- t.Errorf("%s(%d): expected one of: %v", file, line, equiv)
- }
- }
-}
-
-// Tests that an iterator returns all non-cyclic paths that reach an object.
-func TestDuplicatePaths(t *testing.T) {
- st := state.New(rootPublicID)
- sn := st.MutableSnapshot()
-
- // Add some objects
- put(t, sn, rootPublicID, "/", "")
- put(t, sn, rootPublicID, "/teams", "")
- put(t, sn, rootPublicID, "/teams/cardinals", "")
- put(t, sn, rootPublicID, "/players", "")
- mattID := put(t, sn, rootPublicID, "/players/matt", "")
-
- // Add some hard links
- put(t, sn, rootPublicID, "/teams/cardinals/mvp", mattID)
-
- checkAcyclicIterator(t, sn, rootPublicID, nil, []string{
- "",
- "teams",
- "players",
- "teams/cardinals",
- "players/matt",
- "teams/cardinals/mvp",
- })
- checkUniqueObjectsIterator(t, sn, rootPublicID, nil, [][]string{
- {""},
- {"teams"},
- {"players"},
- {"teams/cardinals"},
- {"players/matt", "teams/cardinals/mvp"},
- })
-
- // Test that the iterator does not revisit objects on previously rejected paths.
- rejected := false
- rejectMatt := func(fullPath *refs.FullPath, path *refs.Path) (bool, bool) {
- name := fullPath.Append(path.Suffix(1)).Name().String()
- if !rejected && (name == "players/matt" || name == "teams/cardinals/mvp") {
- rejected = true
- return false, true
- }
- return true, true
- }
- checkUniqueObjectsIterator(t, sn, rootPublicID, rejectMatt, [][]string{
- {""},
- {"teams"},
- {"players"},
- {"teams/cardinals"},
- })
-}
-
-// Test that an iterator doesn't get stuck in cycles.
-func TestCyclicStructure(t *testing.T) {
- st := state.New(rootPublicID)
- sn := st.MutableSnapshot()
-
- // Add some objects
- put(t, sn, rootPublicID, "/", "")
- put(t, sn, rootPublicID, "/teams", "")
- cardinalsID := put(t, sn, rootPublicID, "/teams/cardinals", "")
- put(t, sn, rootPublicID, "/players", "")
- mattID := put(t, sn, rootPublicID, "/players/matt", "")
- put(t, sn, rootPublicID, "/players/joe", "")
-
- // Add some hard links
- put(t, sn, rootPublicID, "/players/matt/team", cardinalsID)
- put(t, sn, rootPublicID, "/teams/cardinals/mvp", mattID)
-
- checkAcyclicIterator(t, sn, rootPublicID, nil, []string{
- "",
- "teams",
- "players",
- "players/joe",
- "players/matt",
- "teams/cardinals/mvp",
- "teams/cardinals",
- "players/matt/team",
- })
- checkUniqueObjectsIterator(t, sn, rootPublicID, nil, [][]string{
- {""},
- {"teams"},
- {"players"},
- {"players/joe"},
- {"players/matt", "teams/cardinals/mvp"},
- {"teams/cardinals", "players/matt/team"},
- })
-}
diff --git a/services/store/memstore/state/log.go b/services/store/memstore/state/log.go
deleted file mode 100644
index 426a751..0000000
--- a/services/store/memstore/state/log.go
+++ /dev/null
@@ -1,111 +0,0 @@
-package state
-
-import (
- "veyron/services/store/memstore/refs"
- "veyron/services/store/raw"
-
- "veyron2/storage"
- "veyron2/verror"
- "veyron2/vom"
-)
-
-type LogVersion uint32
-
-const (
- CurrentLogVersion LogVersion = 1
-)
-
-var (
- errUnsupportedLogVersion = verror.Internalf("unsupported log version")
-)
-
-// header contains the meta-information for a log file.
-type header struct {
- // Version is the version of the log file.
- Version LogVersion
-
- // RootID is the ID for the root value in the initial snapshot.
- RootID storage.ID
-
- // StateLen is the number of entries in the initial snapshot.
- StateLen uint32
-
- // Timestamp is the timestamp of the snapshot, in nanoseconds since the epoch.
- Timestamp uint64
-}
-
-// value is a single entry in the initial state snapshot. It corresponds to the
-// <cell> type.
-type value struct {
- ID storage.ID
- Value interface{}
- Dir []*raw.DEntry
- Version raw.Version
-}
-
-func (st *State) Write(enc *vom.Encoder) error {
- // Write the header.
- sn := st.snapshot
- h := header{
- Version: CurrentLogVersion,
- StateLen: uint32(sn.idTable.Len()),
- RootID: sn.rootID,
- Timestamp: st.timestamp,
- }
- if err := enc.Encode(&h); err != nil {
- return err
- }
-
- // Write the values.
- var err error
- sn.idTable.Iter(func(it interface{}) bool {
- c := it.(*Cell)
- v := value{ID: c.ID, Value: c.Value, Dir: refs.FlattenDir(c.Dir), Version: c.Version}
- err = enc.Encode(v)
- return err == nil
- })
- return err
-}
-
-func (st *State) Read(dec *vom.Decoder) error {
- sn := st.snapshot
-
- var header header
- if err := dec.Decode(&header); err != nil {
- return err
- }
- if header.Version != CurrentLogVersion {
- return errUnsupportedLogVersion
- }
-
- // Create the state without refcounts.
- t := emptyIDTable
- for i := uint32(0); i < header.StateLen; i++ {
- var v value
- if err := dec.Decode(&v); err != nil {
- return err
- }
- d := refs.BuildDir(v.Dir)
-
- // Calculate refs.
- r := refs.NewBuilder()
- r.AddValue(v.Value)
- r.AddDir(d)
-
- // Add the cell.
- c := &Cell{ID: v.ID, Value: v.Value, Dir: d, Version: v.Version, refs: r.Get(), inRefs: refs.Empty}
- t = t.Put(c)
- }
-
- sn.idTable = t
- sn.rootID = header.RootID
- st.timestamp = header.Timestamp
-
- // Update refcounts.
- t.Iter(func(it interface{}) bool {
- c := it.(*Cell)
- sn.addRefs(c.ID, c.refs)
- return true
- })
- return nil
-}
diff --git a/services/store/memstore/state/log_test.go b/services/store/memstore/state/log_test.go
deleted file mode 100644
index 54708b0..0000000
--- a/services/store/memstore/state/log_test.go
+++ /dev/null
@@ -1,176 +0,0 @@
-package state
-
-import (
- "io/ioutil"
- "os"
- "path/filepath"
- "runtime"
- "testing"
-
- "veyron/services/store/memstore/refs"
-
- "veyron/runtimes/google/lib/functional"
- "veyron2/vom"
-)
-
-type Data struct {
- Comment string
-}
-
-func init() {
- vom.Register(&Data{})
-}
-
-func newData(c string) *Data {
- return &Data{Comment: c}
-}
-
-func expectEqDir(t *testing.T, file string, line int, d1, d2 *Dir) {
- for name, id1 := range d1.Entries {
- id2, ok := d2.Entries[name]
- if !ok {
- t.Errorf("%s(%d): does not exist: %s", file, line, name)
- } else if id2 != id1 {
- t.Errorf("%s(%d): expected ID %s, got %s", file, line, id1, id2)
- }
- }
- for name, _ := range d2.Entries {
- _, ok := d1.Entries[name]
- if !ok {
- t.Errorf("%s(%d): should not exist: %s", file, line, name)
- }
- }
-}
-
-func expectEqData(t *testing.T, file string, line int, d1, d2 *Data) {
- if d1.Comment != d2.Comment {
- t.Errorf("%s(%d): expected %q, got %q", d1.Comment, d2.Comment)
- }
-}
-
-// expectEqValues compares two items. They are equal only if they have the same
-// type, and their contents are equal.
-func expectEqValues(t *testing.T, file string, line int, v1, v2 interface{}) {
- switch x1 := v1.(type) {
- case *Dir:
- x2, ok := v2.(*Dir)
- if !ok {
- t.Errorf("%s(%d): not a Dir: %v", file, line, v2)
- } else {
- expectEqDir(t, file, line, x1, x2)
- }
- case *Data:
- x2, ok := v2.(*Data)
- if !ok {
- t.Errorf("%s(%d): not a Data: %v", file, line, v2)
- } else {
- expectEqData(t, file, line, x1, x2)
- }
- default:
- t.Errorf("Unknown type: %T, %v", v1, v1)
- }
-}
-
-// expectEqImplicitDir compares two directories.
-func expectEqImplicitDir(t *testing.T, file string, line int, d1, d2 refs.Dir) {
- l1 := refs.FlattenDir(d1)
- l2 := refs.FlattenDir(d2)
- i1 := 0
- i2 := 0
- for i1 < len(l1) && i2 < len(l2) {
- e1 := l1[i1]
- e2 := l2[i2]
- if e1.Name == e2.Name {
- if e1.ID != e2.ID {
- t.Errorf("%s(%d): expected id %s, got %s", file, line, e1.ID, e2.ID)
- }
- i1++
- i2++
- } else if e1.Name < e2.Name {
- t.Errorf("%s(%d): missing directory %s", file, line, e1.Name)
- i1++
- } else {
- t.Errorf("%s(%d): unexpected directory %s", file, line, e2.Name)
- i2++
- }
- }
- for _, e1 := range l1[i1:] {
- t.Errorf("%s(%d): missing directory %s", file, line, e1.Name)
- }
- for _, e2 := range l2[i2:] {
- t.Errorf("%s(%d): unexpected directory %s", file, line, e2.Name)
- }
-}
-
-// expectEqCells compares two *cell values.
-func expectEqCells(t *testing.T, file string, line int, c1, c2 *Cell) {
- expectEqValues(t, file, line, c1.Value, c2.Value)
- expectEqImplicitDir(t, file, line, c1.Dir, c2.Dir)
-}
-
-// expectEqIDTables compares two states.
-func expectEqIDTables(t *testing.T, t1, t2 functional.Set) {
- _, file, line, _ := runtime.Caller(1)
- t1.Iter(func(it1 interface{}) bool {
- c1 := it1.(*Cell)
- it2, ok := t2.Get(c1)
- if !ok {
- t.Errorf("%s(%d): cell does not exist: %v", file, line, c1)
- } else {
- c2 := it2.(*Cell)
- expectEqCells(t, file, line, c1, c2)
- }
- return true
- })
- t2.Iter(func(it2 interface{}) bool {
- _, ok := t1.Get(it2)
- if !ok {
- t.Errorf("%s(%d): cell should not exist: %v", file, line, it2)
- }
- return true
- })
-}
-
-func TestState(t *testing.T) {
- dbName, err := ioutil.TempDir(os.TempDir(), "store")
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
- defer os.Remove(dbName)
-
- dbFile := filepath.Join(dbName, "db")
- ofile, err := os.Create(dbFile)
- if err != nil {
- t.Fatalf("Error opening log file: %s", err)
- }
- defer ofile.Close()
- enc := vom.NewEncoder(ofile)
-
- // Create an initial state.
- st1 := New(rootPublicID)
- sn := st1.MutableSnapshot()
- mkdir(t, sn, "/")
- mkdir(t, sn, "/a")
- mkdir(t, sn, "/a/b")
- mkdir(t, sn, "/a/b/c")
- mkdir(t, sn, "/a/b/c/d")
- if err := st1.ApplyMutations(sn.Mutations()); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- if err := st1.Write(enc); err != nil {
- t.Errorf("Error writing log: %s", err)
- }
-
- ifile, err := os.Open(dbFile)
- if err != nil {
- t.Fatalf("Error opening log file: %s")
- }
- defer ifile.Close()
- dec := vom.NewDecoder(ifile)
- st2 := New(rootPublicID)
- if err := st2.Read(dec); err != nil {
- t.Fatalf("Can't read state: %s", err)
- }
-
- expectEqIDTables(t, st1.snapshot.idTable, st2.snapshot.idTable)
-}
diff --git a/services/store/memstore/state/mutable_snapshot.go b/services/store/memstore/state/mutable_snapshot.go
deleted file mode 100644
index 2ac285b..0000000
--- a/services/store/memstore/state/mutable_snapshot.go
+++ /dev/null
@@ -1,492 +0,0 @@
-package state
-
-import (
- "fmt"
-
- "veyron/services/store/memstore/field"
- "veyron/services/store/memstore/refs"
- "veyron/services/store/raw"
-
- "veyron/runtimes/google/lib/functional"
- "veyron2/security"
- "veyron2/storage"
- "veyron2/verror"
-)
-
-// MutableSnapshot is a mutable version of the snapshot. It contains a Snapshot
-// and a Mutations set.
-//
-// Reference counting is used to collect garbage that is no longer reachable
-// using a pathname. References can be cyclic, so the reference counting
-// includes cycle detection.
-//
-// References never dangle. This restricts what operations can be performed.
-// It isn't allowed to add an object to the state that has a dangling reference;
-// so if you want to set up a cycle atomically, you should add the objects to
-// the state without references then mutate the objects to add the cyclic
-// references. This can be done in a single transaction, so that the
-// intermediate state is not observable.
-//
-// TODO(jyh): Alternatively, we could relax the object operations so that
-// objects can be added to a transaction with dangling references. The
-// references would still be checked at Commit time, aborting the transaction if
-// dangling references are detected. However, it would mean that intermediate
-// states in the transaction would be inconsistent. This might be fine, but we
-// should decide whether transaction operations like Search() are allowed on
-// these inconsistent states while a transaction is being constructed. In the
-// meantime, we keep the strict approach, where intermediate states are
-// consistent.
-type MutableSnapshot struct {
- snapshot
-
- // gcRoots contains the nodes that should be considered for garbage
- // collection.
- gcRoots map[storage.ID]struct{}
-
- // mutations is the current set of changes.
- mutations *Mutations
-
- // deletions is the current set of deletions. The version is at
- // the point of deletion.
- deletions map[storage.ID]raw.Version
-}
-
-// Mutations represents a set of mutations to the state. This is used to
-// collect the operations in a transaction.
-type Mutations struct {
- // Timestamp corresponds to the time that the mutations were applied to the
- // state. It is set when applyMutations() was called. The value is based
- // on Unix time, the number of nanoseconds elapsed since January 1, 1970
- // UTC. However, it is monotonically increasing so that subsequent
- // mutations have increasing timestamps that differ by at least one.
- Timestamp uint64
-
- // RootID is the storage.ID of the root value. Valid only if SetRootID is true.
- RootID storage.ID
- SetRootID bool
-
- // Preconditions is the set of expected versions.
- Preconditions map[storage.ID]raw.Version
-
- // Delta is the set of changes.
- Delta map[storage.ID]*Mutation
-
- // Deletions contains the IDs for values that have been deleted. The
- // version is taken from the time of deletion. It is like a weak
- // precondition, where *if* the value exists, it should have the specified
- // version. The target snapshot is allowed to perform garbage collection
- // too, so the deleted value is not required to exist.
- Deletions map[storage.ID]raw.Version
-}
-
-// mutation is an update to a single value in the state.
-type Mutation struct {
- // Postcondition is the version after the mutation.
- Postcondition raw.Version
-
- // Value is the new value.
- Value interface{}
-
- // Dir is the set of new directory entries.
- //
- // TODO(jyh): Replace this with a delta, to support large directories.
- Dir []*raw.DEntry
-
- // Refs are the set of references in the Value and Dir.
- refs refs.Set
-}
-
-var (
- errBadPath = verror.BadArgf("malformed path")
- errBadRef = verror.BadArgf("value has dangling references")
- errBadValue = verror.BadArgf("value has the wrong type")
- errDuplicatePutMutation = verror.BadArgf("duplicate calls to PutMutation for the same ID")
- errNotFound = verror.NotFoundf("not found")
- errPreconditionFailed = verror.Abortedf("precondition failed")
-
- nullID storage.ID
-)
-
-// newMutations returns a fresh Mutations set.
-func newMutations() *Mutations {
- var m Mutations
- m.reset()
- return &m
-}
-
-// reset resets the Mutations state.
-func (m *Mutations) reset() {
- m.Preconditions = make(map[storage.ID]raw.Version)
- m.Delta = make(map[storage.ID]*Mutation)
- m.Deletions = make(map[storage.ID]raw.Version)
-}
-
-// addPrecondition adds a precondition if it does not already exisn.
-func (m *Mutations) addPrecondition(c *Cell) {
- // Set the precondition if not already set. For cells that have been
- // created in the current Mutations/transaction, the value store in
- // m.Preconditions[c.id] will be zero, but c.version is the initial non-zero
- // version number, so we guard against the override.
- if _, ok := m.Preconditions[c.ID]; !ok {
- m.Preconditions[c.ID] = c.Version
- }
-}
-
-// UpdateRefs updates the refs field in the Mutation.
-func (m *Mutation) UpdateRefs() {
- r := refs.NewBuilder()
- r.AddValue(m.Value)
- r.AddDEntries(m.Dir)
- m.refs = r.Get()
-}
-
-// newSnapshot returns an empty snapshot.
-func newMutableSnapshot(admin security.PublicID) *MutableSnapshot {
- return &MutableSnapshot{
- snapshot: newSnapshot(admin),
- gcRoots: make(map[storage.ID]struct{}),
- mutations: newMutations(),
- deletions: make(map[storage.ID]raw.Version),
- }
-}
-
-// Mutations returns the set of mutations in the snapshot.
-func (sn *MutableSnapshot) Mutations() *Mutations {
- return sn.mutations
-}
-
-// GetSnapshot create a readonly copy of the snapshot.
-func (sn *MutableSnapshot) GetSnapshot() Snapshot {
- // Perform a GC to clear out gcRoots.
- sn.gc()
- cp := sn.snapshot
- return &cp
-}
-
-// deepCopy creates a copy of the snapshot. Mutations to the copy do not affect
-// the original, and vice versa.
-func (sn *MutableSnapshot) deepCopy() *MutableSnapshot {
- // Perform a GC to clear out gcRoots.
- sn.gc()
- cp := *sn
- cp.mutations = newMutations()
- cp.gcRoots = make(map[storage.ID]struct{})
- return &cp
-}
-
-// deref performs a lookup based on storage.ID, panicing if the cell is not found.
-// This is used internally during garbage collection when we can assume that
-// there are no dangling references.
-func (sn *MutableSnapshot) deref(id storage.ID) *Cell {
- c := sn.Find(id)
- if c == nil {
- panic(fmt.Sprintf("Dangling reference: %s", id))
- }
-
- // Copy the cell to ensure the original state is not modified.
- //
- // TODO(jyh): This can be avoided if the cell has already been copied in the
- // current transaction.
- cp := *c
- sn.idTable = sn.idTable.Put(&cp)
- return &cp
-}
-
-// delete removes the cell from the state.
-func (sn *MutableSnapshot) delete(c *Cell) {
- sn.idTable = sn.idTable.Remove(c)
- sn.deletions[c.ID] = c.Version
-}
-
-// put adds a cell to the state, also adding the new value to the Mutations set.
-func (sn *MutableSnapshot) put(c *Cell) {
- mu := sn.mutations
- d := refs.FlattenDir(c.Dir)
- m, ok := mu.Delta[c.ID]
- if ok {
- m.Value = c.Value
- m.refs = c.refs
- m.Dir = d
- } else {
- mu.Preconditions[c.ID] = c.Version
- m = &Mutation{
- Postcondition: raw.NewVersion(),
- Value: c.Value,
- Dir: d,
- refs: c.refs,
- }
- mu.Delta[c.ID] = m
- }
- c.Version = m.Postcondition
- sn.idTable = sn.idTable.Put(c)
-}
-
-// add adds a new Value to the state, updating reference counts. Fails if the
-// new value contains dangling references.
-func (sn *MutableSnapshot) add(id storage.ID, v interface{}) (*Cell, error) {
- c := sn.Find(id)
- if c == nil {
- // There is no current value, so create a new cell for the value and add
- // it.
- c = &Cell{
- ID: id,
- refcount: 0,
- Value: v,
- Dir: refs.EmptyDir,
- inRefs: refs.Empty,
- Version: raw.NoVersion,
- }
- c.setRefs()
- if !sn.refsExist(c.refs) {
- return nil, errBadRef
- }
- sn.put(c)
- sn.addRefs(id, c.refs)
- return c, nil
- }
-
- // There is already a value in the state, so replace it with the new value.
- return sn.replaceValue(c, v)
-}
-
-// replaceValue updates the cell.value.
-func (sn *MutableSnapshot) replaceValue(c *Cell, v interface{}) (*Cell, error) {
- cp := *c
- cp.Value = v
- cp.setRefs()
- if !sn.refsExist(cp.refs) {
- return nil, errBadRef
- }
- sn.put(&cp)
- sn.updateRefs(c.ID, c.refs, cp.refs)
- return &cp, nil
-}
-
-// replaceDir updates the cell.dir.
-func (sn *MutableSnapshot) replaceDir(c *Cell, d functional.Set) (*Cell, error) {
- cp := *c
- cp.Dir = d
- cp.setRefs()
- if !sn.refsExist(cp.refs) {
- return nil, errBadRef
- }
- sn.put(&cp)
- sn.updateRefs(c.ID, c.refs, cp.refs)
- return &cp, nil
-}
-
-// Get returns the value for a path.
-func (sn *MutableSnapshot) Get(pid security.PublicID, path storage.PathName) (*storage.Entry, error) {
- cell, suffix, v := sn.resolveCell(path, sn.mutations)
- if cell == nil {
- return nil, errNotFound
- }
- var e *storage.Entry
- if len(suffix) == 0 {
- e = cell.GetEntry()
- } else {
- e = newSubfieldEntry(v)
- }
- return e, nil
-}
-
-// Put adds a new value to the state or replaces an existing one. Returns
-// the *Stat for the enclosing *cell.
-func (sn *MutableSnapshot) Put(pid security.PublicID, path storage.PathName, v interface{}) (*storage.Stat, error) {
- c, err := sn.putValueByPath(path, v)
- if err != nil {
- return nil, err
- }
- return c.getStat(), nil
-}
-
-func (sn *MutableSnapshot) putValueByPath(path storage.PathName, v interface{}) (*Cell, error) {
- v = deepcopy(v)
-
- if path.IsRoot() {
- return sn.putRoot(v)
- }
- return sn.putValue(path, v)
-}
-
-// putValue is called for a normal Put() operation, where a new value is being
-// added, and as a consequence the containing "parent" value is being modified.
-// There are two cases: 1) the value <v> is written directly into the parent, or
-// 2) the field has type storage.ID. In the latter case, the <id> is assigned
-// into the parent, and the value id->v is added to the idTable.
-func (sn *MutableSnapshot) putValue(path storage.PathName, v interface{}) (*Cell, error) {
- // Find the parent object.
- c, suffix, _ := sn.resolveCell(path[:len(path)-1], sn.mutations)
- if c == nil {
- return nil, errNotFound
- }
- value := deepcopy(c.Value)
- p, s := field.Get(makeInnerReference(value), suffix)
- if len(s) != 0 {
- return nil, errNotFound
- }
-
- // Add value to the parent.
- name := path[len(path)-1]
- result, id := field.Set(p, name, v)
- switch result {
- case field.SetFailedNotFound:
- if len(suffix) != 0 {
- return nil, errNotFound
- }
- return sn.putDirEntry(c, name, v)
- case field.SetFailedWrongType:
- return nil, errBadValue
- case field.SetAsID:
- nc, err := sn.add(id, v)
- if err != nil {
- return nil, err
- }
- // The sn.add may have modified the cell, so fetch it again.
- if _, err = sn.replaceValue(sn.Find(c.ID), value); err != nil {
- return nil, err
- }
- return nc, nil
- case field.SetAsValue:
- return sn.replaceValue(c, value)
- }
- panic("not reached")
-}
-
-// putDirEntry replaces or adds a directory entry.
-func (sn *MutableSnapshot) putDirEntry(c *Cell, name string, v interface{}) (*Cell, error) {
- r := &refs.Ref{Path: refs.NewSingletonPath(name)}
- if id, ok := v.(storage.ID); ok {
- ncell := sn.Find(id)
- if ncell == nil {
- return nil, errNotFound
- }
- r.ID = id
- dir := c.Dir.Put(r)
- if _, err := sn.replaceDir(c, dir); err != nil {
- return nil, err
- }
- return ncell, nil
- }
-
- x, ok := c.Dir.Get(r)
- if !ok {
- // The entry does not exist yet; create it.
- id := storage.NewID()
- ncell, err := sn.add(id, v)
- if err != nil {
- return nil, err
- }
- r.ID = id
- // The sn.add may have modified the cell, so fetch it again.
- c = sn.Find(c.ID)
- dir := c.Dir.Put(r)
- if _, err := sn.replaceDir(c, dir); err != nil {
- return nil, err
- }
- return ncell, nil
- }
-
- // Replace the existing value.
- return sn.add(x.(*refs.Ref).ID, v)
-}
-
-// putRoot replaces the root.
-func (sn *MutableSnapshot) putRoot(v interface{}) (*Cell, error) {
- id := sn.rootID
- c := sn.Find(id)
- if c == nil {
- id = storage.NewID()
- }
-
- // Add the new element.
- ncell, err := sn.add(id, v)
- if err != nil {
- return nil, err
- }
-
- // Redirect the rootID.
- if c == nil {
- sn.ref(id)
- sn.rootID = id
- sn.mutations.RootID = id
- sn.mutations.SetRootID = true
- }
- return ncell, nil
-}
-
-// Remove removes a value.
-func (sn *MutableSnapshot) Remove(pid security.PublicID, path storage.PathName) error {
- if path.IsRoot() {
- sn.unref(sn.rootID)
- sn.rootID = nullID
- sn.mutations.RootID = nullID
- sn.mutations.SetRootID = true
- return nil
- }
-
- // Split the names into directory and field parts.
- cell, suffix, _ := sn.resolveCell(path[:len(path)-1], sn.mutations)
- if cell == nil {
- return errNotFound
- }
-
- // Remove the field.
- name := path[len(path)-1]
- r := &refs.Ref{Path: refs.NewSingletonPath(name)}
- if cell.Dir.Contains(r) {
- _, err := sn.replaceDir(cell, cell.Dir.Remove(r))
- return err
- }
- value := deepcopy(cell.Value)
- p, _ := field.Get(value, suffix)
- if !field.Remove(p, name) {
- return errNotFound
- }
-
- _, err := sn.replaceValue(cell, value)
- return err
-}
-
-// PutMutation puts an externally constructed mutation. Does not update cells
-// or refs, so regular Puts, Gets and Removes may be inconsistent.
-func (sn *MutableSnapshot) PutMutation(extmu raw.Mutation) error {
- mus := sn.mutations
- id := extmu.ID
- // Check that a mutation has not already been put for this id.
- if _, ok := mus.Delta[id]; ok {
- return errDuplicatePutMutation
- }
- // If the object has no version, it was deleted.
- if extmu.Version == raw.NoVersion {
- mus.Deletions[id] = extmu.PriorVersion
- if extmu.IsRoot {
- mus.SetRootID = true
- mus.RootID = nullID
- }
- return nil
- }
- if extmu.IsRoot {
- mus.SetRootID = true
- mus.RootID = id
- }
- mus.Preconditions[id] = extmu.PriorVersion
- mu := &Mutation{
- Postcondition: extmu.Version,
- Value: extmu.Value,
- Dir: unflattenDir(extmu.Dir),
- }
- mu.UpdateRefs()
- mus.Delta[id] = mu
- return nil
-}
-
-// TODO(tilaks): revisit when vsync.Mutation.Dir is of type []*raw.DEntry
-// (once we support optional structs in the idl).
-func unflattenDir(fdir []raw.DEntry) []*raw.DEntry {
- pdir := make([]*raw.DEntry, len(fdir))
- for i, _ := range fdir {
- pdir[i] = &fdir[i]
- }
- return pdir
-}
diff --git a/services/store/memstore/state/mutable_snapshot_test.go b/services/store/memstore/state/mutable_snapshot_test.go
deleted file mode 100644
index df567ba..0000000
--- a/services/store/memstore/state/mutable_snapshot_test.go
+++ /dev/null
@@ -1,383 +0,0 @@
-package state
-
-import (
- "runtime"
- "testing"
-
- "veyron/services/store/memstore/refs"
-
- "veyron2/storage"
- "veyron2/verror"
- "veyron2/vom"
-)
-
-// Dir is a simple directory.
-type Dir struct {
- Entries map[string]storage.ID
-}
-
-// Value is a simple value.
-type Value struct {
- X int
-}
-
-// Nest is a struct that nests a Value.
-type Nest struct {
- V Value
-}
-
-var (
- root = &Dir{}
- rootPath = storage.ParsePath("/")
-)
-
-func init() {
- vom.Register(&Dir{})
-}
-
-func mkdir(t *testing.T, sn *MutableSnapshot, path string) (storage.ID, interface{}) {
- _, file, line, _ := runtime.Caller(1)
- dir := &Dir{}
- stat, err := sn.Put(rootPublicID, storage.ParsePath(path), dir)
- if err != nil || stat == nil {
- t.Errorf("%s(%d): mkdir %s: %s", file, line, path, err)
- return storage.ID{}, dir
- }
- m, ok := sn.mutations.Delta[stat.ID]
- if !ok {
- t.Errorf("%s(%d): Expected Mutation: %v %v", file, line, stat, sn.mutations)
- } else if _, ok := m.Value.(*Dir); !ok {
- t.Fatalf("%s(%d): %s: not a directory: %v -> %v", file, line, path, stat, m.Value)
- }
- return stat.ID, dir
-}
-
-func expectExists(t *testing.T, sn *MutableSnapshot, path string, id storage.ID) {
- _, file, line, _ := runtime.Caller(1)
- if !sn.idTable.Contains(&Cell{ID: id}) {
- t.Errorf("%s(%d): does not exist: %s", file, line, id)
- }
- e, err := sn.Get(rootPublicID, storage.ParsePath(path))
- if err != nil {
- t.Errorf("%s(%d): does not exist: %s", file, line, path)
- }
- if e.Stat.ID != id {
- t.Errorf("%s(%d): expected id to be %v, but was %v", file, line, id, e.Stat.ID)
- }
-}
-
-func expectNotExists(t *testing.T, sn *MutableSnapshot, id storage.ID) {
- if sn.idTable.Contains(&Cell{ID: id}) {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): should not exist: %s", file, line, id)
- }
-}
-
-func expectValue(t *testing.T, sn *MutableSnapshot, path string, v interface{}) {
- _, file, line, _ := runtime.Caller(1)
- cell, _, _ := sn.resolveCell(storage.ParsePath(path), nil)
- if cell == nil {
- t.Errorf("%s(%d): path does not exist: %s", file, line, path)
- }
- if cell.Value == nil {
- t.Errorf("%s(%d): cell has a nil value: %s", file, line, path)
- }
-}
-
-func checkInRefs(t *testing.T, sn *MutableSnapshot) {
- _, file, line, _ := runtime.Caller(1)
-
- sn.idTable.Iter(func(it interface{}) bool {
- c1 := it.(*Cell)
-
- // Check that each out-ref has an in-ref.
- c1.refs.Iter(func(it interface{}) bool {
- r := it.(*refs.Ref)
- c2 := sn.Find(r.ID)
- if c2 == nil {
- t.Errorf("%s(%d): dangling reference: %s", file, line, r.ID)
- } else if !c2.inRefs.Contains(&refs.Ref{ID: c1.ID, Path: r.Path}) {
- t.Errorf("%s(%d): inRef does not exist: %s <- %s", file, line, c1.ID, c2.ID)
- }
- return true
- })
-
- // Check that each in-ref has an out-ref.
- c1.inRefs.Iter(func(it interface{}) bool {
- r := it.(*refs.Ref)
- c2 := sn.Find(r.ID)
- if c2 == nil {
- t.Errorf("%s(%d): dangling reference: %s", file, line, r.ID)
- } else if !c2.refs.Contains(&refs.Ref{ID: c1.ID, Path: r.Path}) {
- t.Errorf("%s(%d): inRef does not exist: %s -> %s", file, line, c2.ID, c1.ID)
- }
- return true
- })
- return true
- })
-}
-
-// Set up a root directory.
-func TestRoot(t *testing.T) {
- sn := newMutableSnapshot(rootPublicID)
-
- // There should be no root.
- v, err := sn.Get(rootPublicID, rootPath)
- if v != nil {
- t.Errorf("Expected nil for /: %v", v)
- }
- if err == nil {
- t.Errorf("Expected error")
- }
-
- // Add the root object.
- stat, err := sn.Put(rootPublicID, rootPath, root)
- if err != nil {
- t.Errorf("Error adding root: %s", err)
- }
- if sn.mutations.RootID != stat.ID {
- t.Errorf("Expected root update")
- }
- {
- p, ok := sn.mutations.Preconditions[sn.mutations.RootID]
- if !ok {
- t.Errorf("Error fetching root")
- }
- if p != 0 {
- t.Errorf("Expected 0 precondition: %d", p)
- }
- }
-
- // Fetch the root object, and compare.
- v, err = sn.Get(rootPublicID, rootPath)
- if err != nil {
- t.Errorf("Error fetching root: %s", err)
- }
-
- checkInRefs(t, sn)
-}
-
-// Make a directory tree.
-func TestDirTree(t *testing.T) {
- sn := newMutableSnapshot(rootPublicID)
-
- id1, d1 := mkdir(t, sn, "/")
- id2, d2 := mkdir(t, sn, "/Entries/a")
- id3, d3 := mkdir(t, sn, "/Entries/a/Entries/b")
- id4, d4 := mkdir(t, sn, "/Entries/a/Entries/b/Entries/c")
- id5, d5 := mkdir(t, sn, "/Entries/a/Entries/b/Entries/d")
- expectExists(t, sn, "/", id1)
- expectExists(t, sn, "/Entries/a", id2)
- expectExists(t, sn, "/Entries/a/Entries/b", id3)
- expectExists(t, sn, "/Entries/a/Entries/b/Entries/c", id4)
- expectExists(t, sn, "/Entries/a/Entries/b/Entries/d", id5)
-
- // Parent directory has to exist.
- d := &Dir{}
- if _, err := sn.Put(rootPublicID, storage.ParsePath("/a/c/e"), d); err == nil {
- t.Errorf("Expected error")
- }
-
- expectValue(t, sn, "/", d1)
- expectValue(t, sn, "/Entries/a", d2)
- expectValue(t, sn, "/Entries/a/Entries/b", d3)
- expectValue(t, sn, "/Entries/a/Entries/b/Entries/c", d4)
- expectValue(t, sn, "/Entries/a/Entries/b/Entries/d", d5)
- checkInRefs(t, sn)
-
- // Remove part of the tree.
- if err := sn.Remove(rootPublicID, storage.ParsePath("/Entries/a/Entries/b")); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- sn.gc()
- expectExists(t, sn, "/", id1)
- expectExists(t, sn, "/Entries/a", id2)
- expectNotExists(t, sn, id3)
- expectNotExists(t, sn, id4)
- expectNotExists(t, sn, id5)
- checkInRefs(t, sn)
-}
-
-// Make some references.
-func TestRef(t *testing.T) {
- sn := newMutableSnapshot(rootPublicID)
-
- rootID, _ := mkdir(t, sn, "/")
- ePath := storage.ParsePath("/Entries/a")
-
- // Not possible to create a Dir with a dangling reference.
- d := &Dir{Entries: map[string]storage.ID{"ref": storage.NewID()}}
- if _, err := sn.Put(rootPublicID, ePath, d); !verror.Is(err, verror.BadArg) {
- t.Errorf("Error should be %v: got %v", verror.BadArg, err)
- }
-
- // Set the Ref to refer to the root.
- d.Entries["ref"] = rootID
- stat, err := sn.Put(rootPublicID, ePath, d)
- if err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- expectExists(t, sn, "/Entries/a", stat.ID)
- checkInRefs(t, sn)
-
- // Change the ref to refer to itself.
- d.Entries["ref"] = stat.ID
- if stat2, err := sn.Put(rootPublicID, ePath, d); err != nil || stat2.ID != stat.ID {
- t.Errorf("Unexpected error: %s", err)
- }
- sn.gc()
- expectExists(t, sn, "/Entries/a", stat.ID)
- checkInRefs(t, sn)
-
- // Remove it.
- if err := sn.Remove(rootPublicID, ePath); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- sn.gc()
- expectNotExists(t, sn, stat.ID)
- checkInRefs(t, sn)
-}
-
-// Make an implicit directory tree.
-func TestImplicitDirTree(t *testing.T) {
- sn := newMutableSnapshot(rootPublicID)
-
- id1, d1 := mkdir(t, sn, "/")
- id2, d2 := mkdir(t, sn, "/a")
- id3, d3 := mkdir(t, sn, "/a/b")
- id4, d4 := mkdir(t, sn, "/a/b/c")
- id5, d5 := mkdir(t, sn, "/a/b/c/d")
- expectExists(t, sn, "/", id1)
- expectExists(t, sn, "/a", id2)
- expectExists(t, sn, "/a/b", id3)
- expectExists(t, sn, "/a/b/c", id4)
- expectExists(t, sn, "/a/b/c/d", id5)
- checkInRefs(t, sn)
-
- // Parent directory has to exisn.
- d := &Dir{}
- if _, err := sn.Put(rootPublicID, storage.ParsePath("/a/c/e"), d); err == nil {
- t.Errorf("Expected error")
- }
-
- expectValue(t, sn, "/", d1)
- expectValue(t, sn, "/a", d2)
- expectValue(t, sn, "/a/b", d3)
- expectValue(t, sn, "/a/b/c", d4)
- expectValue(t, sn, "/a/b/c/d", d5)
- checkInRefs(t, sn)
-
- // Remove part of the tree.
- if err := sn.Remove(rootPublicID, storage.ParsePath("/a/b")); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- sn.gc()
- expectExists(t, sn, "/", id1)
- expectExists(t, sn, "/a", id2)
- expectNotExists(t, sn, id3)
- expectNotExists(t, sn, id4)
- expectNotExists(t, sn, id5)
- checkInRefs(t, sn)
-}
-
-// Tests that nil maps are converted to empty maps.
-func TestPutToNilMap(t *testing.T) {
- sn := newMutableSnapshot(rootPublicID)
-
- var m map[string]interface{}
- if _, err := sn.Put(rootPublicID, storage.PathName{}, m); err != nil {
- t.Error("failure during nil map put: ", err)
- }
- if _, err := sn.Put(rootPublicID, storage.PathName{"z"}, "z"); err != nil {
- t.Error("failure during put of child of nil map: ", err)
- }
-}
-
-// Tests that slices are settable so that we can append.
-func TestAppendToSlice(t *testing.T) {
- sn := newMutableSnapshot(rootPublicID)
- if _, err := sn.Put(rootPublicID, storage.ParsePath("/"), []int{}); err != nil {
- t.Error("failure during put of empty slice: ", err)
- }
- if _, err := sn.Put(rootPublicID, storage.ParsePath("/@"), 1); err != nil {
- t.Error("failure during append to slice: ", err)
- }
-}
-
-// Replace a struct value with a hard link.
-func TestReplaceStructWithLink(t *testing.T) {
- sn := newMutableSnapshot(rootPublicID)
-
- mkdir(t, sn, "/")
- x := &Value{X: 1}
- stat, err := sn.Put(rootPublicID, storage.ParsePath("/a"), x)
- if err != nil {
- t.Errorf("/a: %s", err)
- }
- x.X = 2
- if _, err := sn.Put(rootPublicID, storage.ParsePath("/b"), x); err != nil {
- t.Errorf("/b: %s", err)
- }
- if v, err := sn.Get(rootPublicID, storage.ParsePath("/a")); err != nil || v.Value.(*Value).X != 1 {
- t.Errorf("Expected 1, got %v", v)
- }
- if v, err := sn.Get(rootPublicID, storage.ParsePath("/b")); err != nil || v.Value.(*Value).X != 2 {
- t.Errorf("Expected 2, got %v", v)
- }
-
- // Create a link.
- if _, err := sn.Put(rootPublicID, storage.ParsePath("/b"), stat.ID); err != nil {
- t.Errorf("/b: %s", err)
- }
- if v, err := sn.Get(rootPublicID, storage.ParsePath("/b")); err != nil || v.Value.(*Value).X != 1 {
- t.Errorf("Expected 1, got %v", v)
- }
- x.X = 3
- if _, err := sn.Put(rootPublicID, storage.ParsePath("/b"), x); err != nil {
- t.Errorf("/b: %s", err)
- }
- if v, err := sn.Get(rootPublicID, storage.ParsePath("/a")); err != nil || v.Value.(*Value).X != 3 {
- t.Errorf("Expected 3, got %v", v)
- }
-}
-
-// Put a value of the wrong type to a subfield.
-func TestPutWrongType(t *testing.T) {
- sn := newMutableSnapshot(rootPublicID)
-
- _, err := sn.Put(rootPublicID, rootPath, Value{7})
- if err != nil {
- t.Errorf("/: %v", err)
- }
- _, err = sn.Put(rootPublicID, storage.ParsePath("/X"), "string")
- if !verror.Is(err, verror.BadArg) {
- t.Errorf("/X: %v", err)
- }
- v, err := sn.Get(rootPublicID, storage.ParsePath("/X"))
- if err != nil {
- t.Errorf("/X: %v", err)
- }
- if v.Value != 7 {
- t.Errorf("Expected 7, got %v", v.Value)
- }
-
- _, err = sn.Put(rootPublicID, storage.ParsePath("/a"), Nest{Value{42}})
- if err != nil {
- t.Errorf("/a: %v", err)
- }
- _, err = sn.Put(rootPublicID, storage.ParsePath("/a/V"), "string")
- if !verror.Is(err, verror.BadArg) {
- t.Errorf("/a/V: %v", err)
- }
- _, err = sn.Put(rootPublicID, storage.ParsePath("/a/V/X"), "string")
- if !verror.Is(err, verror.BadArg) {
- t.Errorf("/a/V/X: %v", err)
- }
- v, err = sn.Get(rootPublicID, storage.ParsePath("/a/V/X"))
- if err != nil {
- t.Errorf("/a/V/X: %v", err)
- }
- if v.Value != 42 {
- t.Errorf("Expected 42, got %v", v.Value)
- }
-}
diff --git a/services/store/memstore/state/perm.go b/services/store/memstore/state/perm.go
deleted file mode 100644
index 7bf2df5..0000000
--- a/services/store/memstore/state/perm.go
+++ /dev/null
@@ -1 +0,0 @@
-package state
diff --git a/services/store/memstore/state/refs.go b/services/store/memstore/state/refs.go
deleted file mode 100644
index 77c8560..0000000
--- a/services/store/memstore/state/refs.go
+++ /dev/null
@@ -1,80 +0,0 @@
-package state
-
-import (
- "veyron/services/store/memstore/refs"
-
- "veyron2/storage"
-)
-
-// refsExist returns true iff there is a value in the state for each reference.
-func (sn *snapshot) refsExist(ids refs.Set) bool {
- for it := ids.Iterator(); it.IsValid(); it.Next() {
- id := it.Get().(*refs.Ref).ID
- if !sn.idTable.Contains(&Cell{ID: id}) {
- return false
- }
- }
- return true
-}
-
-// updateRefs takes a set of references from <before> to <after>. Any reference
-// in (<before> - <after>) should be decremented, and any reference in (<after>
-// - <before>) should be incremented.
-func (sn *MutableSnapshot) updateRefs(id storage.ID, beforeRefs, afterRefs refs.Set) {
- it1 := beforeRefs.Iterator()
- it2 := afterRefs.Iterator()
- for it1.IsValid() && it2.IsValid() {
- r1 := it1.Get().(*refs.Ref)
- r2 := it2.Get().(*refs.Ref)
- cmp := storage.CompareIDs(r1.ID, r2.ID)
- switch {
- case cmp < 0:
- sn.removeRef(id, r1)
- it1.Next()
- case cmp > 0:
- sn.addRef(id, r2)
- it2.Next()
- case cmp == 0:
- it1.Next()
- it2.Next()
- }
- }
- for ; it1.IsValid(); it1.Next() {
- sn.removeRef(id, it1.Get().(*refs.Ref))
- }
- for ; it2.IsValid(); it2.Next() {
- sn.addRef(id, it2.Get().(*refs.Ref))
- }
-}
-
-func (sn *MutableSnapshot) addRefs(id storage.ID, r refs.Set) {
- r.Iter(func(it interface{}) bool {
- sn.addRef(id, it.(*refs.Ref))
- return true
- })
-}
-
-func (sn *MutableSnapshot) removeRefs(id storage.ID, r refs.Set) {
- r.Iter(func(it interface{}) bool {
- sn.removeRef(id, it.(*refs.Ref))
- return true
- })
-}
-
-func (sn *MutableSnapshot) addRef(id storage.ID, r *refs.Ref) {
- // Update refcount.
- sn.ref(r.ID)
-
- // Add the inverse link.
- c := sn.Find(r.ID)
- c.inRefs = c.inRefs.Put(&refs.Ref{ID: id, Path: r.Path})
-}
-
-func (sn *MutableSnapshot) removeRef(id storage.ID, r *refs.Ref) {
- // Remove the inverse link.
- c := sn.deref(r.ID)
- c.inRefs = c.inRefs.Remove(&refs.Ref{ID: id, Path: r.Path})
-
- // Update refcount.
- sn.unref(r.ID)
-}
diff --git a/services/store/memstore/state/snapshot.go b/services/store/memstore/state/snapshot.go
deleted file mode 100644
index 80f6372..0000000
--- a/services/store/memstore/state/snapshot.go
+++ /dev/null
@@ -1,133 +0,0 @@
-package state
-
-import (
- "reflect"
-
- "veyron/services/store/memstore/field"
- "veyron/services/store/memstore/refs"
-
- "veyron2/security"
- "veyron2/storage"
-)
-
-type Snapshot interface {
- // NewIterator returns an Iterator that starts with the value at <path>.
- // pathFilter is used to automatically limit traversal of certain paths.
- // If filter is given, it is used to limit traversal beneath certain paths
- // and limit the results of the iteration. If filter is nil, all decendents
- // of the specified path are returned.
- NewIterator(pid security.PublicID, path storage.PathName, pathFilter PathFilter, filter IterFilter) Iterator
-
- // Find performs a lookup based on storage.ID, returning nil if the cell is not found.
- Find(id storage.ID) *Cell
-
- // Get returns the value for a path.
- Get(pid security.PublicID, path storage.PathName) (*storage.Entry, error)
-}
-
-// Snapshot keeps the state for the store. The snapshot contains a dictionary
-// and a root,
-//
-// idTable : storage.ID -> storage.Value
-// rootID : storage.ID
-//
-// Snapshots support isolation by using a functional/immutable dictionary for the
-// idTable.
-//
-// Paths are resolved by traversing the snapshot from the root, using reflection to
-// traverse fields within each of the values. For example, to resolve a path
-// /a/b/c/d/e/f/g/h, we perform the following steps.
-//
-// id1 := idTable[rootID].a.b.c
-// id2 := idTable[id1].d.e
-// id3 := idTable[id2].f.g.h
-// return id3
-//
-// If any of those resolution steps fails (if the idTable doesn't contain an
-// entry, or a field path like .a.b.c doesn't exist), then the resolution fails.
-type snapshot struct {
- // idTable is the dictionary of values. We use functional sets to make it
- // easy to perform snapshotting.
- idTable cellSet
-
- // rootID is the identifier of the root object.
- rootID storage.ID
-}
-
-// newSnapshot returns an empty snapshot.
-func newSnapshot(admin security.PublicID) snapshot {
- sn := snapshot{
- idTable: emptyIDTable,
- }
- return sn
-}
-
-// Find performs a lookup based on storage.ID, returning nil if the cell is not found.
-func (sn *snapshot) Find(id storage.ID) *Cell {
- v, ok := sn.idTable.Get(&Cell{ID: id})
- if !ok {
- return nil
- }
- return v.(*Cell)
-}
-
-// Get implements the Snapshot method.
-func (sn *snapshot) Get(pid security.PublicID, path storage.PathName) (*storage.Entry, error) {
- // Pass nil for 'mutations' since the snapshot is immutable.
- cell, suffix, v := sn.resolveCell(path, nil)
- if cell == nil {
- return nil, errNotFound
- }
- var e *storage.Entry
- if len(suffix) == 0 {
- e = cell.GetEntry()
- } else {
- e = newSubfieldEntry(v)
- }
- return e, nil
-}
-
-// resolveCell performs a path-based lookup, traversing the state from the root.
-//
-// Returns (cell, suffix, v), where cell contains the value, suffix is the path
-// to the value, v is the value itself. If the operation failed, the returned
-// cell is nil.
-func (sn *snapshot) resolveCell(path storage.PathName, mu *Mutations) (*Cell, storage.PathName, interface{}) {
- cell := sn.Find(sn.rootID)
- if cell == nil {
- return nil, nil, nil
- }
- for {
- if mu != nil {
- mu.addPrecondition(cell)
- }
- var v reflect.Value
- var suffix storage.PathName
- v, suffix = field.Get(cell.Value, path)
- x := v.Interface()
- if id, ok := x.(storage.ID); ok {
- // Always dereference IDs.
- cell = sn.Find(id)
- path = suffix
- continue
- }
- switch len(suffix) {
- case 0:
- // The path is fully resolved. We're done.
- return cell, path, x
- case len(path):
- // The path couldn't be resolved at all. It must be an entry in the
- // implicit directory.
- r, ok := cell.Dir.Get(&refs.Ref{Path: refs.NewSingletonPath(path[0])})
- if !ok {
- return nil, nil, nil
- }
- cell = sn.Find(r.(*refs.Ref).ID)
- path = path[1:]
- default:
- // The path is partially resolved, but it does not resolve to a
- // storage.ID. This is an error.
- return nil, nil, nil
- }
- }
-}
diff --git a/services/store/memstore/state/snapshot_test.go b/services/store/memstore/state/snapshot_test.go
deleted file mode 100644
index c5976cd..0000000
--- a/services/store/memstore/state/snapshot_test.go
+++ /dev/null
@@ -1,51 +0,0 @@
-package state
-
-import (
- "runtime"
- "testing"
- "veyron2/storage"
-)
-
-func expectExistsImmutable(t *testing.T, sn Snapshot, path string) {
- _, file, line, _ := runtime.Caller(1)
- if _, err := sn.Get(rootPublicID, storage.ParsePath(path)); err != nil {
- t.Errorf("%s(%d): does not exist: %s", file, line, path)
- }
-}
-
-func expectNotExistsImmutable(t *testing.T, sn Snapshot, path string) {
- _, file, line, _ := runtime.Caller(1)
- if _, err := sn.Get(rootPublicID, storage.ParsePath(path)); err == nil {
- t.Errorf("%s(%d): should not exist: %s", file, line, path)
- }
-}
-
-// TestImmutableGet is very similar to TestImplicitDirTree in
-// mutable_snapshot_test.go except that it uses the immutable Snapshot type
-// instead of MutableSnapshot. The implementations of Get are different
-// between the two types.
-func TestImmutableGet(t *testing.T) {
- sn := newMutableSnapshot(rootPublicID)
-
- mkdir(t, sn, "/")
- mkdir(t, sn, "/Entries/a")
- mkdir(t, sn, "/Entries/a/Entries/b")
- mkdir(t, sn, "/Entries/a/Entries/b/Entries/c")
- mkdir(t, sn, "/Entries/a/Entries/b/Entries/d")
- expectExistsImmutable(t, &sn.snapshot, "/")
- expectExistsImmutable(t, &sn.snapshot, "/Entries/a")
- expectExistsImmutable(t, &sn.snapshot, "/Entries/a/Entries/b")
- expectExistsImmutable(t, &sn.snapshot, "/Entries/a/Entries/b/Entries/c")
- expectExistsImmutable(t, &sn.snapshot, "/Entries/a/Entries/b/Entries/d")
-
- // Remove part of the tree.
- if err := sn.Remove(rootPublicID, storage.ParsePath("/Entries/a/Entries/b")); err != nil {
- t.Errorf("Unexpected error: %s", err)
- }
- sn.gc()
- expectExistsImmutable(t, &sn.snapshot, "/")
- expectExistsImmutable(t, &sn.snapshot, "/Entries/a")
- expectNotExistsImmutable(t, &sn.snapshot, "/Entries/a/Entries/b")
- expectNotExistsImmutable(t, &sn.snapshot, "/Entries/a/Entries/b/Entries/c")
- expectNotExistsImmutable(t, &sn.snapshot, "/Entries/a/Entries/b/Entries/d")
-}
diff --git a/services/store/memstore/state/state.go b/services/store/memstore/state/state.go
deleted file mode 100644
index e6212a8..0000000
--- a/services/store/memstore/state/state.go
+++ /dev/null
@@ -1,165 +0,0 @@
-package state
-
-import (
- "time"
-
- "veyron/services/store/memstore/refs"
- "veyron/services/store/raw"
-
- "veyron2/security"
- "veyron2/storage"
-)
-
-type State struct {
- snapshot *MutableSnapshot
-
- // timestamp is the time of the last mutation applied, in nanoseconds since
- // the epoch. See comment for snapshot.Mutations.Timestamp.
- timestamp uint64
-}
-
-// refUpdate represents a reference change to a value.
-type refUpdate struct {
- id storage.ID
- before refs.Set
- after refs.Set
-}
-
-// New returns an empty State.
-func New(admin security.PublicID) *State {
- return &State{snapshot: newMutableSnapshot(admin)}
-}
-
-// Timestamp returns the timestamp of the latest mutation to the state in
-// nanoseconds.
-func (st *State) Timestamp() uint64 {
- return st.timestamp
-}
-
-// DeepCopy creates a copy of the state. Mutations to the copy do not affect
-// the original, and vice versa.
-func (st *State) DeepCopy() *State {
- return &State{st.MutableSnapshot(), st.timestamp}
-}
-
-// GC performs a manual garbage collection.
-func (st *State) GC() {
- st.snapshot.gc()
-}
-
-// Snapshot returns a read-only copy of the state.
-func (st *State) Snapshot() Snapshot {
- return st.snapshot.GetSnapshot()
-}
-
-// MutableSnapshot creates a copy of the state. Mutations to the copy do not
-// affect the original, and vice versa.
-func (st *State) MutableSnapshot() *MutableSnapshot {
- return st.snapshot.deepCopy()
-}
-
-// Deletions returns the set of IDs for values that have been deleted from
-// the state. Returns nil iff there have been no deletions.
-func (st *State) Deletions() *Mutations {
- if len(st.snapshot.deletions) == 0 {
- return nil
- }
-
- // Package the deletions into a transaction.
- var mu Mutations
- ts := st.timestamp + 1
- mu.Timestamp = ts
- mu.Deletions = st.snapshot.deletions
- st.timestamp = ts
- st.snapshot.deletions = make(map[storage.ID]raw.Version)
- return &mu
-}
-
-// ApplyMutations applies a set of mutations atomically.
-func (st *State) ApplyMutations(mu *Mutations) error {
- // Assign a timestamp.
- ts := uint64(time.Now().UnixNano())
- if ts <= st.timestamp {
- ts = st.timestamp + 1
- }
- mu.Timestamp = ts
-
- if err := st.snapshot.applyMutations(mu); err != nil {
- return err
- }
-
- st.timestamp = ts
- return nil
-}
-
-func (sn *MutableSnapshot) applyMutations(mu *Mutations) error {
- // Check the preconditions.
- table := sn.idTable
- for id, pre := range mu.Preconditions {
- c, ok := table.Get(&Cell{ID: id})
- // If the precondition is 0, it means that the cell is being created,
- // and it must not already exist. We get a precondition failure if pre
- // is 0 and the cell already exists, or pre is not 0 and the cell does
- // not already exist or have the expected version.
- if pre == 0 && ok || pre != 0 && (!ok || c.(*Cell).Version != pre) {
- return errPreconditionFailed
- }
- }
- for id, pre := range mu.Deletions {
- c, ok := table.Get(&Cell{ID: id})
- // The target is not required to exist.
- if ok && c.(*Cell).Version != pre {
- return errPreconditionFailed
- }
- }
-
- // Changes to the state begin now. These changes should not fail,
- // as we don't support rollback.
-
- // Apply the mutations.
- updates := make([]*refUpdate, 0, len(mu.Delta))
- for id, m := range mu.Delta {
- d := refs.BuildDir(m.Dir)
- cl, ok := table.Get(&Cell{ID: id})
- if !ok {
- c := &Cell{
- ID: id,
- Version: m.Postcondition,
- Value: m.Value,
- Dir: d,
- refs: m.refs,
- inRefs: refs.Empty,
- }
- table = table.Put(c)
- updates = append(updates, &refUpdate{id: c.ID, before: refs.Empty, after: c.refs})
- } else {
- c := cl.(*Cell)
- cp := *c
- cp.Version = m.Postcondition
- cp.Value = m.Value
- cp.Dir = d
- cp.refs = m.refs
- table = table.Put(&cp)
- updates = append(updates, &refUpdate{id: c.ID, before: c.refs, after: cp.refs})
- }
- }
- sn.idTable = table
-
- // Add the refs.
- for _, u := range updates {
- sn.updateRefs(u.id, u.before, u.after)
- }
-
- // Redirect the rootID.
- if mu.SetRootID && mu.RootID != sn.rootID {
- if mu.RootID != nullID {
- sn.ref(mu.RootID)
- }
- if sn.rootID != nullID {
- sn.unref(sn.rootID)
- }
- sn.rootID = mu.RootID
- }
-
- return nil
-}
diff --git a/services/store/memstore/state/state_test.go b/services/store/memstore/state/state_test.go
deleted file mode 100644
index 7a8045f..0000000
--- a/services/store/memstore/state/state_test.go
+++ /dev/null
@@ -1,141 +0,0 @@
-package state_test
-
-import (
- "runtime"
- "testing"
-
- "veyron/services/store/memstore/state"
-
- "veyron2/security"
- "veyron2/storage"
-)
-
-type Node struct {
- E map[string]storage.ID
-}
-
-var (
- rootPublicID security.PublicID = security.FakePublicID("root")
-)
-
-// makeParentNodes creates the parent nodes if they do not already exist.
-func makeParentNodes(t *testing.T, sn *state.MutableSnapshot, user security.PublicID, path string) {
- _, file, line, _ := runtime.Caller(2)
- pl := storage.ParsePath(path)
- for i := 0; i < len(pl); i++ {
- name := pl[:i]
- if _, err := sn.Get(user, name); err != nil {
- if _, err := sn.Put(user, name, &Node{}); err != nil {
- t.Fatalf("%s(%d): can't put %s: %s", file, line, name, err)
- }
- }
- }
-}
-
-// mkdir creates an empty directory. Creates the parent directories if they do
-// not already exist.
-func mkdir(t *testing.T, sn *state.MutableSnapshot, user security.PublicID, path string) storage.ID {
- makeParentNodes(t, sn, user, path)
- name := storage.ParsePath(path)
- stat, err := sn.Put(user, name, &Node{})
- if err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): can't put %s: %s", file, line, path, err)
- }
- return stat.ID
-}
-
-// link adds a hard link to another store value. Creates the parent directories
-// if they do not already exist.
-func link(t *testing.T, sn *state.MutableSnapshot, user security.PublicID, path string, id storage.ID) {
- makeParentNodes(t, sn, user, path)
- name := storage.ParsePath(path)
- if _, err := sn.Put(user, name, id); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): can't put %s: %s", file, line, path, err)
- }
-}
-
-// putPath adds a value to the store. Creates the parent directories if they do
-// not already exist.
-func putPath(t *testing.T, sn *state.MutableSnapshot, user security.PublicID, path string, v interface{}) storage.ID {
- makeParentNodes(t, sn, user, path)
- name := storage.ParsePath(path)
- stat, err := sn.Put(user, name, v)
- if err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): can't put %s: %s", file, line, path, err)
- }
- return stat.ID
-}
-
-// put adds a value to the store.
-func put(t *testing.T, sn *state.MutableSnapshot, user security.PublicID, path string, v interface{}) storage.ID {
- name := storage.ParsePath(path)
- stat, err := sn.Put(user, name, v)
- if err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): can't put %s: %s", file, line, path, err)
- }
- return stat.ID
-}
-
-// maybePut tries to add a value to the store.
-func maybePut(sn *state.MutableSnapshot, user security.PublicID, path string, v interface{}) (storage.ID, error) {
- name := storage.ParsePath(path)
- stat, err := sn.Put(user, name, v)
- if err != nil {
- return storage.ID{}, err
- }
- return stat.ID, nil
-}
-
-// maybeGet try to fetch a value from the store.
-func maybeGet(sn *state.MutableSnapshot, user security.PublicID, path string) (interface{}, error) {
- name := storage.ParsePath(path)
- e, err := sn.Get(user, name)
- if err != nil {
- return nil, err
- }
- return e.Value, nil
-}
-
-// commit commits the state.
-func commit(t *testing.T, st *state.State, sn *state.MutableSnapshot) {
- if err := st.ApplyMutations(sn.Mutations()); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): can't commit: %s", file, line, err)
- }
-}
-
-// Simple DAG based on teams and players.
-func TestState(t *testing.T) {
- st := state.New(rootPublicID)
-
- // Add a Node under /a/b/c.
- sn1 := st.MutableSnapshot()
- nid := putPath(t, sn1, rootPublicID, "/a/b/c", &Node{})
- commit(t, st, sn1)
-
- // Create a link.
- {
- sn := st.MutableSnapshot()
- link(t, sn, rootPublicID, "/a/b/d", nid)
- commit(t, st, sn)
- }
-
- // Add a shared Entry.
- {
- sn := st.MutableSnapshot()
- mkdir(t, sn, rootPublicID, "/a/b/c/Entries/foo")
- commit(t, st, sn)
- }
-
- // Check that it exists.
- {
- sn := st.MutableSnapshot()
- if _, err := maybeGet(sn, rootPublicID, "/a/b/d/Entries/foo"); err != nil {
- t.Errorf("entry should exist")
- }
- }
-}
diff --git a/services/store/memstore/state/util.go b/services/store/memstore/state/util.go
deleted file mode 100644
index cb72db1..0000000
--- a/services/store/memstore/state/util.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package state
-
-import (
- "bytes"
- "fmt"
- "reflect"
- "sort"
-
- "veyron2/storage"
-)
-
-// This function returns an interface with a pointer to the element in val.
-// Note that this is different from returning &val which returns a pointer to
-// val. This is required temporarily before we switch to VOM value so that
-// we can append to slices.
-// TODO(bprosnitz) This is hacky -- remove this when we switch to VOM Value
-func makeInnerReference(val interface{}) interface{} {
- rv := reflect.ValueOf(val)
- ptr := reflect.New(rv.Type())
- ptr.Elem().Set(rv)
- return ptr.Interface()
-}
-
-func deepcopyReflect(rv reflect.Value) reflect.Value {
- switch rv.Kind() {
- case reflect.Array:
- arr := reflect.New(rv.Type()).Elem()
- for i := 0; i < rv.Len(); i++ {
- valcopy := deepcopyReflect(rv.Index(i))
- arr.Index(i).Set(valcopy)
- }
- return arr
- case reflect.Slice:
- s := reflect.MakeSlice(rv.Type(), rv.Len(), rv.Cap())
- for i := 0; i < rv.Len(); i++ {
- valcopy := deepcopyReflect(rv.Index(i))
- s.Index(i).Set(valcopy)
- }
- ptr := reflect.New(rv.Type())
- ptr.Elem().Set(s)
- return ptr.Elem()
- case reflect.Map:
- m := reflect.MakeMap(rv.Type())
- keys := rv.MapKeys()
- for _, key := range keys {
- val := rv.MapIndex(key)
- keycopy := deepcopyReflect(key)
- valcopy := deepcopyReflect(val)
- m.SetMapIndex(keycopy, valcopy)
- }
- return m
- case reflect.Struct:
- s := reflect.New(rv.Type()).Elem()
- for i := 0; i < rv.NumField(); i++ {
- valcopy := deepcopyReflect(rv.Field(i))
- s.Field(i).Set(valcopy)
- }
- return s
- case reflect.Ptr:
- ptr := reflect.New(rv.Type()).Elem()
- elem := reflect.New(rv.Type().Elem())
- ptr.Set(elem)
- ptr.Elem().Set(deepcopyReflect(rv.Elem()))
- return ptr
- case reflect.Interface:
- intr := reflect.New(rv.Type()).Elem()
- intr.Set(deepcopyReflect(rv.Elem()))
- return intr
- case reflect.Chan, reflect.Func, reflect.UnsafePointer:
- panic(fmt.Sprintf("deepcopy of kind %v not supported", rv.Kind()))
- default:
- // Primitives (copy it so we can't set the original)
- return reflect.ValueOf(rv.Interface())
- }
-}
-
-// deepcopy performs a deep copy of a value. We need this to simulate secondary
-// storage where each time a value is stored, it is copied to secondary storage;
-// and when it is retrieved, it is copied out of secondary storage.
-func deepcopy(v interface{}) interface{} {
- if v == nil {
- return nil
- }
- return deepcopyReflect(reflect.ValueOf(v)).Interface()
-}
-
-// addIDToSort adds a storage.ID to a sorted array.
-func addIDToSort(id storage.ID, refs []storage.ID) []storage.ID {
- i := findIDInSort(id, refs)
- newRefs := make([]storage.ID, len(refs)+1)
- copy(newRefs[0:i], refs[0:i])
- newRefs[i] = id
- copy(newRefs[i+1:], refs[i:])
- return newRefs
-}
-
-// removeIDFromSort removes a storage.ID from a sorted array.
-func removeIDFromSort(id storage.ID, refs []storage.ID) []storage.ID {
- i := findIDInSort(id, refs)
- if i < len(refs) && refs[i] == id {
- newRefs := make([]storage.ID, len(refs)-1)
- copy(newRefs[0:i], refs[0:i])
- copy(newRefs[i:], refs[i+1:])
- return newRefs
- }
- return refs
-}
-
-func findIDInSort(id storage.ID, refs []storage.ID) int {
- return sort.Search(len(refs), func(i int) bool {
- return bytes.Compare(refs[i][:], id[:]) >= 0
- })
-}
diff --git a/services/store/memstore/state/util_test.go b/services/store/memstore/state/util_test.go
deleted file mode 100644
index 38653ef..0000000
--- a/services/store/memstore/state/util_test.go
+++ /dev/null
@@ -1,116 +0,0 @@
-// Modeled after veyron2/storage/vstore/blackbox/photoalbum_test.go.
-//
-// TODO(sadovsky): Maybe migrate this to be part of the public store API, to
-// help with writing tests that use storage.
-
-package state
-
-import (
- "reflect"
- "runtime"
- "testing"
-
- "veyron2/security"
- "veyron2/storage"
-)
-
-var (
- rootPublicID security.PublicID = security.FakePublicID("root")
-)
-
-func get(t *testing.T, sn *MutableSnapshot, path string) interface{} {
- _, file, line, _ := runtime.Caller(1)
- name := storage.ParsePath(path)
- e, err := sn.Get(rootPublicID, name)
- if err != nil {
- t.Fatalf("%s(%d): can't get %s: %s", file, line, path, err)
- }
- return e.Value
-}
-
-func put(t *testing.T, sn *MutableSnapshot, path string, v interface{}) storage.ID {
- _, file, line, _ := runtime.Caller(1)
- name := storage.ParsePath(path)
- stat, err := sn.Put(rootPublicID, name, v)
- if err != nil {
- t.Errorf("%s(%d): can't put %s: %s", file, line, path, err)
- }
- if _, err := sn.Get(rootPublicID, name); err != nil {
- t.Errorf("%s(%d): can't get %s: %s", file, line, path, err)
- }
- if stat != nil {
- return stat.ID
- }
- return storage.ID{}
-}
-
-func remove(t *testing.T, sn *MutableSnapshot, path string) {
- name := storage.ParsePath(path)
- if err := sn.Remove(rootPublicID, name); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): can't remove %s: %s", file, line, path, err)
- }
-}
-
-func TestDeepcopy(t *testing.T) {
- type basicTestStruct struct {
- X int
- }
- type compositeHoldingTestStruct struct {
- A [2]int
- Sl []string
- M map[string]int
- St basicTestStruct
- }
-
- var intr interface{} = []string{"X"}
-
- tests := []interface{}{
- nil,
- 0,
- true,
- "str",
- [3]int{4, 5, 6},
- []string{"A", "B"},
- map[string]int{"A": 4, "B": 3},
- basicTestStruct{7},
- &basicTestStruct{7},
- compositeHoldingTestStruct{
- [2]int{3, 4},
- []string{"A"},
- map[string]int{"A": 5},
- basicTestStruct{X: 3},
- },
- intr,
- }
-
- for _, test := range tests {
- copiedVal := deepcopy(test)
- if !reflect.DeepEqual(copiedVal, test) {
- t.Errorf("failure in deepcopy. Expected %v, got %v", test, copiedVal)
- }
- }
-}
-
-func TestDeepcopySliceSettability(t *testing.T) {
- rvSliceCopy := deepcopyReflect(reflect.ValueOf([]int{3, 4}))
- if !rvSliceCopy.CanSet() {
- t.Errorf("can't set slice. This is required for appending to slices")
- }
-}
-
-func TestDeepcopyNilMap(t *testing.T) {
- var nilMap map[int]int
- mapCopy := deepcopy(nilMap)
- if !reflect.DeepEqual(mapCopy, map[int]int{}) {
- t.Errorf("expected an empty map, got %v", mapCopy)
- }
-
- type structWithMap struct {
- M map[int]int
- }
- s := deepcopy(&structWithMap{})
- if !reflect.DeepEqual(s, &structWithMap{map[int]int{}}) {
- t.Errorf("expected an empty map in the struct, got %v", s)
- }
-}
diff --git a/services/store/memstore/store.go b/services/store/memstore/store.go
deleted file mode 100644
index f1090d1..0000000
--- a/services/store/memstore/store.go
+++ /dev/null
@@ -1,178 +0,0 @@
-package memstore
-
-import (
- "veyron/runtimes/google/lib/sync"
-
- "veyron/services/store/memstore/state"
- "veyron/services/store/raw"
-
- "veyron2/ipc"
- "veyron2/security"
- "veyron2/storage"
- "veyron2/verror"
-)
-
-// Store is the in-memory state of the store.
-type Store struct {
- sync.DebugMutex
-
- // state holds the current state of the store.
- State *state.State
-
- // log persists the state to disk and any committed transactions.
- // An ephemeral state has a nil log, and does not persist to disk.
- log *wlog
-}
-
-var (
- ErrRequestCancelled = verror.Abortedf("request cancelled")
-)
-
-// New creates a new store. admin is the public ID of the administrator, dbName
-// is the path of the database directory, to which logs are written.
-func New(admin security.PublicID, dbName string) (*Store, error) {
- rlog, err := openDB(dbName, false)
- if err != nil {
- return nil, err
- }
- if rlog == nil {
- return newState(admin, dbName)
- }
- st, err := readAndCloseDB(admin, rlog)
- if err != nil {
- return nil, err
- }
- // Rename the log file by date.
- if err := backupLog(dbName); err != nil {
- return nil, err
- }
- if err := st.setLog(dbName); err != nil {
- return nil, err
- }
- return st, nil
-}
-
-// newState returns an empty state. dbName is the path of the database directory,
-// to which logs are written.
-func newState(admin security.PublicID, dbName string) (*Store, error) {
- st := &Store{State: state.New(admin)}
- if err := st.setLog(dbName); err != nil {
- return nil, err
- }
- return st, nil
-}
-
-// setLog creates a fresh log file and writes to it.
-func (st *Store) setLog(dbName string) error {
- if dbName != "" {
- log, err := createLog(dbName)
- if err != nil {
- return err
- }
- err = log.writeState(st)
- if err != nil {
- log.close()
- return err
- }
- st.log = log
- }
- return nil
-}
-
-// Bind returns an Object representing a value in the store. The value need not
-// exist; the Put method can be used to add the value if it doesn't already
-// exist.
-func (st *Store) Bind(path string) *Object {
- return &Object{path: storage.ParsePath(path), store: st}
-}
-
-func (st *Store) Close() error {
- st.Lock()
- st.closeLocked()
- st.Unlock()
- return nil
-}
-
-func (st *Store) closeLocked() {
- st.State = nil
- if st.log != nil {
- st.log.close()
- }
- st.log = nil
-}
-
-// GC performs a manual garbage collection.
-func (st *Store) GC() error {
- st.Lock()
- defer st.Unlock()
- st.State.GC()
-
- // Append a transaction containing deletions, if there are any.
- mu := st.State.Deletions()
- if st.log != nil && mu != nil {
- if err := st.log.appendTransaction(mu); err != nil {
- // We can't continue because the log failed. The state has already been updated,
- // but access to the state is blocked because we have the lock. Close the state
- // to ensure that it is never used again.
- st.closeLocked()
- return err
- }
- }
- return nil
-}
-
-// Snapshot returns a read-only state.
-func (st *Store) Snapshot() state.Snapshot {
- st.Lock()
- defer st.Unlock()
- return st.State.Snapshot()
-}
-
-// ApplyMutations applies the mutations to the state atomically.
-func (st *Store) ApplyMutations(mu *state.Mutations) error {
- st.Lock()
- defer st.Unlock()
- if err := st.State.ApplyMutations(mu); err != nil {
- return err
- }
- if st.log != nil {
- // Append the transaction to the log.
- if err := st.log.appendTransaction(mu); err != nil {
- // We can't continue because the log failed. The state has already been updated,
- // but access to the state is blocked because we have the lock. Close the state
- // to ensure that it is never used again.
- st.closeLocked()
- return err
- }
- }
- return nil
-}
-
-// PutMutations atomically commits a stream of Mutations when the stream is
-// closed. Mutations are not committed if the request is cancelled before the
-// stream has been closed.
-func (st *Store) PutMutations(ctx ipc.ServerContext, stream raw.StoreServicePutMutationsStream) error {
- tr := st.newNilTransaction()
- rStream := stream.RecvStream()
- for rStream.Advance() {
- mu := rStream.Value()
-
- if err := tr.snapshot.PutMutation(mu); err != nil {
- tr.Abort()
- return err
- }
- }
- err := rStream.Err()
- if err != nil {
- tr.Abort()
- return err
- }
-
- select {
- case <-ctx.Done():
- tr.Abort()
- return ErrRequestCancelled
- default:
- return tr.Commit()
- }
-}
diff --git a/services/store/memstore/store_test.go b/services/store/memstore/store_test.go
deleted file mode 100644
index 000dceb..0000000
--- a/services/store/memstore/store_test.go
+++ /dev/null
@@ -1,379 +0,0 @@
-package memstore
-
-import (
- "io/ioutil"
- "os"
- "testing"
-
- storetesting "veyron/services/store/memstore/testing"
- "veyron/services/store/raw"
-
- "veyron2/rt"
- "veyron2/storage"
- "veyron2/verror"
-)
-
-func TestLogWrite(t *testing.T) {
- dbName, err := ioutil.TempDir(os.TempDir(), "vstore")
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
- defer os.RemoveAll(dbName)
-
- // Create the state. This should also initialize the log.
- st, err := New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("newState() failed: %v", err)
- }
-
- // Open the log for reading.
- log, err := OpenLog(dbName, true)
- if err != nil {
- t.Fatalf("OpenLog() failed: %v", err)
- }
-
- // The log should be sync'ed, test reading the initial state.
- logst, err := log.ReadState(rootPublicID)
- if err != nil {
- t.Fatalf("ReadState() failed: %v", err)
- }
-
- // Construct a transaction.
- v1 := "v1"
- v2 := "v2"
- v3 := "v3"
- tr := NewTransaction()
- put(t, st, tr, "/", v1)
- put(t, st, tr, "/a", v2)
- put(t, st, tr, "/a/b", v3)
- commit(t, tr)
-
- // Check that the mutations were applied to the state.
- expectValue(t, st, nil, "/", v1)
- expectValue(t, st, nil, "/a", v2)
- expectValue(t, st, nil, "/a/b", v3)
-
- // The log should be sync'ed, test reading the transaction.
- logmu, err := log.ReadTransaction()
- if err != nil {
- t.Fatalf("ReadTransaction() failed: %v", err)
- }
- logst.ApplyMutations(logmu)
- expectValue(t, logst, nil, "/", v1)
- expectValue(t, logst, nil, "/a", v2)
- expectValue(t, logst, nil, "/a/b", v3)
-}
-
-func TestFailedLogWrite(t *testing.T) {
- dbName, err := ioutil.TempDir(os.TempDir(), "vstore")
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
-
- // Create the state. This should also initialize the log.
- st, err := New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("newState() failed: %v", err)
- }
-
- // Construct a transaction.
- v1 := "v1"
- tr := NewTransaction()
- put(t, st, tr, "/", v1)
- v2 := "v2"
- put(t, st, tr, "/a", v2)
-
- // Close the log file. Subsequent writes to the log should fail.
- st.log.close()
-
- // Commit the state. The call should fail.
- if err := st.log.appendTransaction(nil); !verror.Is(err, verror.Aborted) {
- t.Errorf("Expected error %v, got %v", verror.Aborted, err)
- }
-}
-
-func TestRecoverFromLog(t *testing.T) {
- dbName, err := ioutil.TempDir(os.TempDir(), "vstore")
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
- defer os.RemoveAll(dbName)
-
- // Create the state. This should also initialize the log.
- st, err := New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("newState() failed: %v", err)
- }
-
- // Construct a transaction.
- v1 := "v1"
- v2 := "v2"
- v3 := "v3"
- tr := NewTransaction()
- put(t, st, tr, "/", v1)
- put(t, st, tr, "/a", v2)
- put(t, st, tr, "/a/b", v3)
- commit(t, tr)
-
- // Recover state from the log.
- recoverst, err := New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("newState() failed: %v", err)
- }
- expectValue(t, recoverst, nil, "/", v1)
- expectValue(t, recoverst, nil, "/a", v2)
- expectValue(t, recoverst, nil, "/a/b", v3)
-}
-
-func TestPutMutations(t *testing.T) {
- rt.Init()
- dbName, err := ioutil.TempDir(os.TempDir(), "vstore")
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
- defer os.RemoveAll(dbName)
-
- // Create the state.
- st, err := New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
-
- // Add /, /a, /a/b
- id1, id2, id3 := storage.NewID(), storage.NewID(), storage.NewID()
- pre1, pre2, pre3 := raw.NoVersion, raw.NoVersion, raw.NoVersion
- post1, post2, post3 := raw.NewVersion(), raw.NewVersion(), raw.NewVersion()
- v1, v2, v3 := "v1", "v2", "v3"
-
- storetesting.PutMutationsBatch(t, rootPublicID, st.PutMutations, []raw.Mutation{
- raw.Mutation{
- ID: id1,
- PriorVersion: pre1,
- Version: post1,
- IsRoot: true,
- Value: v1,
- Dir: dir("a", id2),
- },
- raw.Mutation{
- ID: id2,
- PriorVersion: pre2,
- Version: post2,
- IsRoot: false,
- Value: v2,
- Dir: dir("b", id3),
- },
- raw.Mutation{
- ID: id3,
- PriorVersion: pre3,
- Version: post3,
- IsRoot: false,
- Value: v3,
- Dir: empty,
- },
- })
-
- expectValue(t, st, nil, "/", v1)
- expectValue(t, st, nil, "/a", v2)
- expectValue(t, st, nil, "/a/b", v3)
-
- // Remove /a/b
- pre1, pre2, pre3 = post1, post2, post3
- post2 = raw.NewVersion()
-
- storetesting.PutMutationsBatch(t, rootPublicID, st.PutMutations, []raw.Mutation{
- raw.Mutation{
- ID: id2, PriorVersion: pre2,
- Version: post2,
- IsRoot: false,
- Value: v2,
- Dir: empty,
- }})
-
- expectValue(t, st, nil, "/", v1)
- expectValue(t, st, nil, "/a", v2)
- expectNotExists(t, st, nil, "a/b")
-
- // Garbage-collect /a/b
- post3 = raw.NoVersion
-
- storetesting.PutMutationsBatch(t, rootPublicID, st.PutMutations, []raw.Mutation{
- raw.Mutation{
- ID: id3,
- PriorVersion: pre3,
- Version: post3,
- IsRoot: false,
- }})
-
- expectValue(t, st, nil, "/", v1)
- expectValue(t, st, nil, "/a", v2)
- expectNotExists(t, st, nil, "a/b")
-
- // Remove /
- pre1, pre2, pre3 = post1, post2, post3
- post1 = raw.NoVersion
-
- storetesting.PutMutationsBatch(t, rootPublicID, st.PutMutations, []raw.Mutation{
- raw.Mutation{
- ID: id1,
- PriorVersion: pre1,
- Version: post1,
- IsRoot: true,
- }})
-
- expectNotExists(t, st, nil, "/")
- expectNotExists(t, st, nil, "/a")
- expectNotExists(t, st, nil, "a/b")
-
- // Garbage-collect /a
- post2 = raw.NoVersion
-
- storetesting.PutMutationsBatch(t, rootPublicID, st.PutMutations, []raw.Mutation{
- raw.Mutation{
- ID: id2,
- PriorVersion: pre2,
- Version: post2,
- IsRoot: false,
- }})
-
- expectNotExists(t, st, nil, "/")
- expectNotExists(t, st, nil, "/a")
- expectNotExists(t, st, nil, "a/b")
-}
-
-func TestPutConflictingMutations(t *testing.T) {
- dbName, err := ioutil.TempDir(os.TempDir(), "vstore")
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
- defer os.RemoveAll(dbName)
-
- // Create the state.
- st, err := New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
-
- // Add /, /a
- id1, id2 := storage.NewID(), storage.NewID()
- pre1, pre2 := raw.NoVersion, raw.NoVersion
- post1, post2 := raw.NewVersion(), raw.NewVersion()
- v1, v2 := "v1", "v2"
-
- storetesting.PutMutationsBatch(t, rootPublicID, st.PutMutations, []raw.Mutation{
- raw.Mutation{
- ID: id1,
- PriorVersion: pre1,
- Version: post1,
- IsRoot: true,
- Value: v1,
- Dir: dir("a", id2),
- },
- raw.Mutation{
- ID: id2,
- PriorVersion: pre2,
- Version: post2,
- IsRoot: false,
- Value: v2,
- Dir: empty,
- },
- })
-
- expectValue(t, st, nil, "/", v1)
- expectValue(t, st, nil, "/a", v2)
-
- // Attempt to update /a with a bad precondition
- pre2 = raw.NewVersion()
- post2 = raw.NewVersion()
- v2 = "v4"
-
- s := storetesting.PutMutations(rootPublicID, st.PutMutations)
- s.SendStream().Send(raw.Mutation{
- ID: id2,
- PriorVersion: pre2,
- Version: post2,
- IsRoot: true,
- Value: v2,
- Dir: empty,
- })
- if err := s.Finish(); !verror.Is(err, verror.Aborted) {
- t.Errorf("Error should be %v: got %v", verror.Aborted, err)
- }
-
-}
-
-func TestPutDuplicateMutations(t *testing.T) {
- dbName, err := ioutil.TempDir(os.TempDir(), "vstore")
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
- defer os.RemoveAll(dbName)
-
- // Create the state.
- st, err := New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
-
- id := storage.NewID()
- s := storetesting.PutMutations(rootPublicID, st.PutMutations)
- s.SendStream().Send(raw.Mutation{
- ID: id,
- PriorVersion: raw.NoVersion,
- Version: raw.NewVersion(),
- IsRoot: true,
- Value: "v1",
- Dir: empty,
- })
- s.SendStream().Send(raw.Mutation{
- ID: id,
- PriorVersion: raw.NoVersion,
- Version: raw.NewVersion(),
- IsRoot: true,
- Value: "v2",
- Dir: empty,
- })
- if err := s.Finish(); !verror.Is(err, verror.BadArg) {
- t.Errorf("Error should be %v: got %v", verror.BadArg, err)
- }
-}
-
-func TestCancelPutMutation(t *testing.T) {
- dbName, err := ioutil.TempDir(os.TempDir(), "vstore")
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
- defer os.RemoveAll(dbName)
-
- // Create the state.
- st, err := New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
-
- s := storetesting.PutMutations(rootPublicID, st.PutMutations)
- s.SendStream().Send(raw.Mutation{
- ID: storage.NewID(),
- PriorVersion: raw.NoVersion,
- Version: raw.NewVersion(),
- IsRoot: true,
- Value: "v1",
- Dir: empty,
- })
- s.Cancel()
- if err := s.Finish(); !verror.Is(err, verror.Aborted) {
- t.Errorf("Error should be %v: got %v", verror.Aborted, err)
- }
-
- expectNotExists(t, st, nil, "/")
-}
-
-var (
- empty = []raw.DEntry{}
-)
-
-func dir(name string, id storage.ID) []raw.DEntry {
- return []raw.DEntry{raw.DEntry{
- Name: name,
- ID: id,
- }}
-}
diff --git a/services/store/memstore/testing/util.go b/services/store/memstore/testing/util.go
deleted file mode 100644
index be31dc9..0000000
--- a/services/store/memstore/testing/util.go
+++ /dev/null
@@ -1,431 +0,0 @@
-package testing
-
-import (
- "io"
- "runtime"
- "testing"
-
- "veyron/services/store/raw"
-
- "veyron2/context"
- "veyron2/ipc"
- "veyron2/naming"
- "veyron2/rt"
- "veyron2/security"
- "veyron2/services/watch"
- "veyron2/services/watch/types"
- "veyron2/storage"
-)
-
-// FakeServerContext implements ipc.ServerContext.
-type FakeServerContext struct {
- context.T
- cancel context.CancelFunc
- id security.PublicID
-}
-
-func NewFakeServerContext(id security.PublicID) *FakeServerContext {
- ctx, cancel := rt.R().NewContext().WithCancel()
-
- return &FakeServerContext{
- T: ctx,
- cancel: cancel,
- id: id,
- }
-}
-
-func (*FakeServerContext) Server() ipc.Server { return nil }
-func (*FakeServerContext) Method() string { return "" }
-func (*FakeServerContext) Name() string { return "" }
-func (*FakeServerContext) Suffix() string { return "" }
-func (*FakeServerContext) Label() (l security.Label) { return }
-func (*FakeServerContext) Discharges() map[string]security.Discharge { return nil }
-func (ctx *FakeServerContext) LocalID() security.PublicID { return ctx.id }
-func (ctx *FakeServerContext) RemoteID() security.PublicID { return ctx.id }
-func (*FakeServerContext) Blessing() security.PublicID { return nil }
-func (*FakeServerContext) LocalEndpoint() naming.Endpoint { return nil }
-func (*FakeServerContext) RemoteEndpoint() naming.Endpoint { return nil }
-func (ctx *FakeServerContext) Cancel() { ctx.cancel() }
-
-// Utilities for PutMutations.
-
-// storeServicePutMutationsStream implements raw.StoreServicePutMutationsStream
-type storeServicePutMutationsStream struct {
- mus <-chan raw.Mutation
- value raw.Mutation
-}
-
-func (s *storeServicePutMutationsStream) RecvStream() interface {
- Advance() bool
- Value() raw.Mutation
- Err() error
-} {
- return s
-}
-
-func (s *storeServicePutMutationsStream) Advance() bool {
- var ok bool
- s.value, ok = <-s.mus
- return ok
-}
-
-func (s *storeServicePutMutationsStream) Value() raw.Mutation {
- return s.value
-}
-
-func (s *storeServicePutMutationsStream) Err() error {
- return nil
-}
-
-// storePutMutationsStream implements raw.StorePutMutationsStream
-type storePutMutationsStream struct {
- closed bool
- mus chan<- raw.Mutation
-}
-
-func (s *storePutMutationsStream) Send(mu raw.Mutation) error {
- s.mus <- mu
- return nil
-}
-
-func (s *storePutMutationsStream) Close() error {
- if !s.closed {
- s.closed = true
- close(s.mus)
- }
- return nil
-}
-
-type storePutMutationsCall struct {
- ctx ipc.ServerContext
- stream storePutMutationsStream
- err <-chan error
-}
-
-func (s *storePutMutationsCall) SendStream() interface {
- Send(mu raw.Mutation) error
- Close() error
-} {
- return &s.stream
-}
-
-func (s *storePutMutationsCall) Finish() error {
- s.stream.Close()
- return <-s.err
-}
-
-func (s *storePutMutationsCall) Cancel() {
- s.ctx.(*FakeServerContext).Cancel()
- s.stream.Close()
-}
-
-func PutMutations(id security.PublicID, putMutationsFn func(ipc.ServerContext, raw.StoreServicePutMutationsStream) error) raw.StorePutMutationsCall {
- ctx := NewFakeServerContext(id)
- mus := make(chan raw.Mutation)
- err := make(chan error)
- go func() {
- err <- putMutationsFn(ctx, &storeServicePutMutationsStream{mus: mus})
- close(err)
- }()
- return &storePutMutationsCall{
- ctx: ctx,
- err: err,
- stream: storePutMutationsStream{
- mus: mus,
- },
- }
-}
-
-func PutMutationsBatch(t *testing.T, id security.PublicID, putMutationsFn func(ipc.ServerContext, raw.StoreServicePutMutationsStream) error, mus []raw.Mutation) {
- storePutMutationsStream := PutMutations(id, putMutationsFn)
- for _, mu := range mus {
- storePutMutationsStream.SendStream().Send(mu)
- }
- if err := storePutMutationsStream.Finish(); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): can't put mutations %s: %s", file, line, mus, err)
- }
-}
-
-// Utilities for Watch.
-
-// watcherServiceWatchStreamSender implements watch.WatcherServiceWatchStreamSender
-type watcherServiceWatchStreamSender struct {
- ctx ipc.ServerContext
- output chan<- types.Change
-}
-
-func (s *watcherServiceWatchStreamSender) Send(cb types.Change) error {
- select {
- case s.output <- cb:
- return nil
- case <-s.ctx.Done():
- return io.EOF
- }
-}
-
-// watcherServiceWatchStream implements watch.WatcherServiceWatchStream
-type watcherServiceWatchStream struct {
- watcherServiceWatchStreamSender
-}
-
-func (s *watcherServiceWatchStream) SendStream() interface {
- Send(cb types.Change) error
-} {
- return s
-}
-func (*watcherServiceWatchStream) Cancel() {}
-
-// watcherWatchStream implements watch.WatcherWatchStream.
-type watcherWatchStream struct {
- ctx *FakeServerContext
- value types.Change
- input <-chan types.Change
- err <-chan error
-}
-
-func (s *watcherWatchStream) Advance() bool {
- var ok bool
- s.value, ok = <-s.input
- return ok
-}
-
-func (s *watcherWatchStream) Value() types.Change {
- return s.value
-}
-
-func (*watcherWatchStream) Err() error {
- return nil
-}
-
-func (s *watcherWatchStream) Finish() error {
- <-s.input
- return <-s.err
-}
-
-func (s *watcherWatchStream) Cancel() {
- s.ctx.Cancel()
-}
-
-func (s *watcherWatchStream) RecvStream() interface {
- Advance() bool
- Value() types.Change
- Err() error
-} {
- return s
-}
-
-func watchImpl(id security.PublicID, watchFn func(ipc.ServerContext, *watcherServiceWatchStream) error) *watcherWatchStream {
- ctx := NewFakeServerContext(id)
- outputc := make(chan types.Change)
- inputc := make(chan types.Change)
- // This goroutine ensures that inputs will eventually stop going through
- // once the context is done. Send could handle this, but running a separate
- // goroutine is easier as we do not control invocations of Send.
- go func() {
- for {
- select {
- case change := <-outputc:
- inputc <- change
- case <-ctx.Done():
- close(inputc)
- return
- }
- }
- }()
- errc := make(chan error, 1)
- go func() {
- stream := &watcherServiceWatchStream{
- watcherServiceWatchStreamSender{
- ctx: ctx,
- output: outputc,
- },
- }
- err := watchFn(ctx, stream)
- errc <- err
- close(errc)
- ctx.Cancel()
- }()
- return &watcherWatchStream{
- ctx: ctx,
- input: inputc,
- err: errc,
- }
-}
-
-func WatchRaw(id security.PublicID, watchFn func(ipc.ServerContext, raw.Request, raw.StoreServiceWatchStream) error, req raw.Request) raw.StoreWatchCall {
- return watchImpl(id, func(ctx ipc.ServerContext, stream *watcherServiceWatchStream) error {
- return watchFn(ctx, req, stream)
- })
-}
-
-func WatchGlob(id security.PublicID, watchFn func(ipc.ServerContext, types.GlobRequest, watch.GlobWatcherServiceWatchGlobStream) error, req types.GlobRequest) watch.GlobWatcherWatchGlobCall {
- return watchImpl(id, func(ctx ipc.ServerContext, iterator *watcherServiceWatchStream) error {
- return watchFn(ctx, req, iterator)
- })
-}
-
-func WatchGlobOnPath(id security.PublicID, watchFn func(ipc.ServerContext, storage.PathName, types.GlobRequest, watch.GlobWatcherServiceWatchGlobStream) error, path storage.PathName, req types.GlobRequest) watch.GlobWatcherWatchGlobCall {
- return watchImpl(id, func(ctx ipc.ServerContext, stream *watcherServiceWatchStream) error {
- return watchFn(ctx, path, req, stream)
- })
-}
-
-func ExpectInitialStateSkipped(t *testing.T, change types.Change) {
- if change.Name != "" {
- t.Fatalf("Expect Name to be \"\" but was: %v", change.Name)
- }
- if change.State != types.InitialStateSkipped {
- t.Fatalf("Expect State to be InitialStateSkipped but was: %v", change.State)
- }
- if len(change.ResumeMarker) != 0 {
- t.Fatalf("Expect no ResumeMarker but was: %v", change.ResumeMarker)
- }
-}
-
-func ExpectEntryExists(t *testing.T, changes []types.Change, name string, id storage.ID, value string) {
- change := findEntry(t, changes, name)
- if change.State != types.Exists {
- t.Fatalf("Expected name to exist: %v", name)
- }
- cv, ok := change.Value.(*storage.Entry)
- if !ok {
- t.Fatalf("Expected an Entry")
- }
- if cv.Stat.ID != id {
- t.Fatalf("Expected ID to be %v, but was: %v", id, cv.Stat.ID)
- }
- if cv.Value != value {
- t.Fatalf("Expected Value to be %v, but was: %v", value, cv.Value)
- }
-}
-
-func ExpectEntryExistsNameOnly(t *testing.T, changes []types.Change, name string) {
- change := findEntry(t, changes, name)
- if change.State != types.Exists {
- t.Fatalf("Expected name to exist: %v", name)
- }
- _, ok := change.Value.(*storage.Entry)
- if !ok {
- t.Fatalf("Expected an Entry")
- }
-}
-
-func ExpectEntryDoesNotExist(t *testing.T, changes []types.Change, name string) {
- change := findEntry(t, changes, name)
- if change.State != types.DoesNotExist {
- t.Fatalf("Expected name to not exist: %v", name)
- }
- if change.Value != nil {
- t.Fatalf("Expected entry to be nil")
- }
-}
-
-func findEntry(t *testing.T, changes []types.Change, name string) types.Change {
- for _, change := range changes {
- if change.Name == name {
- return change
- }
- }
- t.Fatalf("Expected a change for name: %v", name)
- panic("Should not reach here")
-}
-
-var (
- EmptyDir = []raw.DEntry{}
-)
-
-func DirOf(name string, id storage.ID) []raw.DEntry {
- return []raw.DEntry{raw.DEntry{
- Name: name,
- ID: id,
- }}
-}
-
-func ExpectMutationExists(t *testing.T, changes []types.Change, id storage.ID, pre, post raw.Version, isRoot bool, value string, dir []raw.DEntry) {
- change := findMutation(t, changes, id)
- if change.State != types.Exists {
- t.Fatalf("Expected id to exist: %v", id)
- }
- cv := change.Value.(*raw.Mutation)
- if cv.PriorVersion != pre {
- t.Fatalf("Expected PriorVersion to be %v, but was: %v", pre, cv.PriorVersion)
- }
- if cv.Version != post {
- t.Fatalf("Expected Version to be %v, but was: %v", post, cv.Version)
- }
- if cv.IsRoot != isRoot {
- t.Fatalf("Expected IsRoot to be: %v, but was: %v", isRoot, cv.IsRoot)
- }
- if cv.Value != value {
- t.Fatalf("Expected Value to be: %v, but was: %v", value, cv.Value)
- }
- expectDirEquals(t, cv.Dir, dir)
-}
-
-func ExpectMutationDoesNotExist(t *testing.T, changes []types.Change, id storage.ID, pre raw.Version, isRoot bool) {
- change := findMutation(t, changes, id)
- if change.State != types.DoesNotExist {
- t.Fatalf("Expected id to not exist: %v", id)
- }
- cv := change.Value.(*raw.Mutation)
- if cv.PriorVersion != pre {
- t.Fatalf("Expected PriorVersion to be %v, but was: %v", pre, cv.PriorVersion)
- }
- if cv.Version != raw.NoVersion {
- t.Fatalf("Expected Version to be NoVersion, but was: %v", cv.Version)
- }
- if cv.IsRoot != isRoot {
- t.Fatalf("Expected IsRoot to be: %v, but was: %v", isRoot, cv.IsRoot)
- }
- if cv.Value != nil {
- t.Fatalf("Expected Value to be nil")
- }
- if cv.Dir != nil {
- t.Fatalf("Expected Dir to be nil")
- }
-}
-
-func ExpectMutationExistsNoVersionCheck(t *testing.T, changes []types.Change, id storage.ID, value string) {
- change := findMutation(t, changes, id)
- if change.State != types.Exists {
- t.Fatalf("Expected id to exist: %v", id)
- }
- cv := change.Value.(*raw.Mutation)
- if cv.Value != value {
- t.Fatalf("Expected Value to be: %v, but was: %v", value, cv.Value)
- }
-}
-
-func ExpectMutationDoesNotExistNoVersionCheck(t *testing.T, changes []types.Change, id storage.ID) {
- change := findMutation(t, changes, id)
- if change.State != types.DoesNotExist {
- t.Fatalf("Expected id to not exist: %v", id)
- }
-}
-
-func findMutation(t *testing.T, changes []types.Change, id storage.ID) types.Change {
- for _, change := range changes {
- cv, ok := change.Value.(*raw.Mutation)
- if !ok {
- t.Fatalf("Expected a Mutation")
- }
- if cv.ID == id {
- return change
- }
- }
- t.Fatalf("Expected a change for id: %v", id)
- panic("Should not reach here")
-}
-
-func expectDirEquals(t *testing.T, actual, expected []raw.DEntry) {
- if len(actual) != len(expected) {
- t.Fatalf("Expected Dir to have %v refs, but had %v", len(expected), len(actual))
- }
- for i, e := range expected {
- a := actual[i]
- if a != e {
- t.Fatalf("Expected Dir entry %v to be %v, but was %v", i, e, a)
- }
- }
-}
diff --git a/services/store/memstore/transaction.go b/services/store/memstore/transaction.go
deleted file mode 100644
index cf01271..0000000
--- a/services/store/memstore/transaction.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package memstore
-
-import (
- "sync"
-
- "veyron/services/store/memstore/state"
-)
-
-// Transaction is the type of transactions. Each transaction has a snapshot of
-// the entire state of the store; <store> is the shared *Store, assigned when
-// the transaction was first used, and <snapshot> is MutableSnapshot that
-// includes any changes in the transaction.
-//
-// Transactions are initially empty. The <store> and <snapshot> fields are
-// set when the transaction is first used.
-type Transaction struct {
- mutex sync.Mutex
- store *Store
- snapshot *state.MutableSnapshot
-}
-
-// newNilTransaction is used when nil is passed in as the transaction for an
-// object operation. This means that the operation is to be performed on the
-// state <st>.
-func (st *Store) newNilTransaction() *Transaction {
- st.Lock()
- defer st.Unlock()
- return &Transaction{store: st, snapshot: st.State.MutableSnapshot()}
-}
-
-// getTransaction returns the *Transaction value for the service.Transaction.
-// Returns bool commit==true iff the transaction argument is nil, which means
-// that the transaction lifetime is the duration of the operation (so the
-// transaction should be committed immediately after the operation that uses it
-// is performed).
-func (st *Store) getTransaction(tr *Transaction) (*Transaction, bool, error) {
- if tr == nil {
- return st.newNilTransaction(), true, nil
- }
- tr.useState(st)
- return tr, false, nil
-}
-
-// GetTransactionSnapshot returns a read-only snapshot from the transaction.
-func (st *Store) GetTransactionSnapshot(tr *Transaction) (state.Snapshot, error) {
- t, _, err := st.getTransaction(tr)
- if err != nil {
- return nil, err
- }
- return t.Snapshot(), nil
-}
-
-// NewTransaction returns a fresh transaction containing no changes.
-func NewTransaction() *Transaction {
- return &Transaction{}
-}
-
-// useState sets the state in the transaction if it hasn't been set already.
-func (t *Transaction) useState(st *Store) {
- t.mutex.Lock()
- defer t.mutex.Unlock()
- if t.store == nil {
- t.store = st
- t.snapshot = st.State.MutableSnapshot()
- }
-}
-
-// Commit commits the transaction. The commit may abort if there have been
-// concurrent changes to the store that conflict with the changes in the
-// transaction. If so, Commit returns an error and leaves the store unchanged.
-// The Commit is atomic -- all of the changes in the transaction are applied to
-// the state, or none of them are.
-func (t *Transaction) Commit() error {
- t.mutex.Lock()
- st, sn := t.store, t.snapshot
- t.mutex.Unlock()
- if st == nil || sn == nil {
- return nil
- }
- err := st.ApplyMutations(sn.Mutations())
- // Log deleted objects via garbage collection. This occurs within the
- // transaction boundary, i.e. before the state lock is released.
- // TODO(tilaks): separate discovery and collection, collect lazily.
- st.maybeGC(err)
- return err
-}
-
-// maybeGC will run a garbage collection if the committed transaction may result
-// in an orphaned object.
-// For now, the heuristic is simple - if the transaction succeeded, run GC().
-func (st *Store) maybeGC(err error) {
- if err == nil {
- st.GC()
- }
-}
-
-// Abort discards a transaction. This is an optimization; transactions
-// eventually time out and get discarded. However, live transactions
-// consume resources, so it is good practice to clean up.
-func (t *Transaction) Abort() error {
- t.mutex.Lock()
- t.store = nil
- t.snapshot = nil
- t.mutex.Unlock()
- return nil
-}
-
-// Snapshot returns a read-only snapshot.
-func (t *Transaction) Snapshot() state.Snapshot {
- t.mutex.Lock()
- defer t.mutex.Unlock()
- return t.snapshot.GetSnapshot()
-}
diff --git a/services/store/memstore/util_test.go b/services/store/memstore/util_test.go
deleted file mode 100644
index a7571ee..0000000
--- a/services/store/memstore/util_test.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package memstore
-
-import (
- "runtime"
- "testing"
-
- "veyron2/storage"
-)
-
-func mkdir(t *testing.T, st *Store, tr *Transaction, path string) (storage.ID, interface{}) {
- _, file, line, _ := runtime.Caller(1)
- dir := &Dir{}
- stat, err := st.Bind(path).Put(rootPublicID, tr, dir)
- if err != nil || stat == nil {
- t.Errorf("%s(%d): mkdir %s: %s", file, line, path, err)
- }
- return stat.ID, dir
-}
-
-func get(t *testing.T, st *Store, tr *Transaction, path string) interface{} {
- _, file, line, _ := runtime.Caller(1)
- e, err := st.Bind(path).Get(rootPublicID, tr)
- if err != nil {
- t.Fatalf("%s(%d): can't get %s: %s", file, line, path, err)
- }
- return e.Value
-}
-
-func put(t *testing.T, st *Store, tr *Transaction, path string, v interface{}) storage.ID {
- _, file, line, _ := runtime.Caller(1)
- stat, err := st.Bind(path).Put(rootPublicID, tr, v)
- if err != nil {
- t.Errorf("%s(%d): can't put %s: %s", file, line, path, err)
- }
- if _, err := st.Bind(path).Get(rootPublicID, tr); err != nil {
- t.Errorf("%s(%d): can't get %s: %s", file, line, path, err)
- }
- if stat != nil {
- return stat.ID
- }
- return storage.ID{}
-}
-
-func remove(t *testing.T, st *Store, tr *Transaction, path string) {
- if err := st.Bind(path).Remove(rootPublicID, tr); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): can't remove %s: %s", file, line, path, err)
- }
-}
-
-func commit(t *testing.T, tr *Transaction) {
- if err := tr.Commit(); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): Transaction aborted: %s", file, line, err)
- }
-}
-
-func expectExists(t *testing.T, st *Store, tr *Transaction, path string) {
- _, file, line, _ := runtime.Caller(1)
- if ok, _ := st.Bind(path).Exists(rootPublicID, tr); !ok {
- t.Errorf("%s(%d): does not exist: %s", file, line, path)
- }
-}
-
-func expectNotExists(t *testing.T, st *Store, tr *Transaction, path string) {
- if e, err := st.Bind(path).Get(rootPublicID, tr); err == nil {
- _, file, line, _ := runtime.Caller(1)
- t.Errorf("%s(%d): should not exist: %s: got %+v", file, line, path, e.Value)
- }
-}
-
-func expectValue(t *testing.T, st *Store, tr *Transaction, path string, v interface{}) {
- _, file, line, _ := runtime.Caller(1)
- e, err := st.Bind(path).Get(rootPublicID, tr)
- if err != nil {
- t.Errorf("%s(%d): does not exist: %s", file, line, path)
- return
- }
- if e.Value != v {
- t.Errorf("%s(%d): expected %+v, got %+v", file, line, e.Value, v)
- }
-
-}
diff --git a/services/store/memstore/watch/glob_processor.go b/services/store/memstore/watch/glob_processor.go
deleted file mode 100644
index 500e256..0000000
--- a/services/store/memstore/watch/glob_processor.go
+++ /dev/null
@@ -1,189 +0,0 @@
-package watch
-
-import (
- iquery "veyron/services/store/memstore/query"
- "veyron/services/store/memstore/state"
-
- "veyron2/security"
- "veyron2/services/watch/types"
- "veyron2/storage"
-)
-
-// globProcessor processes log entries into storage entries that match a pattern.
-type globProcessor struct {
- // hasProcessedState is true iff the initial state has been processed.
- hasProcessedState bool
- // pid is the identity of the client watching for changes.
- pid security.PublicID
- // path on which the watch is placed. Returned names are rooted at this path.
- path storage.PathName
- // pattern that the returned names match.
- pattern string
- // st is the store state as of the last processed event.
- st *state.State
- // matches is a map of each matching name to the id of the object at that
- // name, as of the last processed event.
- matches map[string]storage.ID
-}
-
-func newGlobProcessor(pid security.PublicID, path storage.PathName,
- pattern string) (reqProcessor, error) {
-
- return &globProcessor{
- hasProcessedState: false,
- pid: pid,
- path: path,
- pattern: pattern,
- }, nil
-}
-
-func (p *globProcessor) processState(st *state.State) ([]types.Change, error) {
- // Check that the initial state has not already been processed.
- if p.hasProcessedState {
- return nil, errInitialStateAlreadyProcessed
- }
- p.hasProcessedState = true
-
- // Find all names that match the pattern.
- sn := st.MutableSnapshot()
- matches, err := glob(sn, p.pid, p.path, p.pattern)
- if err != nil {
- return nil, err
- }
- p.st = st
- p.matches = matches
-
- var changes []types.Change
-
- // Create a change for every matching name.
- for name, id := range matches {
- cell := sn.Find(id)
- entry := cell.GetEntry()
- change := types.Change{
- Name: name,
- State: types.Exists,
- Value: entry,
- }
- // TODO(tilaks): don't clone change.
- changes = append(changes, change)
- }
-
- return changes, nil
-}
-
-func (p *globProcessor) processTransaction(mus *state.Mutations) ([]types.Change, error) {
- // Ensure that the initial state has been processed.
- if !p.hasProcessedState {
- return nil, errInitialStateNotProcessed
- }
-
- previousMatches := p.matches
- // Apply the transaction to the state.
- if err := p.st.ApplyMutations(mus); err != nil {
- return nil, err
- }
- // Find all names that match the pattern in the new state.
- sn := p.st.MutableSnapshot()
- newMatches, err := glob(sn, p.pid, p.path, p.pattern)
- if err != nil {
- return nil, err
- }
- p.matches = newMatches
-
- var changes []types.Change
-
- removed, updated := diffMatches(previousMatches, newMatches, mus.Delta)
-
- // Create a change for every matching name that was removed.
- for name := range removed {
- change := types.Change{
- Name: name,
- State: types.DoesNotExist,
- }
- // TODO(tilaks): don't clone change
- changes = append(changes, change)
- }
-
- // Create a change for every matching name that was updated.
- for name := range updated {
- id := newMatches[name]
- cell := sn.Find(id)
- entry := cell.GetEntry()
- change := types.Change{
- Name: name,
- State: types.Exists,
- Value: entry,
- }
- // TODO(tilaks): don't clone change.
- changes = append(changes, change)
- }
-
- return changes, nil
-}
-
-// diffMatches returns the names that have been removed or updated.
-//
-// A name is removed if it can no longer be resolved, or if the object at that
-// name is no longer accessible.
-//
-// A name is updated if
-// 1) it is newly added.
-// 2) the object at that name is now accessible.
-// 3) the object at the name has a new value or new references.
-// 4) the object at that name replaced a previous object.
-func diffMatches(previousMatches, newMatches map[string]storage.ID,
- delta map[storage.ID]*state.Mutation) (removed, updated map[string]struct{}) {
-
- removed = make(map[string]struct{})
- updated = make(map[string]struct{})
- present := struct{}{}
-
- for name, previousID := range previousMatches {
- if newID, ok := newMatches[name]; !ok {
- // There is no longer an object at this name.
- removed[name] = present
- } else if newID != previousID {
- // The object at this name was replaced.
- updated[name] = present
- }
- }
-
- for name, newID := range newMatches {
- if _, ok := previousMatches[name]; !ok {
- // An object was added at this name.
- updated[name] = present
- continue
- }
- if _, ok := delta[newID]; ok {
- // The value or implicit directory of the object at this name was
- // updated.
- updated[name] = present
- }
- }
-
- return
-}
-
-// glob returns all names in a snapshot that match a pattern. Each name maps to
-// the id of the object in the snapshot at that name.
-func glob(sn state.Snapshot, pid security.PublicID, path storage.PathName,
- pattern string) (map[string]storage.ID, error) {
-
- matches := make(map[string]storage.ID)
-
- it, err := iquery.GlobIterator(sn, pid, path, pattern)
- if err != nil {
- return nil, err
- }
-
- for it.IsValid() {
- name := it.Name()
- matchName := append(path, storage.ParsePath(name)...).String()
- entry := it.Get()
- id := entry.Stat.ID
- matches[matchName] = id
- it.Next()
- }
-
- return matches, nil
-}
diff --git a/services/store/memstore/watch/glob_processor_test.go b/services/store/memstore/watch/glob_processor_test.go
deleted file mode 100644
index 02f73c7..0000000
--- a/services/store/memstore/watch/glob_processor_test.go
+++ /dev/null
@@ -1,405 +0,0 @@
-package watch
-
-import (
- "testing"
-
- "veyron/services/store/memstore"
- watchtesting "veyron/services/store/memstore/testing"
-
- "veyron2/storage"
-)
-
-func TestGlobProcessState(t *testing.T) {
- // Create a new store.
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- // Put /, /a, /a/b.
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- id2 := put(t, st, tr, "/a", "val2")
- put(t, st, tr, "/a/b", "val3")
- id4 := put(t, st, tr, "/a/c", "val4")
- // Test duplicate paths to the same object.
- put(t, st, tr, "/a/d", id4)
- commit(t, tr)
-
- // Remove /a/b.
- tr = memstore.NewTransaction()
- remove(t, st, tr, "/a/b")
- commit(t, tr)
- gc(t, st)
-
- if err := st.Close(); err != nil {
- t.Fatalf("Close() failed: %v", err)
- }
-
- // Re-create a new store. This should compress the log, creating an initial
- // state containing / and /a.
- st, cleanup = openStore(t, dbName)
- defer cleanup()
-
- log, cleanup := openLog(t, dbName)
- defer cleanup()
-
- rootRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/"), "...")
- rootListProcessor := createGlobProcessor(t, storage.ParsePath("/"), "*")
- aRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "...")
- aListProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "*")
-
- // Expect initial state that contains /, /a, /a/c and /a/d.
- logst := readState(t, log)
-
- changes := processState(t, rootRecursiveProcessor, logst, 4)
- watchtesting.ExpectEntryExists(t, changes, "", id1, "val1")
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
- watchtesting.ExpectEntryExists(t, changes, "a/c", id4, "val4")
- watchtesting.ExpectEntryExists(t, changes, "a/d", id4, "val4")
-
- changes = processState(t, rootListProcessor, logst, 1)
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
-
- changes = processState(t, aRecursiveProcessor, logst, 3)
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
- watchtesting.ExpectEntryExists(t, changes, "a/c", id4, "val4")
- watchtesting.ExpectEntryExists(t, changes, "a/d", id4, "val4")
-
- processState(t, aListProcessor, logst, 2)
- watchtesting.ExpectEntryExists(t, changes, "a/c", id4, "val4")
- watchtesting.ExpectEntryExists(t, changes, "a/d", id4, "val4")
-}
-
-func TestGlobProcessTransactionAdd(t *testing.T) {
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- log, cleanup := openLog(t, dbName)
- defer cleanup()
-
- rootRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/"), "...")
- rootListProcessor := createGlobProcessor(t, storage.ParsePath("/"), "*")
- aRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "...")
- aListProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "*")
- bRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/a/b"), "...")
-
- logst := readState(t, log)
- processState(t, rootRecursiveProcessor, logst.DeepCopy(), 0)
- processState(t, rootListProcessor, logst.DeepCopy(), 0)
- processState(t, aRecursiveProcessor, logst.DeepCopy(), 0)
- processState(t, aListProcessor, logst.DeepCopy(), 0)
- processState(t, bRecursiveProcessor, logst.DeepCopy(), 0)
-
- // First transaction, put /, /a, /a/b.
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- id2 := put(t, st, tr, "/a", "val2")
- id3 := put(t, st, tr, "/a/b", "val3")
- id4 := put(t, st, tr, "/a/c", "val4")
- // Test duplicate paths to the same object.
- put(t, st, tr, "/a/d", id4)
- commit(t, tr)
-
- // Expect transaction that adds /, /a and /a/b.
- mus := readTransaction(t, log)
-
- changes := processTransaction(t, rootRecursiveProcessor, mus, 5)
- watchtesting.ExpectEntryExists(t, changes, "", id1, "val1")
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
- watchtesting.ExpectEntryExists(t, changes, "a/b", id3, "val3")
- watchtesting.ExpectEntryExists(t, changes, "a/c", id4, "val4")
- watchtesting.ExpectEntryExists(t, changes, "a/d", id4, "val4")
-
- changes = processTransaction(t, rootListProcessor, mus, 1)
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
-
- changes = processTransaction(t, aRecursiveProcessor, mus, 4)
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
- watchtesting.ExpectEntryExists(t, changes, "a/b", id3, "val3")
- watchtesting.ExpectEntryExists(t, changes, "a/c", id4, "val4")
- watchtesting.ExpectEntryExists(t, changes, "a/d", id4, "val4")
-
- changes = processTransaction(t, aListProcessor, mus, 3)
- watchtesting.ExpectEntryExists(t, changes, "a/b", id3, "val3")
- watchtesting.ExpectEntryExists(t, changes, "a/c", id4, "val4")
- watchtesting.ExpectEntryExists(t, changes, "a/d", id4, "val4")
-
- changes = processTransaction(t, bRecursiveProcessor, mus, 1)
- watchtesting.ExpectEntryExists(t, changes, "a/b", id3, "val3")
-}
-
-func TestGlobProcessTransactionEmptyPath(t *testing.T) {
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- log, cleanup := openLog(t, dbName)
- defer cleanup()
-
- bRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/a/b"), "...")
-
- expectState(t, log, bRecursiveProcessor, 0)
-
- // First transaction, put /, /a.
- tr := memstore.NewTransaction()
- put(t, st, tr, "/", "val1")
- put(t, st, tr, "/a", "val2")
- commit(t, tr)
-
- // Expect no change.
- expectTransaction(t, log, bRecursiveProcessor, 0)
-
- // Next transaction, put /a/b.
- tr = memstore.NewTransaction()
- id3 := put(t, st, tr, "/a/b", "val3")
- commit(t, tr)
-
- // Expect transaction that adds /a/b.
- changes := expectTransaction(t, log, bRecursiveProcessor, 1)
- watchtesting.ExpectEntryExists(t, changes, "a/b", id3, "val3")
-}
-
-func TestGlobProcessTransactionUpdate(t *testing.T) {
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- log, cleanup := openLog(t, dbName)
- defer cleanup()
-
- rootRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/"), "...")
- rootListProcessor := createGlobProcessor(t, storage.ParsePath("/"), "*")
- aRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "...")
- aListProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "*")
- bRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/a/b"), "...")
-
- logst := readState(t, log)
- processState(t, rootRecursiveProcessor, logst.DeepCopy(), 0)
- processState(t, rootListProcessor, logst.DeepCopy(), 0)
- processState(t, aRecursiveProcessor, logst.DeepCopy(), 0)
- processState(t, aListProcessor, logst.DeepCopy(), 0)
- processState(t, bRecursiveProcessor, logst.DeepCopy(), 0)
-
- // First transaction, put /, /a, /a/b.
- tr := memstore.NewTransaction()
- put(t, st, tr, "/", "val1")
- put(t, st, tr, "/a", "val2")
- id3 := put(t, st, tr, "/a/b", "val3")
- commit(t, tr)
-
- mus := readTransaction(t, log)
- changes := processTransaction(t, rootRecursiveProcessor, mus, 3)
- changes = processTransaction(t, rootListProcessor, mus, 1)
- changes = processTransaction(t, aRecursiveProcessor, mus, 2)
- changes = processTransaction(t, aListProcessor, mus, 1)
- changes = processTransaction(t, bRecursiveProcessor, mus, 1)
-
- // Next transaction, remove /a/b.
- tr = memstore.NewTransaction()
- put(t, st, tr, "/a/b", "val4")
- commit(t, tr)
-
- // Expect transaction that updates /a/b.
- mus = readTransaction(t, log)
-
- changes = processTransaction(t, rootRecursiveProcessor, mus, 1)
- watchtesting.ExpectEntryExists(t, changes, "a/b", id3, "val4")
-
- processTransaction(t, rootListProcessor, mus, 0)
-
- changes = processTransaction(t, aRecursiveProcessor, mus, 1)
- watchtesting.ExpectEntryExists(t, changes, "a/b", id3, "val4")
-
- processTransaction(t, aListProcessor, mus, 1)
- watchtesting.ExpectEntryExists(t, changes, "a/b", id3, "val4")
-
- processTransaction(t, bRecursiveProcessor, mus, 1)
- watchtesting.ExpectEntryExists(t, changes, "a/b", id3, "val4")
-}
-
-func TestGlobProcessTransactionRemove(t *testing.T) {
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- log, cleanup := openLog(t, dbName)
- defer cleanup()
-
- rootRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/"), "...")
- rootListProcessor := createGlobProcessor(t, storage.ParsePath("/"), "*")
- aRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "...")
- aListProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "*")
- bRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/a/b"), "...")
-
- logst := readState(t, log)
- processState(t, rootRecursiveProcessor, logst.DeepCopy(), 0)
- processState(t, rootListProcessor, logst.DeepCopy(), 0)
- processState(t, aRecursiveProcessor, logst.DeepCopy(), 0)
- processState(t, aListProcessor, logst.DeepCopy(), 0)
- processState(t, bRecursiveProcessor, logst.DeepCopy(), 0)
-
- // First transaction, put /, /a, /a/b.
- tr := memstore.NewTransaction()
- put(t, st, tr, "/", "val1")
- id2 := put(t, st, tr, "/a", "val2")
- put(t, st, tr, "/a/b", "val3")
- commit(t, tr)
-
- mus := readTransaction(t, log)
- processTransaction(t, rootRecursiveProcessor, mus, 3)
- processTransaction(t, rootListProcessor, mus, 1)
- processTransaction(t, aRecursiveProcessor, mus, 2)
- processTransaction(t, aListProcessor, mus, 1)
- processTransaction(t, bRecursiveProcessor, mus, 1)
-
- // Next transaction, remove /a/b.
- tr = memstore.NewTransaction()
- remove(t, st, tr, "/a/b")
- commit(t, tr)
-
- // Expect transaction that updates /a and removes /a/b.
- mus = readTransaction(t, log)
-
- changes := processTransaction(t, rootRecursiveProcessor, mus, 2)
- // TODO(tilaks): Should we report implicit directory changes?
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
- watchtesting.ExpectEntryDoesNotExist(t, changes, "a/b")
-
- changes = processTransaction(t, rootListProcessor, mus, 1)
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
-
- changes = processTransaction(t, aRecursiveProcessor, mus, 2)
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
- watchtesting.ExpectEntryDoesNotExist(t, changes, "a/b")
-
- changes = processTransaction(t, aListProcessor, mus, 1)
- watchtesting.ExpectEntryDoesNotExist(t, changes, "a/b")
-
- changes = processTransaction(t, bRecursiveProcessor, mus, 1)
- watchtesting.ExpectEntryDoesNotExist(t, changes, "a/b")
-
- // Garbage-collect the node at /a/b.
- gc(t, st)
-
- // Expect no change.
- mus = readTransaction(t, log)
- processTransaction(t, rootRecursiveProcessor, mus, 0)
- processTransaction(t, rootListProcessor, mus, 0)
- processTransaction(t, aRecursiveProcessor, mus, 0)
- processTransaction(t, aListProcessor, mus, 0)
- processTransaction(t, bRecursiveProcessor, mus, 0)
-}
-
-func TestGlobProcessTransactionRemoveRecursive(t *testing.T) {
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- log, cleanup := openLog(t, dbName)
- defer cleanup()
-
- rootRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/"), "...")
- rootListProcessor := createGlobProcessor(t, storage.ParsePath("/"), "*")
- aRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "...")
- aListProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "*")
- bRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/a/b"), "...")
-
- logst := readState(t, log)
- processState(t, rootRecursiveProcessor, logst.DeepCopy(), 0)
- processState(t, rootListProcessor, logst.DeepCopy(), 0)
- processState(t, aRecursiveProcessor, logst.DeepCopy(), 0)
- processState(t, aListProcessor, logst.DeepCopy(), 0)
- processState(t, bRecursiveProcessor, logst.DeepCopy(), 0)
-
- // First transaction, put /, /a, /a/b.
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- put(t, st, tr, "/a", "val2")
- put(t, st, tr, "/a/b", "val3")
- commit(t, tr)
-
- mus := readTransaction(t, log)
- processTransaction(t, rootRecursiveProcessor, mus, 3)
- processTransaction(t, rootListProcessor, mus, 1)
- processTransaction(t, aRecursiveProcessor, mus, 2)
- processTransaction(t, aListProcessor, mus, 1)
- processTransaction(t, bRecursiveProcessor, mus, 1)
-
- // Next transaction, remove /a.
- tr = memstore.NewTransaction()
- remove(t, st, tr, "/a")
- commit(t, tr)
-
- // Expect transaction that removes /a and /a/b.
- mus = readTransaction(t, log)
-
- changes := processTransaction(t, rootRecursiveProcessor, mus, 3)
- // TODO(tilaks): Should we report implicit directory changes?
- watchtesting.ExpectEntryExists(t, changes, "", id1, "val1")
- watchtesting.ExpectEntryDoesNotExist(t, changes, "a")
- watchtesting.ExpectEntryDoesNotExist(t, changes, "a/b")
-
- changes = processTransaction(t, rootListProcessor, mus, 1)
- watchtesting.ExpectEntryDoesNotExist(t, changes, "a")
-
- changes = processTransaction(t, aRecursiveProcessor, mus, 2)
- watchtesting.ExpectEntryDoesNotExist(t, changes, "a")
- watchtesting.ExpectEntryDoesNotExist(t, changes, "a/b")
-
- changes = processTransaction(t, aListProcessor, mus, 1)
- watchtesting.ExpectEntryDoesNotExist(t, changes, "a/b")
-
- changes = processTransaction(t, bRecursiveProcessor, mus, 1)
- watchtesting.ExpectEntryDoesNotExist(t, changes, "a/b")
-}
-
-func TestGlobProcessTransactionReplace(t *testing.T) {
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- log, cleanup := openLog(t, dbName)
- defer cleanup()
-
- rootRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/"), "...")
- rootListProcessor := createGlobProcessor(t, storage.ParsePath("/"), "*")
- aRecursiveProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "...")
- aListProcessor := createGlobProcessor(t, storage.ParsePath("/a"), "*")
-
- logst := readState(t, log)
-
- processState(t, rootRecursiveProcessor, logst.DeepCopy(), 0)
- processState(t, rootListProcessor, logst.DeepCopy(), 0)
- processState(t, aRecursiveProcessor, logst.DeepCopy(), 0)
- processState(t, aListProcessor, logst.DeepCopy(), 0)
-
- // First transaction, put /, /a, /a/b.
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- put(t, st, tr, "/a", "val2")
- commit(t, tr)
-
- mus := readTransaction(t, log)
- processTransaction(t, rootRecursiveProcessor, mus, 2)
- processTransaction(t, rootListProcessor, mus, 1)
- processTransaction(t, aRecursiveProcessor, mus, 1)
- processTransaction(t, aListProcessor, mus, 0)
-
- // Next transaction, replace /a.
- tr = memstore.NewTransaction()
- remove(t, st, tr, "/a")
- id2 := put(t, st, tr, "/a", "val2")
- commit(t, tr)
-
- // Expect transaction that updates /a.
- mus = readTransaction(t, log)
-
- changes := processTransaction(t, rootRecursiveProcessor, mus, 2)
- // TODO(tilaks): Should we report implicit directory changes?
- watchtesting.ExpectEntryExists(t, changes, "", id1, "val1")
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
-
- changes = processTransaction(t, rootListProcessor, mus, 1)
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
-
- changes = processTransaction(t, aRecursiveProcessor, mus, 1)
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
-
- processTransaction(t, aListProcessor, mus, 0)
-}
-
-// TODO(tilaks): test ACL update.
diff --git a/services/store/memstore/watch/processor.go b/services/store/memstore/watch/processor.go
deleted file mode 100644
index a3779e6..0000000
--- a/services/store/memstore/watch/processor.go
+++ /dev/null
@@ -1,30 +0,0 @@
-package watch
-
-import (
- "veyron/services/store/memstore/state"
- "veyron2/services/watch/types"
- "veyron2/verror"
-)
-
-var (
- errInitialStateAlreadyProcessed = verror.Internalf("cannot process state after processing the initial state")
- errInitialStateNotProcessed = verror.Internalf("cannot process a transaction before processing the initial state")
-)
-
-// reqProcessor processes log entries into watch changes. At first,
-// processState() must be called with the initial state recorded in the log.
-// Subsequently, processTransaction() may be called with transactions recorded
-// consecutively in the log.
-type reqProcessor interface {
- // processState returns a set of changes that represent the initial state of
- // the store. The returned changes need not be the sequence of changes that
- // originally created the initial state (e.g. in the case of compress), but
- // are sufficient to re-construct the state viewable within the request.
- // processState may modify its input.
- processState(st *state.State) ([]types.Change, error)
-
- // processTransaction returns the set of changes made in some transaction.
- // The changes are returned in no specific order.
- // processTransaction may modify its input.
- processTransaction(mu *state.Mutations) ([]types.Change, error)
-}
diff --git a/services/store/memstore/watch/raw_processor.go b/services/store/memstore/watch/raw_processor.go
deleted file mode 100644
index a5c012e..0000000
--- a/services/store/memstore/watch/raw_processor.go
+++ /dev/null
@@ -1,206 +0,0 @@
-package watch
-
-import (
- "veyron/services/store/memstore/refs"
- "veyron/services/store/memstore/state"
- "veyron/services/store/raw"
-
- "veyron2/security"
- "veyron2/services/watch/types"
- "veyron2/storage"
- "veyron2/verror"
-)
-
-var (
- rootPath = storage.ParsePath("/")
- nullID storage.ID
-)
-
-// rawProcessor processes log entries into mutations.
-type rawProcessor struct {
- // st is true iff the initial state has been processed.
- hasProcessedState bool
- // pid is the identity of the client watching for changes.
- pid security.PublicID
- // rootID is the id of the root object after processing a change.
- rootID storage.ID
- // rootVersion is the version of the store root after processing a change.
- rootVersion raw.Version
- // preparedDeletions is the set of ids for which deletion changes have been
- // sent by watch, but deleted entries have not been processed from the log.
- // This set consists of deleted store roots, because
- // 1) A root deletion is propagated as a deletion change on the root.
- // 2) A root deletion must be propagated immediately.
- // 3) GC is lazy, so we aggressively create a deletion change for the root.
- // An id is removed from preparedDeletions when the corresponding deleted
- // entry is processed from the log.
- preparedDeletions map[storage.ID]bool
-}
-
-func newRawProcessor(pid security.PublicID) (reqProcessor, error) {
- return &rawProcessor{
- hasProcessedState: false,
- pid: pid,
- preparedDeletions: make(map[storage.ID]bool),
- }, nil
-}
-
-func (p *rawProcessor) processState(st *state.State) ([]types.Change, error) {
- // Check that the initial state has not already been processed.
- if p.hasProcessedState {
- return nil, errInitialStateAlreadyProcessed
- }
- p.hasProcessedState = true
-
- sn := st.MutableSnapshot()
-
- rootID, err := rootID(p.pid, sn)
- if err != nil {
- return nil, err
- }
- p.rootID = rootID
-
- var changes []types.Change
-
- // Create a change for each id in the state. In each change, the object
- // exists, has no PriorVersion, has the Version of the new cell, and
- // has the Value, Tags and Dir of the new cell.
- for it := sn.NewIterator(p.pid, nil,
- state.ListObjects, state.RecursiveFilter); it.IsValid(); it.Next() {
-
- entry := it.Get()
- id := entry.Stat.ID
- // Retrieve Value, Tags and Dir from the corresponding cell.
- cell := sn.Find(id)
- // If this object is the root, update rootVersion.
- isRoot := id == p.rootID
- if isRoot {
- p.rootVersion = cell.Version
- }
- value := &raw.Mutation{
- ID: id,
- PriorVersion: raw.NoVersion,
- Version: cell.Version,
- IsRoot: isRoot,
- Value: cell.Value,
- Dir: flattenDir(refs.FlattenDir(cell.Dir)),
- }
- change := types.Change{
- State: types.Exists,
- Value: value,
- }
- // TODO(tilaks): don't clone change
- changes = append(changes, change)
- }
- return changes, nil
-}
-
-func (p *rawProcessor) processTransaction(mus *state.Mutations) ([]types.Change, error) {
- // Ensure that the initial state has been processed.
- if !p.hasProcessedState {
- return nil, errInitialStateNotProcessed
- }
-
- // If the root was deleted, add extra space for a prepared deletion.
- extra := 0
- if mus.SetRootID && !mus.RootID.IsValid() {
- extra = 1
- }
- changes := make([]types.Change, 0, len(mus.Delta)+len(mus.Deletions)+extra)
-
- if mus.SetRootID {
- if mus.RootID.IsValid() {
- p.rootID = mus.RootID
- } else {
- // The root was deleted, prepare a deletion change.
- value := &raw.Mutation{
- ID: p.rootID,
- PriorVersion: p.rootVersion,
- Version: raw.NoVersion,
- IsRoot: true,
- }
- // TODO(tilaks): don't clone value.
- change := types.Change{
- State: types.DoesNotExist,
- Value: value,
- }
- changes = append(changes, change)
-
- p.preparedDeletions[p.rootID] = true
- p.rootID = nullID
- p.rootVersion = raw.NoVersion
- }
- }
-
- // Create a change for each mutation. In each change, the object exists,
- // has the PriorVersion, Version, Value, Tags and Dir specified in
- // the mutation.
- for id, mu := range mus.Delta {
- // If this object is the root, update rootVersion.
- isRoot := id == p.rootID
- if isRoot {
- p.rootVersion = mu.Postcondition
- }
- value := &raw.Mutation{
- ID: id,
- PriorVersion: mus.Preconditions[id],
- Version: mu.Postcondition,
- IsRoot: isRoot,
- Value: mu.Value,
- Dir: flattenDir(mu.Dir),
- }
- // TODO(tilaks): don't clone value.
- change := types.Change{
- State: types.Exists,
- Value: value,
- }
- // TODO(tilaks): don't clone change.
- changes = append(changes, change)
- }
- // Create a change for each deletion (if one has not already been prepared).
- // In each change, the object does not exist, has the specified PriorVersion,
- // has no Version, and has nil Value, Tags and Dir.
- for id, precondition := range mus.Deletions {
- if p.preparedDeletions[id] {
- delete(p.preparedDeletions, id)
- continue
- }
- value := &raw.Mutation{
- ID: id,
- PriorVersion: precondition,
- Version: raw.NoVersion,
- IsRoot: false,
- }
- // TODO(tilaks): don't clone value.
- change := types.Change{
- State: types.DoesNotExist,
- Value: value,
- }
- // TODO(tilaks): don't clone change.
- changes = append(changes, change)
- }
- return changes, nil
-}
-
-// TODO(tilaks): revisit when raw.Mutation.Dir is of type []*raw.DEntry
-// (once we support optional structs in the idl).
-func flattenDir(pdir []*raw.DEntry) []raw.DEntry {
- fdir := make([]raw.DEntry, len(pdir))
- for i, p := range pdir {
- fdir[i] = *p
- }
- return fdir
-}
-
-// rootID returns the id of the root object in the snapshot. If the snapshot
-// does not have a root, nullID is returned.
-func rootID(pid security.PublicID, sn *state.MutableSnapshot) (storage.ID, error) {
- entry, err := sn.Get(pid, rootPath)
- if verror.Is(err, verror.NotFound) {
- return nullID, nil
- }
- if err != nil {
- return nullID, err
- }
- return entry.Stat.ID, nil
-}
diff --git a/services/store/memstore/watch/raw_processor_test.go b/services/store/memstore/watch/raw_processor_test.go
deleted file mode 100644
index f1a94bd..0000000
--- a/services/store/memstore/watch/raw_processor_test.go
+++ /dev/null
@@ -1,218 +0,0 @@
-package watch
-
-import (
- "testing"
-
- "veyron/services/store/memstore"
- watchtesting "veyron/services/store/memstore/testing"
- "veyron/services/store/raw"
-)
-
-func TestRawProcessState(t *testing.T) {
- // Create a new store.
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- // Put /, /a, /a/b
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- id2 := put(t, st, tr, "/a", "val2")
- put(t, st, tr, "/a/b", "val3")
- commit(t, tr)
-
- // Remove /a/b
- tr = memstore.NewTransaction()
- remove(t, st, tr, "/a/b")
- commit(t, tr)
- gc(t, st)
-
- if err := st.Close(); err != nil {
- t.Fatalf("Close() failed: %v", err)
- }
-
- // Re-create a new store. This should compress the log, creating an initial
- // state containing / and /a.
- st, cleanup = openStore(t, dbName)
- defer cleanup()
-
- log, cleanup := openLog(t, dbName)
- defer cleanup()
- processor := createRawProcessor(t)
-
- post1 := st.Snapshot().Find(id1).Version
- post2 := st.Snapshot().Find(id2).Version
-
- // Expect initial state that
- // 1) Contains / with value val1 and implicit directory entry /a
- // 2) Contains /a with value val2
- changes := expectState(t, log, processor, 2)
- watchtesting.ExpectMutationExists(t, changes, id1, raw.NoVersion, post1, true, "val1", watchtesting.DirOf("a", id2))
- watchtesting.ExpectMutationExists(t, changes, id2, raw.NoVersion, post2, false, "val2", watchtesting.EmptyDir)
-}
-
-func TestRawProcessTransactionAddRemove(t *testing.T) {
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- log, cleanup := openLog(t, dbName)
- defer cleanup()
- processor := createRawProcessor(t)
-
- expectState(t, log, processor, 0)
-
- // First transaction, put /, /a, /a/b
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- id2 := put(t, st, tr, "/a", "val2")
- id3 := put(t, st, tr, "/a/b", "val3")
- commit(t, tr)
-
- post1 := st.Snapshot().Find(id1).Version
- post2 := st.Snapshot().Find(id2).Version
- post3 := st.Snapshot().Find(id3).Version
-
- // Expect transaction that
- // 1) Adds / with value val1 and implicit directory entry /a
- // 2) Adds /a with value val2 and implicit directory entry /a/b
- // 3) Adds /a/b with value val3
- changes := expectTransaction(t, log, processor, 3)
- watchtesting.ExpectMutationExists(t, changes, id1, raw.NoVersion, post1, true, "val1", watchtesting.DirOf("a", id2))
- watchtesting.ExpectMutationExists(t, changes, id2, raw.NoVersion, post2, false, "val2", watchtesting.DirOf("b", id3))
- watchtesting.ExpectMutationExists(t, changes, id3, raw.NoVersion, post3, false, "val3", watchtesting.EmptyDir)
-
- // Next transaction, remove /a/b
- tr = memstore.NewTransaction()
- remove(t, st, tr, "/a/b")
- commit(t, tr)
-
- pre2 := post2
- pre3 := post3
- post2 = st.Snapshot().Find(id2).Version
-
- // Expect transaction that removes implicit dir entry /a/b from /a
- changes = expectTransaction(t, log, processor, 1)
- watchtesting.ExpectMutationExists(t, changes, id2, pre2, post2, false, "val2", watchtesting.EmptyDir)
-
- // Garbage-collect the node at /a/b
- gc(t, st)
-
- // Expect transaction that deletes the node at /a/b
- changes = expectTransaction(t, log, processor, 1)
- watchtesting.ExpectMutationDoesNotExist(t, changes, id3, pre3, false)
-}
-
-func TestRawProcessTransactionRemoveRecursive(t *testing.T) {
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- log, cleanup := openLog(t, dbName)
- defer cleanup()
- processor := createRawProcessor(t)
-
- processor, err := newRawProcessor(rootPublicID)
- if err != nil {
- t.Fatalf("newRawProcessor() failed: %v", err)
- }
-
- expectState(t, log, processor, 0)
-
- // First transaction, put /, /a, /a/b
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- id2 := put(t, st, tr, "/a", "val2")
- id3 := put(t, st, tr, "/a/b", "val3")
- commit(t, tr)
-
- post1 := st.Snapshot().Find(id1).Version
- post2 := st.Snapshot().Find(id2).Version
- post3 := st.Snapshot().Find(id3).Version
-
- // Assume the first transaction
- // 1) Adds / with value val1 and implicit directory entry /a
- // 2) Adds /a with value val2 and implicit directory entry /a/b
- // 3) Adds /a/b with value val3
- expectTransaction(t, log, processor, 3)
-
- // Next transaction, remove /a
- tr = memstore.NewTransaction()
- remove(t, st, tr, "/a")
- commit(t, tr)
-
- pre1 := post1
- pre2 := post2
- pre3 := post3
- post1 = st.Snapshot().Find(id1).Version
-
- // Expect transaction that removes implicit dir entry /a from /
- changes := expectTransaction(t, log, processor, 1)
- watchtesting.ExpectMutationExists(t, changes, id1, pre1, post1, true, "val1", watchtesting.EmptyDir)
-
- // Garbage-collect the nodes at /a and /a/b
- gc(t, st)
-
- // Expect transaction that deletes the nodes at /a and /a/b
- changes = expectTransaction(t, log, processor, 2)
- watchtesting.ExpectMutationDoesNotExist(t, changes, id2, pre2, false)
- watchtesting.ExpectMutationDoesNotExist(t, changes, id3, pre3, false)
-}
-
-func TestRawProcessTransactionUpdateRemoveRoot(t *testing.T) {
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- log, cleanup := openLog(t, dbName)
- defer cleanup()
- processor := createRawProcessor(t)
-
- processor, err := newRawProcessor(rootPublicID)
- if err != nil {
- t.Fatalf("newRawProcessor() failed: %v", err)
- }
-
- expectState(t, log, processor, 0)
-
- // First transaction, put /, /a
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- id2 := put(t, st, tr, "/a", "val2")
- commit(t, tr)
-
- post1 := st.Snapshot().Find(id1).Version
- post2 := st.Snapshot().Find(id2).Version
-
- // Assume the first transaction
- // 1) Adds / with value val1 and implicit directory entry /a
- // 2) Adds /a with value val2
- expectTransaction(t, log, processor, 2)
-
- // Next transaction, update /
- tr = memstore.NewTransaction()
- put(t, st, tr, "/", "val3")
- commit(t, tr)
-
- pre1 := post1
- post1 = st.Snapshot().Find(id1).Version
-
- // Expect transaction that updates / with value val3
- changes := expectTransaction(t, log, processor, 1)
- watchtesting.ExpectMutationExists(t, changes, id1, pre1, post1, true, "val3", watchtesting.DirOf("a", id2))
-
- // Next transaction, remove /
- tr = memstore.NewTransaction()
- remove(t, st, tr, "/")
- commit(t, tr)
-
- pre1 = post1
- pre2 := post2
-
- // Expect a transaction that deletes /
- changes = expectTransaction(t, log, processor, 1)
- watchtesting.ExpectMutationDoesNotExist(t, changes, id1, pre1, true)
-
- // Garbage-collect the nodes at / and /a
- gc(t, st)
-
- // Expect transaction that deletes the nodes at / and /a
- changes = expectTransaction(t, log, processor, 1)
- watchtesting.ExpectMutationDoesNotExist(t, changes, id2, pre2, false)
-}
diff --git a/services/store/memstore/watch/test_util.go b/services/store/memstore/watch/test_util.go
deleted file mode 100644
index 32735bc..0000000
--- a/services/store/memstore/watch/test_util.go
+++ /dev/null
@@ -1,178 +0,0 @@
-package watch
-
-import (
- "io/ioutil"
- "os"
- "runtime"
- "testing"
-
- "veyron/services/store/memstore"
- "veyron/services/store/memstore/state"
-
- "veyron2/security"
- "veyron2/services/watch/types"
- "veyron2/storage"
-)
-
-var (
- rootPublicID security.PublicID = security.FakePublicID("root")
-)
-
-func get(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) interface{} {
- _, file, line, _ := runtime.Caller(1)
- e, err := st.Bind(path).Get(rootPublicID, tr)
- if err != nil {
- t.Fatalf("%s(%d): can't get %s: %s", file, line, path, err)
- }
- return e.Value
-}
-
-func put(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string, v interface{}) storage.ID {
- _, file, line, _ := runtime.Caller(1)
- stat, err := st.Bind(path).Put(rootPublicID, tr, v)
- if err != nil {
- t.Fatalf("%s(%d): can't put %s: %s", file, line, path, err)
- }
- if _, err := st.Bind(path).Get(rootPublicID, tr); err != nil {
- t.Fatalf("%s(%d): can't get %s: %s", file, line, path, err)
- }
- if stat != nil {
- return stat.ID
- }
- return storage.ID{}
-}
-
-func remove(t *testing.T, st *memstore.Store, tr *memstore.Transaction, path string) {
- if err := st.Bind(path).Remove(rootPublicID, tr); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): can't remove %s: %s", file, line, path, err)
- }
-}
-
-func commit(t *testing.T, tr *memstore.Transaction) {
- if err := tr.Commit(); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): Transaction aborted: %s", file, line, err)
- }
-}
-
-func gc(t *testing.T, st *memstore.Store) {
- if err := st.GC(); err != nil {
- _, file, line, _ := runtime.Caller(1)
- t.Fatalf("%s(%d): can't gc: %s", file, line, err)
- }
-}
-
-func createStore(t *testing.T) (string, *memstore.Store, func()) {
- dbName, err := ioutil.TempDir(os.TempDir(), "vstore")
- if err != nil {
- t.Fatalf("ioutil.TempDir() failed: %v", err)
- }
- cleanup := func() {
- os.RemoveAll(dbName)
- }
-
- st, err := memstore.New(rootPublicID, dbName)
- if err != nil {
- cleanup()
- t.Fatalf("memstore.New() failed: %v", err)
- }
-
- return dbName, st, cleanup
-}
-
-func openStore(t *testing.T, dbName string) (*memstore.Store, func()) {
- st, err := memstore.New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("memstore.New() failed: %v", err)
- }
-
- return st, func() {
- os.RemoveAll(dbName)
- }
-}
-
-func openLog(t *testing.T, dbName string) (*memstore.RLog, func()) {
- log, err := memstore.OpenLog(dbName, true)
- if err != nil {
- t.Fatalf("openLog() failed: %v", err)
- }
-
- return log, func() {
- log.Close()
- }
-}
-
-func createRawProcessor(t *testing.T) reqProcessor {
- processor, err := newRawProcessor(rootPublicID)
- if err != nil {
- t.Fatalf("newRawProcessor() failed: %v", err)
- }
- return processor
-}
-
-func createGlobProcessor(t *testing.T, path storage.PathName, pattern string) reqProcessor {
- processor, err := newGlobProcessor(rootPublicID, path, pattern)
- if err != nil {
- t.Fatalf("newGlobProcessor() failed: %v", err)
- }
- return processor
-}
-
-func createWatcher(t *testing.T, dbName string) (*Watcher, func()) {
- w, err := New(rootPublicID, dbName)
- if err != nil {
- t.Fatalf("New() failed: %v", err)
- }
- return w, func() {
- w.Close()
- }
-}
-
-func expectState(t *testing.T, log *memstore.RLog, processor reqProcessor, numChanges int) []types.Change {
- st := readState(t, log)
- return processState(t, processor, st, numChanges)
-}
-
-func readState(t *testing.T, log *memstore.RLog) *state.State {
- st, err := log.ReadState(rootPublicID)
- if err != nil {
- t.Fatalf("ReadState() failed: %v", err)
- }
- return st.State
-}
-
-func processState(t *testing.T, processor reqProcessor, st *state.State, numChanges int) []types.Change {
- changes, err := processor.processState(st)
- if err != nil {
- t.Fatalf("processState() failed: %v", err)
- }
- if len(changes) != numChanges {
- t.Fatalf("Expected state to have %d changes, got %d", numChanges, len(changes))
- }
- return changes
-}
-
-func expectTransaction(t *testing.T, log *memstore.RLog, processor reqProcessor, numChanges int) []types.Change {
- mus := readTransaction(t, log)
- return processTransaction(t, processor, mus, numChanges)
-}
-
-func readTransaction(t *testing.T, log *memstore.RLog) *state.Mutations {
- mus, err := log.ReadTransaction()
- if err != nil {
- t.Fatalf("ReadTransaction() failed: %v", err)
- }
- return mus
-}
-
-func processTransaction(t *testing.T, processor reqProcessor, mus *state.Mutations, numChanges int) []types.Change {
- changes, err := processor.processTransaction(mus)
- if err != nil {
- t.Fatalf("processTransaction() failed: %v", err)
- }
- if len(changes) != numChanges {
- t.Fatalf("Expected transaction to have %d changes, got %d", numChanges, len(changes))
- }
- return changes
-}
diff --git a/services/store/memstore/watch/watcher.go b/services/store/memstore/watch/watcher.go
deleted file mode 100644
index 5896f5e..0000000
--- a/services/store/memstore/watch/watcher.go
+++ /dev/null
@@ -1,326 +0,0 @@
-package watch
-
-import (
- "bytes"
- "encoding/binary"
- "io"
- "time"
-
- "veyron/runtimes/google/lib/sync"
- "veyron/services/store/memstore"
- "veyron/services/store/raw"
-
- "veyron2/ipc"
- "veyron2/security"
- "veyron2/services/watch"
- "veyron2/services/watch/types"
- "veyron2/storage"
- "veyron2/verror"
-)
-
-var (
- errWatchClosed = io.EOF
- errUnknownResumeMarker = verror.BadArgf("Unknown ResumeMarker")
- nowResumeMarker = []byte("now") // UTF-8 conversion.
- initialStateSkippedChange = types.Change{
- Name: "",
- State: types.InitialStateSkipped,
- }
-)
-
-type Watcher struct {
- // admin is the public id of the store administrator.
- admin security.PublicID
- // dbName is the name of the store's database directory.
- dbName string
- // closed is a channel that is closed when the watcher is closed.
- // Watch invocations finish as soon as possible once the channel is closed.
- closed chan struct{}
- // pending records the number of Watch invocations on this watcher that
- // have not yet finished.
- pending sync.WaitGroup
-}
-
-// New returns a new watcher. The returned watcher supports repeated and
-// concurrent invocations of Watch until it is closed.
-// admin is the public id of the store administrator. dbName is the name of the
-// of the store's database directory.
-func New(admin security.PublicID, dbName string) (*Watcher, error) {
- return &Watcher{
- admin: admin,
- dbName: dbName,
- closed: make(chan struct{}),
- }, nil
-}
-
-// WatchRaw returns a stream of all changes.
-func (w *Watcher) WatchRaw(ctx ipc.ServerContext, req raw.Request,
- stream raw.StoreServiceWatchStream) error {
-
- processor, err := newRawProcessor(ctx.RemoteID())
- if err != nil {
- return err
- }
- return w.Watch(ctx, processor, req.ResumeMarker, stream.SendStream())
-}
-
-// WatchGlob returns a stream of changes that match a pattern.
-func (w *Watcher) WatchGlob(ctx ipc.ServerContext, path storage.PathName,
- req types.GlobRequest, stream watch.GlobWatcherServiceWatchGlobStream) error {
-
- processor, err := newGlobProcessor(ctx.RemoteID(), path, req.Pattern)
- if err != nil {
- return err
- }
- return w.Watch(ctx, processor, req.ResumeMarker, stream.SendStream())
-}
-
-// WatchQuery returns a stream of changes that satisfy a query.
-func (w *Watcher) WatchQuery(ctx ipc.ServerContext, path storage.PathName,
- req types.QueryRequest, stream watch.QueryWatcherServiceWatchQueryStream) error {
-
- return verror.Internalf("WatchQuery not yet implemented")
-}
-
-// WatchStream is the interface for streaming responses of Watch methods.
-type WatchStream interface {
- // Send places the item onto the output stream, blocking if there is no
- // buffer space available.
- Send(item types.Change) error
-}
-
-// Watch handles the specified request, processing records in the store log and
-// sending changes to the specified watch stream. If the call is cancelled or
-// otherwise closed early, Watch will terminate and return an error.
-// Watch implements the service.Watcher interface.
-func (w *Watcher) Watch(ctx ipc.ServerContext, processor reqProcessor,
- resumeMarker types.ResumeMarker, stream WatchStream) error {
-
- // Closing cancel terminates processRequest.
- cancel := make(chan struct{})
- defer close(cancel)
-
- done := make(chan error, 1)
-
- if !w.pending.TryAdd() {
- return errWatchClosed
- }
- // This goroutine does not leak because processRequest is always terminated.
- go func() {
- defer w.pending.Done()
- done <- w.processRequest(cancel, processor, resumeMarker, stream)
- close(done)
- }()
-
- select {
- case err := <-done:
- return err
- // Close cancel and terminate processRequest if:
- // 1) The watcher has been closed.
- // 2) The call closes. This is signalled on the context's closed channel.
- case <-w.closed:
- case <-ctx.Done():
- }
- return errWatchClosed
-}
-
-func (w *Watcher) processRequest(cancel <-chan struct{}, processor reqProcessor,
- resumeMarker types.ResumeMarker, stream WatchStream) error {
-
- log, err := memstore.OpenLog(w.dbName, true)
- if err != nil {
- return err
- }
- // This goroutine does not leak because cancel is always closed.
- go func() {
- <-cancel
-
- // Closing the log terminates any ongoing read, and processRequest
- // returns an error.
- log.Close()
-
- // stream.Send() is automatically cancelled when the call completes,
- // so we don't explicitly cancel sendChanges.
-
- // TODO(tilaks): cancel processState(), processTransaction().
- }()
-
- filter, err := newChangeFilter(resumeMarker)
- if err != nil {
- return err
- }
-
- if isNowResumeMarker(resumeMarker) {
- sendChanges(stream, []types.Change{initialStateSkippedChange})
- }
-
- // Process initial state.
- store, err := log.ReadState(w.admin)
- if err != nil {
- return err
- }
- st := store.State
- // Save timestamp as processState may modify st.
- timestamp := st.Timestamp()
- changes, err := processor.processState(st)
- if err != nil {
- return err
- }
- if send, err := filter.shouldProcessChanges(timestamp); err != nil {
- return err
- } else if send {
- if err := processChanges(stream, changes, timestamp); err != nil {
- return err
- }
- }
-
- for {
- // Process transactions.
- mu, err := log.ReadTransaction()
- if err != nil {
- return err
- }
- // Save timestamp as processTransaction may modify mu.
- timestamp := mu.Timestamp
- changes, err = processor.processTransaction(mu)
- if err != nil {
- return err
- }
- if send, err := filter.shouldProcessChanges(timestamp); err != nil {
- return err
- } else if send {
- if err := processChanges(stream, changes, timestamp); err != nil {
- return err
- }
- }
- }
-}
-
-// Close implements the service.Watcher interface.
-func (w *Watcher) Close() error {
- close(w.closed)
- w.pending.Wait()
- return nil
-}
-
-// IsClosed returns true iff the watcher has been closed.
-func (w *Watcher) isClosed() bool {
- select {
- case <-w.closed:
- return true
- default:
- return false
- }
-}
-
-type changeFilter interface {
- // shouldProcessChanges determines whether to process changes with the given
- // timestamp. Changes should appear in the sequence of the store log, and
- // timestamps should be monotonically increasing.
- shouldProcessChanges(timestamp uint64) (bool, error)
-}
-
-type baseFilter struct {
- // initialTimestamp is the minimum timestamp of the first change sent.
- initialTimestamp uint64
- // crossedInitialTimestamp is true if a change with timestamp >=
- // initialTimestamp has already been sent.
- crossedInitialTimestamp bool
-}
-
-// onOrAfterFilter accepts any change with timestamp >= initialTimestamp.
-type onOrAfterFilter struct {
- baseFilter
-}
-
-// onAndAfterFilter accepts any change with timestamp >= initialTimestamp, but
-// requires the first change to have timestamp = initialTimestamp.
-type onAndAfterFilter struct {
- baseFilter
-}
-
-// newChangeFilter creates a changeFilter that processes changes only
-// at or after the requested resumeMarker.
-func newChangeFilter(resumeMarker []byte) (changeFilter, error) {
- if len(resumeMarker) == 0 {
- return &onOrAfterFilter{baseFilter{0, false}}, nil
- }
- if isNowResumeMarker(resumeMarker) {
- // TODO(tilaks): Get the current resume marker from the log.
- return &onOrAfterFilter{baseFilter{uint64(time.Now().UnixNano()), false}}, nil
- }
- if len(resumeMarker) != 8 {
- return nil, errUnknownResumeMarker
- }
- return &onAndAfterFilter{baseFilter{binary.BigEndian.Uint64(resumeMarker), false}}, nil
-}
-
-func (f *onOrAfterFilter) shouldProcessChanges(timestamp uint64) (bool, error) {
- // Bypass checks if a change with timestamp >= initialTimestamp has already
- // been sent.
- if !f.crossedInitialTimestamp {
- if timestamp < f.initialTimestamp {
- return false, nil
- }
- }
- f.crossedInitialTimestamp = true
- return true, nil
-}
-
-func (f *onAndAfterFilter) shouldProcessChanges(timestamp uint64) (bool, error) {
- // Bypass checks if a change with timestamp >= initialTimestamp has already
- // been sent.
- if !f.crossedInitialTimestamp {
- if timestamp < f.initialTimestamp {
- return false, nil
- }
- if timestamp > f.initialTimestamp {
- return false, errUnknownResumeMarker
- }
- // TODO(tilaks): if the most recent timestamp in the log is less than
- // initialTimestamp, return ErrUnknownResumeMarker.
- }
- f.crossedInitialTimestamp = true
- return true, nil
-}
-
-func processChanges(stream WatchStream, changes []types.Change, timestamp uint64) error {
- addContinued(changes)
- addResumeMarkers(changes, timestampToResumeMarker(timestamp))
- return sendChanges(stream, changes)
-}
-
-func sendChanges(stream WatchStream, changes []types.Change) error {
- for _, change := range changes {
- if err := stream.Send(change); err != nil {
- return err
- }
- }
- return nil
-}
-
-func addContinued(changes []types.Change) {
- // Last change marks the end of the processed atomic group.
- for i, _ := range changes {
- changes[i].Continued = true
- }
- if len(changes) > 0 {
- changes[len(changes)-1].Continued = false
- }
-}
-
-func addResumeMarkers(changes []types.Change, resumeMarker []byte) {
- for i, _ := range changes {
- changes[i].ResumeMarker = resumeMarker
- }
-}
-
-func isNowResumeMarker(resumeMarker []byte) bool {
- return bytes.Equal(resumeMarker, nowResumeMarker)
-}
-
-func timestampToResumeMarker(timestamp uint64) []byte {
- buf := make([]byte, 8)
- binary.BigEndian.PutUint64(buf, timestamp)
- return buf
-}
diff --git a/services/store/memstore/watch/watcher_test.go b/services/store/memstore/watch/watcher_test.go
deleted file mode 100644
index 46132a1..0000000
--- a/services/store/memstore/watch/watcher_test.go
+++ /dev/null
@@ -1,585 +0,0 @@
-package watch
-
-import (
- "bytes"
- "io"
- "sync"
- "testing"
- "time"
-
- "veyron/services/store/memstore"
- watchtesting "veyron/services/store/memstore/testing"
- "veyron/services/store/raw"
-
- "veyron2/rt"
- "veyron2/services/watch/types"
- "veyron2/storage"
- "veyron2/verror"
-)
-
-func TestWatchRaw(t *testing.T) {
- rt.Init()
-
- // Create a new store.
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- // Create a new watcher.
- w, cleanup := createWatcher(t, dbName)
- defer cleanup()
-
- // Put /
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- commit(t, tr)
-
- post1 := st.Snapshot().Find(id1).Version
-
- // Start a watch request.
- req := raw.Request{}
- ws := watchtesting.WatchRaw(rootPublicID, w.WatchRaw, req)
-
- rStream := ws.RecvStream()
-
- // Check that watch detects the changes in the first transaction.
- changes := []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatal("Expected change to be the last in this transaction")
- }
- watchtesting.ExpectMutationExists(t, changes, id1, raw.NoVersion, post1, true, "val1", watchtesting.EmptyDir)
-
- // Put /a
- tr = memstore.NewTransaction()
- id2 := put(t, st, tr, "/a", "val2")
- commit(t, tr)
-
- pre1 := post1
- post1 = st.Snapshot().Find(id1).Version
- post2 := st.Snapshot().Find(id2).Version
-
- // Check that watch detects the changes in the second transaction.
- changes = []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if !change.Continued {
- t.Fatal("Expected change to continue the transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatal("Expected change to be the last in this transaction")
- }
- watchtesting.ExpectMutationExists(t, changes, id1, pre1, post1, true, "val1", watchtesting.DirOf("a", id2))
- watchtesting.ExpectMutationExists(t, changes, id2, raw.NoVersion, post2, false, "val2", watchtesting.EmptyDir)
-}
-
-func TestWatchGlob(t *testing.T) {
- rt.Init()
-
- // Create a new store.
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- // Create a new watcher.
- w, cleanup := createWatcher(t, dbName)
- defer cleanup()
-
- // Put /
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- commit(t, tr)
-
- // Start a watch request.
- path := storage.ParsePath("/")
- req := types.GlobRequest{Pattern: "..."}
- ws := watchtesting.WatchGlobOnPath(rootPublicID, w.WatchGlob, path, req)
-
- rStream := ws.RecvStream()
-
- // Check that watch detects the changes in the first transaction.
- changes := []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatal("Expected change to be the last in this transaction")
- }
- watchtesting.ExpectEntryExists(t, changes, "", id1, "val1")
-
- // Put /a
- tr = memstore.NewTransaction()
- id2 := put(t, st, tr, "/a", "val2")
- commit(t, tr)
-
- // Check that watch detects the changes in the second transaction.
- changes = []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if !change.Continued {
- t.Fatal("Expected change to continue the transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatal("Expected change to be the last in this transaction")
- }
- watchtesting.ExpectEntryExists(t, changes, "", id1, "val1")
- watchtesting.ExpectEntryExists(t, changes, "a", id2, "val2")
-}
-
-func TestWatchCancellation(t *testing.T) {
- rt.Init()
-
- // Create a new store.
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- // Create a new watcher.
- w, cleanup := createWatcher(t, dbName)
- defer cleanup()
-
- // Start a watch request.
- req := raw.Request{}
- ws := watchtesting.WatchRaw(rootPublicID, w.WatchRaw, req)
-
- // Commit a transaction.
- tr := memstore.NewTransaction()
- put(t, st, tr, "/", "val1")
- commit(t, tr)
-
- // Check that watch processed the first transaction.
- rStream := ws.RecvStream()
- if !rStream.Advance() {
- t.Fatal("Expected a change.")
- }
-
- // Cancel the watch request.
- ws.Cancel()
- // Give watch some time to process the cancellation.
- time.Sleep(time.Second)
-
- // Commit a second transaction.
- tr = memstore.NewTransaction()
- put(t, st, tr, "/a", "val2")
- commit(t, tr)
-
- // Check that watch did not processed the second transaction.
- if rStream.Advance() || rStream.Err() != nil {
- t.Errorf("Unexpected error: %v", rStream.Err())
- }
-
- // Check that io.EOF was returned.
- if err := ws.Finish(); err != io.EOF {
- t.Errorf("Unexpected error: %v", err)
- }
-}
-
-func TestWatchClosed(t *testing.T) {
- rt.Init()
-
- // Create a new store.
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- // Create a new watcher.
- w, cleanup := createWatcher(t, dbName)
- var once sync.Once
- defer once.Do(cleanup)
-
- // Start a watch request.
- req := raw.Request{}
- ws := watchtesting.WatchRaw(rootPublicID, w.WatchRaw, req)
-
- // Commit a transaction.
- tr := memstore.NewTransaction()
- put(t, st, tr, "/", "val1")
- commit(t, tr)
-
- // Check that watch processed the first transaction.
- if !ws.RecvStream().Advance() {
- t.Fatal("Expected a change.")
- }
-
- // Close the watcher, check that io.EOF was returned.
- once.Do(cleanup)
- if err := ws.Finish(); err != io.EOF {
- t.Errorf("Unexpected error: %v", err)
- }
-}
-
-func TestStateResumeMarker(t *testing.T) {
- rt.Init()
-
- // Create a new store.
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- // Put /
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- commit(t, tr)
-
- post11 := st.Snapshot().Find(id1).Version
-
- if err := st.Close(); err != nil {
- t.Fatalf("Close() failed: %v", err)
- }
-
- // Re-create a new store. This should compress the log, creating an initial
- // state containing / and /a.
- st, cleanup = openStore(t, dbName)
- defer cleanup()
-
- // Create a new watcher.
- w, cleanup := createWatcher(t, dbName)
- defer cleanup()
-
- // Put /a
- tr = memstore.NewTransaction()
- id2 := put(t, st, tr, "/a", "val2")
- commit(t, tr)
-
- pre21 := post11
- post21 := st.Snapshot().Find(id1).Version
- post22 := st.Snapshot().Find(id2).Version
-
- // Start a watch request.
- req := raw.Request{}
- ws := watchtesting.WatchRaw(rootPublicID, w.WatchRaw, req)
-
- // Retrieve the resume marker for the initial state.
- rStream := ws.RecvStream()
-
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- resumeMarker1 := change.ResumeMarker
-
- // Cancel the watch request.
- ws.Cancel()
- ws.Finish()
-
- // Start a watch request after the initial state.
- req = raw.Request{ResumeMarker: resumeMarker1}
- ws = watchtesting.WatchRaw(rootPublicID, w.WatchRaw, req)
-
- rStream = ws.RecvStream()
-
- // Check that watch detects the changes in the state and the transaction.
- changes := []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatal("Expected change to be the last in this transaction")
- }
- watchtesting.ExpectMutationExists(t, changes, id1, raw.NoVersion, post11, true, "val1", watchtesting.EmptyDir)
-
- // Check that watch detects the changes in the state and the transaction.
- changes = []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if !change.Continued {
- t.Fatal("Expected change to continue the transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatal("Expected change to be the last in this transaction")
- }
- watchtesting.ExpectMutationExists(t, changes, id1, pre21, post21, true, "val1", watchtesting.DirOf("a", id2))
- watchtesting.ExpectMutationExists(t, changes, id2, raw.NoVersion, post22, false, "val2", watchtesting.EmptyDir)
-}
-
-func TestTransactionResumeMarker(t *testing.T) {
- rt.Init()
-
- // Create a new store.
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- // Create a new watcher.
- w, cleanup := createWatcher(t, dbName)
- defer cleanup()
-
- // Put /
- tr := memstore.NewTransaction()
- id1 := put(t, st, tr, "/", "val1")
- commit(t, tr)
-
- post11 := st.Snapshot().Find(id1).Version
-
- // Put /a
- tr = memstore.NewTransaction()
- id2 := put(t, st, tr, "/a", "val2")
- commit(t, tr)
-
- pre21 := post11
- post21 := st.Snapshot().Find(id1).Version
- post22 := st.Snapshot().Find(id2).Version
-
- // Start a watch request.
- req := raw.Request{}
- ws := watchtesting.WatchRaw(rootPublicID, w.WatchRaw, req)
-
- // Retrieve the resume marker for the first transaction.
- rStream := ws.RecvStream()
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- resumeMarker1 := change.ResumeMarker
-
- // Cancel the watch request.
- ws.Cancel()
- ws.Finish()
-
- // Start a watch request after the first transaction.
- req = raw.Request{ResumeMarker: resumeMarker1}
- ws = watchtesting.WatchRaw(rootPublicID, w.WatchRaw, req)
-
- rStream = ws.RecvStream()
-
- // Check that watch detects the changes in the first transaction.
- changes := []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatal("Expected change to be the last in this transaction")
- }
- watchtesting.ExpectMutationExists(t, changes, id1, raw.NoVersion, post11, true, "val1", watchtesting.EmptyDir)
-
- // Check that watch detects the changes in the second transaction.
- changes = []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if !change.Continued {
- t.Fatal("Expected change to continue the transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatal("Expected change to be the last in this transaction")
- }
- resumeMarker2 := change.ResumeMarker
- watchtesting.ExpectMutationExists(t, changes, id1, pre21, post21, true, "val1", watchtesting.DirOf("a", id2))
- watchtesting.ExpectMutationExists(t, changes, id2, raw.NoVersion, post22, false, "val2", watchtesting.EmptyDir)
-
- // Cancel the watch request.
- ws.Cancel()
- ws.Finish()
-
- // Start a watch request at the second transaction.
- req = raw.Request{ResumeMarker: resumeMarker2}
- ws = watchtesting.WatchRaw(rootPublicID, w.WatchRaw, req)
-
- rStream = ws.RecvStream()
-
- // Check that watch detects the changes in the second transaction.
- changes = []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if !change.Continued {
- t.Fatal("Expected change to continue the transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatal("Expected change to be the last in this transaction")
- }
- watchtesting.ExpectMutationExists(t, changes, id1, pre21, post21, true, "val1", watchtesting.DirOf("a", id2))
- watchtesting.ExpectMutationExists(t, changes, id2, raw.NoVersion, post22, false, "val2", watchtesting.EmptyDir)
-}
-
-func TestNowResumeMarker(t *testing.T) {
- rt.Init()
-
- // Create a new store.
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- // Create a new watcher.
- w, cleanup := createWatcher(t, dbName)
- defer cleanup()
-
- // Put /
- tr := memstore.NewTransaction()
- put(t, st, tr, "/", "val1")
- commit(t, tr)
-
- // Put /a
- tr = memstore.NewTransaction()
- id2 := put(t, st, tr, "/a", "val2")
- commit(t, tr)
-
- post22 := st.Snapshot().Find(id2).Version
-
- // Pass some time so that the second transaction happened before "now".
- time.Sleep(time.Millisecond)
-
- // Start a watch request with the "now" resume marker.
- req := raw.Request{ResumeMarker: nowResumeMarker}
- ws := watchtesting.WatchRaw(rootPublicID, w.WatchRaw, req)
-
- // Check that watch announces that the initial state was skipped.
- // This also ensures that the third transaction happens after "now".
- rStream := ws.RecvStream()
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- watchtesting.ExpectInitialStateSkipped(t, change)
-
- // Put /a/b
- tr = memstore.NewTransaction()
- id3 := put(t, st, tr, "/a/b", "val3")
- commit(t, tr)
-
- pre32 := post22
- post32 := st.Snapshot().Find(id2).Version
- post33 := st.Snapshot().Find(id3).Version
-
- // Check that watch detects the changes in the third transaction.
- changes := []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if !change.Continued {
- t.Fatalf("Expected change to continue the transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- watchtesting.ExpectMutationExists(t, changes, id2, pre32, post32, false, "val2", watchtesting.DirOf("b", id3))
- watchtesting.ExpectMutationExists(t, changes, id3, raw.NoVersion, post33, false, "val3", watchtesting.EmptyDir)
-}
-
-func TestUnknownResumeMarkers(t *testing.T) {
- rt.Init()
-
- // Create a new store.
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- // Create a new watcher.
- w, cleanup := createWatcher(t, dbName)
- defer cleanup()
-
- // Put /
- tr := memstore.NewTransaction()
- put(t, st, tr, "/", "val1")
- commit(t, tr)
-
- // Start a watch request with a resume marker that's too early.
- resumeMarker := timestampToResumeMarker(1)
- req := raw.Request{ResumeMarker: resumeMarker}
- ws := watchtesting.WatchRaw(rootPublicID, w.WatchRaw, req)
-
- // The resume marker should be unknown.
- if err := ws.Finish(); !verror.Is(err, verror.BadArg) {
- t.Errorf("Error should be %v: got %v", verror.BadArg, err)
- }
-
- // Start a watch request with a resume marker that's too late.
- resumeMarker = timestampToResumeMarker(2 ^ 63 - 1)
- req = raw.Request{ResumeMarker: resumeMarker}
- ws = watchtesting.WatchRaw(rootPublicID, w.WatchRaw, req)
-
- // The resume marker should be unknown.
- if err := ws.Finish(); !verror.Is(err, verror.BadArg) {
- t.Errorf("Error should be %v: got %v", verror.BadArg, err)
- }
-}
-
-func TestConsistentResumeMarkers(t *testing.T) {
- rt.Init()
-
- // Create a new store.
- dbName, st, cleanup := createStore(t)
- defer cleanup()
-
- // Create a new watcher.
- w, cleanup := createWatcher(t, dbName)
- defer cleanup()
-
- // Put /
- tr := memstore.NewTransaction()
- put(t, st, tr, "/", "val1")
- commit(t, tr)
-
- // Start a watch request.
- path := storage.ParsePath("/")
- req := types.GlobRequest{Pattern: "..."}
- ws := watchtesting.WatchGlobOnPath(rootPublicID, w.WatchGlob, path, req)
-
- rStream := ws.RecvStream()
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- // Save the ResumeMarker of the change.
- r := change.ResumeMarker
-
- // Start another watch request.
- ws = watchtesting.WatchGlobOnPath(rootPublicID, w.WatchGlob, path, req)
-
- rStream = ws.RecvStream()
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- // Expect the same ResumeMarker.
- if !bytes.Equal(r, change.ResumeMarker) {
- t.Fatal("Inconsistent ResumeMarker.")
- }
-}
diff --git a/services/store/raw/service.vdl b/services/store/raw/service.vdl
deleted file mode 100644
index 4d18639..0000000
--- a/services/store/raw/service.vdl
+++ /dev/null
@@ -1,79 +0,0 @@
-// Package raw defines a raw interface for the Veyron store.
-//
-// The raw interface supports synchronizing with remote stores by transporting
-// Mutations.
-
-package raw
-
-import (
- "veyron2/storage"
- "veyron2/services/watch/types"
-)
-
-const (
- // The raw Store has Object name "<mount>/.store.raw", where <mount> is the
- // Object name of the mount point.
- RawStoreSuffix = ".store.raw"
-
- // NoVersion means the entry is not present in the store.
- NoVersion = Version(0)
-)
-
-// Version identifies the value in the store for a key at some point in time.
-// The version is a numeric identifier that is globally unique within the space
-// of a single ID, meaning that if two stores contain an entry with the same ID
-// and version, then the entries represent the same thing, at the same point in
-// time (as agreed upon by the two stores).
-type Version uint64
-
-// DEntry is a directory entry.
-type DEntry struct {
- Name string
- ID storage.ID
-}
-
-// Mutation represents an update to an entry in the store, and contains enough
-// information for a privileged service to replicate the update elsewhere.
-type Mutation struct {
- // ID is the key that identifies the entry.
- ID storage.ID
-
- // The version of the entry immediately before the update. For new entries,
- // the PriorVersion is NoVersion.
- PriorVersion Version
-
- // The version of the entry immediately after the update. For deleted entries,
- // the Version is NoVersion.
- Version Version
-
- // IsRoot is true if
- // 1) The entry was the store root immediately before being deleted, or
- // 2) The entry is the store root immediately after the update.
- IsRoot bool
-
- // Value is value stored at this entry.
- Value any
-
- // Dir is the implicit directory of this entry, and may contain references
- // to other entries in the store.
- Dir []DEntry
-}
-
-// Request specifies how to resume from a previous Watch call.
-type Request struct {
- // ResumeMarker specifies how to resume from a previous Watch call.
- // See the ResumeMarker type for detailed comments.
- ResumeMarker types.ResumeMarker
-}
-
-// Store defines a raw interface for the Veyron store. Mutations can be received
-// via the Watcher interface, and committed via PutMutation.
-type Store interface {
- // Watch returns a stream of all changes.
- Watch(Req Request) stream<_, types.Change> error
-
- // PutMutations atomically commits a stream of Mutations when the stream is
- // closed. Mutations are not committed if the request is cancelled before
- // the stream has been closed.
- PutMutations() stream<Mutation, _> error
-}
diff --git a/services/store/raw/service.vdl.go b/services/store/raw/service.vdl.go
deleted file mode 100644
index 5e75600..0000000
--- a/services/store/raw/service.vdl.go
+++ /dev/null
@@ -1,577 +0,0 @@
-// This file was auto-generated by the veyron vdl tool.
-// Source: service.vdl
-
-package raw
-
-import (
- "veyron2/services/watch/types"
-
- "veyron2/storage"
-
- // The non-user imports are prefixed with "_gen_" to prevent collisions.
- _gen_io "io"
- _gen_veyron2 "veyron2"
- _gen_context "veyron2/context"
- _gen_ipc "veyron2/ipc"
- _gen_naming "veyron2/naming"
- _gen_vdlutil "veyron2/vdl/vdlutil"
- _gen_wiretype "veyron2/wiretype"
-)
-
-// Version identifies the value in the store for a key at some point in time.
-// The version is a numeric identifier that is globally unique within the space
-// of a single ID, meaning that if two stores contain an entry with the same ID
-// and version, then the entries represent the same thing, at the same point in
-// time (as agreed upon by the two stores).
-type Version uint64
-
-// DEntry is a directory entry.
-type DEntry struct {
- Name string
- ID storage.ID
-}
-
-// Mutation represents an update to an entry in the store, and contains enough
-// information for a privileged service to replicate the update elsewhere.
-type Mutation struct {
- // ID is the key that identifies the entry.
- ID storage.ID
- // The version of the entry immediately before the update. For new entries,
- // the PriorVersion is NoVersion.
- PriorVersion Version
- // The version of the entry immediately after the update. For deleted entries,
- // the Version is NoVersion.
- Version Version
- // IsRoot is true if
- // 1) The entry was the store root immediately before being deleted, or
- // 2) The entry is the store root immediately after the update.
- IsRoot bool
- // Value is value stored at this entry.
- Value _gen_vdlutil.Any
- // Dir is the implicit directory of this entry, and may contain references
- // to other entries in the store.
- Dir []DEntry
-}
-
-// Request specifies how to resume from a previous Watch call.
-type Request struct {
- // ResumeMarker specifies how to resume from a previous Watch call.
- // See the ResumeMarker type for detailed comments.
- ResumeMarker types.ResumeMarker
-}
-
-// The raw Store has Object name "<mount>/.store.raw", where <mount> is the
-// Object name of the mount point.
-const RawStoreSuffix = ".store.raw"
-
-// NoVersion means the entry is not present in the store.
-const NoVersion = Version(0)
-
-// TODO(bprosnitz) Remove this line once signatures are updated to use typevals.
-// It corrects a bug where _gen_wiretype is unused in VDL pacakges where only bootstrap types are used on interfaces.
-const _ = _gen_wiretype.TypeIDInvalid
-
-// Store defines a raw interface for the Veyron store. Mutations can be received
-// via the Watcher interface, and committed via PutMutation.
-// Store is the interface the client binds and uses.
-// Store_ExcludingUniversal is the interface without internal framework-added methods
-// to enable embedding without method collisions. Not to be used directly by clients.
-type Store_ExcludingUniversal interface {
- // Watch returns a stream of all changes.
- Watch(ctx _gen_context.T, Req Request, opts ..._gen_ipc.CallOpt) (reply StoreWatchCall, err error)
- // PutMutations atomically commits a stream of Mutations when the stream is
- // closed. Mutations are not committed if the request is cancelled before
- // the stream has been closed.
- PutMutations(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply StorePutMutationsCall, err error)
-}
-type Store interface {
- _gen_ipc.UniversalServiceMethods
- Store_ExcludingUniversal
-}
-
-// StoreService is the interface the server implements.
-type StoreService interface {
-
- // Watch returns a stream of all changes.
- Watch(context _gen_ipc.ServerContext, Req Request, stream StoreServiceWatchStream) (err error)
- // PutMutations atomically commits a stream of Mutations when the stream is
- // closed. Mutations are not committed if the request is cancelled before
- // the stream has been closed.
- PutMutations(context _gen_ipc.ServerContext, stream StoreServicePutMutationsStream) (err error)
-}
-
-// StoreWatchCall is the interface for call object of the method
-// Watch in the service interface Store.
-type StoreWatchCall interface {
- // RecvStream returns the recv portion of the stream
- RecvStream() interface {
- // Advance stages an element so the client can retrieve it
- // with Value. Advance returns true iff there is an
- // element to retrieve. The client must call Advance before
- // calling Value. Advance may block if an element is not
- // immediately available.
- Advance() bool
-
- // Value returns the element that was staged by Advance.
- // Value may panic if Advance returned false or was not
- // called at all. Value does not block.
- Value() types.Change
-
- // Err returns a non-nil error iff the stream encountered
- // any errors. Err does not block.
- Err() error
- }
-
- // Finish blocks until the server is done and returns the positional
- // return values for call.
- //
- // If Cancel has been called, Finish will return immediately; the output of
- // Finish could either be an error signalling cancelation, or the correct
- // positional return values from the server depending on the timing of the
- // call.
- //
- // Calling Finish is mandatory for releasing stream resources, unless Cancel
- // has been called or any of the other methods return an error.
- // Finish should be called at most once.
- Finish() (err error)
-
- // Cancel cancels the RPC, notifying the server to stop processing. It
- // is safe to call Cancel concurrently with any of the other stream methods.
- // Calling Cancel after Finish has returned is a no-op.
- Cancel()
-}
-
-type implStoreWatchStreamIterator struct {
- clientCall _gen_ipc.Call
- val types.Change
- err error
-}
-
-func (c *implStoreWatchStreamIterator) Advance() bool {
- c.val = types.Change{}
- c.err = c.clientCall.Recv(&c.val)
- return c.err == nil
-}
-
-func (c *implStoreWatchStreamIterator) Value() types.Change {
- return c.val
-}
-
-func (c *implStoreWatchStreamIterator) Err() error {
- if c.err == _gen_io.EOF {
- return nil
- }
- return c.err
-}
-
-// Implementation of the StoreWatchCall interface that is not exported.
-type implStoreWatchCall struct {
- clientCall _gen_ipc.Call
- readStream implStoreWatchStreamIterator
-}
-
-func (c *implStoreWatchCall) RecvStream() interface {
- Advance() bool
- Value() types.Change
- Err() error
-} {
- return &c.readStream
-}
-
-func (c *implStoreWatchCall) Finish() (err error) {
- if ierr := c.clientCall.Finish(&err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (c *implStoreWatchCall) Cancel() {
- c.clientCall.Cancel()
-}
-
-type implStoreServiceWatchStreamSender struct {
- serverCall _gen_ipc.ServerCall
-}
-
-func (s *implStoreServiceWatchStreamSender) Send(item types.Change) error {
- return s.serverCall.Send(item)
-}
-
-// StoreServiceWatchStream is the interface for streaming responses of the method
-// Watch in the service interface Store.
-type StoreServiceWatchStream interface {
- // SendStream returns the send portion of the stream.
- SendStream() interface {
- // Send places the item onto the output stream, blocking if there is no buffer
- // space available. If the client has canceled, an error is returned.
- Send(item types.Change) error
- }
-}
-
-// Implementation of the StoreServiceWatchStream interface that is not exported.
-type implStoreServiceWatchStream struct {
- writer implStoreServiceWatchStreamSender
-}
-
-func (s *implStoreServiceWatchStream) SendStream() interface {
- // Send places the item onto the output stream, blocking if there is no buffer
- // space available. If the client has canceled, an error is returned.
- Send(item types.Change) error
-} {
- return &s.writer
-}
-
-// StorePutMutationsCall is the interface for call object of the method
-// PutMutations in the service interface Store.
-type StorePutMutationsCall interface {
-
- // SendStream returns the send portion of the stream
- SendStream() interface {
- // Send places the item onto the output stream, blocking if there is no
- // buffer space available. Calls to Send after having called Close
- // or Cancel will fail. Any blocked Send calls will be unblocked upon
- // calling Cancel.
- Send(item Mutation) error
-
- // Close indicates to the server that no more items will be sent;
- // server Recv calls will receive io.EOF after all sent items. This is
- // an optional call - it's used by streaming clients that need the
- // server to receive the io.EOF terminator before the client calls
- // Finish (for example, if the client needs to continue receiving items
- // from the server after having finished sending).
- // Calls to Close after having called Cancel will fail.
- // Like Send, Close blocks when there's no buffer space available.
- Close() error
- }
-
- // Finish performs the equivalent of SendStream().Close, then blocks until the server
- // is done, and returns the positional return values for call.
- // If Cancel has been called, Finish will return immediately; the output of
- // Finish could either be an error signalling cancelation, or the correct
- // positional return values from the server depending on the timing of the
- // call.
- //
- // Calling Finish is mandatory for releasing stream resources, unless Cancel
- // has been called or any of the other methods return an error.
- // Finish should be called at most once.
- Finish() (err error)
-
- // Cancel cancels the RPC, notifying the server to stop processing. It
- // is safe to call Cancel concurrently with any of the other stream methods.
- // Calling Cancel after Finish has returned is a no-op.
- Cancel()
-}
-
-type implStorePutMutationsStreamSender struct {
- clientCall _gen_ipc.Call
-}
-
-func (c *implStorePutMutationsStreamSender) Send(item Mutation) error {
- return c.clientCall.Send(item)
-}
-
-func (c *implStorePutMutationsStreamSender) Close() error {
- return c.clientCall.CloseSend()
-}
-
-// Implementation of the StorePutMutationsCall interface that is not exported.
-type implStorePutMutationsCall struct {
- clientCall _gen_ipc.Call
- writeStream implStorePutMutationsStreamSender
-}
-
-func (c *implStorePutMutationsCall) SendStream() interface {
- Send(item Mutation) error
- Close() error
-} {
- return &c.writeStream
-}
-
-func (c *implStorePutMutationsCall) Finish() (err error) {
- if ierr := c.clientCall.Finish(&err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (c *implStorePutMutationsCall) Cancel() {
- c.clientCall.Cancel()
-}
-
-type implStoreServicePutMutationsStreamIterator struct {
- serverCall _gen_ipc.ServerCall
- val Mutation
- err error
-}
-
-func (s *implStoreServicePutMutationsStreamIterator) Advance() bool {
- s.val = Mutation{}
- s.err = s.serverCall.Recv(&s.val)
- return s.err == nil
-}
-
-func (s *implStoreServicePutMutationsStreamIterator) Value() Mutation {
- return s.val
-}
-
-func (s *implStoreServicePutMutationsStreamIterator) Err() error {
- if s.err == _gen_io.EOF {
- return nil
- }
- return s.err
-}
-
-// StoreServicePutMutationsStream is the interface for streaming responses of the method
-// PutMutations in the service interface Store.
-type StoreServicePutMutationsStream interface {
- // RecvStream returns the recv portion of the stream
- RecvStream() interface {
- // Advance stages an element so the client can retrieve it
- // with Value. Advance returns true iff there is an
- // element to retrieve. The client must call Advance before
- // calling Value. Advance may block if an element is not
- // immediately available.
- Advance() bool
-
- // Value returns the element that was staged by Advance.
- // Value may panic if Advance returned false or was not
- // called at all. Value does not block.
- Value() Mutation
-
- // Err returns a non-nil error iff the stream encountered
- // any errors. Err does not block.
- Err() error
- }
-}
-
-// Implementation of the StoreServicePutMutationsStream interface that is not exported.
-type implStoreServicePutMutationsStream struct {
- reader implStoreServicePutMutationsStreamIterator
-}
-
-func (s *implStoreServicePutMutationsStream) RecvStream() interface {
- // Advance stages an element so the client can retrieve it
- // with Value. Advance returns true iff there is an
- // element to retrieve. The client must call Advance before
- // calling Value. The client must call Cancel if it does
- // not iterate through all elements (i.e. until Advance
- // returns false). Advance may block if an element is not
- // immediately available.
- Advance() bool
-
- // Value returns the element that was staged by Advance.
- // Value may panic if Advance returned false or was not
- // called at all. Value does not block.
- Value() Mutation
-
- // Err returns a non-nil error iff the stream encountered
- // any errors. Err does not block.
- Err() error
-} {
- return &s.reader
-}
-
-// BindStore returns the client stub implementing the Store
-// interface.
-//
-// If no _gen_ipc.Client is specified, the default _gen_ipc.Client in the
-// global Runtime is used.
-func BindStore(name string, opts ..._gen_ipc.BindOpt) (Store, error) {
- var client _gen_ipc.Client
- switch len(opts) {
- case 0:
- // Do nothing.
- case 1:
- if clientOpt, ok := opts[0].(_gen_ipc.Client); opts[0] == nil || ok {
- client = clientOpt
- } else {
- return nil, _gen_vdlutil.ErrUnrecognizedOption
- }
- default:
- return nil, _gen_vdlutil.ErrTooManyOptionsToBind
- }
- stub := &clientStubStore{defaultClient: client, name: name}
-
- return stub, nil
-}
-
-// NewServerStore creates a new server stub.
-//
-// It takes a regular server implementing the StoreService
-// interface, and returns a new server stub.
-func NewServerStore(server StoreService) interface{} {
- return &ServerStubStore{
- service: server,
- }
-}
-
-// clientStubStore implements Store.
-type clientStubStore struct {
- defaultClient _gen_ipc.Client
- name string
-}
-
-func (__gen_c *clientStubStore) client(ctx _gen_context.T) _gen_ipc.Client {
- if __gen_c.defaultClient != nil {
- return __gen_c.defaultClient
- }
- return _gen_veyron2.RuntimeFromContext(ctx).Client()
-}
-
-func (__gen_c *clientStubStore) Watch(ctx _gen_context.T, Req Request, opts ..._gen_ipc.CallOpt) (reply StoreWatchCall, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "Watch", []interface{}{Req}, opts...); err != nil {
- return
- }
- reply = &implStoreWatchCall{clientCall: call, readStream: implStoreWatchStreamIterator{clientCall: call}}
- return
-}
-
-func (__gen_c *clientStubStore) PutMutations(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply StorePutMutationsCall, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "PutMutations", nil, opts...); err != nil {
- return
- }
- reply = &implStorePutMutationsCall{clientCall: call, writeStream: implStorePutMutationsStreamSender{clientCall: call}}
- return
-}
-
-func (__gen_c *clientStubStore) UnresolveStep(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply []string, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "UnresolveStep", nil, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubStore) Signature(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply _gen_ipc.ServiceSignature, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "Signature", nil, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubStore) GetMethodTags(ctx _gen_context.T, method string, opts ..._gen_ipc.CallOpt) (reply []interface{}, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "GetMethodTags", []interface{}{method}, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-// ServerStubStore wraps a server that implements
-// StoreService and provides an object that satisfies
-// the requirements of veyron2/ipc.ReflectInvoker.
-type ServerStubStore struct {
- service StoreService
-}
-
-func (__gen_s *ServerStubStore) GetMethodTags(call _gen_ipc.ServerCall, method string) ([]interface{}, error) {
- // TODO(bprosnitz) GetMethodTags() will be replaces with Signature().
- // Note: This exhibits some weird behavior like returning a nil error if the method isn't found.
- // This will change when it is replaced with Signature().
- switch method {
- case "Watch":
- return []interface{}{}, nil
- case "PutMutations":
- return []interface{}{}, nil
- default:
- return nil, nil
- }
-}
-
-func (__gen_s *ServerStubStore) Signature(call _gen_ipc.ServerCall) (_gen_ipc.ServiceSignature, error) {
- result := _gen_ipc.ServiceSignature{Methods: make(map[string]_gen_ipc.MethodSignature)}
- result.Methods["PutMutations"] = _gen_ipc.MethodSignature{
- InArgs: []_gen_ipc.MethodArgument{},
- OutArgs: []_gen_ipc.MethodArgument{
- {Name: "", Type: 68},
- },
- InStream: 75,
- }
- result.Methods["Watch"] = _gen_ipc.MethodSignature{
- InArgs: []_gen_ipc.MethodArgument{
- {Name: "Req", Type: 67},
- },
- OutArgs: []_gen_ipc.MethodArgument{
- {Name: "", Type: 68},
- },
-
- OutStream: 70,
- }
-
- result.TypeDefs = []_gen_vdlutil.Any{
- _gen_wiretype.NamedPrimitiveType{Type: 0x32, Name: "byte", Tags: []string(nil)}, _gen_wiretype.SliceType{Elem: 0x41, Name: "veyron2/services/watch/types.ResumeMarker", Tags: []string(nil)}, _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x42, Name: "ResumeMarker"},
- },
- "veyron/services/store/raw.Request", []string(nil)},
- _gen_wiretype.NamedPrimitiveType{Type: 0x1, Name: "error", Tags: []string(nil)}, _gen_wiretype.NamedPrimitiveType{Type: 0x1, Name: "anydata", Tags: []string(nil)}, _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x3, Name: "Name"},
- _gen_wiretype.FieldType{Type: 0x24, Name: "State"},
- _gen_wiretype.FieldType{Type: 0x45, Name: "Value"},
- _gen_wiretype.FieldType{Type: 0x42, Name: "ResumeMarker"},
- _gen_wiretype.FieldType{Type: 0x2, Name: "Continued"},
- },
- "veyron2/services/watch/types.Change", []string(nil)},
- _gen_wiretype.ArrayType{Elem: 0x41, Len: 0x10, Name: "veyron2/storage.ID", Tags: []string(nil)}, _gen_wiretype.NamedPrimitiveType{Type: 0x35, Name: "veyron/services/store/raw.Version", Tags: []string(nil)}, _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x3, Name: "Name"},
- _gen_wiretype.FieldType{Type: 0x47, Name: "ID"},
- },
- "veyron/services/store/raw.DEntry", []string(nil)},
- _gen_wiretype.SliceType{Elem: 0x49, Name: "", Tags: []string(nil)}, _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x47, Name: "ID"},
- _gen_wiretype.FieldType{Type: 0x48, Name: "PriorVersion"},
- _gen_wiretype.FieldType{Type: 0x48, Name: "Version"},
- _gen_wiretype.FieldType{Type: 0x2, Name: "IsRoot"},
- _gen_wiretype.FieldType{Type: 0x45, Name: "Value"},
- _gen_wiretype.FieldType{Type: 0x4a, Name: "Dir"},
- },
- "veyron/services/store/raw.Mutation", []string(nil)},
- }
-
- return result, nil
-}
-
-func (__gen_s *ServerStubStore) UnresolveStep(call _gen_ipc.ServerCall) (reply []string, err error) {
- if unresolver, ok := __gen_s.service.(_gen_ipc.Unresolver); ok {
- return unresolver.UnresolveStep(call)
- }
- if call.Server() == nil {
- return
- }
- var published []string
- if published, err = call.Server().Published(); err != nil || published == nil {
- return
- }
- reply = make([]string, len(published))
- for i, p := range published {
- reply[i] = _gen_naming.Join(p, call.Name())
- }
- return
-}
-
-func (__gen_s *ServerStubStore) Watch(call _gen_ipc.ServerCall, Req Request) (err error) {
- stream := &implStoreServiceWatchStream{writer: implStoreServiceWatchStreamSender{serverCall: call}}
- err = __gen_s.service.Watch(call, Req, stream)
- return
-}
-
-func (__gen_s *ServerStubStore) PutMutations(call _gen_ipc.ServerCall) (err error) {
- stream := &implStoreServicePutMutationsStream{reader: implStoreServicePutMutationsStreamIterator{serverCall: call}}
- err = __gen_s.service.PutMutations(call, stream)
- return
-}
diff --git a/services/store/raw/version.go b/services/store/raw/version.go
deleted file mode 100644
index aa5d385..0000000
--- a/services/store/raw/version.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package raw
-
-import (
- "math/rand"
- "time"
-)
-
-var rng *rand.Rand
-
-func init() {
- rng = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
-}
-
-// NewVersion returns a new version number.
-//
-// TODO(jyh): Choose a better version generator.
-func NewVersion() Version {
- for {
- if v := Version(rng.Int63()); v != 0 {
- return v
- }
- }
-}
diff --git a/services/store/server/server.go b/services/store/server/server.go
deleted file mode 100644
index e979c5a..0000000
--- a/services/store/server/server.go
+++ /dev/null
@@ -1,373 +0,0 @@
-// Package server implements a storage service.
-package server
-
-// This file defines Server, which implements the server-side Store API from
-// veyron2/services/store/service.vdl.
-
-import (
- "fmt"
- "math/rand"
- "reflect"
- "strconv"
- "strings"
- "sync"
- "time"
-
- "veyron/services/store/memstore"
- memwatch "veyron/services/store/memstore/watch"
- "veyron/services/store/raw"
-
- "veyron2/ipc"
- "veyron2/security"
- "veyron2/verror"
-)
-
-const (
- // transactionMaxLifetime is the maximum duration before a transaction will
- // be garbage collected.
- //
- // TODO(jyh): This should probably be a configuration parameter.
- transactionMaxLifetime = 30 * time.Second
-)
-
-var (
- errNestedTransaction = verror.BadArgf("cannot create a nested Transaction")
- // Triggers if client calls commit/abort on a name that's not part of a
- // transaction.
- errNoTransaction = verror.NotFoundf("no transaction")
- // Note, this can happen e.g. due to expiration.
- errTransactionDoesNotExist = verror.NotFoundf("transaction does not exist")
- // Transaction exists, but may not be used by the caller.
- errPermissionDenied = verror.NotAuthorizedf("permission denied")
-
- rng = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
-
- nullTransactionID transactionID
-)
-
-// Server implements Store and uses memstore internally.
-type Server struct {
- mutex sync.RWMutex
-
- // store is the actual store implementation.
- store *memstore.Store
-
- // transactions is the set of active transactions.
- transactions map[transactionID]*transaction
-
- // Transaction garbage collection.
- pending sync.WaitGroup
- ticker *time.Ticker
- closed chan struct{}
-
- // watcher is the actual store watcher implementation.
- watcher *memwatch.Watcher
-}
-
-// transactionID is an internal transaction identifier chosen by the server.
-//
-// TODO(jyh): Consider using a larger identifier space to reduce chance of
-// collisions. (Note, createTransaction handles collisions when generating
-// transactionIDs.)
-type transactionID uint64
-
-// transactionContext defines the context in which a transaction is used. A
-// transaction may be used only in the context that created it.
-// transactionContext weakly identifies a session by the local and remote
-// principals involved in the RPC.
-// TODO(tilaks): Use the local and remote addresses to identify the session.
-// Does a session with a mobile device break if the remote address changes?
-type transactionContext interface {
- // LocalID returns the PublicID of the principal at the local end of the
- // request.
- LocalID() security.PublicID
- // RemoteID returns the PublicID of the principal at the remote end of the
- // request.
- RemoteID() security.PublicID
-}
-
-type transaction struct {
- trans *memstore.Transaction
- expires time.Time
- creatorCtx transactionContext
-}
-
-// ServerConfig provides the parameters needed to construct a Server.
-type ServerConfig struct {
- Admin security.PublicID // Administrator.
- DBName string // DBName is the name if the database directory.
-}
-
-// New creates a new server.
-func New(config ServerConfig) (*Server, error) {
- mstore, err := memstore.New(config.Admin, config.DBName)
- if err != nil {
- return nil, err
- }
- mwatcher, err := memwatch.New(config.Admin, config.DBName)
- if err != nil {
- return nil, err
- }
- s := &Server{
- store: mstore,
- transactions: make(map[transactionID]*transaction),
- ticker: time.NewTicker(time.Second),
- closed: make(chan struct{}),
- watcher: mwatcher,
- }
- s.pending.Add(1)
- go s.gcLoop()
- // Start with an empty directory at root.
- rootDir := &thing{name: "", obj: s.store.Bind(""), tid: nullTransactionID, server: s}
- if err := rootDir.makeInternal(config.Admin, nil); err != nil {
- return nil, err
- }
- return s, nil
-}
-
-func (s *Server) Close() {
- close(s.closed)
- s.ticker.Stop()
- s.pending.Wait()
- s.store.Close()
- s.watcher.Close()
-}
-
-func (s *Server) String() string {
- return "StoreServer"
-}
-
-// Attributes returns the server status.
-func (s *Server) Attributes(arg string) map[string]string {
- return map[string]string{
- "health": "ok",
- "servertype": s.String(),
- }
-}
-
-// findTransactionComponent returns the (begin, end) offsets of the "$tid.*"
-// component in the given object name, or (-1, -1) if oname does not contain a
-// transaction component.
-func findTransactionComponent(oname string) (int, int) {
- begin := 0
- if !strings.HasPrefix(oname, "$tid") {
- begin = strings.Index(oname, "/$tid")
- }
- if begin == -1 {
- return -1, -1
- }
- end := strings.Index(oname[begin+1:], "/")
- if end == -1 {
- end = len(oname)
- } else {
- end += begin + 1
- }
- return begin, end
-}
-
-// TODO(sadovsky): One of the following:
-// - Reserve prefix string "$tid." for internal use.
-// - Reserve prefix char "$" for internal use.
-// - Require users to escape prefix char "$" when they are referring to their
-// own data, e.g. "\$foo".
-func makeTransactionComponent(id transactionID) string {
- return fmt.Sprintf("$tid.%d", id)
-}
-
-// stripTransactionComponent returns the given object name with its "$tid.*"
-// component removed, and also returns the stripped transactionID.
-// Examples:
-// "/foo/$tid.123/bar" => {"/foo/bar", transactionID(123)}
-// "/foo/bar" => {"/foo/bar", nullTransactionID}
-func stripTransactionComponent(oname string) (string, transactionID, error) {
- begin, end := findTransactionComponent(oname)
- if begin == -1 {
- return oname, nullTransactionID, nil
- }
- tc := oname[begin:end]
- id, err := strconv.ParseInt(tc[strings.LastIndex(tc, ".")+1:], 10, 64)
- if err != nil {
- return "", nullTransactionID, fmt.Errorf("Failed to extract id from %q", tc)
- }
- return oname[:begin] + oname[end:], transactionID(id), nil
-}
-
-// NOTE(sadovsky): The transaction's scope should be limited to oname's subtree
-// and its parent, but for now we expand it to the entire store (and don't use
-// oname below).
-func (s *Server) createTransaction(ctx transactionContext, oname string) (string, error) {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- var id transactionID
- for {
- id = transactionID(rng.Int63())
- _, ok := s.transactions[id]
- if !ok {
- break
- }
- }
- tdata := &transaction{
- trans: memstore.NewTransaction(),
- expires: time.Now().Add(transactionMaxLifetime),
- creatorCtx: ctx,
- }
- s.transactions[id] = tdata
- return makeTransactionComponent(id), nil
-}
-
-// findTransaction returns the transaction for the given transaction ID.
-func (s *Server) findTransaction(ctx transactionContext, id transactionID) (*memstore.Transaction, error) {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
- return s.findTransactionLocked(ctx, id)
-}
-
-func (s *Server) findTransactionLocked(ctx transactionContext, id transactionID) (*memstore.Transaction, error) {
- if id == nullTransactionID {
- return nil, nil
- }
- tdata, ok := s.transactions[id]
- if !ok {
- return nil, errTransactionDoesNotExist
- }
- // A transaction may be used only by the session (and therefore client)
- // that created it.
- if !tdata.matchesContext(ctx) {
- return nil, errPermissionDenied
- }
- return tdata.trans, nil
-}
-
-// Commit commits the changes in the transaction to the store. The
-// operation is atomic, so all mutations are performed, or none. Returns an
-// error if the transaction aborted.
-func (s *Server) commitTransaction(ctx transactionContext, id transactionID) error {
- s.mutex.Lock()
- defer s.mutex.Unlock()
- t, err := s.findTransactionLocked(ctx, id)
- if err != nil {
- return err
- }
- if t == nil {
- return errNoTransaction
- }
- err = t.Commit()
- delete(s.transactions, id)
- return err
-}
-
-// Abort discards a transaction. This is an optimization; transactions
-// eventually time out and get discarded. However, live transactions
-// consume resources, so if you know that you won't be using a transaction
-// anymore, you should discard it explicitly.
-func (s *Server) abortTransaction(ctx transactionContext, id transactionID) error {
- s.mutex.Lock()
- defer s.mutex.Unlock()
- t, err := s.findTransactionLocked(ctx, id)
- if err != nil {
- return err
- }
- if t == nil {
- return errNoTransaction
- }
- err = t.Abort()
- delete(s.transactions, id)
- return err
-}
-
-func (t *transaction) matchesContext(ctx transactionContext) bool {
- creatorCtx := t.creatorCtx
- return membersEqual(creatorCtx.LocalID().Names(), ctx.LocalID().Names()) &&
- membersEqual(creatorCtx.RemoteID().Names(), ctx.RemoteID().Names())
-}
-
-// membersEquals checks whether two slices of strings have the same set of
-// members, regardless of order.
-func membersEqual(slice1, slice2 []string) bool {
- set1 := make(map[string]bool, len(slice1))
- for _, s := range slice1 {
- set1[s] = true
- }
- set2 := make(map[string]bool, len(slice2))
- for _, s := range slice2 {
- set2[s] = true
- }
- // DeepEqual tests keys for == equality, which is sufficient for strings.
- return reflect.DeepEqual(set1, set2)
-}
-
-// gcLoop drops transactions that have expired.
-func (s *Server) gcLoop() {
- for {
- select {
- case <-s.closed:
- s.pending.Done()
- return
- case <-s.ticker.C:
- }
-
- s.mutex.Lock()
- now := time.Now()
- for id, tdata := range s.transactions {
- if now.After(tdata.expires) {
- tdata.trans.Abort()
- delete(s.transactions, id)
- }
- }
- s.mutex.Unlock()
- }
-}
-
-// Watch returns a stream of all changes.
-func (s *Server) Watch(ctx ipc.ServerContext, req raw.Request, stream raw.StoreServiceWatchStream) error {
- return s.watcher.WatchRaw(ctx, req, stream)
-}
-
-// PutMutations atomically commits a stream of Mutations when the stream is
-// closed. Mutations are not committed if the request is cancelled before the
-// stream has been closed.
-func (s *Server) PutMutations(ctx ipc.ServerContext, stream raw.StoreServicePutMutationsStream) error {
- return s.store.PutMutations(ctx, stream)
-}
-
-type storeDispatcher struct {
- s *Server
- auth security.Authorizer
-}
-
-// NewStoreDispatcher returns an object dispatcher.
-func NewStoreDispatcher(s *Server, auth security.Authorizer) ipc.Dispatcher {
- return &storeDispatcher{s: s, auth: auth}
-}
-
-func (d *storeDispatcher) Lookup(suffix, method string) (ipc.Invoker, security.Authorizer, error) {
- serv, err := d.lookupServer(suffix)
- if err != nil {
- return nil, nil, err
- }
- return ipc.ReflectInvoker(serv), d.auth, nil
-}
-
-func (d *storeDispatcher) lookupServer(suffix string) (interface{}, error) {
- // Strip leading "/" if present so that server internals can assume a
- // particular form.
- suffix = strings.TrimPrefix(suffix, "/")
- if strings.HasSuffix(suffix, raw.RawStoreSuffix) {
- return raw.NewServerStore(d.s), nil
- } else {
- t, err := d.s.lookupThing(suffix)
- if err != nil {
- return nil, err
- }
- return NewServerstoreThing(t), nil
- }
-}
-
-func (s *Server) lookupThing(name string) (*thing, error) {
- name, tid, err := stripTransactionComponent(name)
- if err != nil {
- return nil, err
- }
- return &thing{name: name, obj: s.store.Bind(name), tid: tid, server: s}, nil
-}
diff --git a/services/store/server/server_test.go b/services/store/server/server_test.go
deleted file mode 100644
index a0f3ca7..0000000
--- a/services/store/server/server_test.go
+++ /dev/null
@@ -1,672 +0,0 @@
-package server
-
-import (
- "fmt"
- "io/ioutil"
- "log"
- "os"
- "reflect"
- "runtime"
- "testing"
-
- _ "veyron/lib/testutil" // initialize vlog
- storetest "veyron/services/store/memstore/testing"
- "veyron/services/store/raw"
-
- "veyron2/ipc"
- "veyron2/naming"
- "veyron2/rt"
- "veyron2/security"
- "veyron2/services/watch/types"
- "veyron2/storage"
- _ "veyron2/vlog"
- "veyron2/vom"
-)
-
-var (
- rootPublicID security.PublicID = security.FakePublicID("root")
- rootName = fmt.Sprintf("%s", rootPublicID)
- blessedPublicId security.PublicID = security.FakePublicID("root/blessed")
-)
-
-// Dir is a simple directory.
-type Dir struct {
- Entries map[string]storage.ID
-}
-
-func init() {
- vom.Register(&Dir{})
-}
-
-func newValue() interface{} {
- return &Dir{}
-}
-
-func closeTest(config ServerConfig, s *Server) {
- s.Close()
- os.Remove(config.DBName)
-}
-
-func newServer() (*Server, func()) {
- dbName, err := ioutil.TempDir(os.TempDir(), "test_server_test.db")
- if err != nil {
- log.Fatal("ioutil.TempDir() failed: ", err)
- }
- config := ServerConfig{
- Admin: rootPublicID,
- DBName: dbName,
- }
- s, err := New(config)
- if err != nil {
- log.Fatal("server.New() failed: ", err)
- }
- closer := func() { closeTest(config, s) }
- return s, closer
-}
-
-func lookupThingOrDie(s *Server, name string) *thing {
- t, err := s.lookupThing(name)
- if err != nil {
- panic(err)
- }
- return t
-}
-
-// createTransaction creates a new transaction and returns its name relative to
-// the root of the store.
-func createTransaction(t *testing.T, s *Server, ctx ipc.ServerContext, name string) string {
- _, file, line, _ := runtime.Caller(1)
- tid, err := lookupThingOrDie(s, name).NewTransaction(ctx, nil)
- if err != nil {
- t.Fatalf("%s(%d): can't create transaction %s: %s", file, line, name, err)
- }
- return naming.Join(name, tid)
-}
-
-func TestLookupInvalidTransactionName(t *testing.T) {
- s, c := newServer()
- defer c()
-
- _, err := s.lookupThing("/$tid.bad/foo")
- if err == nil {
- t.Fatalf("lookupThing should've failed, but didn't")
- }
-}
-
-func TestNestedTransactionError(t *testing.T) {
- rt.Init()
- s, c := newServer()
- defer c()
-
- rootCtx := storetest.NewFakeServerContext(rootPublicID)
- tname := createTransaction(t, s, rootCtx, "/")
- if _, err := lookupThingOrDie(s, tname).NewTransaction(rootCtx, nil); err == nil {
- t.Fatalf("creating nested transaction at %s should've failed, but didn't", tname)
- }
- // Try again with a valid object in between the two $tid components;
- // CreateTransaction should still fail.
- lookupThingOrDie(s, tname).Put(rootCtx, newValue())
- foo := naming.Join(tname, "foo")
- if _, err := lookupThingOrDie(s, foo).NewTransaction(rootCtx, nil); err == nil {
- t.Fatalf("creating nested transaction at %s should've failed, but didn't", foo)
- }
-}
-
-func TestPutGetRemoveObject(t *testing.T) {
- s, c := newServer()
- defer c()
-
- testPutGetRemove(t, s, "a")
-}
-
-func testPutGetRemove(t *testing.T, s *Server, name string) {
- rt.Init()
- rootCtx := storetest.NewFakeServerContext(rootPublicID)
- value := newValue()
- {
- // Check that the object does not exist.
- tobj := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name))
- if ok, err := tobj.Exists(rootCtx); ok || err != nil {
- t.Fatalf("Should not exist: %s", err)
- }
- if v, err := tobj.Get(rootCtx); v.Stat.ID.IsValid() && err == nil {
- t.Fatalf("Should not exist: %v, %s", v, err)
- }
- }
-
- {
- // Add the object.
- tobj1 := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name))
- if _, err := tobj1.Put(rootCtx, value); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- if ok, err := tobj1.Exists(rootCtx); !ok || err != nil {
- t.Fatalf("Should exist: %s", err)
- }
- if _, err := tobj1.Get(rootCtx); err != nil {
- t.Fatalf("Object should exist: %s", err)
- }
-
- // Transactions are isolated.
- tobj2 := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name))
- if ok, err := tobj2.Exists(rootCtx); ok || err != nil {
- t.Fatalf("Should not exist: %s", err)
- }
- if v, err := tobj2.Get(rootCtx); v.Stat.ID.IsValid() && err == nil {
- t.Fatalf("Should not exist: %v, %s", v, err)
- }
-
- // Apply tobj1.
- if err := tobj1.Commit(rootCtx); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
-
- // tobj2 is still isolated.
- if ok, err := tobj2.Exists(rootCtx); ok || err != nil {
- t.Fatalf("Should not exist: %s", err)
- }
- if v, err := tobj2.Get(rootCtx); v.Stat.ID.IsValid() && err == nil {
- t.Fatalf("Should not exist: %v, %s", v, err)
- }
-
- // tobj3 observes the commit.
- tobj3 := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name))
- if ok, err := tobj3.Exists(rootCtx); !ok || err != nil {
- t.Fatalf("Should exist: %s", err)
- }
- if _, err := tobj3.Get(rootCtx); err != nil {
- t.Fatalf("Object should exist: %s", err)
- }
- }
-
- {
- // Remove the object.
- tobj1 := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name))
- if err := tobj1.Remove(rootCtx); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- if ok, err := tobj1.Exists(rootCtx); ok || err != nil {
- t.Fatalf("Should not exist: %s", err)
- }
- if v, err := tobj1.Get(rootCtx); v.Stat.ID.IsValid() || err == nil {
- t.Fatalf("Object should not exist: %T, %v, %s", v, v, err)
- }
-
- // The removal is isolated.
- tobj2 := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name))
- if ok, err := tobj2.Exists(rootCtx); !ok || err != nil {
- t.Fatalf("Should exist: %s", err)
- }
- if _, err := tobj2.Get(rootCtx); err != nil {
- t.Fatalf("Object should exist: %s", err)
- }
-
- // Apply tobj1.
- if err := tobj1.Commit(rootCtx); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
-
- // The removal is isolated.
- if ok, err := tobj2.Exists(rootCtx); !ok || err != nil {
- t.Fatalf("Should exist: %s", err)
- }
- if _, err := tobj2.Get(rootCtx); err != nil {
- t.Fatalf("Object should exist: %s", err)
- }
- }
-
- {
- // Check that the object does not exist.
- tobj1 := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name))
- if ok, err := tobj1.Exists(rootCtx); ok || err != nil {
- t.Fatalf("Should not exist")
- }
- if v, err := tobj1.Get(rootCtx); v.Stat.ID.IsValid() && err == nil {
- t.Fatalf("Should not exist: %v, %s", v, err)
- }
- }
-}
-
-// TODO(sadovsky): Add more test cases for Commit/Abort:
-// - expired transaction: server should return errTransactionDoesNotExist
-// - no transaction: server should return errNoTransaction
-
-func TestWatchGlob(t *testing.T) {
- rt.Init()
- rootCtx := storetest.NewFakeServerContext(rootPublicID)
-
- s, c := newServer()
- defer c()
-
- dirname, objname := "/a", "/a/b"
- dir, obj := lookupThingOrDie(s, dirname), lookupThingOrDie(s, objname)
-
- // Before the watch request has been made, commit a transaction that makes
- // directory /a.
- {
- tdir := lookupThingOrDie(s, createTransaction(t, s, rootCtx, dirname))
- err := tdir.Make(rootCtx)
- if err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- if err := tdir.Commit(rootCtx); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- }
-
- // Start watch requests on /a and /a/b.
- req := types.GlobRequest{Pattern: "..."}
- wdir := storetest.WatchGlob(rootPublicID, dir.WatchGlob, req)
- wobj := storetest.WatchGlob(rootPublicID, obj.WatchGlob, req)
-
- rStreamDir := wdir.RecvStream()
- rStreamObj := wobj.RecvStream()
-
- // The watch on /a should send a change on /a.
- {
- if !rStreamDir.Advance() {
- t.Fatalf("Advance() failed: %v", rStreamDir.Err())
- }
- change := rStreamDir.Value()
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- }
- // The watch on /a/b should send no change. The first change it sends is
- // verified below.
-
- value := "v"
- var id storage.ID
-
- // Commit a second transaction that puts /a/b.
- {
- tobj := lookupThingOrDie(s, createTransaction(t, s, rootCtx, objname))
- st, err := tobj.Put(rootCtx, value)
- if err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- id = st.ID
- if err := tobj.Commit(rootCtx); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- }
-
- // The watch on /a should send changes on /a and /a/b.
- {
- changes := []types.Change{}
- if !rStreamDir.Advance() {
- t.Fatalf("Advance() failed: %v", rStreamDir.Err())
- }
- change := rStreamDir.Value()
- changes = append(changes, change)
- if !change.Continued {
- t.Fatalf("Expected change to NOT be the last in this transaction")
- }
- if !rStreamDir.Advance() {
- t.Fatalf("Advance() failed: %v", rStreamDir.Err())
- }
- change = rStreamDir.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- storetest.ExpectEntryExistsNameOnly(t, changes, "a")
- storetest.ExpectEntryExists(t, changes, "a/b", id, value)
- }
- // The watch on /a/b should send a change on /a/b.
- {
- changes := []types.Change{}
- if !rStreamObj.Advance() {
- t.Fatalf("Advance() failed: %v", rStreamObj.Err())
- }
- change := rStreamObj.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- storetest.ExpectEntryExists(t, changes, "a/b", id, value)
- }
-}
-
-func TestRawWatch(t *testing.T) {
- rt.Init()
- rootCtx := storetest.NewFakeServerContext(rootPublicID)
-
- s, c := newServer()
- defer c()
-
- name1 := "/a"
- value1 := "v1"
- var id1 storage.ID
-
- // Before the watch request has been made, commit a transaction that puts /a.
- {
- tobj := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name1))
- st, err := tobj.Put(rootCtx, value1)
- if err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- id1 = st.ID
- if err := tobj.Commit(rootCtx); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- }
-
- // Start a watch request.
- req := raw.Request{}
- ws := storetest.WatchRaw(rootPublicID, s.Watch, req)
-
- rStream := ws.RecvStream()
- // Check that watch detects the changes in the first transaction.
- {
- changes := []types.Change{}
- // First change is making the root dir (in server.go), second is updating
- // the root dir (adding a dir entry), third is putting /a.
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if !change.Continued {
- t.Fatalf("Expected change to NOT be the last in this transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- storetest.ExpectMutationExistsNoVersionCheck(t, changes, id1, value1)
- }
-
- name2 := "/b"
- value2 := "v2"
- var id2 storage.ID
-
- // Commit a second transaction that puts /b.
- {
- tobj := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name2))
- st, err := tobj.Put(rootCtx, value2)
- if err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- id2 = st.ID
- if err := tobj.Commit(rootCtx); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- }
-
- // Check that watch detects the changes in the second transaction.
- {
- changes := []types.Change{}
- // First change is updating the root dir (adding a dir entry), second is
- // putting /b.
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- changes = append(changes, change)
- if !change.Continued {
- t.Fatalf("Expected change to NOT be the last in this transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- // Note, we don't know the ID of the root dir so we can't check that it
- // exists in 'changes'.
- storetest.ExpectMutationExistsNoVersionCheck(t, changes, id2, value2)
- }
-}
-
-// Note, this test is identical to TestRawWatch up until the removal of /b.
-func TestGarbageCollectionOnCommit(t *testing.T) {
- rt.Init()
- rootCtx := storetest.NewFakeServerContext(rootPublicID)
-
- s, c := newServer()
- defer c()
-
- name1 := "/a"
- value1 := "v1"
- var id1 storage.ID
-
- // Before the watch request has been made, commit a transaction that puts /a.
- {
- tobj := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name1))
- st, err := tobj.Put(rootCtx, value1)
- if err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- id1 = st.ID
- if err := tobj.Commit(rootCtx); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- }
-
- // Start a watch request.
- req := raw.Request{}
- ws := storetest.WatchRaw(rootPublicID, s.Watch, req)
-
- rStream := ws.RecvStream()
- // Check that watch detects the changes in the first transaction.
- {
- changes := []types.Change{}
- // First change is making the root dir (in server.go), second is updating
- // the root dir (adding a dir entry), third is putting /a.
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if !change.Continued {
- t.Fatalf("Expected change to NOT be the last in this transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- storetest.ExpectMutationExistsNoVersionCheck(t, changes, id1, value1)
- }
-
- name2 := "/b"
- value2 := "v2"
- var id2 storage.ID
-
- // Commit a second transaction that puts /b.
- {
- tobj := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name2))
- st, err := tobj.Put(rootCtx, value2)
- if err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- id2 = st.ID
- if err := tobj.Commit(rootCtx); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- }
-
- // Check that watch detects the changes in the second transaction.
- {
- changes := []types.Change{}
- // First change is updating the root dir (adding a dir entry), second is
- // putting /b.
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- changes = append(changes, change)
- if !change.Continued {
- t.Fatalf("Expected change to NOT be the last in this transaction")
- }
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change = rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- // Note, we don't know the ID of the root dir so we can't check that it
- // exists in 'changes'.
- storetest.ExpectMutationExistsNoVersionCheck(t, changes, id2, value2)
- }
-
- // Commit a third transaction that removes /b.
- {
- tobj := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name2))
- if err := tobj.Remove(rootCtx); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- if err := tobj.Commit(rootCtx); err != nil {
- t.Fatalf("Unexpected error: %s", err)
- }
- }
-
- // Check that watch detects the changes in the third transaction.
- {
- changes := []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- // Note, we don't know the ID of the root dir so we can't check that it
- // exists in 'changes'.
- }
-
- // Check that watch detects the garbage collection of /b.
- {
- changes := []types.Change{}
- if !rStream.Advance() {
- t.Fatalf("Advance() failed: %v", rStream.Err())
- }
- change := rStream.Value()
- changes = append(changes, change)
- if change.Continued {
- t.Fatalf("Expected change to be the last in this transaction")
- }
- storetest.ExpectMutationDoesNotExistNoVersionCheck(t, changes, id2)
- }
-}
-
-func TestTransactionSecurity(t *testing.T) {
- rt.Init()
- rootCtx := storetest.NewFakeServerContext(rootPublicID)
- blessedCtx := storetest.NewFakeServerContext(blessedPublicId)
-
- s, c := newServer()
- defer c()
-
- // Create a root.
- name := "/"
- value := newValue()
-
- // Create a transaction in the root's session.
- tobj := lookupThingOrDie(s, createTransaction(t, s, rootCtx, name))
-
- // Check that the transaction cannot be accessed by the blessee.
- if _, err := tobj.Exists(blessedCtx); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
- if _, err := tobj.Get(blessedCtx); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
- if _, err := tobj.Put(blessedCtx, value); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
- if err := tobj.Remove(blessedCtx); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
- if err := tobj.Abort(blessedCtx); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
- if err := tobj.Commit(blessedCtx); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
-
- // Create a transaction in the blessee's session.
- tobj = lookupThingOrDie(s, createTransaction(t, s, blessedCtx, name))
-
- // Check that the transaction cannot be accessed by the root.
- if _, err := tobj.Exists(rootCtx); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
- if _, err := tobj.Get(rootCtx); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
- if _, err := tobj.Put(rootCtx, value); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
- if err := tobj.Remove(rootCtx); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
- if err := tobj.Abort(rootCtx); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
- if err := tobj.Commit(rootCtx); err != errPermissionDenied {
- t.Fatalf("Unexpected error: %v", err)
- }
-}
-
-func TestStoreDispatcher(t *testing.T) {
- rawType := reflect.PtrTo(reflect.TypeOf(raw.ServerStubStore{}))
- thingType := reflect.PtrTo(reflect.TypeOf(ServerStubstoreThing{}))
-
- tests := []struct {
- name string
- t reflect.Type
- }{
- {raw.RawStoreSuffix, rawType},
- {"a/b/" + raw.RawStoreSuffix, rawType},
- {"a/b/c" + raw.RawStoreSuffix, rawType},
- {"", thingType},
- {"a/b/", thingType},
- {"a/b/c", thingType},
- }
-
- s, c := newServer()
- defer c()
-
- // TODO(bprosnitz): Switch this to use just exported methods (using signature)
- // once signature stabilizes.
- d := NewStoreDispatcher(s, nil).(*storeDispatcher)
- for _, test := range tests {
- serv, err := d.lookupServer(test.name)
- if err != nil {
- t.Fatalf("error looking up %s: %s", test.name, err)
- }
- if reflect.TypeOf(serv) != test.t {
- t.Fatalf("error looking up %s. got %T, expected %v", test.name, serv, test.t)
- }
- }
-}
diff --git a/services/store/server/service.vdl b/services/store/server/service.vdl
deleted file mode 100644
index f4648ee..0000000
--- a/services/store/server/service.vdl
+++ /dev/null
@@ -1,14 +0,0 @@
-package server
-
-import (
- "veyron2/services/store"
-)
-
-// Named 'storeThing' instead of 'thing' so that the struct in thing.go can be
-// named 'thing'.
-type storeThing interface {
- store.DirSpecific
- store.ObjectSpecific
- store.DirOrObject
- store.Transaction
-}
diff --git a/services/store/server/service.vdl.go b/services/store/server/service.vdl.go
deleted file mode 100644
index f596c17..0000000
--- a/services/store/server/service.vdl.go
+++ /dev/null
@@ -1,408 +0,0 @@
-// This file was auto-generated by the veyron vdl tool.
-// Source: service.vdl
-
-package server
-
-import (
- "veyron2/services/store"
-
- // The non-user imports are prefixed with "_gen_" to prevent collisions.
- _gen_veyron2 "veyron2"
- _gen_context "veyron2/context"
- _gen_ipc "veyron2/ipc"
- _gen_naming "veyron2/naming"
- _gen_vdlutil "veyron2/vdl/vdlutil"
- _gen_wiretype "veyron2/wiretype"
-)
-
-// TODO(bprosnitz) Remove this line once signatures are updated to use typevals.
-// It corrects a bug where _gen_wiretype is unused in VDL pacakges where only bootstrap types are used on interfaces.
-const _ = _gen_wiretype.TypeIDInvalid
-
-// Named 'storeThing' instead of 'thing' so that the struct in thing.go can be
-// named 'thing'.
-// storeThing is the interface the client binds and uses.
-// storeThing_ExcludingUniversal is the interface without internal framework-added methods
-// to enable embedding without method collisions. Not to be used directly by clients.
-type storeThing_ExcludingUniversal interface {
- store.DirSpecific_ExcludingUniversal
- store.ObjectSpecific_ExcludingUniversal
- store.DirOrObject_ExcludingUniversal
- store.Transaction_ExcludingUniversal
-}
-type storeThing interface {
- _gen_ipc.UniversalServiceMethods
- storeThing_ExcludingUniversal
-}
-
-// storeThingService is the interface the server implements.
-type storeThingService interface {
- store.DirSpecificService
- store.ObjectSpecificService
- store.DirOrObjectService
- store.TransactionService
-}
-
-// BindstoreThing returns the client stub implementing the storeThing
-// interface.
-//
-// If no _gen_ipc.Client is specified, the default _gen_ipc.Client in the
-// global Runtime is used.
-func BindstoreThing(name string, opts ..._gen_ipc.BindOpt) (storeThing, error) {
- var client _gen_ipc.Client
- switch len(opts) {
- case 0:
- // Do nothing.
- case 1:
- if clientOpt, ok := opts[0].(_gen_ipc.Client); opts[0] == nil || ok {
- client = clientOpt
- } else {
- return nil, _gen_vdlutil.ErrUnrecognizedOption
- }
- default:
- return nil, _gen_vdlutil.ErrTooManyOptionsToBind
- }
- stub := &clientStubstoreThing{defaultClient: client, name: name}
- stub.DirSpecific_ExcludingUniversal, _ = store.BindDirSpecific(name, client)
- stub.ObjectSpecific_ExcludingUniversal, _ = store.BindObjectSpecific(name, client)
- stub.DirOrObject_ExcludingUniversal, _ = store.BindDirOrObject(name, client)
- stub.Transaction_ExcludingUniversal, _ = store.BindTransaction(name, client)
-
- return stub, nil
-}
-
-// NewServerstoreThing creates a new server stub.
-//
-// It takes a regular server implementing the storeThingService
-// interface, and returns a new server stub.
-func NewServerstoreThing(server storeThingService) interface{} {
- return &ServerStubstoreThing{
- ServerStubDirSpecific: *store.NewServerDirSpecific(server).(*store.ServerStubDirSpecific),
- ServerStubObjectSpecific: *store.NewServerObjectSpecific(server).(*store.ServerStubObjectSpecific),
- ServerStubDirOrObject: *store.NewServerDirOrObject(server).(*store.ServerStubDirOrObject),
- ServerStubTransaction: *store.NewServerTransaction(server).(*store.ServerStubTransaction),
- service: server,
- }
-}
-
-// clientStubstoreThing implements storeThing.
-type clientStubstoreThing struct {
- store.DirSpecific_ExcludingUniversal
- store.ObjectSpecific_ExcludingUniversal
- store.DirOrObject_ExcludingUniversal
- store.Transaction_ExcludingUniversal
-
- defaultClient _gen_ipc.Client
- name string
-}
-
-func (__gen_c *clientStubstoreThing) client(ctx _gen_context.T) _gen_ipc.Client {
- if __gen_c.defaultClient != nil {
- return __gen_c.defaultClient
- }
- return _gen_veyron2.RuntimeFromContext(ctx).Client()
-}
-
-func (__gen_c *clientStubstoreThing) UnresolveStep(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply []string, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "UnresolveStep", nil, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubstoreThing) Signature(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply _gen_ipc.ServiceSignature, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "Signature", nil, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubstoreThing) GetMethodTags(ctx _gen_context.T, method string, opts ..._gen_ipc.CallOpt) (reply []interface{}, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "GetMethodTags", []interface{}{method}, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-// ServerStubstoreThing wraps a server that implements
-// storeThingService and provides an object that satisfies
-// the requirements of veyron2/ipc.ReflectInvoker.
-type ServerStubstoreThing struct {
- store.ServerStubDirSpecific
- store.ServerStubObjectSpecific
- store.ServerStubDirOrObject
- store.ServerStubTransaction
-
- service storeThingService
-}
-
-func (__gen_s *ServerStubstoreThing) GetMethodTags(call _gen_ipc.ServerCall, method string) ([]interface{}, error) {
- // TODO(bprosnitz) GetMethodTags() will be replaces with Signature().
- // Note: This exhibits some weird behavior like returning a nil error if the method isn't found.
- // This will change when it is replaced with Signature().
- if resp, err := __gen_s.ServerStubDirSpecific.GetMethodTags(call, method); resp != nil || err != nil {
- return resp, err
- }
- if resp, err := __gen_s.ServerStubObjectSpecific.GetMethodTags(call, method); resp != nil || err != nil {
- return resp, err
- }
- if resp, err := __gen_s.ServerStubDirOrObject.GetMethodTags(call, method); resp != nil || err != nil {
- return resp, err
- }
- if resp, err := __gen_s.ServerStubTransaction.GetMethodTags(call, method); resp != nil || err != nil {
- return resp, err
- }
- return nil, nil
-}
-
-func (__gen_s *ServerStubstoreThing) Signature(call _gen_ipc.ServerCall) (_gen_ipc.ServiceSignature, error) {
- result := _gen_ipc.ServiceSignature{Methods: make(map[string]_gen_ipc.MethodSignature)}
-
- result.TypeDefs = []_gen_vdlutil.Any{}
- var ss _gen_ipc.ServiceSignature
- var firstAdded int
- ss, _ = __gen_s.ServerStubDirSpecific.Signature(call)
- firstAdded = len(result.TypeDefs)
- for k, v := range ss.Methods {
- for i, _ := range v.InArgs {
- if v.InArgs[i].Type >= _gen_wiretype.TypeIDFirst {
- v.InArgs[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- for i, _ := range v.OutArgs {
- if v.OutArgs[i].Type >= _gen_wiretype.TypeIDFirst {
- v.OutArgs[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- if v.InStream >= _gen_wiretype.TypeIDFirst {
- v.InStream += _gen_wiretype.TypeID(firstAdded)
- }
- if v.OutStream >= _gen_wiretype.TypeIDFirst {
- v.OutStream += _gen_wiretype.TypeID(firstAdded)
- }
- result.Methods[k] = v
- }
- //TODO(bprosnitz) combine type definitions from embeded interfaces in a way that doesn't cause duplication.
- for _, d := range ss.TypeDefs {
- switch wt := d.(type) {
- case _gen_wiretype.SliceType:
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.ArrayType:
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.MapType:
- if wt.Key >= _gen_wiretype.TypeIDFirst {
- wt.Key += _gen_wiretype.TypeID(firstAdded)
- }
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.StructType:
- for i, fld := range wt.Fields {
- if fld.Type >= _gen_wiretype.TypeIDFirst {
- wt.Fields[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- d = wt
- // NOTE: other types are missing, but we are upgrading anyways.
- }
- result.TypeDefs = append(result.TypeDefs, d)
- }
- ss, _ = __gen_s.ServerStubObjectSpecific.Signature(call)
- firstAdded = len(result.TypeDefs)
- for k, v := range ss.Methods {
- for i, _ := range v.InArgs {
- if v.InArgs[i].Type >= _gen_wiretype.TypeIDFirst {
- v.InArgs[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- for i, _ := range v.OutArgs {
- if v.OutArgs[i].Type >= _gen_wiretype.TypeIDFirst {
- v.OutArgs[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- if v.InStream >= _gen_wiretype.TypeIDFirst {
- v.InStream += _gen_wiretype.TypeID(firstAdded)
- }
- if v.OutStream >= _gen_wiretype.TypeIDFirst {
- v.OutStream += _gen_wiretype.TypeID(firstAdded)
- }
- result.Methods[k] = v
- }
- //TODO(bprosnitz) combine type definitions from embeded interfaces in a way that doesn't cause duplication.
- for _, d := range ss.TypeDefs {
- switch wt := d.(type) {
- case _gen_wiretype.SliceType:
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.ArrayType:
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.MapType:
- if wt.Key >= _gen_wiretype.TypeIDFirst {
- wt.Key += _gen_wiretype.TypeID(firstAdded)
- }
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.StructType:
- for i, fld := range wt.Fields {
- if fld.Type >= _gen_wiretype.TypeIDFirst {
- wt.Fields[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- d = wt
- // NOTE: other types are missing, but we are upgrading anyways.
- }
- result.TypeDefs = append(result.TypeDefs, d)
- }
- ss, _ = __gen_s.ServerStubDirOrObject.Signature(call)
- firstAdded = len(result.TypeDefs)
- for k, v := range ss.Methods {
- for i, _ := range v.InArgs {
- if v.InArgs[i].Type >= _gen_wiretype.TypeIDFirst {
- v.InArgs[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- for i, _ := range v.OutArgs {
- if v.OutArgs[i].Type >= _gen_wiretype.TypeIDFirst {
- v.OutArgs[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- if v.InStream >= _gen_wiretype.TypeIDFirst {
- v.InStream += _gen_wiretype.TypeID(firstAdded)
- }
- if v.OutStream >= _gen_wiretype.TypeIDFirst {
- v.OutStream += _gen_wiretype.TypeID(firstAdded)
- }
- result.Methods[k] = v
- }
- //TODO(bprosnitz) combine type definitions from embeded interfaces in a way that doesn't cause duplication.
- for _, d := range ss.TypeDefs {
- switch wt := d.(type) {
- case _gen_wiretype.SliceType:
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.ArrayType:
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.MapType:
- if wt.Key >= _gen_wiretype.TypeIDFirst {
- wt.Key += _gen_wiretype.TypeID(firstAdded)
- }
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.StructType:
- for i, fld := range wt.Fields {
- if fld.Type >= _gen_wiretype.TypeIDFirst {
- wt.Fields[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- d = wt
- // NOTE: other types are missing, but we are upgrading anyways.
- }
- result.TypeDefs = append(result.TypeDefs, d)
- }
- ss, _ = __gen_s.ServerStubTransaction.Signature(call)
- firstAdded = len(result.TypeDefs)
- for k, v := range ss.Methods {
- for i, _ := range v.InArgs {
- if v.InArgs[i].Type >= _gen_wiretype.TypeIDFirst {
- v.InArgs[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- for i, _ := range v.OutArgs {
- if v.OutArgs[i].Type >= _gen_wiretype.TypeIDFirst {
- v.OutArgs[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- if v.InStream >= _gen_wiretype.TypeIDFirst {
- v.InStream += _gen_wiretype.TypeID(firstAdded)
- }
- if v.OutStream >= _gen_wiretype.TypeIDFirst {
- v.OutStream += _gen_wiretype.TypeID(firstAdded)
- }
- result.Methods[k] = v
- }
- //TODO(bprosnitz) combine type definitions from embeded interfaces in a way that doesn't cause duplication.
- for _, d := range ss.TypeDefs {
- switch wt := d.(type) {
- case _gen_wiretype.SliceType:
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.ArrayType:
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.MapType:
- if wt.Key >= _gen_wiretype.TypeIDFirst {
- wt.Key += _gen_wiretype.TypeID(firstAdded)
- }
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.StructType:
- for i, fld := range wt.Fields {
- if fld.Type >= _gen_wiretype.TypeIDFirst {
- wt.Fields[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- d = wt
- // NOTE: other types are missing, but we are upgrading anyways.
- }
- result.TypeDefs = append(result.TypeDefs, d)
- }
-
- return result, nil
-}
-
-func (__gen_s *ServerStubstoreThing) UnresolveStep(call _gen_ipc.ServerCall) (reply []string, err error) {
- if unresolver, ok := __gen_s.service.(_gen_ipc.Unresolver); ok {
- return unresolver.UnresolveStep(call)
- }
- if call.Server() == nil {
- return
- }
- var published []string
- if published, err = call.Server().Published(); err != nil || published == nil {
- return
- }
- reply = make([]string, len(published))
- for i, p := range published {
- reply[i] = _gen_naming.Join(p, call.Name())
- }
- return
-}
diff --git a/services/store/server/thing.go b/services/store/server/thing.go
deleted file mode 100644
index 37c0d73..0000000
--- a/services/store/server/thing.go
+++ /dev/null
@@ -1,394 +0,0 @@
-package server
-
-// This file defines thing, which implements the server-side Thing API from
-// veyron2/services/store/service.vdl.
-
-import (
- "veyron/services/store/memstore"
-
- "veyron2/ipc"
- "veyron2/query"
- "veyron2/security"
- "veyron2/services/mounttable"
- mttypes "veyron2/services/mounttable/types"
- "veyron2/services/store"
- "veyron2/services/watch"
- watchtypes "veyron2/services/watch/types"
- "veyron2/storage"
- "veyron2/vdl/vdlutil"
- "veyron2/verror"
-)
-
-const (
- // Large random value, used to indicate that this memstore object represents
- // a directory.
- dirValue int64 = 9380751577234
-)
-
-type thing struct {
- name string // will never contain a transaction id
- obj *memstore.Object
- tid transactionID // may be nullTransactionID
- server *Server
-}
-
-var (
- // TODO(sadovsky): Make sure all of these error cases are covered by tests.
- // TODO(sadovsky): Revisit these error types.
- errMakeDirObjectExists = verror.Existsf("Object exists in Dir.Make path")
- errCalledObjectMethodOnDir = verror.BadArgf("called Object method on Dir")
- errPutObjectInObject = verror.BadArgf("put Object in another Object")
- errCannotRemoveRootDir = verror.BadArgf("cannot remove root Dir")
-
- _ storeThingService = (*thing)(nil)
-
- nullEntry storage.Entry
- nullStat storage.Stat
-)
-
-func isDir(entry *storage.Entry) bool {
- value, ok := entry.Value.(int64)
- return ok && value == dirValue
-}
-
-func isObject(entry *storage.Entry) bool {
- return !isDir(entry)
-}
-
-func (t *thing) String() string {
- return t.name
-}
-
-func (t *thing) Attributes(arg string) map[string]string {
- return map[string]string{
- "health": "ok",
- "servertype": t.String(),
- }
-}
-
-////////////////////////////////////////
-// DirOrObject methods
-
-// Remove removes this thing.
-func (t *thing) Remove(ctx ipc.ServerContext) error {
- tx, err := t.server.findTransaction(ctx, t.tid)
- if err != nil {
- return err
- }
- path := storage.ParsePath(t.name)
- if len(path) == 0 {
- return errCannotRemoveRootDir
- }
- return t.obj.Remove(ctx.RemoteID(), tx)
-}
-
-// Query returns a sequence of objects that match the given query.
-func (t *thing) Query(ctx ipc.ServerContext, q query.Query, stream store.DirOrObjectServiceQueryStream) error {
- tx, err := t.server.findTransaction(ctx, t.tid)
- if err != nil {
- return err
- }
- it, err := t.obj.Query(ctx.RemoteID(), tx, q)
- if err != nil {
- return err
- }
- for it.Next() {
- if err := stream.SendStream().Send(*it.Get()); err != nil {
- it.Abort()
- return err
- }
- }
- return it.Err()
-}
-
-// Stat returns information about this thing.
-func (t *thing) Stat(ctx ipc.ServerContext) (storage.Stat, error) {
- tx, err := t.server.findTransaction(ctx, t.tid)
- if err != nil {
- return nullStat, err
- }
- s, err := t.obj.Stat(ctx.RemoteID(), tx)
- if err != nil {
- return nullStat, err
- }
- // Determine the Kind.
- entry, err := t.obj.Get(ctx.RemoteID(), tx)
- if err != nil {
- // TODO(sadovsky): Is this the right thing to return here? If obj.Get()
- // returns state.errNotFound, it probably makes more sense to return
- // nullStat and nil error. (Note, for now t.obj.Stat() is not implemented
- // and always returns an error, so we never actually get here.)
- return nullStat, err
- }
- if isDir(entry) {
- s.Kind = storage.DirKind
- } else {
- s.Kind = storage.ObjectKind
- }
- return *s, err
-}
-
-// Exists returns true iff this thing is present in the store.
-func (t *thing) Exists(ctx ipc.ServerContext) (bool, error) {
- tx, err := t.server.findTransaction(ctx, t.tid)
- if err != nil {
- return false, err
- }
- return t.obj.Exists(ctx.RemoteID(), tx)
-}
-
-// NewTransaction creates a transaction with the given options. It returns the
-// name of the transaction relative to this thing's name.
-func (t *thing) NewTransaction(ctx ipc.ServerContext, opts []vdlutil.Any) (string, error) {
- if t.tid != nullTransactionID {
- return "", errNestedTransaction
- }
- return t.server.createTransaction(ctx, t.name)
-}
-
-// Glob streams a series of names that match the given pattern.
-func (t *thing) Glob(ctx ipc.ServerContext, pattern string, stream mounttable.GlobbableServiceGlobStream) error {
- tx, err := t.server.findTransaction(ctx, t.tid)
- if err != nil {
- return err
- }
- it, err := t.obj.Glob(ctx.RemoteID(), tx, pattern)
- if err != nil {
- return err
- }
- gsa := &globStreamAdapter{stream}
- for ; it.IsValid(); it.Next() {
- if err := gsa.SendStream().Send(it.Name()); err != nil {
- return err
- }
- }
- return nil
-}
-
-// WatchGlob returns a stream of changes that match a pattern.
-func (t *thing) WatchGlob(ctx ipc.ServerContext, req watchtypes.GlobRequest, stream watch.GlobWatcherServiceWatchGlobStream) error {
- return t.server.watcher.WatchGlob(ctx, storage.ParsePath(t.name), req, stream)
-}
-
-// WatchQuery returns a stream of changes that satisfy a query.
-func (t *thing) WatchQuery(ctx ipc.ServerContext, req watchtypes.QueryRequest, stream watch.QueryWatcherServiceWatchQueryStream) error {
- return t.server.watcher.WatchQuery(ctx, storage.ParsePath(t.name), req, stream)
-}
-
-////////////////////////////////////////
-// Dir-only methods
-
-// Called by Make(), and also called directly by server.go with a nil
-// transaction to create the root directory.
-func (t *thing) makeInternal(remoteID security.PublicID, tx *memstore.Transaction) error {
- // Make dirs from the top down. Return error if we encounter an Object.
- parts := storage.PathName{""}
- parts = append(parts, storage.ParsePath(t.name)...)
- // Set to true once we encounter a path component that doesn't already
- // exist. Create new dirs from there on down.
- newTree := false
- for i, _ := range parts {
- obj := t.server.store.Bind(parts[:i+1].String())
- if !newTree {
- // Note, obj.Get() returns state.errNotFound if the entry does not exist.
- // TODO(sadovsky): Check for that specific error type and propagate all
- // other errors.
- entry, err := obj.Get(remoteID, tx)
- if err != nil {
- newTree = true
- } else if isObject(entry) {
- return errMakeDirObjectExists
- }
- }
- if newTree {
- obj.Put(remoteID, tx, dirValue)
- }
- }
- return nil
-}
-
-func (t *thing) Make(ctx ipc.ServerContext) error {
- tx, err := t.server.findTransaction(ctx, t.tid)
- if err != nil {
- return err
- }
- return t.makeInternal(ctx.RemoteID(), tx)
-}
-
-////////////////////////////////////////
-// Object-only methods
-
-// Get returns the value for the Object. The value returned is from the
-// most recent mutation of the entry in the Transaction, or from the
-// Transaction's snapshot if there is no mutation.
-func (t *thing) Get(ctx ipc.ServerContext) (storage.Entry, error) {
- tx, err := t.server.findTransaction(ctx, t.tid)
- if err != nil {
- return nullEntry, err
- }
- // Note, obj.Get() returns state.errNotFound if the entry does not exist.
- entry, err := t.obj.Get(ctx.RemoteID(), tx)
- if err != nil {
- return nullEntry, err
- }
- if isDir(entry) {
- return nullEntry, errCalledObjectMethodOnDir
- }
- return *entry, err
-}
-
-// Put modifies the value of the Object.
-func (t *thing) Put(ctx ipc.ServerContext, val vdlutil.Any) (storage.Stat, error) {
- tx, err := t.server.findTransaction(ctx, t.tid)
- if err != nil {
- return nullStat, err
- }
- // Verify that this entry either doesn't exist or exists and is an Object.
- // Note, obj.Get() returns state.errNotFound if the entry does not exist.
- // TODO(sadovsky): Check for that specific error type and propagate all
- // other errors.
- entry, err := t.obj.Get(ctx.RemoteID(), tx)
- if err == nil && isDir(entry) {
- return nullStat, errCalledObjectMethodOnDir
- }
- // Verify that the parent already exists and is a Dir.
- // Note, at this point we know t.name isn't the root of the store, b/c if it
- // were, the check above would've failed.
- path := storage.ParsePath(t.name)
- path = path[:len(path)-1] // path to parent
- // Note, we don't return an error here if path doesn't exist -- we let the
- // memstore Put() code handle that case.
- entry, err = t.server.store.Bind(path.String()).Get(ctx.RemoteID(), tx)
- if err == nil && isObject(entry) {
- return nullStat, errPutObjectInObject
- }
- s, err := t.obj.Put(ctx.RemoteID(), tx, interface{}(val))
- if err != nil {
- return nullStat, err
- }
- // TODO(sadovsky): Add test for this.
- s.Kind = storage.ObjectKind
- return *s, err
-}
-
-////////////////////////////////////////
-// Transaction methods
-
-func (t *thing) Commit(ctx ipc.ServerContext) error {
- return t.server.commitTransaction(ctx, t.tid)
-}
-
-func (t *thing) Abort(ctx ipc.ServerContext) error {
- return t.server.abortTransaction(ctx, t.tid)
-}
-
-////////////////////////////////////////
-// SyncGroup methods.
-// See veyron2/services/store/service.vdl for detailed comments.
-// TODO(hpucha): Actually implement these methods.
-
-// GetSyncGroupNames returns the global names of all SyncGroups attached
-// to this directory.
-func (t *thing) GetSyncGroupNames(ctx ipc.ServerContext) (names []string, err error) {
- return nil, verror.Internalf("GetSyncGroupNames not yet implemented")
-}
-
-// CreateSyncGroup creates a new SyncGroup.
-func (t *thing) CreateSyncGroup(ctx ipc.ServerContext, name string, config store.SyncGroupConfig) error {
- // One approach is that all syncgroup operations are
- // serialized with a single lock. This enables easily checking
- // nesting of syncgroups.
-
- // sgop.Lock(), defer sgop.Unlock()
- // acl check
- // sanity checks
- // sgname in the dir does not exist
- // syncgroup is not nested
- // call syncd
- // if err != nil return err
- // update dir->sgname mapping
-
- return verror.Internalf("CreateSyncGroup not yet implemented")
-}
-
-// JoinSyncGroup joins a SyncGroup with the specified global Veyron name.
-func (t *thing) JoinSyncGroup(ctx ipc.ServerContext, name string) error {
- // sgop.Lock(), defer sgop.Unlock()
- // acl check (parentdir or dir)
- // sanity checks
- // if dir exists
- // sgname in the dir does not exist
- // syncgroup is not nested
- // call syncd
- // if err != nil return err
- // if err == nil && rootoid does not exist && dir does not exist
- // mkdir(rootoid) and setacl
- // update dir->sgname mapping
- // if err == nil && rootoid exists && dir exists && oid(dir) == rootoid
- // update dir->sgname mapping
- // All other cases are error, call syncd.leave(sg) and return err
-
- return verror.Internalf("JoinSyncGroup not yet implemented")
-}
-
-// LeaveSyncGroup leaves the SyncGroup.
-func (t *thing) LeaveSyncGroup(ctx ipc.ServerContext, name string) error {
- // sgop.Lock(), defer sgop.Unlock()
- // acl check
- // sanity checks (sgname in the dir exists)
- // call syncd
- // if err != nil return err
- // update dir->sgname mapping
- // if sglist in dir is empty, rewrite oids.
-
- return verror.Internalf("LeaveSyncGroup not yet implemented")
-}
-
-// DestroySyncGroup destroys the SyncGroup.
-func (t *thing) DestroySyncGroup(ctx ipc.ServerContext, name string) error {
- return verror.Internalf("DestroySyncGroup not yet implemented")
-}
-
-// EjectFromSyncGroup ejects a member from the SyncGroup.
-func (t *thing) EjectFromSyncGroup(ctx ipc.ServerContext, name, member string) error {
- return verror.Internalf("EjectFromSyncGroup not yet implemented")
-}
-
-// GetSyncGroupConfig gets the config info of the SyncGroup.
-func (t *thing) GetSyncGroupConfig(ctx ipc.ServerContext, name string) (config store.SyncGroupConfig, eTag string, err error) {
- return store.SyncGroupConfig{}, "", verror.Internalf("GetSyncGroupConfig not yet implemented")
-}
-
-// SetSyncGroupConfig sets the config info of the SyncGroup.
-func (t *thing) SetSyncGroupConfig(ctx ipc.ServerContext, name string, config store.SyncGroupConfig, eTag string) error {
- return verror.Internalf("SetSyncGroupConfig not yet implemented")
-}
-
-// GetMembersOfSyncGroup gets the Veyron names of the Stores that joined
-// this SyncGroup.
-func (t *thing) GetMembersOfSyncGroup(ctx ipc.ServerContext, name string) ([]string, error) {
- return nil, verror.Internalf("GetMembersOfSyncGroup not yet implemented")
-}
-
-////////////////////////////////////////
-// Internals
-
-type globStreamSenderAdapter struct {
- stream interface {
- Send(entry mttypes.MountEntry) error
- }
-}
-
-func (a *globStreamSenderAdapter) Send(item string) error {
- return a.stream.Send(mttypes.MountEntry{Name: item})
-}
-
-type globStreamAdapter struct {
- stream mounttable.GlobbableServiceGlobStream
-}
-
-func (a *globStreamAdapter) SendStream() interface {
- Send(item string) error
-} {
- return &globStreamSenderAdapter{a.stream.SendStream()}
-}
diff --git a/services/store/stored/main.go b/services/store/stored/main.go
deleted file mode 100644
index 9c19912..0000000
--- a/services/store/stored/main.go
+++ /dev/null
@@ -1,99 +0,0 @@
-// stored is a storage server.
-//
-// Usage:
-//
-// stored [--name=<mount>] [--db=<dbName>]
-//
-// - <name> is the Veyron mount point name, default /global/vstore/<hostname>/<username>.
-// - <dbName> is the filename in which to store the data.
-//
-// The store service has Object name, <name>/.store.
-// The raw store service has Object name, <name>/.store.raw.
-// Individual values with path <path> have name <name>/<path>.
-package main
-
-import (
- "flag"
- "log"
- "os"
- "os/user"
-
- vflag "veyron/security/flag"
- "veyron/services/store/server"
-
- "veyron2/rt"
-
- _ "veyron/services/store/typeregistryhack"
-)
-
-var (
- mountName string
-
- // TODO(rthellend): Remove the protocol and address flags when the config
- // manager is working.
- protocol = flag.String("protocol", "tcp", "protocol to listen on")
- address = flag.String("address", ":0", "address to listen on")
-
- dbName = flag.String("db", "/var/tmp/veyron_store.db",
- "Metadata database")
- viewerPort = flag.Int("viewerPort", 5000,
- "IPV4 port to serve viewer from, or 0 to disable viewer")
-)
-
-func init() {
- username := "unknown"
- if u, err := user.Current(); err == nil {
- username = u.Username
- }
- hostname := "unknown"
- if h, err := os.Hostname(); err == nil {
- hostname = h
- }
- dir := "global/vstore/" + hostname + "/" + username
- flag.StringVar(&mountName, "name", dir, "Mount point for media")
-}
-
-// main starts the store service, taking args from command line flags.
-func main() {
- r := rt.Init()
-
- // Create a new server instance.
- s, err := r.NewServer()
- if err != nil {
- log.Fatal("r.NewServer() failed: ", err)
- }
-
- // Create a new StoreService.
- storeService, err := server.New(
- server.ServerConfig{Admin: r.Identity().PublicID(), DBName: *dbName})
- if err != nil {
- log.Fatal("server.New() failed: ", err)
- }
- defer storeService.Close()
-
- // Create the authorizer.
- auth := vflag.NewAuthorizerOrDie()
-
- // Register the services.
- storeDisp := server.NewStoreDispatcher(storeService, auth)
- // Create an endpoint and start listening.
- ep, err := s.Listen(*protocol, *address)
- if err != nil {
- log.Fatal("s.Listen() failed: ", err)
- }
- // Publish the service in the mount table.
- log.Printf("Mounting store on %s, endpoint /%s", mountName, ep)
- if err := s.Serve(mountName, storeDisp); err != nil {
- log.Fatal("s.Serve() failed: ", err)
- }
-
- // Run viewer if requested.
- if *viewerPort > 0 {
- // TODO(kash): Port the viewer to the new dir/object store api.
- //go viewer.ListenAndServe(r, fmt.Sprintf(":%d", *viewerPort), mountName, vstore.New())
- }
-
- // Wait forever.
- done := make(chan struct{})
- <-done
-}
diff --git a/services/store/testutil/store.go b/services/store/testutil/store.go
deleted file mode 100644
index 9190f09..0000000
--- a/services/store/testutil/store.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package testutil
-
-import (
- "crypto/rand"
- "io/ioutil"
- "os"
- "testing"
-
- istore "veyron/services/store/server"
- _ "veyron/services/store/typeregistryhack"
-
- "veyron2/ipc"
- "veyron2/naming"
- "veyron2/security"
-)
-
-// NewStore creates a new testing instance of the store server and returns
-// an object name that identifies the instance and a closure that can
-// be used to terminate the instance and clean up.
-func NewStore(t *testing.T, server ipc.Server, id security.PublicID) (string, func()) {
- // Create a temporary directory for the store server.
- prefix := "vstore-test-db"
- dbName, err := ioutil.TempDir("", prefix)
- if err != nil {
- t.Fatalf("TempDir(%v, %v) failed: %v", "", prefix, err)
- }
-
- // Create a new StoreService.
- config := istore.ServerConfig{Admin: id, DBName: dbName}
- storeService, err := istore.New(config)
- if err != nil {
- t.Fatalf("New(%v) failed: %v", config, err)
- }
-
- var buf [16]byte
- if _, err := rand.Read(buf[:]); err != nil {
- t.Fatalf("rand.Read() failed: %v", err)
- }
-
- // Register the services.
- storeDispatcher := istore.NewStoreDispatcher(storeService, nil)
- if err := server.Serve("", storeDispatcher); err != nil {
- t.Fatalf("Register(%v) failed: %v", storeDispatcher, err)
- }
-
- // Create an endpoint and start listening.
- protocol, hostname := "tcp", "127.0.0.1:0"
- ep, err := server.Listen(protocol, hostname)
- if err != nil {
- t.Fatalf("Listen(%v, %v) failed: %v", protocol, hostname, err)
- }
-
- name := naming.JoinAddressName(ep.String(), "")
-
- // Create a closure that cleans things up.
- cleanup := func() {
- server.Stop()
- os.RemoveAll(dbName)
- }
-
- return name, cleanup
-}
diff --git a/services/store/typeregistryhack/init.go b/services/store/typeregistryhack/init.go
deleted file mode 100644
index bea5d02..0000000
--- a/services/store/typeregistryhack/init.go
+++ /dev/null
@@ -1,34 +0,0 @@
-// Package typeregistryhack registers types that client send to the store, for
-// VOM cannot decode objects of unknown type.
-//
-// TODO(tilaks): use val.Value to decode unknown types.
-package typeregistryhack
-
-import (
- // Register boxes types.
- "veyron/examples/boxes"
- // Register mdb types.
- _ "veyron/examples/mdb/schema"
- // Register todos types.
- _ "veyron/examples/todos/schema"
- // Register bank types.
- _ "veyron/examples/bank/schema"
- // Register stfortune types.
- _ "veyron/examples/stfortune/schema"
- // Register profile types.
- "veyron/services/mgmt/profile"
- // Register application types.
- "veyron2/services/mgmt/application"
- // Register build types.
- _ "veyron2/services/mgmt/build"
-
- "veyron2/vom"
-)
-
-func init() {
- // Register profile types.
- vom.Register(&struct{}{}) // directories have type struct{}.
- vom.Register(&profile.Specification{})
- vom.Register(&application.Envelope{})
- vom.Register(&boxes.Box{})
-}
diff --git a/services/store/viewer/entry.go b/services/store/viewer/entry.go
deleted file mode 100644
index a5e3991..0000000
--- a/services/store/viewer/entry.go
+++ /dev/null
@@ -1,77 +0,0 @@
-// TODO(kash): Rewrite this to use the new dir/object store api.
-// +build ignore
-
-package viewer
-
-import (
- "path/filepath"
- "sort"
- "time"
-
- "veyron2/context"
- "veyron2/naming"
- "veyron2/storage"
-)
-
-// Entry is the type used to pass store values to user-defined templates.
-type Entry struct {
- ctx context.T
- storeRoot string
- store storage.Store
- Name string
- Value interface{}
-}
-
-// EntryForRawTemplate is the type used to pass store values to the "raw"
-// template defined in viewer.go.
-type EntryForRawTemplate struct {
- *Entry
- Subdirs []string // relative names
- RawSubdirs bool // whether to add "?raw" to subdir hrefs
-}
-
-// abspath returns the absolute path from a path relative to this value.
-func (e *Entry) abspath(path string) string {
- return naming.Join(e.storeRoot, e.Name, path)
-}
-
-// Date performs a Time conversion, given an integer argument that represents a
-// time in nanoseconds since the Unix epoch.
-func (e *Entry) Date(ns int64) time.Time {
- return time.Unix(0, ns)
-}
-
-// Join joins the path elements.
-func (e *Entry) Join(elem ...string) string {
- return filepath.ToSlash(filepath.Join(elem...))
-}
-
-// Base returns the last element of the path.
-func (e *Entry) Base(path string) string {
- return filepath.Base(path)
-}
-
-// Glob performs a glob expansion of the pattern. The results are sorted.
-func (e *Entry) Glob(pattern string) ([]string, error) {
- results := e.store.Bind(e.abspath("")).Glob(e.ctx, pattern)
- names := []string{}
- rStream := results.RecvStream()
- for rStream.Advance() {
- names = append(names, rStream.Value())
- }
- if err := rStream.Err(); err != nil {
- return nil, err
- }
- sort.Strings(names)
- return names, nil
-}
-
-// Get fetches a value from the store, where path is relative to this value.
-// The result is nil if the value does not exist.
-func (e *Entry) Get(path string) interface{} {
- en, err := e.store.Bind(e.abspath(path)).Get(e.ctx)
- if err != nil {
- return nil
- }
- return en.Value
-}
diff --git a/services/store/viewer/reflect.go b/services/store/viewer/reflect.go
deleted file mode 100644
index c7b4359..0000000
--- a/services/store/viewer/reflect.go
+++ /dev/null
@@ -1,150 +0,0 @@
-package viewer
-
-import (
- "bytes"
- "fmt"
- "reflect"
-
- "veyron2/storage"
-)
-
-// printer defines a pretty printer for Go values, using reflection to traverse
-// the values.
-type printer struct {
- buf bytes.Buffer
-}
-
-var (
- tyID = reflect.TypeOf(storage.ID{})
-)
-
-// stringTypePointers removes the outer Ptr and Interface types.
-func stripTypePointers(ty reflect.Type) reflect.Type {
- kind := ty.Kind()
- for kind == reflect.Ptr || kind == reflect.Interface {
- ty = ty.Elem()
- kind = ty.Kind()
- }
- return ty
-}
-
-// templatePath returns the path to the templates value, defined as
-// /templates/<pkgPath>/<typeName>.
-func templatePath(v interface{}) string {
- ty := stripTypePointers(reflect.TypeOf(v))
- pkgPath := ty.PkgPath()
- tyName := ty.Name()
- if pkgPath == "" || tyName == "" {
- return ""
- }
- return fmt.Sprintf("templates/%s/%s", pkgPath, tyName)
-}
-
-// print formats the argument.
-func (p *printer) print(v interface{}) {
- p.printValue(0, reflect.ValueOf(v))
-}
-
-// printType formats the type.
-func (p *printer) printType(indent int, v reflect.Type) {
- p.printString(v.Name())
-}
-
-// printValue is the main pretty-printer method. It formats the value at the
-// indentation level specified by the argument.
-func (p *printer) printValue(indent int, v reflect.Value) {
- if !v.IsValid() {
- p.printString("<invalid>")
- return
- }
- ty := v.Type()
- if ty == tyID {
- p.printString(v.Interface().(storage.ID).String())
- return
- }
- switch ty.Kind() {
- case reflect.Ptr:
- p.printString("&")
- fallthrough
- case reflect.Interface:
- p.printValue(indent, v.Elem())
- case reflect.Array, reflect.Slice:
- p.printSliceValue(indent, v)
- case reflect.Map:
- p.printMapValue(indent, v)
- case reflect.Struct:
- p.printStructValue(indent, v)
- case reflect.String:
- fmt.Fprintf(&p.buf, "%q", v.Interface())
- default:
- fmt.Fprintf(&p.buf, "%+v", v.Interface())
- }
-}
-
-// printSliceValue formats a slice or array.
-func (p *printer) printSliceValue(indent int, v reflect.Value) {
- p.printType(indent, v.Type())
- p.printString("{")
- l := v.Len()
- for i := 0; i < l; i++ {
- p.printIndent(indent + 1)
- p.printValue(indent+1, v.Index(i))
- p.printString(",")
- }
- p.printIndent(indent)
- p.printString("}")
-}
-
-// printMapValue formats a map.
-func (p *printer) printMapValue(indent int, v reflect.Value) {
- p.printType(indent, v.Type())
- p.printString("{")
- for _, key := range v.MapKeys() {
- p.printIndent(indent + 1)
- p.printValue(indent+1, key)
- p.printString(": ")
- p.printValue(indent+2, v.MapIndex(key))
- p.printString(",")
- }
- p.printIndent(indent)
- p.printString("}")
-}
-
-// printStructValue formats a struct.
-func (p *printer) printStructValue(indent int, v reflect.Value) {
- ty := v.Type()
- p.printType(indent, ty)
- p.printString("{")
- l := ty.NumField()
- for i := 0; i < l; i++ {
- field := ty.Field(i)
- if field.PkgPath != "" {
- continue
- }
- p.printIndent(indent + 1)
- p.printString(field.Name)
- p.printString(": ")
- p.printValue(indent+1, v.Field(i))
- p.printString(",")
- }
- p.printIndent(indent)
- p.printString("}")
-}
-
-// printString adds a string to the output.
-func (p *printer) printString(s string) {
- p.buf.WriteString(s)
-}
-
-// printIndent prints a newline and then indents to the specified indentation.
-func (p *printer) printIndent(indent int) {
- p.buf.WriteRune('\n')
- for i := 0; i < indent; i++ {
- p.buf.WriteString(" ")
- }
-}
-
-// String returns the formatted text.
-func (p *printer) String() string {
- return p.buf.String()
-}
diff --git a/services/store/viewer/viewer.go b/services/store/viewer/viewer.go
deleted file mode 100644
index dd222c9..0000000
--- a/services/store/viewer/viewer.go
+++ /dev/null
@@ -1,180 +0,0 @@
-// TODO(kash): Rewrite this to use the new dir/object store api.
-// +build ignore
-
-// package viewer exports a store through an HTTP server, with the following
-// features.
-//
-// URL paths correspond to store paths. For example, if the URL is
-// http://myhost/a/b/c, the store value that is fetched is /a/b/c.
-//
-// Values can be formatted using html templates (documented in the
-// text/templates package). Templates are named according to the type of the
-// value that they format, using a path /templates/<pkgPath>/<typeName>. For
-// example, suppose we are viewing the page for /movies/Inception, and it
-// contains a value of type *examples/store/mdb/schema.Movie. We fetch the
-// template /templates/examples/store/mdb/schema/Movie, which must be a string
-// in html/template format. If it exists, the template is compiled and used to
-// print the value. If the template does not exist, the value is formatted in
-// raw form.
-//
-// String values that have a path ending with suffix .css are printed in raw
-// form.
-package viewer
-
-import (
- "fmt"
- "html"
- "html/template"
- "io"
- "net/http"
- "path/filepath"
-
- "veyron2"
- "veyron2/context"
- "veyron2/naming"
- "veyron2/storage"
- "veyron2/vlog"
-)
-
-// server is the HTTP server handler.
-type server struct {
- storeRoot string
- store storage.Store
- runtime veyron2.Runtime
-}
-
-var _ http.Handler = (*server)(nil)
-
-const (
- // rawTemplateText is used to format the output in a raw textual form.
- rawTemplateText = `<!DOCTYPE html>
-<html>
-{{$entry := .}}
-{{$prefix := $entry.Name}}
-{{$rawSubdirs := $entry.RawSubdirs}}
-<head>
-<title>{{.Name}}</title>
-</head>
-<body>
-<h1>{{.Name}}</h1>
-<pre>{{.Value}}</pre>
-{{with .Subdirs}}
-<h3>Subdirectories</h3>
-{{range .}}
-{{$name := $entry.Join $prefix .}}
-<p><a href="{{$name}}{{if $rawSubdirs}}?raw{{end}}">{{.}}</a></p>
-{{end}}
-{{end}}
-</body>
-</html>`
-)
-
-var (
- rawTemplate = mustParse("raw", rawTemplateText)
-)
-
-// mustParse parses the template text. It panics on error.
-func mustParse(name, text string) *template.Template {
- tmpl, err := template.New(name).Parse(text)
- if err != nil {
- panic(fmt.Sprintf("Error parsing template %q: %s", text, err))
- }
- return tmpl
-}
-
-// abspath returns the absolute path from a path relative to the store root.
-func (s *server) abspath(path string) string {
- return naming.Join(s.storeRoot, path)
-}
-
-// loadTemplate fetches the template for the value from the store. The template
-// is based on the type of the value, under /template/<pkgPath>/<typeName>.
-func (s *server) loadTemplate(ctx context.T, v interface{}) *template.Template {
- path := templatePath(v)
- en, err := s.store.Bind(s.abspath(path)).Get(ctx)
- if err != nil {
- return nil
- }
- str, ok := en.Value.(string)
- if !ok {
- return nil
- }
- tmpl, err := template.New(path).Parse(str)
- if err != nil {
- vlog.Infof("Template error: %s: %s", path, err)
- return nil
- }
- return tmpl
-}
-
-// printRawValuePage prints the value in raw format.
-func (s *server) printRawValuePage(ctx context.T, w http.ResponseWriter, path string, v interface{}, rawSubdirs bool) {
- var p printer
- p.print(v)
- x := &EntryForRawTemplate{&Entry{ctx: ctx, storeRoot: s.storeRoot, store: s.store, Name: path, Value: p.String()}, []string{}, rawSubdirs}
- x.Subdirs, _ = x.Glob("*")
- if err := rawTemplate.Execute(w, x); err != nil {
- w.Write([]byte(html.EscapeString(err.Error())))
- }
-}
-
-// printValuePage prints the value using a template if possible. If a template
-// is not found, the value is printed in raw format instead.
-func (s *server) printValuePage(ctx context.T, w http.ResponseWriter, path string, v interface{}) {
- if tmpl := s.loadTemplate(ctx, v); tmpl != nil {
- x := &Entry{ctx: ctx, storeRoot: s.storeRoot, store: s.store, Name: path, Value: v}
- if err := tmpl.Execute(w, x); err != nil {
- w.Write([]byte(html.EscapeString(err.Error())))
- }
- return
- }
- s.printRawValuePage(ctx, w, path, v, false)
-}
-
-// printRawPage prints a string value directly, without processing.
-func (s *server) printRawPage(w http.ResponseWriter, v interface{}) {
- str, ok := v.(string)
- if !ok {
- fmt.Fprintf(w, "%s", v)
- } else {
- io.WriteString(w, str)
- }
-}
-
-// ServeHTTP is the main HTTP handler.
-func (s *server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
- path := req.URL.Path
- ctx := s.runtime.NewContext()
- en, err := s.store.Bind(s.abspath(path)).Get(ctx)
- if err != nil {
- msg := fmt.Sprintf(
- "<html><body><h1>%s</h1><h2>Error: %s</h2></body></html>",
- html.EscapeString(path),
- html.EscapeString(err.Error()))
- w.WriteHeader(http.StatusNotFound)
- w.Write([]byte(msg))
- return
- }
-
- q := req.URL.Query()
- switch filepath.Ext(path) {
- case ".css":
- w.Header().Set("Content-Type", "text/css; charset=utf-8")
- s.printRawPage(w, en.Value)
- default:
- w.Header().Set("Content-Type", "text/html; charset=utf-8")
- if q["raw"] != nil {
- s.printRawValuePage(ctx, w, path, en.Value, true)
- } else {
- s.printValuePage(ctx, w, path, en.Value)
- }
- }
-}
-
-// ListenAndServe is the main entry point. It serves store at the specified
-// network address.
-func ListenAndServe(runtime veyron2.Runtime, addr string, storeRoot string, st storage.Store) error {
- s := &server{storeRoot: storeRoot, store: st, runtime: runtime}
- vlog.Infof("Viewer running at http://localhost%s", addr)
- return http.ListenAndServe(addr, s)
-}
diff --git a/services/syncgroup/id.go b/services/syncgroup/id.go
deleted file mode 100644
index 1c1189a..0000000
--- a/services/syncgroup/id.go
+++ /dev/null
@@ -1,47 +0,0 @@
-package syncgroup
-
-import (
- "encoding/hex"
- "veyron2/storage"
- "veyron2/verror"
-)
-
-// The current implementation of SyncGroupID assumes that an ID is a storage.ID.
-// ID is defined in syncgroup.vdl.
-
-// NewID returns a new ID.
-func NewID() ID {
- return ID(storage.NewID())
-}
-
-// String returns id as a hex string.
-func (id ID) String() string {
- return storage.ID(id).String()
-}
-
-// ParseID returns an ID formed from the string str, which will normally have
-// been created with ID.String().
-func ParseID(str string) (id ID, err error) {
- var tmp []byte
- tmp, err = hex.DecodeString(str)
- if err != nil {
- return id, err
- }
- if len(tmp) == 0 || len(tmp) > len(id) {
- return id, verror.BadProtocolf("string \"%v\" has wrong length to be a syncgroup.ID", str)
- }
- for i := 0; i != len(tmp); i++ {
- id[i] = tmp[i]
- }
- return id, err
-}
-
-// IsValid returns whether id is not the zero id.
-func (id ID) IsValid() bool {
- return storage.ID(id).IsValid()
-}
-
-// CompareIDs compares two IDs lexicographically.
-func CompareIDs(id1 ID, id2 ID) int {
- return storage.CompareIDs(storage.ID(id1), storage.ID(id2))
-}
diff --git a/services/syncgroup/id_test.go b/services/syncgroup/id_test.go
deleted file mode 100644
index 06fff56..0000000
--- a/services/syncgroup/id_test.go
+++ /dev/null
@@ -1,85 +0,0 @@
-package syncgroup
-
-// This file tests syncgroup/id.go
-
-import (
- "testing"
-)
-
-// signum returns -1, 0, or 1 according to whether its argument is -ve, 0, or
-// +ve, respectively.
-func signum(x int) (result int) {
- if x < 0 {
- result = -1
- } else if x > 0 {
- result = 1
- }
- return result
-}
-
-// NewID is tested as a side-effect of the other routines.
-
-// TestIsValid tests IsValid().
-func TestIsValid(t *testing.T) {
- var zeroID ID
- var nonZeroID ID = NewID()
-
- if zeroID.IsValid() {
- t.Errorf("IsValid(%v) == true", zeroID)
- }
- if !nonZeroID.IsValid() {
- t.Errorf("IsValid(%v) == false", nonZeroID)
- }
-}
-
-// TestCompareIDs tests CompareIDs().
-func TestCompareIDs(t *testing.T) {
- ids := []ID{ID{}, NewID(), NewID()}
- for i := 0; i != len(ids); i++ {
- for j := 0; j != len(ids); j++ {
- if (i == j) != (CompareIDs(ids[i], ids[j]) == 0) {
- t.Errorf("(CompareIDs(%v, %v) == 0) == %v, expected %v",
- ids[i], ids[j],
- CompareIDs(ids[i], ids[j]) == 0,
- i == j)
- }
- if signum(CompareIDs(ids[i], ids[j])) !=
- -signum(CompareIDs(ids[j], ids[i])) {
- t.Errorf("(signum(CompareIDs(%v, %v)) == %v != -(%v == signum(CompareIDs(%v, %v)))",
- ids[i], ids[j], signum(CompareIDs(ids[i], ids[j])),
- signum(CompareIDs(ids[j], ids[i])), ids[j], ids[i])
- }
- }
- }
-}
-
-// TestStringAndParseID tests String() and ParseID().
-func TestStringAndParseID(t *testing.T) {
- var lenOfString int
- for i := 0; i != 10; i++ {
- var id ID
- var str string
- if i == 0 { // First time through, use the zero id, and remember the length.
- str = id.String()
- lenOfString = len(str)
- if lenOfString <= 0 {
- t.Errorf("len(id.String()) == %v <= 0", len(str))
- }
- } else { // Subsequently, use a new id.
- id = NewID()
- str = id.String()
- }
- if len(str) != lenOfString {
- t.Errorf("len(id.String()) == %v != %v", len(str), lenOfString)
- }
- var parsedID ID
- var err error
- parsedID, err = ParseID(str)
- if err != nil {
- t.Errorf("ParseID(%v) yields %v", str, err)
- }
- if CompareIDs(id, parsedID) != 0 {
- t.Errorf("ParseID(%v) == %v != %v", str, parsedID, id)
- }
- }
-}
diff --git a/services/syncgroup/syncgroup.vdl b/services/syncgroup/syncgroup.vdl
deleted file mode 100644
index 0531567..0000000
--- a/services/syncgroup/syncgroup.vdl
+++ /dev/null
@@ -1,164 +0,0 @@
-// Package syncgroup provides the means for Stores to set up SyncGroups for
-// subsequent synchronization of objects between them.
-//
-// The intent is that SyncGroup objects are created and administered by
-// SyncGroup servers, even though they are subsequently mirrored among SyncGroup
-// members by the normal synchronization mechanisms.
-//
-// SyncGroupServer also aids in discovering members of a particular SyncGroup.
-// SyncGroupServer maintains the names of joiners for a SyncGroup that are in
-// turn accessible to all members. Each SyncGroup is also associated with a
-// set of mount tables. A Store that joins a SyncGroup must advertise its name
-// (and optionally its SyncGroups) in these mount tables. Stores are expected
-// also optionally to advertise the SyncGroups they join in the local
-// neighbourhood.
-package syncgroup
-
-import "veyron2/security"
-import "veyron2/services/security/access"
-// import "veyron2/services/watch" // Watch call is not yet implemented
-import "veyron2/storage"
-
-// An ID is a globally unique identifier for a SyncGroup.
-type ID storage.ID
-
-// A SyncGroupInfo is the conceptual state of a SyncGroup object.
-type SyncGroupInfo struct {
- Name string // Global Veyron name of object.
- Config SyncGroupConfig // Configuration parameters of this SyncGroup.
- RootOID storage.ID // ID of object at root of SyncGroup's tree.
- ETag string // Version ID for concurrency control.
-
- // The SyncGroup's object ID, which is chosen by the creating SyncGroupServer
- // and is globally unique.
- SGOID ID
-
- // A map from joiner names to the associated metaData for devices that
- // have called Join() or Create() and not subsequently called Leave()
- // or had Eject() called on them. The map returned by the calls below
- // may contain only a subset of joiners if the number is large.
- Joiners map[NameIdentity]JoinerMetaData
-}
-
-// A SyncGroupConfig contains some fields of SyncGroupInfo that
-// are passed at create time, but which can be changed later.
-type SyncGroupConfig struct {
- Desc string // Human readable description.
- PathPatterns []string // Global path patterns.
- Options map[string]any // Options for future evolution.
- ACL security.ACL // The object's ACL.
-
- // Mount tables used to advertise for synchronization.
- // Typically, we will have only one entry. However, an array allows
- // mount tables to be changed over time.
- MountTables []string
-}
-
-// A JoinerMetaData contains the non-name information stored per joiner.
-type JoinerMetaData struct {
- // SyncPriority is a hint to bias the choice of syncing partners.
- // Members of the SyncGroup should choose to synchronize more often
- // with partners with lower values.
- SyncPriority int32
-}
-
-// A NameIdentity gives a Veyron name and identity for a joiner.
-// TODO(m3b): When names include an identity, this should become a single
-// string.
-type NameIdentity struct {
- Name string // Global name of joiner.
- Identity string // Security identity of the joiner.
-}
-
-// SyncGroupServer is the collection of calls on SyncGroup objects at
-// a SyncGroup server. The calls used most often, like Create and Join, are
-// used almost exclusively by the Store. Clients typically call the Store to
-// cause these things to happen.
-//
-// Calls starting with "Set" take an eTag value that may be either empty, or
-// the value of ETag from a recent response to Get(), Watch(), or GetACL().
-type SyncGroupServer interface {
- // Create creates this SyncGroup with the given arguments, and if
- // joiner.Name!="", with {joiner, metaData} in its Joiners map. It is
- // expected that acl will give Read and Write access to any device that
- // the administrator expects to join and sync; if the acl is empty, a
- // default ACL giving access only to the caller is used. On success,
- // Create returns the SyncGroupInfo of the newly created object.
- //
- // Requires: this SyncGroup must not exist;
- // the caller must have write permission at the SyncGroup server;
- // the caller's identity must be a prefix of the SyncGroup's name.
- // Beware that for Create(), the access label is matched against a
- // server-wide ACL; for all other calls the access label is matched
- // against the object's ACL.
- Create(createArgs SyncGroupConfig, rootOID storage.ID,
- joiner NameIdentity, metaData JoinerMetaData) (
- sgInfo SyncGroupInfo, err error) {security.WriteLabel}
-
- // Join adds {joiner, metaData} to the SyncGroup's Joiners map and
- // returns the SyncGroupInfo for this SyncGroup. The act of joining
- // allows other devices to find the caller, which is still required to
- // have read+write access on the SyncGroup to participate in
- // synchronization. A device may call Join again with the same
- // NameIdentity in order to change metaData.
- // For SyncGroups with large numbers of joiners, Join may return
- // a subset of Joiners.
- //
- // Requires: this SyncGroup must exist;
- // the caller must have both read and write permission on the
- // SyncGroup.
- // TODO(m3b): The label should be read and write; can that be expressed?
- Join(joiner NameIdentity, metaData JoinerMetaData) (
- sgInfo SyncGroupInfo, err error) {security.WriteLabel}
-
- // Leave removes the joiner with the given name/identity from the
- // SyncGroup's Joiners map.
- //
- // Requires: this SyncGroup must exist;
- // the caller must assert the identity name.Identity.
- // (Thus, a device that Joined may Leave even if it would no longer
- // have permission to Join() the SyncGroup.)
- Leave(name NameIdentity) error
-
- // Eject is like Leave, but the caller must wield Admin
- // privilege on the group, and need not wield name.Identity.
- //
- // Requires: the SyncGroup must exist;
- // the caller must have admin permission on the SyncGroup.
- Eject(name NameIdentity) error {security.AdminLabel}
-
- // Destroy ejects all devices from the SyncGroup and removes it.
- // Devices that had joined will learn of this when their
- // SyncGroup object disappears.
- //
- // Requires: this SyncGroup must exist;
- // the caller must have admin permission on the SyncGroup.
- Destroy() error {security.AdminLabel}
-
- // Get returns the SyncGroupInfo for this SyncGroup. For SyncGroups
- // with a large number of joiners, Get may return a subset of Joiners.
- //
- // Requires: this SyncGroup must exist;
- // the caller must have read permission on the SyncGroup.
- // TODO(m3b): This call may be removed when Watch is implemented.
- Get() (sgInfo SyncGroupInfo, err error) {security.ReadLabel}
-
- // Watch returns stream of SyncGroupInfo for this SyncGroup as
- // it changes.
- //
- // Requires: this SyncGroup must exist;
- // the caller must have read permission on the SyncGroup.
- // TODO(m3b): This call is not yet implemented.
- // Watch() stream<_, watch.ChangeBatch> error {security.ReadLabel}
-
- // SetConfig sets the Config field of this SyncGroup.
- //
- // Requires: this SyncGroup must exist;
- // if non-empty, the eTag must match the value in the object;
- // the caller must have admin permission on the SyncGroup.
- SetConfig(config SyncGroupConfig, eTag string) error {security.AdminLabel}
-
- // SetACL and GetACL are included from access.Object.
- // TODO(m3b): This inherits AdminLabel for GetACL(); we might prefer ReadLabel.
- access.Object
-}
diff --git a/services/syncgroup/syncgroup.vdl.go b/services/syncgroup/syncgroup.vdl.go
deleted file mode 100644
index 78c38c5..0000000
--- a/services/syncgroup/syncgroup.vdl.go
+++ /dev/null
@@ -1,646 +0,0 @@
-// This file was auto-generated by the veyron vdl tool.
-// Source: syncgroup.vdl
-
-// Package syncgroup provides the means for Stores to set up SyncGroups for
-// subsequent synchronization of objects between them.
-//
-// The intent is that SyncGroup objects are created and administered by
-// SyncGroup servers, even though they are subsequently mirrored among SyncGroup
-// members by the normal synchronization mechanisms.
-//
-// SyncGroupServer also aids in discovering members of a particular SyncGroup.
-// SyncGroupServer maintains the names of joiners for a SyncGroup that are in
-// turn accessible to all members. Each SyncGroup is also associated with a
-// set of mount tables. A Store that joins a SyncGroup must advertise its name
-// (and optionally its SyncGroups) in these mount tables. Stores are expected
-// also optionally to advertise the SyncGroups they join in the local
-// neighbourhood.
-package syncgroup
-
-import (
- "veyron2/security"
-
- "veyron2/services/security/access"
-
- "veyron2/storage"
-
- // The non-user imports are prefixed with "_gen_" to prevent collisions.
- _gen_veyron2 "veyron2"
- _gen_context "veyron2/context"
- _gen_ipc "veyron2/ipc"
- _gen_naming "veyron2/naming"
- _gen_vdlutil "veyron2/vdl/vdlutil"
- _gen_wiretype "veyron2/wiretype"
-)
-
-// An ID is a globally unique identifier for a SyncGroup.
-type ID storage.ID
-
-// A SyncGroupInfo is the conceptual state of a SyncGroup object.
-type SyncGroupInfo struct {
- Name string // Global Veyron name of object.
- Config SyncGroupConfig // Configuration parameters of this SyncGroup.
- RootOID storage.ID // ID of object at root of SyncGroup's tree.
- ETag string // Version ID for concurrency control.
- // The SyncGroup's object ID, which is chosen by the creating SyncGroupServer
- // and is globally unique.
- SGOID ID
- // A map from joiner names to the associated metaData for devices that
- // have called Join() or Create() and not subsequently called Leave()
- // or had Eject() called on them. The map returned by the calls below
- // may contain only a subset of joiners if the number is large.
- Joiners map[NameIdentity]JoinerMetaData
-}
-
-// A SyncGroupConfig contains some fields of SyncGroupInfo that
-// are passed at create time, but which can be changed later.
-type SyncGroupConfig struct {
- Desc string // Human readable description.
- PathPatterns []string // Global path patterns.
- Options map[string]_gen_vdlutil.Any // Options for future evolution.
- ACL security.ACL // The object's ACL.
- // Mount tables used to advertise for synchronization.
- // Typically, we will have only one entry. However, an array allows
- // mount tables to be changed over time.
- MountTables []string
-}
-
-// A JoinerMetaData contains the non-name information stored per joiner.
-type JoinerMetaData struct {
- // SyncPriority is a hint to bias the choice of syncing partners.
- // Members of the SyncGroup should choose to synchronize more often
- // with partners with lower values.
- SyncPriority int32
-}
-
-// A NameIdentity gives a Veyron name and identity for a joiner.
-// TODO(m3b): When names include an identity, this should become a single
-// string.
-type NameIdentity struct {
- Name string // Global name of joiner.
- Identity string // Security identity of the joiner.
-}
-
-// TODO(bprosnitz) Remove this line once signatures are updated to use typevals.
-// It corrects a bug where _gen_wiretype is unused in VDL pacakges where only bootstrap types are used on interfaces.
-const _ = _gen_wiretype.TypeIDInvalid
-
-// SyncGroupServer is the collection of calls on SyncGroup objects at
-// a SyncGroup server. The calls used most often, like Create and Join, are
-// used almost exclusively by the Store. Clients typically call the Store to
-// cause these things to happen.
-//
-// Calls starting with "Set" take an eTag value that may be either empty, or
-// the value of ETag from a recent response to Get(), Watch(), or GetACL().
-// SyncGroupServer is the interface the client binds and uses.
-// SyncGroupServer_ExcludingUniversal is the interface without internal framework-added methods
-// to enable embedding without method collisions. Not to be used directly by clients.
-type SyncGroupServer_ExcludingUniversal interface {
- // Object provides access control for Veyron objects.
- access.Object_ExcludingUniversal
- // Create creates this SyncGroup with the given arguments, and if
- // joiner.Name!="", with {joiner, metaData} in its Joiners map. It is
- // expected that acl will give Read and Write access to any device that
- // the administrator expects to join and sync; if the acl is empty, a
- // default ACL giving access only to the caller is used. On success,
- // Create returns the SyncGroupInfo of the newly created object.
- //
- // Requires: this SyncGroup must not exist;
- // the caller must have write permission at the SyncGroup server;
- // the caller's identity must be a prefix of the SyncGroup's name.
- // Beware that for Create(), the access label is matched against a
- // server-wide ACL; for all other calls the access label is matched
- // against the object's ACL.
- Create(ctx _gen_context.T, createArgs SyncGroupConfig, rootOID storage.ID, joiner NameIdentity, metaData JoinerMetaData, opts ..._gen_ipc.CallOpt) (reply SyncGroupInfo, err error)
- // Join adds {joiner, metaData} to the SyncGroup's Joiners map and
- // returns the SyncGroupInfo for this SyncGroup. The act of joining
- // allows other devices to find the caller, which is still required to
- // have read+write access on the SyncGroup to participate in
- // synchronization. A device may call Join again with the same
- // NameIdentity in order to change metaData.
- // For SyncGroups with large numbers of joiners, Join may return
- // a subset of Joiners.
- //
- // Requires: this SyncGroup must exist;
- // the caller must have both read and write permission on the
- // SyncGroup.
- // TODO(m3b): The label should be read and write; can that be expressed?
- Join(ctx _gen_context.T, joiner NameIdentity, metaData JoinerMetaData, opts ..._gen_ipc.CallOpt) (reply SyncGroupInfo, err error)
- // Leave removes the joiner with the given name/identity from the
- // SyncGroup's Joiners map.
- //
- // Requires: this SyncGroup must exist;
- // the caller must assert the identity name.Identity.
- // (Thus, a device that Joined may Leave even if it would no longer
- // have permission to Join() the SyncGroup.)
- Leave(ctx _gen_context.T, name NameIdentity, opts ..._gen_ipc.CallOpt) (err error)
- // Eject is like Leave, but the caller must wield Admin
- // privilege on the group, and need not wield name.Identity.
- //
- // Requires: the SyncGroup must exist;
- // the caller must have admin permission on the SyncGroup.
- Eject(ctx _gen_context.T, name NameIdentity, opts ..._gen_ipc.CallOpt) (err error)
- // Destroy ejects all devices from the SyncGroup and removes it.
- // Devices that had joined will learn of this when their
- // SyncGroup object disappears.
- //
- // Requires: this SyncGroup must exist;
- // the caller must have admin permission on the SyncGroup.
- Destroy(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (err error)
- // Get returns the SyncGroupInfo for this SyncGroup. For SyncGroups
- // with a large number of joiners, Get may return a subset of Joiners.
- //
- // Requires: this SyncGroup must exist;
- // the caller must have read permission on the SyncGroup.
- // TODO(m3b): This call may be removed when Watch is implemented.
- Get(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply SyncGroupInfo, err error)
- // SetConfig sets the Config field of this SyncGroup.
- //
- // Requires: this SyncGroup must exist;
- // if non-empty, the eTag must match the value in the object;
- // the caller must have admin permission on the SyncGroup.
- SetConfig(ctx _gen_context.T, config SyncGroupConfig, eTag string, opts ..._gen_ipc.CallOpt) (err error)
-}
-type SyncGroupServer interface {
- _gen_ipc.UniversalServiceMethods
- SyncGroupServer_ExcludingUniversal
-}
-
-// SyncGroupServerService is the interface the server implements.
-type SyncGroupServerService interface {
-
- // Object provides access control for Veyron objects.
- access.ObjectService
- // Create creates this SyncGroup with the given arguments, and if
- // joiner.Name!="", with {joiner, metaData} in its Joiners map. It is
- // expected that acl will give Read and Write access to any device that
- // the administrator expects to join and sync; if the acl is empty, a
- // default ACL giving access only to the caller is used. On success,
- // Create returns the SyncGroupInfo of the newly created object.
- //
- // Requires: this SyncGroup must not exist;
- // the caller must have write permission at the SyncGroup server;
- // the caller's identity must be a prefix of the SyncGroup's name.
- // Beware that for Create(), the access label is matched against a
- // server-wide ACL; for all other calls the access label is matched
- // against the object's ACL.
- Create(context _gen_ipc.ServerContext, createArgs SyncGroupConfig, rootOID storage.ID, joiner NameIdentity, metaData JoinerMetaData) (reply SyncGroupInfo, err error)
- // Join adds {joiner, metaData} to the SyncGroup's Joiners map and
- // returns the SyncGroupInfo for this SyncGroup. The act of joining
- // allows other devices to find the caller, which is still required to
- // have read+write access on the SyncGroup to participate in
- // synchronization. A device may call Join again with the same
- // NameIdentity in order to change metaData.
- // For SyncGroups with large numbers of joiners, Join may return
- // a subset of Joiners.
- //
- // Requires: this SyncGroup must exist;
- // the caller must have both read and write permission on the
- // SyncGroup.
- // TODO(m3b): The label should be read and write; can that be expressed?
- Join(context _gen_ipc.ServerContext, joiner NameIdentity, metaData JoinerMetaData) (reply SyncGroupInfo, err error)
- // Leave removes the joiner with the given name/identity from the
- // SyncGroup's Joiners map.
- //
- // Requires: this SyncGroup must exist;
- // the caller must assert the identity name.Identity.
- // (Thus, a device that Joined may Leave even if it would no longer
- // have permission to Join() the SyncGroup.)
- Leave(context _gen_ipc.ServerContext, name NameIdentity) (err error)
- // Eject is like Leave, but the caller must wield Admin
- // privilege on the group, and need not wield name.Identity.
- //
- // Requires: the SyncGroup must exist;
- // the caller must have admin permission on the SyncGroup.
- Eject(context _gen_ipc.ServerContext, name NameIdentity) (err error)
- // Destroy ejects all devices from the SyncGroup and removes it.
- // Devices that had joined will learn of this when their
- // SyncGroup object disappears.
- //
- // Requires: this SyncGroup must exist;
- // the caller must have admin permission on the SyncGroup.
- Destroy(context _gen_ipc.ServerContext) (err error)
- // Get returns the SyncGroupInfo for this SyncGroup. For SyncGroups
- // with a large number of joiners, Get may return a subset of Joiners.
- //
- // Requires: this SyncGroup must exist;
- // the caller must have read permission on the SyncGroup.
- // TODO(m3b): This call may be removed when Watch is implemented.
- Get(context _gen_ipc.ServerContext) (reply SyncGroupInfo, err error)
- // SetConfig sets the Config field of this SyncGroup.
- //
- // Requires: this SyncGroup must exist;
- // if non-empty, the eTag must match the value in the object;
- // the caller must have admin permission on the SyncGroup.
- SetConfig(context _gen_ipc.ServerContext, config SyncGroupConfig, eTag string) (err error)
-}
-
-// BindSyncGroupServer returns the client stub implementing the SyncGroupServer
-// interface.
-//
-// If no _gen_ipc.Client is specified, the default _gen_ipc.Client in the
-// global Runtime is used.
-func BindSyncGroupServer(name string, opts ..._gen_ipc.BindOpt) (SyncGroupServer, error) {
- var client _gen_ipc.Client
- switch len(opts) {
- case 0:
- // Do nothing.
- case 1:
- if clientOpt, ok := opts[0].(_gen_ipc.Client); opts[0] == nil || ok {
- client = clientOpt
- } else {
- return nil, _gen_vdlutil.ErrUnrecognizedOption
- }
- default:
- return nil, _gen_vdlutil.ErrTooManyOptionsToBind
- }
- stub := &clientStubSyncGroupServer{defaultClient: client, name: name}
- stub.Object_ExcludingUniversal, _ = access.BindObject(name, client)
-
- return stub, nil
-}
-
-// NewServerSyncGroupServer creates a new server stub.
-//
-// It takes a regular server implementing the SyncGroupServerService
-// interface, and returns a new server stub.
-func NewServerSyncGroupServer(server SyncGroupServerService) interface{} {
- return &ServerStubSyncGroupServer{
- ServerStubObject: *access.NewServerObject(server).(*access.ServerStubObject),
- service: server,
- }
-}
-
-// clientStubSyncGroupServer implements SyncGroupServer.
-type clientStubSyncGroupServer struct {
- access.Object_ExcludingUniversal
-
- defaultClient _gen_ipc.Client
- name string
-}
-
-func (__gen_c *clientStubSyncGroupServer) client(ctx _gen_context.T) _gen_ipc.Client {
- if __gen_c.defaultClient != nil {
- return __gen_c.defaultClient
- }
- return _gen_veyron2.RuntimeFromContext(ctx).Client()
-}
-
-func (__gen_c *clientStubSyncGroupServer) Create(ctx _gen_context.T, createArgs SyncGroupConfig, rootOID storage.ID, joiner NameIdentity, metaData JoinerMetaData, opts ..._gen_ipc.CallOpt) (reply SyncGroupInfo, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "Create", []interface{}{createArgs, rootOID, joiner, metaData}, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubSyncGroupServer) Join(ctx _gen_context.T, joiner NameIdentity, metaData JoinerMetaData, opts ..._gen_ipc.CallOpt) (reply SyncGroupInfo, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "Join", []interface{}{joiner, metaData}, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubSyncGroupServer) Leave(ctx _gen_context.T, name NameIdentity, opts ..._gen_ipc.CallOpt) (err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "Leave", []interface{}{name}, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubSyncGroupServer) Eject(ctx _gen_context.T, name NameIdentity, opts ..._gen_ipc.CallOpt) (err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "Eject", []interface{}{name}, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubSyncGroupServer) Destroy(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "Destroy", nil, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubSyncGroupServer) Get(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply SyncGroupInfo, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "Get", nil, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubSyncGroupServer) SetConfig(ctx _gen_context.T, config SyncGroupConfig, eTag string, opts ..._gen_ipc.CallOpt) (err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "SetConfig", []interface{}{config, eTag}, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubSyncGroupServer) UnresolveStep(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply []string, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "UnresolveStep", nil, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubSyncGroupServer) Signature(ctx _gen_context.T, opts ..._gen_ipc.CallOpt) (reply _gen_ipc.ServiceSignature, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "Signature", nil, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-func (__gen_c *clientStubSyncGroupServer) GetMethodTags(ctx _gen_context.T, method string, opts ..._gen_ipc.CallOpt) (reply []interface{}, err error) {
- var call _gen_ipc.Call
- if call, err = __gen_c.client(ctx).StartCall(ctx, __gen_c.name, "GetMethodTags", []interface{}{method}, opts...); err != nil {
- return
- }
- if ierr := call.Finish(&reply, &err); ierr != nil {
- err = ierr
- }
- return
-}
-
-// ServerStubSyncGroupServer wraps a server that implements
-// SyncGroupServerService and provides an object that satisfies
-// the requirements of veyron2/ipc.ReflectInvoker.
-type ServerStubSyncGroupServer struct {
- access.ServerStubObject
-
- service SyncGroupServerService
-}
-
-func (__gen_s *ServerStubSyncGroupServer) GetMethodTags(call _gen_ipc.ServerCall, method string) ([]interface{}, error) {
- // TODO(bprosnitz) GetMethodTags() will be replaces with Signature().
- // Note: This exhibits some weird behavior like returning a nil error if the method isn't found.
- // This will change when it is replaced with Signature().
- if resp, err := __gen_s.ServerStubObject.GetMethodTags(call, method); resp != nil || err != nil {
- return resp, err
- }
- switch method {
- case "Create":
- return []interface{}{security.Label(4)}, nil
- case "Join":
- return []interface{}{security.Label(4)}, nil
- case "Leave":
- return []interface{}{}, nil
- case "Eject":
- return []interface{}{security.Label(8)}, nil
- case "Destroy":
- return []interface{}{security.Label(8)}, nil
- case "Get":
- return []interface{}{security.Label(2)}, nil
- case "SetConfig":
- return []interface{}{security.Label(8)}, nil
- default:
- return nil, nil
- }
-}
-
-func (__gen_s *ServerStubSyncGroupServer) Signature(call _gen_ipc.ServerCall) (_gen_ipc.ServiceSignature, error) {
- result := _gen_ipc.ServiceSignature{Methods: make(map[string]_gen_ipc.MethodSignature)}
- result.Methods["Create"] = _gen_ipc.MethodSignature{
- InArgs: []_gen_ipc.MethodArgument{
- {Name: "createArgs", Type: 72},
- {Name: "rootOID", Type: 74},
- {Name: "joiner", Type: 75},
- {Name: "metaData", Type: 76},
- },
- OutArgs: []_gen_ipc.MethodArgument{
- {Name: "sgInfo", Type: 79},
- {Name: "err", Type: 80},
- },
- }
- result.Methods["Destroy"] = _gen_ipc.MethodSignature{
- InArgs: []_gen_ipc.MethodArgument{},
- OutArgs: []_gen_ipc.MethodArgument{
- {Name: "", Type: 80},
- },
- }
- result.Methods["Eject"] = _gen_ipc.MethodSignature{
- InArgs: []_gen_ipc.MethodArgument{
- {Name: "name", Type: 75},
- },
- OutArgs: []_gen_ipc.MethodArgument{
- {Name: "", Type: 80},
- },
- }
- result.Methods["Get"] = _gen_ipc.MethodSignature{
- InArgs: []_gen_ipc.MethodArgument{},
- OutArgs: []_gen_ipc.MethodArgument{
- {Name: "sgInfo", Type: 79},
- {Name: "err", Type: 80},
- },
- }
- result.Methods["Join"] = _gen_ipc.MethodSignature{
- InArgs: []_gen_ipc.MethodArgument{
- {Name: "joiner", Type: 75},
- {Name: "metaData", Type: 76},
- },
- OutArgs: []_gen_ipc.MethodArgument{
- {Name: "sgInfo", Type: 79},
- {Name: "err", Type: 80},
- },
- }
- result.Methods["Leave"] = _gen_ipc.MethodSignature{
- InArgs: []_gen_ipc.MethodArgument{
- {Name: "name", Type: 75},
- },
- OutArgs: []_gen_ipc.MethodArgument{
- {Name: "", Type: 80},
- },
- }
- result.Methods["SetConfig"] = _gen_ipc.MethodSignature{
- InArgs: []_gen_ipc.MethodArgument{
- {Name: "config", Type: 72},
- {Name: "eTag", Type: 3},
- },
- OutArgs: []_gen_ipc.MethodArgument{
- {Name: "", Type: 80},
- },
- }
-
- result.TypeDefs = []_gen_vdlutil.Any{
- _gen_wiretype.NamedPrimitiveType{Type: 0x1, Name: "anydata", Tags: []string(nil)}, _gen_wiretype.MapType{Key: 0x3, Elem: 0x41, Name: "", Tags: []string(nil)}, _gen_wiretype.NamedPrimitiveType{Type: 0x3, Name: "veyron2/security.BlessingPattern", Tags: []string(nil)}, _gen_wiretype.NamedPrimitiveType{Type: 0x34, Name: "veyron2/security.LabelSet", Tags: []string(nil)}, _gen_wiretype.MapType{Key: 0x43, Elem: 0x44, Name: "", Tags: []string(nil)}, _gen_wiretype.MapType{Key: 0x3, Elem: 0x44, Name: "", Tags: []string(nil)}, _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x45, Name: "In"},
- _gen_wiretype.FieldType{Type: 0x46, Name: "NotIn"},
- },
- "veyron2/security.ACL", []string(nil)},
- _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x3, Name: "Desc"},
- _gen_wiretype.FieldType{Type: 0x3d, Name: "PathPatterns"},
- _gen_wiretype.FieldType{Type: 0x42, Name: "Options"},
- _gen_wiretype.FieldType{Type: 0x47, Name: "ACL"},
- _gen_wiretype.FieldType{Type: 0x3d, Name: "MountTables"},
- },
- "veyron/services/syncgroup.SyncGroupConfig", []string(nil)},
- _gen_wiretype.NamedPrimitiveType{Type: 0x32, Name: "byte", Tags: []string(nil)}, _gen_wiretype.ArrayType{Elem: 0x49, Len: 0x10, Name: "veyron2/storage.ID", Tags: []string(nil)}, _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x3, Name: "Name"},
- _gen_wiretype.FieldType{Type: 0x3, Name: "Identity"},
- },
- "veyron/services/syncgroup.NameIdentity", []string(nil)},
- _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x24, Name: "SyncPriority"},
- },
- "veyron/services/syncgroup.JoinerMetaData", []string(nil)},
- _gen_wiretype.ArrayType{Elem: 0x49, Len: 0x10, Name: "veyron/services/syncgroup.ID", Tags: []string(nil)}, _gen_wiretype.MapType{Key: 0x4b, Elem: 0x4c, Name: "", Tags: []string(nil)}, _gen_wiretype.StructType{
- []_gen_wiretype.FieldType{
- _gen_wiretype.FieldType{Type: 0x3, Name: "Name"},
- _gen_wiretype.FieldType{Type: 0x48, Name: "Config"},
- _gen_wiretype.FieldType{Type: 0x4a, Name: "RootOID"},
- _gen_wiretype.FieldType{Type: 0x3, Name: "ETag"},
- _gen_wiretype.FieldType{Type: 0x4d, Name: "SGOID"},
- _gen_wiretype.FieldType{Type: 0x4e, Name: "Joiners"},
- },
- "veyron/services/syncgroup.SyncGroupInfo", []string(nil)},
- _gen_wiretype.NamedPrimitiveType{Type: 0x1, Name: "error", Tags: []string(nil)}}
- var ss _gen_ipc.ServiceSignature
- var firstAdded int
- ss, _ = __gen_s.ServerStubObject.Signature(call)
- firstAdded = len(result.TypeDefs)
- for k, v := range ss.Methods {
- for i, _ := range v.InArgs {
- if v.InArgs[i].Type >= _gen_wiretype.TypeIDFirst {
- v.InArgs[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- for i, _ := range v.OutArgs {
- if v.OutArgs[i].Type >= _gen_wiretype.TypeIDFirst {
- v.OutArgs[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- if v.InStream >= _gen_wiretype.TypeIDFirst {
- v.InStream += _gen_wiretype.TypeID(firstAdded)
- }
- if v.OutStream >= _gen_wiretype.TypeIDFirst {
- v.OutStream += _gen_wiretype.TypeID(firstAdded)
- }
- result.Methods[k] = v
- }
- //TODO(bprosnitz) combine type definitions from embeded interfaces in a way that doesn't cause duplication.
- for _, d := range ss.TypeDefs {
- switch wt := d.(type) {
- case _gen_wiretype.SliceType:
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.ArrayType:
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.MapType:
- if wt.Key >= _gen_wiretype.TypeIDFirst {
- wt.Key += _gen_wiretype.TypeID(firstAdded)
- }
- if wt.Elem >= _gen_wiretype.TypeIDFirst {
- wt.Elem += _gen_wiretype.TypeID(firstAdded)
- }
- d = wt
- case _gen_wiretype.StructType:
- for i, fld := range wt.Fields {
- if fld.Type >= _gen_wiretype.TypeIDFirst {
- wt.Fields[i].Type += _gen_wiretype.TypeID(firstAdded)
- }
- }
- d = wt
- // NOTE: other types are missing, but we are upgrading anyways.
- }
- result.TypeDefs = append(result.TypeDefs, d)
- }
-
- return result, nil
-}
-
-func (__gen_s *ServerStubSyncGroupServer) UnresolveStep(call _gen_ipc.ServerCall) (reply []string, err error) {
- if unresolver, ok := __gen_s.service.(_gen_ipc.Unresolver); ok {
- return unresolver.UnresolveStep(call)
- }
- if call.Server() == nil {
- return
- }
- var published []string
- if published, err = call.Server().Published(); err != nil || published == nil {
- return
- }
- reply = make([]string, len(published))
- for i, p := range published {
- reply[i] = _gen_naming.Join(p, call.Name())
- }
- return
-}
-
-func (__gen_s *ServerStubSyncGroupServer) Create(call _gen_ipc.ServerCall, createArgs SyncGroupConfig, rootOID storage.ID, joiner NameIdentity, metaData JoinerMetaData) (reply SyncGroupInfo, err error) {
- reply, err = __gen_s.service.Create(call, createArgs, rootOID, joiner, metaData)
- return
-}
-
-func (__gen_s *ServerStubSyncGroupServer) Join(call _gen_ipc.ServerCall, joiner NameIdentity, metaData JoinerMetaData) (reply SyncGroupInfo, err error) {
- reply, err = __gen_s.service.Join(call, joiner, metaData)
- return
-}
-
-func (__gen_s *ServerStubSyncGroupServer) Leave(call _gen_ipc.ServerCall, name NameIdentity) (err error) {
- err = __gen_s.service.Leave(call, name)
- return
-}
-
-func (__gen_s *ServerStubSyncGroupServer) Eject(call _gen_ipc.ServerCall, name NameIdentity) (err error) {
- err = __gen_s.service.Eject(call, name)
- return
-}
-
-func (__gen_s *ServerStubSyncGroupServer) Destroy(call _gen_ipc.ServerCall) (err error) {
- err = __gen_s.service.Destroy(call)
- return
-}
-
-func (__gen_s *ServerStubSyncGroupServer) Get(call _gen_ipc.ServerCall) (reply SyncGroupInfo, err error) {
- reply, err = __gen_s.service.Get(call)
- return
-}
-
-func (__gen_s *ServerStubSyncGroupServer) SetConfig(call _gen_ipc.ServerCall, config SyncGroupConfig, eTag string) (err error) {
- err = __gen_s.service.SetConfig(call, config, eTag)
- return
-}
diff --git a/services/wsprd/lib/remove_this.go b/services/wsprd/lib/remove_this.go
index 32be65d..cd02607 100644
--- a/services/wsprd/lib/remove_this.go
+++ b/services/wsprd/lib/remove_this.go
@@ -1,11 +1,11 @@
package lib
import (
+ "veyron.io/store/veyron2/services/store"
+ "veyron.io/store/veyron2/storage"
rps "veyron/examples/rockpaperscissors"
mttypes "veyron2/services/mounttable/types"
- "veyron2/services/store"
watchtypes "veyron2/services/watch/types"
- "veyron2/storage"
"veyron2/vom"
)
diff --git a/tools/qsh/impl/impl.go b/tools/qsh/impl/impl.go
deleted file mode 100644
index d349a92..0000000
--- a/tools/qsh/impl/impl.go
+++ /dev/null
@@ -1,73 +0,0 @@
-package impl
-
-import (
- "fmt"
- "io"
- "os"
- "sort"
-
- "veyron2/context"
- "veyron2/query"
- "veyron2/storage"
-
- // TODO(rjkroege@google.com): Replace with the appropriate vom2 functionality
- // when available.
- _ "veyron/services/store/typeregistryhack"
-)
-
-func indenter(w io.Writer, indent int) {
- for i := 0; i < indent; i++ {
- fmt.Fprintf(w, "\t")
- }
-}
-
-// Prints a single QueryResult to the provided io.Writer.
-func printResult(qr storage.QueryResult, w io.Writer, indent int) {
- // TODO(rjkroege@google.com): Consider permitting the user to provide a Go
- // template to format output.
- if v := qr.Value(); v != nil {
- indenter(w, indent)
- fmt.Fprintf(w, "%s: %#v\n", qr.Name(), v)
- } else {
- // Force fields to be consistently ordered.
- fields := qr.Fields()
- names := make([]string, 0, len(fields))
- for k, _ := range fields {
- names = append(names, k)
- }
- sort.Strings(names)
-
- indenter(w, indent)
- fmt.Fprintf(w, "%s: map[string]interface {}{\n", qr.Name())
- for _, k := range names {
- f := fields[k]
- switch v := f.(type) {
- case storage.QueryStream:
- indenter(w, indent+1)
- fmt.Fprintf(w, "%s: {\n", k)
- printStream(v, w, indent+2)
- indenter(w, indent+1)
- fmt.Fprintf(w, "},\n")
- default:
- indenter(w, indent+1)
- fmt.Fprintf(w, "\"%s\":%#v,\n", k, v)
- }
- }
- indenter(w, indent)
- fmt.Fprintf(w, "},\n")
- }
-}
-
-func printStream(qs storage.QueryStream, w io.Writer, indent int) error {
- for qs.Advance() {
- printResult(qs.Value(), w, indent)
- }
- if err := qs.Err(); err != nil {
- return err
- }
- return nil
-}
-
-func RunQuery(ctx context.T, queryRoot storage.Dir, queryString string) error {
- return printStream(queryRoot.Query(ctx, query.Query{queryString}), os.Stdout, 0)
-}
diff --git a/tools/qsh/impl/impl_test.go b/tools/qsh/impl/impl_test.go
deleted file mode 100644
index 8419b95..0000000
--- a/tools/qsh/impl/impl_test.go
+++ /dev/null
@@ -1,132 +0,0 @@
-package impl
-
-import (
- "bytes"
- "testing"
-
- "veyron2/storage"
-)
-
-type mockQueryResult struct {
- name string
- value interface{}
- fields map[string]interface{}
-}
-
-func (mqr mockQueryResult) Name() string {
- return mqr.name
-}
-
-func (mqr mockQueryResult) Value() interface{} {
- return mqr.value
-}
-
-func (mqr mockQueryResult) Fields() map[string]interface{} {
- return mqr.fields
-}
-
-type mockQueryStream struct {
- index int
- results []mockQueryResult
- error error
-}
-
-func (mqs *mockQueryStream) Advance() bool {
- if mqs.error != nil {
- return false
- }
- // Initialize index to -1
- mqs.index++
- if mqs.index >= len(mqs.results) {
- return false
- }
- return true
-}
-
-func (mqs *mockQueryStream) Value() storage.QueryResult {
- return mqs.results[mqs.index]
-}
-
-func (mqs *mockQueryStream) Err() error {
- return mqs.error
-}
-
-func (mqs *mockQueryStream) Cancel() {
- mqs.index = len(mqs.results) + 1
-}
-
-type testCase struct {
- result mockQueryResult
- expectedOutput string
-}
-
-const (
- result3Out = `result3: map[string]interface {}{
- qs: {
- resultNested1: 10
- resultNested2: 11
- },
-},
-`
-)
-
-func TestPrintResult(t *testing.T) {
- tests := []testCase{
- {
- result: mockQueryResult{
- name: "result1",
- value: 10,
- fields: nil,
- },
- expectedOutput: "result1: 10\n",
- },
-
- {
- result: mockQueryResult{
- name: "result2",
- value: nil,
- fields: map[string]interface{}{"a": 1, "b": 2},
- },
- expectedOutput: `result2: map[string]interface {}{
- "a":1,
- "b":2,
-},
-`,
- },
-
- {
- result: mockQueryResult{
- name: "result3",
- value: nil,
- fields: map[string]interface{}{
- "qs": storage.QueryStream(&mockQueryStream{
- index: -1,
- error: nil,
- results: []mockQueryResult{
- mockQueryResult{
- name: "resultNested1",
- value: 10,
- fields: nil,
- },
- mockQueryResult{
- name: "resultNested2",
- value: 11,
- fields: nil,
- },
- },
- }),
- },
- },
- expectedOutput: result3Out,
- },
- }
-
- for _, d := range tests {
- var b bytes.Buffer
- printResult(d.result, &b, 0)
-
- if got, want := b.String(), d.expectedOutput; got != want {
- t.Errorf("got <%s>, want <%s>", got, want)
- }
- }
-}
diff --git a/tools/qsh/main.go b/tools/qsh/main.go
deleted file mode 100644
index 2588b5e..0000000
--- a/tools/qsh/main.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package main
-
-import (
- "flag"
- "log"
-
- "veyron/tools/qsh/impl"
-
- "veyron2/rt"
- "veyron2/storage/vstore"
-)
-
-var flagQueryRoot = flag.String("queryroot", "",
- "An object name in the store to serve as the root of the query.")
-
-const usage = `
-Synopsis: qsh --queryroot=<object in the store> query...
-
-Runs a given query starting at the given root.
-`
-
-func main() {
- r := rt.Init()
-
- // TODO(rjkroege@google.com): Handle ^c nicely.
- flag.Parse()
- queryStringArgs := flag.Args()
- if len(queryStringArgs) != 1 {
- log.Fatalf("qsh: Expected only one query arg\n" + usage)
- }
-
- queryRoot := *flagQueryRoot
- if queryRoot == "" {
- log.Fatalf("qsh: No queryroot specified\n" + usage)
- }
-
- err := impl.RunQuery(r.NewContext(), vstore.BindDir(queryRoot), queryStringArgs[0])
- if err != nil {
- log.Printf("qsh: When attempting query: \"%s\" experienced an error: ", queryStringArgs[0], err.Error())
- }
-}
