syncbase/sync: remove obsolete sync code.
Remove the obsolete sync code from Syncbase.
Change-Id: I1d7303db725c36bd5dd8488fe0a3bc537ef6e2ae
diff --git a/services/syncbase/sync/dag.go b/services/syncbase/sync/dag.go
deleted file mode 100644
index 72d78cf..0000000
--- a/services/syncbase/sync/dag.go
+++ /dev/null
@@ -1,1183 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Veyron Sync DAG (directed acyclic graph) utility functions.
-// The DAG is used to track the version history of objects in order to
-// detect and resolve conflicts (concurrent changes on different devices).
-//
-// Terminology:
-// * An object is a unique value in the Veyron Store represented by its UID.
-// * As an object mutates, its version number is updated by the Store.
-// * Each (object, version) tuple is represented by a node in the Sync DAG.
-// * The previous version of an object is its parent in the DAG, i.e. the
-// new version is derived from that parent.
-// * When there are no conflicts, the node has a single reference back to
-// a parent node.
-// * When a conflict between two concurrent object versions is resolved,
-// the new version has references back to each of the two parents to
-// indicate that it is derived from both nodes.
-// * During a sync operation from a source device to a target device, the
-// target receives a DAG fragment from the source. That fragment has to
-// be incorporated (grafted) into the target device's DAG. It may be a
-// continuation of the DAG of an object, with the attachment (graft) point
-// being the current head of DAG, in which case there are no conflicts.
-// Or the graft point(s) may be older nodes, which means the new fragment
-// is a divergence in the graph causing a conflict that must be resolved
-// in order to re-converge the two DAG fragments.
-//
-// In the diagrams below:
-// (h) represents the head node in the local device.
-// (nh) represents the new head node received from the remote device.
-// (g) represents a graft node, where new nodes attach to the existing DAG.
-// <- represents a derived-from mutation, i.e. a child-to-parent pointer
-//
-// a- No-conflict example: the new nodes (v3, v4) attach to the head node (v2).
-// In this case the new head becomes the head node, the new DAG fragment
-// being a continuation of the existing DAG.
-//
-// Before:
-// v0 <- v1 <- v2(h)
-//
-// Sync updates applied, no conflict detected:
-// v0 <- v1 <- v2(h,g) <- v3 <- v4 (nh)
-//
-// After:
-// v0 <- v1 <- v2 <- v3 <- v4 (h)
-//
-// b- Conflict example: the new nodes (v3, v4) attach to an old node (v1).
-// The current head node (v2) and the new head node (v4) are divergent
-// (concurrent) mutations that need to be resolved. The conflict
-// resolution function is passed the old head (v2), new head (v4), and
-// the common ancestor (v1) and resolves the conflict with (v5) which
-// is represented in the DAG as derived from both v2 and v4 (2 parents).
-//
-// Before:
-// v0 <- v1 <- v2(h)
-//
-// Sync updates applied, conflict detected (v2 not a graft node):
-// v0 <- v1(g) <- v2(h)
-// <- v3 <- v4 (nh)
-//
-// After, conflict resolver creates v5 having 2 parents (v2, v4):
-// v0 <- v1(g) <- v2 <------- v5(h)
-// <- v3 <- v4 <-
-//
-// Note: the DAG does not grow indefinitely. During a sync operation each
-// device learns what the other device already knows -- where it's at in
-// the version history for the objects. When a device determines that all
-// devices that sync an object (as per the definitions of replication groups
-// in the Veyron Store) have moved past some version for that object, the
-// DAG for that object can be pruned, deleting all prior (ancestor) nodes.
-//
-// The DAG DB contains four tables persisted to disk (nodes, heads, trans,
-// priv) and three in-memory (ephemeral) maps (graft, txSet, txGC):
-// * nodes: one entry per (object, version) with references to the
-// parent node(s) it is derived from, a reference to the
-// log record identifying that change, a reference to its
-// transaction set (or NoTxId if none), and a boolean to
-// indicate whether this change was a deletion of the object.
-// * heads: one entry per object pointing to its most recent version
-// in the nodes table
-// * trans: one entry per transaction ID containing the set of objects
-// that forms the transaction and their versions.
-// * priv: one entry per object ID for objects that are private to the
-// store, not shared through any SyncGroup.
-// * graft: during a sync operation, it tracks the nodes where the new
-// DAG fragments are attached to the existing graph for each
-// mutated object. This map is used to determine whether a
-// conflict happened for an object and, if yes, select the most
-// recent common ancestor from these graft points to use when
-// resolving the conflict. At the end of a sync operation the
-// graft map is destroyed.
-// * txSet: used to incrementally construct the transaction sets that
-// are stored in the "trans" table once all the nodes of a
-// transaction have been added. Multiple transaction sets
-// can be constructed to support the concurrency between the
-// Sync Initiator and Watcher threads.
-// * txGC: used to track the transactions impacted by objects being
-// pruned. At the end of the pruning operation the records
-// of the "trans" table are updated from the txGC information.
-//
-// Note: for regular (no-conflict) changes, a node has a reference to
-// one parent from which it was derived. When a conflict is resolved,
-// the new node has references to the two concurrent parents that triggered
-// the conflict. The states of the parents[] array are:
-// * [] The earliest/first version of an object
-// * [XYZ] Regular non-conflict version derived from XYZ
-// * [XYZ, ABC] Resolution version caused by XYZ-vs-ABC conflict
-
-import (
- "container/list"
- "errors"
- "fmt"
- "math/rand"
- "strconv"
- "time"
-
- "v.io/x/lib/vlog"
- "v.io/x/ref/lib/stats"
-)
-
-const (
- NoTxId = TxId(0)
-)
-
-var (
- errBadDAG = errors.New("invalid DAG")
-)
-
-type dagTxMap map[ObjId]Version
-
-// dagTxState tracks the state of a transaction.
-type dagTxState struct {
- TxMap dagTxMap
- TxCount uint32
-}
-
-type dag struct {
- fname string // file pathname
- store *kvdb // underlying K/V store
- heads *kvtable // pointer to "heads" table in the store
- nodes *kvtable // pointer to "nodes" table in the store
- trans *kvtable // pointer to "trans" table in the store
- priv *kvtable // pointer to "priv" table in the store
- graft map[ObjId]*graftInfo // in-memory state of DAG object grafting
- txSet map[TxId]*dagTxState // in-memory construction of transaction sets
- txGC map[TxId]dagTxMap // in-memory tracking of transaction sets to cleanup
- txGen *rand.Rand // transaction ID random number generator
-
- // DAG stats
- numObj *stats.Integer // number of objects
- numNode *stats.Integer // number of versions across all objects
- numTx *stats.Integer // number of transactions tracked
- numPriv *stats.Integer // number of private objects
-}
-
-type dagNode struct {
- Level uint64 // node distance from root
- Parents []Version // references to parent versions
- Logrec string // reference to log record change
- TxId TxId // ID of a transaction set
- Deleted bool // true if the change was a delete
-}
-
-type graftInfo struct {
- newNodes map[Version]struct{} // set of newly added nodes during a sync
- graftNodes map[Version]uint64 // set of graft nodes and their level
- newHeads map[Version]struct{} // set of candidate new head nodes
-}
-
-type privNode struct {
- //Mutation *raw.Mutation // most recent store mutation for a private (unshared) object
- PathIDs []ObjId // store IDs in the path from the object to the root of the store
- SyncTime int64 // SyncTime is the timestamp of the mutation when it arrives at the Sync server.
- TxId TxId // ID of the transaction in which this mutation was done
- TxCount uint32 // total number of object mutations in that transaction
-}
-
-// openDAG opens or creates a DAG for the given filename.
-func openDAG(filename string) (*dag, error) {
- // Open the file and create it if it does not exist.
- // Also initialize the store and its tables.
- db, tbls, err := kvdbOpen(filename, []string{"heads", "nodes", "trans", "priv"})
- if err != nil {
- return nil, err
- }
-
- d := &dag{
- fname: filename,
- store: db,
- heads: tbls[0],
- nodes: tbls[1],
- trans: tbls[2],
- priv: tbls[3],
- txGen: rand.New(rand.NewSource(time.Now().UTC().UnixNano())),
- txSet: make(map[TxId]*dagTxState),
- numObj: stats.NewInteger(statsNumDagObj),
- numNode: stats.NewInteger(statsNumDagNode),
- numTx: stats.NewInteger(statsNumDagTx),
- numPriv: stats.NewInteger(statsNumDagPrivNode),
- }
-
- // Initialize the stats counters from the tables.
- d.numObj.Set(int64(d.heads.getNumKeys()))
- d.numNode.Set(int64(d.nodes.getNumKeys()))
- d.numTx.Set(int64(d.trans.getNumKeys()))
- d.numPriv.Set(int64(d.priv.getNumKeys()))
-
- d.clearGraft()
- d.clearTxGC()
-
- return d, nil
-}
-
-// close closes the DAG and invalidates its structure.
-func (d *dag) close() {
- if d.store != nil {
- d.store.close() // this also closes the tables
- stats.Delete(statsNumDagObj)
- stats.Delete(statsNumDagNode)
- stats.Delete(statsNumDagTx)
- stats.Delete(statsNumDagPrivNode)
- }
- *d = dag{} // zero out the DAG struct
-}
-
-// flush flushes the DAG store to disk.
-func (d *dag) flush() {
- if d.store != nil {
- d.store.flush()
- }
-}
-
-// clearGraft clears the temporary in-memory grafting maps.
-func (d *dag) clearGraft() {
- if d.store != nil {
- d.graft = make(map[ObjId]*graftInfo)
- }
-}
-
-// clearTxGC clears the temporary in-memory transaction garbage collection maps.
-func (d *dag) clearTxGC() {
- if d.store != nil {
- d.txGC = make(map[TxId]dagTxMap)
- }
-}
-
-// getObjectGraft returns the graft structure for an object ID.
-// The graftInfo struct for an object is ephemeral (in-memory) and it
-// tracks the following information:
-// - newNodes: the set of newly added nodes used to detect the type of
-// edges between nodes (new-node to old-node or vice versa).
-// - newHeads: the set of new candidate head nodes used to detect conflicts.
-// - graftNodes: the set of nodes used to find common ancestors between
-// conflicting nodes.
-//
-// After the received Sync logs are applied, if there are two new heads in
-// the newHeads set, there is a conflict to be resolved for this object.
-// Otherwise if there is only one head, no conflict was triggered and the
-// new head becomes the current version for the object.
-//
-// In case of conflict, the graftNodes set is used to select the common
-// ancestor to pass to the conflict resolver.
-//
-// Note: if an object's graft structure does not exist only create it
-// if the "create" parameter is set to true.
-func (d *dag) getObjectGraft(oid ObjId, create bool) *graftInfo {
- graft := d.graft[oid]
- if graft == nil && create {
- graft = &graftInfo{
- newNodes: make(map[Version]struct{}),
- graftNodes: make(map[Version]uint64),
- newHeads: make(map[Version]struct{}),
- }
-
- // If a current head node exists for this object, initialize
- // the set of candidate new heads to include it.
- head, err := d.getHead(oid)
- if err == nil {
- graft.newHeads[head] = struct{}{}
- }
-
- d.graft[oid] = graft
- }
- return graft
-}
-
-// addNodeTxStart generates a transaction ID and returns it to the
-// caller if a TxId is not specified. This transaction ID is stored
-// as part of each log record. If a TxId is specified by the caller,
-// state corresponding to that TxId is instantiated. TxId is used to
-// track DAG nodes that are part of the same transaction.
-func (d *dag) addNodeTxStart(tid TxId) TxId {
- if d.store == nil {
- return NoTxId
- }
-
- // Check if "tid" already exists.
- if tid != NoTxId {
- if _, ok := d.txSet[tid]; ok {
- return tid
- }
- txSt, err := d.getTransaction(tid)
- if err == nil {
- d.txSet[tid] = txSt
- return tid
- }
- } else {
- // Generate a random 64-bit transaction ID different than NoTxId.
- // Also make sure the ID is not already being used.
- for (tid == NoTxId) || (d.txSet[tid] != nil) {
- // Generate an unsigned 64-bit random value by combining a
- // random 63-bit value and a random 1-bit value.
- tid = (TxId(d.txGen.Int63()) << 1) | TxId(d.txGen.Int63n(2))
- }
- }
-
- // Initialize the in-memory object/version map for that transaction ID.
- d.txSet[tid] = &dagTxState{TxMap: make(dagTxMap), TxCount: 0}
- return tid
-}
-
-// addNodeTxEnd marks the end of a given transaction.
-// The DAG completes its internal tracking of the transaction information.
-func (d *dag) addNodeTxEnd(tid TxId, count uint32) error {
- if d.store == nil {
- return errBadDAG
- }
- if tid == NoTxId || count == 0 {
- return fmt.Errorf("invalid TxState: %v %v", tid, count)
- }
-
- txSt, ok := d.txSet[tid]
- if !ok {
- return fmt.Errorf("unknown transaction ID: %v", tid)
- }
-
- // The first time a transaction (TxId) is ended, TxCount is
- // zero while "count" is not. Subsequently if this TxId is
- // started and ended, TxCount should be the same as the
- // incoming "count".
- if txSt.TxCount != 0 && txSt.TxCount != count {
- return fmt.Errorf("incorrect counts for transaction: %v (%v %v)", tid, txSt.TxCount, count)
- }
-
- // Only save non-empty transactions, i.e. those that have at least
- // one mutation on a shared (non-private) object.
- if len(txSt.TxMap) > 0 {
- txSt.TxCount = count
- if err := d.setTransaction(tid, txSt); err != nil {
- return err
- }
- }
-
- delete(d.txSet, tid)
- return nil
-}
-
-// addNode adds a new node for an object in the DAG, linking it to its parent nodes.
-// It verifies that this node does not exist and that its parent nodes are valid.
-// It also determines the DAG level of the node from its parent nodes (max() + 1).
-//
-// If the node is due to a local change (from the Watcher API), no need to
-// update the grafting structure. Otherwise the node is due to a remote change
-// (from the Sync protocol) being grafted on the DAG:
-// - If a parent node is not new, mark it as a DAG graft point.
-// - Mark this version as a new node.
-// - Update the new head node pointer of the grafted DAG.
-//
-// If the transaction ID is set to NoTxId, this node is not part of a transaction.
-// Otherwise, track its membership in the given transaction ID.
-func (d *dag) addNode(oid ObjId, version Version, remote, deleted bool,
- parents []Version, logrec string, tid TxId) error {
- if d.store == nil {
- return errBadDAG
- }
-
- if parents != nil {
- if len(parents) > 2 {
- return fmt.Errorf("cannot have more than 2 parents, not %d", len(parents))
- }
- if len(parents) == 0 {
- // Replace an empty array with a nil.
- parents = nil
- }
- }
-
- // The new node must not exist.
- if d.hasNode(oid, version) {
- return fmt.Errorf("node %v:%d already exists in the DAG", oid, version)
- }
-
- // A new root node (no parents) is allowed only for new objects.
- if parents == nil {
- _, err := d.getHead(oid)
- if err == nil {
- return fmt.Errorf("cannot add another root node %v:%d for this object in the DAG", oid, version)
- }
- }
-
- // For a remote change, make sure the object has a graft info entry.
- // During a sync operation, each mutated object gets new nodes added
- // in its DAG. These new nodes are either derived from nodes that
- // were previously known on this device (i.e. their parent nodes are
- // pre-existing), or they are derived from other new DAG nodes being
- // discovered during this sync (i.e. their parent nodes were also
- // just added to the DAG).
- //
- // To detect a conflict and find the most recent common ancestor to
- // pass to the conflict resolver callback, the DAG keeps track of the
- // new nodes that have old parent nodes. These old-to-new edges are
- // the points where new DAG fragments are attached (grafted) onto the
- // existing DAG. The old nodes are the "graft nodes" and they form
- // the set of possible common ancestors to use in case of conflict:
- // 1- A conflict happens when the current "head node" for an object
- // is not in the set of graft nodes. It means the object mutations
- // were not derived from what the device knows, but where divergent
- // changes from a prior point (from one of the graft nodes).
- // 2- The most recent common ancestor to use in resolving the conflict
- // is the object graft node with the deepest level (furthest from
- // the origin root node), representing the most up-to-date common
- // knowledge between this device and the divergent changes.
- //
- // Note: at the end of a sync operation between 2 devices, the whole
- // graft info is cleared (Syncd calls clearGraft()) to prepare it for
- // the new pairwise sync operation.
- graft := d.getObjectGraft(oid, remote)
-
- // Verify the parents and determine the node level.
- // Update the graft info in the DAG for this object.
- var level uint64
- for _, parent := range parents {
- node, err := d.getNode(oid, parent)
- if err != nil {
- return err
- }
- if level <= node.Level {
- level = node.Level + 1
- }
- if remote {
- // If this parent is an old node, it's a graft point in the DAG
- // and may be a common ancestor used during conflict resolution.
- if _, ok := graft.newNodes[parent]; !ok {
- graft.graftNodes[parent] = node.Level
- }
-
- // The parent nodes can no longer be candidates for new head versions.
- if _, ok := graft.newHeads[parent]; ok {
- delete(graft.newHeads, parent)
- }
- }
- }
-
- if remote {
- // This new node is a candidate for new head version.
- graft.newNodes[version] = struct{}{}
- graft.newHeads[version] = struct{}{}
- }
-
- // If this node is part of a transaction, add it to that set.
- if tid != NoTxId {
- txSt, ok := d.txSet[tid]
- if !ok {
- return fmt.Errorf("unknown transaction ID: %v", tid)
- }
-
- txSt.TxMap[oid] = version
- }
-
- // Insert the new node in the kvdb.
- node := &dagNode{Level: level, Parents: parents, Logrec: logrec, TxId: tid, Deleted: deleted}
- if err := d.setNode(oid, version, node); err != nil {
- return err
- }
-
- d.numNode.Incr(1)
- if parents == nil {
- d.numObj.Incr(1)
- }
- return nil
-}
-
-// hasNode returns true if the node (oid, version) exists in the DAG DB.
-func (d *dag) hasNode(oid ObjId, version Version) bool {
- if d.store == nil {
- return false
- }
- key := objNodeKey(oid, version)
- return d.nodes.hasKey(key)
-}
-
-// addParent adds to the DAG node (oid, version) linkage to this parent node.
-// If the parent linkage is due to a local change (from conflict resolution
-// by blessing an existing version), no need to update the grafting structure.
-// Otherwise a remote change (from the Sync protocol) updates the graft.
-//
-// TODO(rdaoud): recompute the levels of reachable child-nodes if the new
-// parent's level is greater or equal to the node's current level.
-func (d *dag) addParent(oid ObjId, version, parent Version, remote bool) error {
- if version == parent {
- return fmt.Errorf("addParent: object %v: node %d cannot be its own parent", oid, version)
- }
-
- node, err := d.getNode(oid, version)
- if err != nil {
- return err
- }
-
- pnode, err := d.getNode(oid, parent)
- if err != nil {
- vlog.VI(1).Infof("addParent: object %v, node %d, parent %d: parent node not found", oid, version, parent)
- return err
- }
-
- // Check if the parent is already linked to this node.
- found := false
- for i := range node.Parents {
- if node.Parents[i] == parent {
- found = true
- break
- }
- }
-
- // If the parent is not yet linked (local or remote) add it.
- if !found {
- // Make sure that adding the link does not create a cycle in the DAG.
- // This is done by verifying that the node is not an ancestor of the
- // parent that it is being linked to.
- err = d.ancestorIter(oid, pnode.Parents, func(oid ObjId, v Version, nd *dagNode) error {
- if v == version {
- return fmt.Errorf("addParent: cycle on object %v: node %d is an ancestor of parent node %d",
- oid, version, parent)
- }
- return nil
- })
- if err != nil {
- return err
- }
- node.Parents = append(node.Parents, parent)
- err = d.setNode(oid, version, node)
- if err != nil {
- return err
- }
- }
-
- // For local changes we are done, the grafting structure is not updated.
- if !remote {
- return nil
- }
-
- // If the node and its parent are new/old or old/new then add
- // the parent as a graft point (a potential common ancestor).
- graft := d.getObjectGraft(oid, true)
-
- _, nodeNew := graft.newNodes[version]
- _, parentNew := graft.newNodes[parent]
- if (nodeNew && !parentNew) || (!nodeNew && parentNew) {
- graft.graftNodes[parent] = pnode.Level
- }
-
- // The parent node can no longer be a candidate for a new head version.
- // The addParent() function only removes candidates from newHeads that
- // have become parents. It does not add the child nodes to newHeads
- // because they are not necessarily new-head candidates. If they are
- // new nodes, the addNode() function handles adding them to newHeads.
- // For old nodes, only the current head could be a candidate and it is
- // added to newHeads when the graft struct is initialized.
- if _, ok := graft.newHeads[parent]; ok {
- delete(graft.newHeads, parent)
- }
-
- return nil
-}
-
-// moveHead moves the object head node in the DAG.
-func (d *dag) moveHead(oid ObjId, head Version) error {
- if d.store == nil {
- return errBadDAG
- }
-
- // Verify that the node exists.
- if !d.hasNode(oid, head) {
- return fmt.Errorf("node %v:%d does not exist in the DAG", oid, head)
- }
-
- return d.setHead(oid, head)
-}
-
-// hasConflict determines if there is a conflict for this object between its
-// new and old head nodes.
-// - Yes: return (true, newHead, oldHead, ancestor)
-// - No: return (false, newHead, oldHead, NoVersion)
-// A conflict exists when there are two new-head nodes. It means the newly
-// added object versions are not derived in part from this device's current
-// knowledge. If there is a single new-head, the object changes were applied
-// without triggering a conflict.
-func (d *dag) hasConflict(oid ObjId) (isConflict bool, newHead, oldHead, ancestor Version, err error) {
- oldHead = NoVersion
- newHead = NoVersion
- ancestor = NoVersion
- if d.store == nil {
- err = errBadDAG
- return
- }
-
- graft := d.graft[oid]
- if graft == nil {
- err = fmt.Errorf("node %v has no DAG graft information", oid)
- return
- }
-
- numHeads := len(graft.newHeads)
- if numHeads < 1 || numHeads > 2 {
- err = fmt.Errorf("node %v has invalid number of new head candidates %d: %v", oid, numHeads, graft.newHeads)
- return
- }
-
- // Fetch the current head for this object if it exists. The error from getHead()
- // is ignored because a newly received object is not yet known on this device and
- // will not trigger a conflict.
- oldHead, _ = d.getHead(oid)
-
- // If there is only one new head node there is no conflict.
- // The new head is that single one, even if it might also be the same old node.
- if numHeads == 1 {
- for k := range graft.newHeads {
- newHead = k
- }
- return
- }
-
- // With two candidate head nodes, the new one is the node that is
- // not the current (old) head node.
- for k := range graft.newHeads {
- if k != oldHead {
- newHead = k
- break
- }
- }
-
- // There is a conflict: the best choice ancestor is the graft point
- // node with the largest level (farthest from the root). It is
- // possible in some corner cases to have multiple graft nodes at
- // the same level. This would still be a single conflict, but the
- // multiple same-level graft points representing equivalent conflict
- // resolutions on different devices that are now merging their
- // resolutions. In such a case it does not matter which node is
- // chosen as the ancestor because the conflict resolver function
- // is assumed to be convergent. However it's nicer to make that
- // selection deterministic so all devices see the same choice.
- // For this the version number is used as a tie-breaker.
- isConflict = true
- var maxLevel uint64
- for node, level := range graft.graftNodes {
- if maxLevel < level ||
- (maxLevel == level && ancestor < node) {
- maxLevel = level
- ancestor = node
- }
- }
- return
-}
-
-// ancestorIter iterates over the DAG ancestor nodes for an object in a
-// breadth-first traversal starting from given version node(s). In its
-// traversal it invokes the callback function once for each node, passing
-// the object ID, version number and a pointer to the dagNode.
-func (d *dag) ancestorIter(oid ObjId, startVersions []Version,
- cb func(ObjId, Version, *dagNode) error) error {
- visited := make(map[Version]bool)
- queue := list.New()
- for _, version := range startVersions {
- queue.PushBack(version)
- visited[version] = true
- }
-
- for queue.Len() > 0 {
- version := queue.Remove(queue.Front()).(Version)
- node, err := d.getNode(oid, version)
- if err != nil {
- // Ignore it, the parent was previously pruned.
- continue
- }
- for _, parent := range node.Parents {
- if !visited[parent] {
- queue.PushBack(parent)
- visited[parent] = true
- }
- }
- if err = cb(oid, version, node); err != nil {
- return err
- }
- }
-
- return nil
-}
-
-// hasDeletedDescendant returns true if the node (oid, version) exists in the
-// DAG DB and one of its descendants is a deleted node (i.e. has its "Deleted"
-// flag set true). This means that at some object mutation after this version,
-// the object was deleted.
-func (d *dag) hasDeletedDescendant(oid ObjId, version Version) bool {
- if d.store == nil {
- return false
- }
- if !d.hasNode(oid, version) {
- return false
- }
-
- // Do a breadth-first traversal from the object's head node back to
- // the given version. Along the way, track whether a deleted node is
- // traversed. Return true only if a traversal reaches the given version
- // and had seen a deleted node along the way.
-
- // nodeStep tracks a step along a traversal. It stores the node to visit
- // when taking that step and a boolean tracking whether a deleted node
- // was seen so far along that trajectory.
- head, err := d.getHead(oid)
- if err != nil {
- return false
- }
-
- type nodeStep struct {
- node Version
- deleted bool
- }
-
- visited := make(map[nodeStep]struct{})
- queue := list.New()
-
- step := nodeStep{node: head, deleted: false}
- queue.PushBack(&step)
- visited[step] = struct{}{}
-
- for queue.Len() > 0 {
- step := queue.Remove(queue.Front()).(*nodeStep)
- if step.node == version {
- if step.deleted {
- return true
- }
- continue
- }
- node, err := d.getNode(oid, step.node)
- if err != nil {
- // Ignore it, the parent was previously pruned.
- continue
- }
- nextDel := step.deleted || node.Deleted
-
- for _, parent := range node.Parents {
- nextStep := nodeStep{node: parent, deleted: nextDel}
- if _, ok := visited[nextStep]; !ok {
- queue.PushBack(&nextStep)
- visited[nextStep] = struct{}{}
- }
- }
- }
-
- return false
-}
-
-// prune trims the DAG of an object at a given version (node) by deleting
-// all its ancestor nodes, making it the new root node. For each deleted
-// node it calls the given callback function to delete its log record.
-// This function should only be called when Sync determines that all devices
-// that know about the object have gotten past this version.
-// Also track any transaction sets affected by deleting DAG objects that
-// have transaction IDs. This is later used to do garbage collection
-// on transaction sets when pruneDone() is called.
-func (d *dag) prune(oid ObjId, version Version, delLogRec func(logrec string) error) error {
- if d.store == nil {
- return errBadDAG
- }
-
- // Get the node at the pruning point and set its parents to nil.
- // It will become the oldest DAG node (root) for the object.
- node, err := d.getNode(oid, version)
- if err != nil {
- return err
- }
- if node.Parents == nil {
- // Nothing to do, this node is already the root.
- return nil
- }
-
- iterVersions := node.Parents
-
- node.Parents = nil
- if err = d.setNode(oid, version, node); err != nil {
- return err
- }
-
- // Delete all ancestor nodes and their log records.
- // Delete as many as possible and track the error counts.
- // Keep track of objects deleted from transaction in order
- // to cleanup transaction sets when pruneDone() is called.
- numNodeErrs, numLogErrs := 0, 0
- err = d.ancestorIter(oid, iterVersions, func(oid ObjId, v Version, node *dagNode) error {
- nodeErrs, logErrs, err := d.removeNode(oid, v, node, delLogRec)
- numNodeErrs += nodeErrs
- numLogErrs += logErrs
- return err
- })
- if err != nil {
- return err
- }
- if numNodeErrs != 0 || numLogErrs != 0 {
- return fmt.Errorf("prune failed to delete %d nodes and %d log records", numNodeErrs, numLogErrs)
- }
- return nil
-}
-
-// removeNode removes the state associated with a DAG node.
-func (d *dag) removeNode(oid ObjId, v Version, node *dagNode, delLogRec func(logrec string) error) (int, int, error) {
- numNodeErrs, numLogErrs := 0, 0
- if tid := node.TxId; tid != NoTxId {
- if d.txGC[tid] == nil {
- d.txGC[tid] = make(dagTxMap)
- }
- d.txGC[tid][oid] = v
- }
-
- if err := delLogRec(node.Logrec); err != nil {
- numLogErrs++
- }
- if err := d.delNode(oid, v); err != nil {
- numNodeErrs++
- }
- d.numNode.Incr(-1)
- return numNodeErrs, numLogErrs, nil
-}
-
-// pruneAll prunes the entire DAG state corresponding to an object,
-// including the head.
-func (d *dag) pruneAll(oid ObjId, delLogRec func(logrec string) error) error {
- vers, err := d.getHead(oid)
- if err != nil {
- return err
- }
- node, err := d.getNode(oid, vers)
- if err != nil {
- return err
- }
-
- if err := d.prune(oid, vers, delLogRec); err != nil {
- return err
- }
-
- // Clean up the head.
- numNodeErrs, numLogErrs, err := d.removeNode(oid, vers, node, delLogRec)
- if err != nil {
- return err
- }
- if numNodeErrs != 0 || numLogErrs != 0 {
- return fmt.Errorf("pruneAll failed to delete %d nodes and %d log records", numNodeErrs, numLogErrs)
- }
-
- return d.delHead(oid)
-}
-
-// pruneDone is called when object pruning is finished within a single pass
-// of the Sync garbage collector. It updates the transaction sets affected
-// by the objects deleted by the prune() calls.
-func (d *dag) pruneDone() error {
- if d.store == nil {
- return errBadDAG
- }
-
- // Update transaction sets by removing from them the objects that
- // were pruned. If the resulting set is empty, delete it.
- for tid, txMapGC := range d.txGC {
- txSt, err := d.getTransaction(tid)
- if err != nil {
- return err
- }
-
- for oid := range txMapGC {
- delete(txSt.TxMap, oid)
- }
-
- if len(txSt.TxMap) > 0 {
- err = d.setTransaction(tid, txSt)
- } else {
- err = d.delTransaction(tid)
- }
- if err != nil {
- return err
- }
- }
-
- d.clearTxGC()
- return nil
-}
-
-// getLogrec returns the log record information for a given object version.
-func (d *dag) getLogrec(oid ObjId, version Version) (string, error) {
- node, err := d.getNode(oid, version)
- if err != nil {
- return "", err
- }
- return node.Logrec, nil
-}
-
-// objNodeKey returns the key used to access the object node (oid, version)
-// in the DAG DB.
-func objNodeKey(oid ObjId, version Version) string {
- return fmt.Sprintf("%v:%d", oid, version)
-}
-
-// setNode stores the dagNode structure for the object node (oid, version)
-// in the DAG DB.
-func (d *dag) setNode(oid ObjId, version Version, node *dagNode) error {
- if d.store == nil {
- return errBadDAG
- }
- key := objNodeKey(oid, version)
- return d.nodes.set(key, node)
-}
-
-// getNode retrieves the dagNode structure for the object node (oid, version)
-// from the DAG DB.
-func (d *dag) getNode(oid ObjId, version Version) (*dagNode, error) {
- if d.store == nil {
- return nil, errBadDAG
- }
- var node dagNode
- key := objNodeKey(oid, version)
- if err := d.nodes.get(key, &node); err != nil {
- return nil, err
- }
- return &node, nil
-}
-
-// delNode deletes the object node (oid, version) from the DAG DB.
-func (d *dag) delNode(oid ObjId, version Version) error {
- if d.store == nil {
- return errBadDAG
- }
- key := objNodeKey(oid, version)
- return d.nodes.del(key)
-}
-
-// objHeadKey returns the key used to access the object head in the DAG DB.
-func objHeadKey(oid ObjId) string {
- return oid.String()
-}
-
-// setHead stores version as the object head in the DAG DB.
-func (d *dag) setHead(oid ObjId, version Version) error {
- if d.store == nil {
- return errBadDAG
- }
- key := objHeadKey(oid)
- return d.heads.set(key, version)
-}
-
-// getHead retrieves the object head from the DAG DB.
-func (d *dag) getHead(oid ObjId) (Version, error) {
- var version Version
- if d.store == nil {
- return version, errBadDAG
- }
- key := objHeadKey(oid)
- err := d.heads.get(key, &version)
- if err != nil {
- version = NoVersion
- }
- return version, err
-}
-
-// delHead deletes the object head from the DAG DB.
-func (d *dag) delHead(oid ObjId) error {
- if d.store == nil {
- return errBadDAG
- }
- key := objHeadKey(oid)
- if err := d.heads.del(key); err != nil {
- return err
- }
- d.numObj.Incr(-1)
- return nil
-}
-
-// dagTransactionKey returns the key used to access the transaction in the DAG DB.
-func dagTransactionKey(tid TxId) string {
- return fmt.Sprintf("%v", tid)
-}
-
-// setTransaction stores the transaction object/version map in the DAG DB.
-func (d *dag) setTransaction(tid TxId, txSt *dagTxState) error {
- if d.store == nil {
- return errBadDAG
- }
- if tid == NoTxId {
- return fmt.Errorf("invalid TxId: %v", tid)
- }
- key := dagTransactionKey(tid)
- exists := d.trans.hasKey(key)
-
- if err := d.trans.set(key, txSt); err != nil {
- return err
- }
-
- if !exists {
- d.numTx.Incr(1)
- }
- return nil
-}
-
-// getTransaction retrieves the transaction object/version map from the DAG DB.
-func (d *dag) getTransaction(tid TxId) (*dagTxState, error) {
- if d.store == nil {
- return nil, errBadDAG
- }
- if tid == NoTxId {
- return nil, fmt.Errorf("invalid TxId: %v", tid)
- }
- var txSt dagTxState
- key := dagTransactionKey(tid)
- if err := d.trans.get(key, &txSt); err != nil {
- return nil, err
- }
- return &txSt, nil
-}
-
-// delTransaction deletes the transation object/version map from the DAG DB.
-func (d *dag) delTransaction(tid TxId) error {
- if d.store == nil {
- return errBadDAG
- }
- if tid == NoTxId {
- return fmt.Errorf("invalid TxId: %v", tid)
- }
- key := dagTransactionKey(tid)
- if err := d.trans.del(key); err != nil {
- return err
- }
- d.numTx.Incr(-1)
- return nil
-}
-
-// objPrivNodeKey returns the key used to access a private (unshared) node in the DAG DB.
-func objPrivNodeKey(oid ObjId) string {
- return oid.String()
-}
-
-// setPrivNode stores the privNode structure for a private (unshared) object in the DAG DB.
-func (d *dag) setPrivNode(oid ObjId, priv *privNode) error {
- if d.store == nil {
- return errBadDAG
- }
- key := objPrivNodeKey(oid)
- exists := d.priv.hasKey(key)
-
- if err := d.priv.set(key, priv); err != nil {
- return err
- }
-
- if !exists {
- d.numPriv.Incr(1)
- }
- return nil
-}
-
-// getPrivNode retrieves the privNode structure for a private (unshared) object from the DAG DB.
-func (d *dag) getPrivNode(oid ObjId) (*privNode, error) {
- if d.store == nil {
- return nil, errBadDAG
- }
- var priv privNode
- key := objPrivNodeKey(oid)
- if err := d.priv.get(key, &priv); err != nil {
- return nil, err
- }
- return &priv, nil
-}
-
-// delPrivNode deletes a private (unshared) object from the DAG DB.
-func (d *dag) delPrivNode(oid ObjId) error {
- if d.store == nil {
- return errBadDAG
- }
- key := objPrivNodeKey(oid)
- if err := d.priv.del(key); err != nil {
- return err
- }
- d.numPriv.Incr(-1)
- return nil
-}
-
-// getParentMap is a testing and debug helper function that returns for
-// an object a map of all the object version in the DAG and their parents.
-// The map represents the graph of the object version history.
-func (d *dag) getParentMap(oid ObjId) map[Version][]Version {
- parentMap := make(map[Version][]Version)
- var iterVersions []Version
-
- if head, err := d.getHead(oid); err == nil {
- iterVersions = append(iterVersions, head)
- }
- if graft := d.graft[oid]; graft != nil {
- for k := range graft.newHeads {
- iterVersions = append(iterVersions, k)
- }
- }
-
- // Breadth-first traversal starting from the object head.
- d.ancestorIter(oid, iterVersions, func(oid ObjId, v Version, node *dagNode) error {
- parentMap[v] = node.Parents
- return nil
- })
-
- return parentMap
-}
-
-// getGraftNodes is a testing and debug helper function that returns for
-// an object the graft information built and used during a sync operation.
-// The newHeads map identifies the candidate head nodes based on the data
-// reported by the other device during a sync operation. The graftNodes map
-// identifies the set of old nodes where the new DAG fragments were attached
-// and their depth level in the DAG.
-func (d *dag) getGraftNodes(oid ObjId) (map[Version]struct{}, map[Version]uint64) {
- if d.store != nil {
- if ginfo := d.graft[oid]; ginfo != nil {
- return ginfo.newHeads, ginfo.graftNodes
- }
- }
- return nil, nil
-}
-
-// strToTxId converts from a string to a transaction ID.
-func strToTxId(txStr string) (TxId, error) {
- tx, err := strconv.ParseUint(txStr, 10, 64)
- if err != nil {
- return NoTxId, err
- }
- return TxId(tx), nil
-}
-
-// dump writes to the log file information on all DAG entries.
-func (d *dag) dump() {
- if d.store == nil {
- return
- }
-
- // Dump the head and ancestor information for DAG objects.
- d.heads.keyIter(func(oidStr string) {
- oid, err := strToObjId(oidStr)
- if err != nil {
- return
- }
-
- head, err := d.getHead(oid)
- if err != nil {
- return
- }
-
- vlog.VI(1).Infof("DUMP: DAG oid %v: head %v", oid, head)
- start := []Version{head}
- d.ancestorIter(oid, start, func(oid ObjId, v Version, node *dagNode) error {
- vlog.VI(1).Infof("DUMP: DAG node %v:%v: tx %v, del %t, logrec %s --> %v",
- oid, v, node.TxId, node.Deleted, node.Logrec, node.Parents)
- return nil
- })
- })
-
- // Dump the transactions.
- d.trans.keyIter(func(tidStr string) {
- tid, err := strToTxId(tidStr)
- if err != nil {
- return
- }
-
- txSt, err := d.getTransaction(tid)
- if err != nil {
- return
- }
-
- vlog.VI(1).Infof("DUMP: DAG tx %v: count %d, elem %v", tid, txSt.TxCount, txSt.TxMap)
- })
-}
diff --git a/services/syncbase/sync/dag_test.go b/services/syncbase/sync/dag_test.go
deleted file mode 100644
index 3ea5865..0000000
--- a/services/syncbase/sync/dag_test.go
+++ /dev/null
@@ -1,2128 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Tests for the Veyron Sync DAG component.
-
-import (
- "errors"
- "fmt"
- "os"
- "reflect"
- "testing"
- "time"
-
- "v.io/x/ref/lib/stats"
-)
-
-// dagFilename generates a filename for a temporary (per unit test) DAG file.
-// Do not replace this function with TempFile because TempFile creates the new
-// file and the tests must verify that the DAG can create a non-existing file.
-func dagFilename() string {
- return fmt.Sprintf("%s/sync_dag_test_%d_%d", os.TempDir(), os.Getpid(), time.Now().UnixNano())
-}
-
-// TestDAGOpen tests the creation of a DAG, closing and re-opening it. It also
-// verifies that its backing file is created and that a 2nd close is safe.
-func TestDAGOpen(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- fsize := getFileSize(dagfile)
- if fsize < 0 {
- //t.Fatalf("DAG file %s not created", dagfile)
- }
-
- dag.flush()
- oldfsize := fsize
- fsize = getFileSize(dagfile)
- if fsize <= oldfsize {
- //t.Fatalf("DAG file %s not flushed", dagfile)
- }
-
- dag.close()
-
- dag, err = openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot re-open existing DAG file %s", dagfile)
- }
-
- oldfsize = fsize
- fsize = getFileSize(dagfile)
- if fsize != oldfsize {
- t.Fatalf("DAG file %s size changed across re-open", dagfile)
- }
-
- dag.close()
- dag.close() // multiple closes should be a safe NOP
-
- fsize = getFileSize(dagfile)
- if fsize != oldfsize {
- t.Fatalf("DAG file %s size changed across close", dagfile)
- }
-
- // Fail opening a DAG in a non-existent directory.
- _, err = openDAG("/not/really/there/junk.dag")
- if err == nil {
- //t.Fatalf("openDAG() did not fail when using a bad pathname")
- }
-}
-
-// TestInvalidDAG tests using DAG methods on an invalid (closed) DAG.
-func TestInvalidDAG(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- dag.close()
-
- oid, err := strToObjId("6789")
- if err != nil {
- t.Error(err)
- }
-
- validateError := func(err error, funcName string) {
- if err == nil || err.Error() != "invalid DAG" {
- t.Errorf("%s() did not fail on a closed DAG: %v", funcName, err)
- }
- }
-
- err = dag.addNode(oid, 4, false, false, []Version{2, 3}, "foobar", NoTxId)
- validateError(err, "addNode")
-
- err = dag.moveHead(oid, 4)
- validateError(err, "moveHead")
-
- _, _, _, _, err = dag.hasConflict(oid)
- validateError(err, "hasConflict")
-
- _, err = dag.getLogrec(oid, 4)
- validateError(err, "getLogrec")
-
- err = dag.prune(oid, 4, func(lr string) error {
- return nil
- })
- validateError(err, "prune")
-
- err = dag.pruneAll(oid, func(lr string) error {
- return nil
- })
- validateError(err, "pruneAll")
-
- err = dag.pruneDone()
- validateError(err, "pruneDone")
-
- node := &dagNode{Level: 15, Parents: []Version{444, 555}, Logrec: "logrec-23"}
- err = dag.setNode(oid, 4, node)
- validateError(err, "setNode")
-
- _, err = dag.getNode(oid, 4)
- validateError(err, "getNode")
-
- err = dag.delNode(oid, 4)
- validateError(err, "delNode")
-
- err = dag.addParent(oid, 4, 2, true)
- validateError(err, "addParent")
-
- err = dag.setHead(oid, 4)
- validateError(err, "setHead")
-
- _, err = dag.getHead(oid)
- validateError(err, "getHead")
-
- err = dag.delHead(oid)
- validateError(err, "delHead")
-
- if tid := dag.addNodeTxStart(NoTxId); tid != NoTxId {
- t.Errorf("addNodeTxStart() did not fail on a closed DAG: TxId %v", tid)
- }
-
- err = dag.addNodeTxEnd(1, 1)
- validateError(err, "addNodeTxEnd")
-
- err = dag.setTransaction(1, nil)
- validateError(err, "setTransaction")
-
- _, err = dag.getTransaction(1)
- validateError(err, "getTransaction")
-
- err = dag.delTransaction(1)
- validateError(err, "delTransaction")
-
- err = dag.setPrivNode(oid, nil)
- validateError(err, "setPrivNode")
-
- _, err = dag.getPrivNode(oid)
- validateError(err, "getPrivNode")
-
- err = dag.delPrivNode(oid)
- validateError(err, "delPrivNode")
-
- // These calls should be harmless NOPs.
- dag.clearGraft()
- dag.clearTxGC()
- dag.dump()
- dag.flush()
- dag.close()
- if dag.hasNode(oid, 4) {
- t.Errorf("hasNode() found an object on a closed DAG")
- }
- if dag.hasDeletedDescendant(oid, 3) {
- t.Errorf("hasDeletedDescendant() returned true on a closed DAG")
- }
- if pmap := dag.getParentMap(oid); len(pmap) != 0 {
- t.Errorf("getParentMap() found data on a closed DAG: %v", pmap)
- }
- if hmap, gmap := dag.getGraftNodes(oid); hmap != nil || gmap != nil {
- t.Errorf("getGraftNodes() found data on a closed DAG: head map: %v, graft map: %v", hmap, gmap)
- }
-}
-
-// TestSetNode tests setting and getting a DAG node across DAG open/close/reopen.
-func TestSetNode(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- version := Version(0)
- oid, err := strToObjId("1111")
- if err != nil {
- t.Fatal(err)
- }
-
- node, err := dag.getNode(oid, version)
- if err == nil || node != nil {
- t.Errorf("Found non-existent object %v:%d in DAG file %s: %v", oid, version, dagfile, node)
- }
-
- if dag.hasNode(oid, version) {
- t.Errorf("hasNode() found non-existent object %v:%d in DAG file %s", oid, version, dagfile)
- }
-
- if logrec, err := dag.getLogrec(oid, version); err == nil || logrec != "" {
- t.Errorf("Non-existent object %v:%d has a logrec in DAG file %s: %v", oid, version, dagfile, logrec)
- }
-
- node = &dagNode{Level: 15, Parents: []Version{444, 555}, Logrec: "logrec-23"}
- if err = dag.setNode(oid, version, node); err != nil {
- t.Fatalf("Cannot set object %v:%d (%v) in DAG file %s", oid, version, node, dagfile)
- }
-
- for i := 0; i < 2; i++ {
- node2, err := dag.getNode(oid, version)
- if err != nil || node2 == nil {
- t.Errorf("Cannot find stored object %v:%d (i=%d) in DAG file %s", oid, version, i, dagfile)
- }
-
- if !dag.hasNode(oid, version) {
- t.Errorf("hasNode() did not find object %v:%d (i=%d) in DAG file %s", oid, version, i, dagfile)
- }
-
- if !reflect.DeepEqual(node, node2) {
- t.Errorf("Object %v:%d has wrong data (i=%d) in DAG file %s: %v instead of %v",
- oid, version, i, dagfile, node2, node)
- }
-
- if logrec, err := dag.getLogrec(oid, version); err != nil || logrec != "logrec-23" {
- t.Errorf("Object %v:%d has wrong logrec (i=%d) in DAG file %s: %v",
- oid, version, i, dagfile, logrec)
- }
-
- if i == 0 {
- dag.flush()
- dag.close()
- dag, err = openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot re-open DAG file %s", dagfile)
- }
- }
- }
-
- dag.close()
-}
-
-// TestDelNode tests deleting a DAG node across DAG open/close/reopen.
-func TestDelNode(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- version := Version(1)
- oid, err := strToObjId("2222")
- if err != nil {
- t.Fatal(err)
- }
-
- node := &dagNode{Level: 123, Parents: []Version{333}, Logrec: "logrec-789"}
- if err = dag.setNode(oid, version, node); err != nil {
- t.Fatalf("Cannot set object %v:%d (%v) in DAG file %s", oid, version, node, dagfile)
- }
-
- dag.flush()
-
- err = dag.delNode(oid, version)
- if err != nil {
- t.Fatalf("Cannot delete object %v:%d in DAG file %s", oid, version, dagfile)
- }
-
- dag.flush()
-
- for i := 0; i < 2; i++ {
- node2, err := dag.getNode(oid, version)
- if err == nil || node2 != nil {
- t.Errorf("Found deleted object %v:%d (%v) (i=%d) in DAG file %s", oid, version, node2, i, dagfile)
- }
-
- if dag.hasNode(oid, version) {
- t.Errorf("hasNode() found deleted object %v:%d (i=%d) in DAG file %s", oid, version, i, dagfile)
- }
-
- if logrec, err := dag.getLogrec(oid, version); err == nil || logrec != "" {
- t.Errorf("Deleted object %v:%d (i=%d) has logrec in DAG file %s: %v", oid, version, i, dagfile, logrec)
- }
-
- if i == 0 {
- dag.close()
- dag, err = openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot re-open DAG file %s", dagfile)
- }
- }
- }
-
- dag.close()
-}
-
-// TestAddParent tests adding parents to a DAG node.
-func TestAddParent(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- version := Version(7)
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- if err = dag.addParent(oid, version, 1, true); err == nil {
- t.Errorf("addParent() did not fail for an unknown object %v:%d in DAG file %s", oid, version, dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- node := &dagNode{Level: 15, Logrec: "logrec-22"}
- if err = dag.setNode(oid, version, node); err != nil {
- t.Fatalf("Cannot set object %v:%d (%v) in DAG file %s", oid, version, node, dagfile)
- }
-
- if err = dag.addParent(oid, version, version, true); err == nil {
- t.Errorf("addParent() did not fail on a self-parent for object %v:%d in DAG file %s", oid, version, dagfile)
- }
-
- for _, parent := range []Version{4, 5, 6} {
- if err = dag.addParent(oid, version, parent, true); err == nil {
- t.Errorf("addParent() did not reject invalid parent %d for object %v:%d in DAG file %s",
- parent, oid, version, dagfile)
- }
-
- pnode := &dagNode{Level: 11, Logrec: fmt.Sprintf("logrec-%d", parent), Parents: []Version{3}}
- if err = dag.setNode(oid, parent, pnode); err != nil {
- t.Fatalf("Cannot set parent object %v:%d (%v) in DAG file %s", oid, parent, pnode, dagfile)
- }
-
- remote := parent%2 == 0
- for i := 0; i < 2; i++ {
- if err = dag.addParent(oid, version, parent, remote); err != nil {
- t.Errorf("addParent() failed on parent %d, remote %t (i=%d) for object %v:%d in DAG file %s: %v",
- parent, remote, i, oid, version, dagfile, err)
- }
- }
- }
-
- node2, err := dag.getNode(oid, version)
- if err != nil || node2 == nil {
- t.Errorf("Cannot find stored object %v:%d in DAG file %s", oid, version, dagfile)
- }
-
- expParents := []Version{4, 5, 6}
- if !reflect.DeepEqual(node2.Parents, expParents) {
- t.Errorf("invalid parents for object %v:%d in DAG file %s: %v instead of %v",
- oid, version, dagfile, node2.Parents, expParents)
- }
-
- // Creating cycles should fail.
- for v := Version(1); v < version; v++ {
- if err = dag.addParent(oid, v, version, false); err == nil {
- t.Errorf("addParent() failed to reject a cycle for object %v: from ancestor %d to node %d in DAG file %s",
- oid, v, version, dagfile)
- }
- }
-
- dag.close()
-}
-
-// TestSetHead tests setting and getting a DAG head node across DAG open/close/reopen.
-func TestSetHead(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- oid, err := strToObjId("3333")
- if err != nil {
- t.Fatal(err)
- }
-
- version, err := dag.getHead(oid)
- if err == nil {
- t.Errorf("Found non-existent object head %v in DAG file %s: %d", oid, dagfile, version)
- }
-
- version = 555
- if err = dag.setHead(oid, version); err != nil {
- t.Fatalf("Cannot set object head %v (%d) in DAG file %s", oid, version, dagfile)
- }
-
- dag.flush()
-
- for i := 0; i < 3; i++ {
- version2, err := dag.getHead(oid)
- if err != nil {
- t.Errorf("Cannot find stored object head %v (i=%d) in DAG file %s", oid, i, dagfile)
- }
- if version != version2 {
- t.Errorf("Object %v has wrong head data (i=%d) in DAG file %s: %d instead of %d",
- oid, i, dagfile, version2, version)
- }
-
- if i == 0 {
- dag.close()
- dag, err = openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot re-open DAG file %s", dagfile)
- }
- } else if i == 1 {
- version = 888
- if err = dag.setHead(oid, version); err != nil {
- t.Fatalf("Cannot set new object head %v (%d) in DAG file %s", oid, version, dagfile)
- }
- dag.flush()
- }
- }
-
- dag.close()
-}
-
-// checkEndOfSync simulates and check the end-of-sync operations: clear the
-// node grafting metadata and verify that it is empty and that HasConflict()
-// detects this case and fails, then close the DAG.
-func checkEndOfSync(d *dag, oid ObjId) error {
- // Clear grafting info; this happens at the end of a sync log replay.
- d.clearGraft()
-
- // There should be no grafting or transaction info, and hasConflict() should fail.
- newHeads, grafts := d.getGraftNodes(oid)
- if newHeads != nil || grafts != nil {
- return fmt.Errorf("Object %v: graft info not cleared: newHeads (%v), grafts (%v)", oid, newHeads, grafts)
- }
-
- if n := len(d.txSet); n != 0 {
- return fmt.Errorf("transaction set not empty: %d entries found", n)
- }
-
- isConflict, newHead, oldHead, ancestor, errConflict := d.hasConflict(oid)
- if errConflict == nil {
- return fmt.Errorf("Object %v: conflict did not fail: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- d.dump()
- d.close()
- return nil
-}
-
-// checkDAGStats verifies the DAG stats counters.
-func checkDAGStats(t *testing.T, which string, numObj, numNode, numTx, numPriv int64) {
- if num, err := stats.Value(statsNumDagObj); err != nil || num != numObj {
- t.Errorf("num-dag-objects (%s): got %v (err: %v) instead of %v", which, num, err, numObj)
- }
- if num, err := stats.Value(statsNumDagNode); err != nil || num != numNode {
- t.Errorf("num-dag-nodes (%s): got %v (err: %v) instead of %v", which, num, err, numNode)
- }
- if num, err := stats.Value(statsNumDagTx); err != nil || num != numTx {
- t.Errorf("num-dag-tx (%s): got %v (err: %v) instead of %v", which, num, err, numTx)
- }
- if num, err := stats.Value(statsNumDagPrivNode); err != nil || num != numPriv {
- t.Errorf("num-dag-privnodes (%s): got %v (err: %v) instead of %v", which, num, err, numPriv)
- }
-}
-
-// TestLocalUpdates tests the sync handling of initial local updates: an object
-// is created (v0) and updated twice (v1, v2) on this device. The DAG should
-// show: v0 -> v1 -> v2 and the head should point to v2.
-func TestLocalUpdates(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must have moved to "v2" and the parent map shows the updated DAG.
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 2 {
- t.Errorf("Invalid object %v head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Make sure an existing node cannot be added again.
- if err = dag.addNode(oid, 1, false, false, []Version{0, 2}, "foobar", NoTxId); err == nil {
- t.Errorf("addNode() did not fail when given an existing node")
- }
-
- // Make sure a new node cannot have more than 2 parents.
- if err = dag.addNode(oid, 3, false, false, []Version{0, 1, 2}, "foobar", NoTxId); err == nil {
- t.Errorf("addNode() did not fail when given 3 parents")
- }
-
- // Make sure a new node cannot have an invalid parent.
- if err = dag.addNode(oid, 3, false, false, []Version{0, 555}, "foobar", NoTxId); err == nil {
- t.Errorf("addNode() did not fail when using an invalid parent")
- }
-
- // Make sure a new root node (no parents) cannot be added once a root exists.
- // For the parents array, check both the "nil" and the empty array as input.
- if err = dag.addNode(oid, 6789, false, false, nil, "foobar", NoTxId); err == nil {
- t.Errorf("Adding a 2nd root node (nil parents) for object %v in DAG file %s did not fail", oid, dagfile)
- }
- if err = dag.addNode(oid, 6789, false, false, []Version{}, "foobar", NoTxId); err == nil {
- t.Errorf("Adding a 2nd root node (empty parents) for object %v in DAG file %s did not fail", oid, dagfile)
- }
-
- checkDAGStats(t, "local-update", 1, 3, 0, 0)
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteUpdates tests the sync handling of initial remote updates:
-// an object is created (v0) and updated twice (v1, v2) on another device and
-// we learn about it during sync. The updated DAG should show: v0 -> v1 -> v2
-// and report no conflicts with the new head pointing at v2.
-func TestRemoteUpdates(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "remote-init-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still undefined) and the parent
- // map shows the newly grafted DAG fragment.
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e == nil {
- t.Errorf("Object %v head found in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[Version]struct{}{2: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %v has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[Version]uint64{}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %v graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be no conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(!isConflict && newHead == 2 && oldHead == 0 && ancestor == 0 && errConflict == nil) {
- t.Errorf("Object %v wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-02" {
- t.Errorf("Invalid logrec for newhead object %v:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
- }
-
- // Make sure an unknown node cannot become the new head.
- if err = dag.moveHead(oid, 55); err == nil {
- t.Errorf("moveHead() did not fail on an invalid node")
- }
-
- // Then we can move the head and clear the grafting data.
- if err = dag.moveHead(oid, newHead); err != nil {
- t.Errorf("Object %v cannot move head to %d in DAG file %s: %v", oid, newHead, dagfile, err)
- }
-
- checkDAGStats(t, "remote-update", 1, 3, 0, 0)
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteNoConflict tests sync of remote updates on top of a local initial
-// state without conflict. An object is created locally and updated twice
-// (v0 -> v1 -> v2). Another device, having gotten this info, makes 3 updates
-// on top of that (v2 -> v3 -> v4 -> v5) and sends this info in a later sync.
-// The updated DAG should show (v0 -> v1 -> v2 -> v3 -> v4 -> v5) and report
-// no conflicts with the new head pointing at v5. It should also report v2 as
-// the graft point on which the new fragment (v3 -> v4 -> v5) gets attached.
-func TestRemoteNoConflict(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-noconf-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 2 {
- t.Errorf("Object %v has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}, 3: {2}, 4: {3}, 5: {4}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[Version]struct{}{5: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %v has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[Version]uint64{2: 2}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %v graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be no conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(!isConflict && newHead == 5 && oldHead == 2 && ancestor == 0 && errConflict == nil) {
- t.Errorf("Object %v wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if logrec, e := dag.getLogrec(oid, oldHead); e != nil || logrec != "logrec-02" {
- t.Errorf("Invalid logrec for oldhead object %v:%d in DAG file %s: %v", oid, oldHead, dagfile, logrec)
- }
- if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-05" {
- t.Errorf("Invalid logrec for newhead object %v:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
- }
-
- // Then we can move the head and clear the grafting data.
- if err = dag.moveHead(oid, newHead); err != nil {
- t.Errorf("Object %v cannot move head to %d in DAG file %s: %v", oid, newHead, dagfile, err)
- }
-
- // Clear the grafting data and verify that hasConflict() fails without it.
- dag.clearGraft()
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if errConflict == nil {
- t.Errorf("hasConflict() on %v did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- checkDAGStats(t, "remote-noconf", 1, 6, 0, 0)
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteConflict tests sync handling remote updates that build on the
-// local initial state and trigger a conflict. An object is created locally
-// and updated twice (v0 -> v1 -> v2). Another device, having only gotten
-// the v0 -> v1 history, makes 3 updates on top of v1 (v1 -> v3 -> v4 -> v5)
-// and sends this info during a later sync. Separately, the local device
-// makes a conflicting (concurrent) update v1 -> v2. The updated DAG should
-// show the branches: (v0 -> v1 -> v2) and (v0 -> v1 -> v3 -> v4 -> v5) and
-// report the conflict between v2 and v5 (current and new heads). It should
-// also report v1 as the graft point and the common ancestor in the conflict.
-// The conflict is resolved locally by creating v6 that is derived from both
-// v2 and v5 and it becomes the new head.
-func TestRemoteConflict(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-conf-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 2 {
- t.Errorf("Object %v has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}, 3: {1}, 4: {3}, 5: {4}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[Version]struct{}{2: struct{}{}, 5: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %v has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[Version]uint64{1: 1}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %v graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be a conflict between v2 and v5 with v1 as ancestor.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(isConflict && newHead == 5 && oldHead == 2 && ancestor == 1 && errConflict == nil) {
- t.Errorf("Object %v wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if logrec, e := dag.getLogrec(oid, oldHead); e != nil || logrec != "logrec-02" {
- t.Errorf("Invalid logrec for oldhead object %v:%d in DAG file %s: %v", oid, oldHead, dagfile, logrec)
- }
- if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-05" {
- t.Errorf("Invalid logrec for newhead object %v:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
- }
- if logrec, e := dag.getLogrec(oid, ancestor); e != nil || logrec != "logrec-01" {
- t.Errorf("Invalid logrec for ancestor object %v:%d in DAG file %s: %v", oid, ancestor, dagfile, logrec)
- }
-
- checkDAGStats(t, "remote-conf-pre", 1, 6, 0, 0)
-
- // Resolve the conflict by adding a new local v6 derived from v2 and v5 (this replay moves the head).
- if err = dagReplayCommands(dag, "local-resolve-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // Verify that the head moved to v6 and the parent map shows the resolution.
- if head, e := dag.getHead(oid); e != nil || head != 6 {
- t.Errorf("Object %v has wrong head after conflict resolution in DAG file %s: %d", oid, dagfile, head)
- }
-
- exp[6] = []Version{2, 5}
- pmap = dag.getParentMap(oid)
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map after conflict resolution in DAG file %s: (%v) instead of (%v)",
- oid, dagfile, pmap, exp)
- }
-
- checkDAGStats(t, "remote-conf-post", 1, 7, 0, 0)
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteConflictTwoGrafts tests sync handling remote updates that build
-// on the local initial state and trigger a conflict with 2 graft points.
-// An object is created locally and updated twice (v0 -> v1 -> v2). Another
-// device, first learns about v0 and makes it own conflicting update v0 -> v3.
-// That remote device later learns about v1 and resolves the v1/v3 confict by
-// creating v4. Then it makes a last v4 -> v5 update -- which will conflict
-// with v2 but it doesn't know that.
-// Now the sync order is reversed and the local device learns all of what
-// happened on the remote device. The local DAG should get be augmented by
-// a subtree with 2 graft points: v0 and v1. It receives this new branch:
-// v0 -> v3 -> v4 -> v5. Note that v4 is also derived from v1 as a remote
-// conflict resolution. This should report a conflict between v2 and v5
-// (current and new heads), with v0 and v1 as graft points, and v1 as the
-// most-recent common ancestor for that conflict. The conflict is resolved
-// locally by creating v6, derived from both v2 and v5, becoming the new head.
-func TestRemoteConflictTwoGrafts(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-conf-01.sync"); err != nil {
- t.Fatal(err)
- }
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 2 {
- t.Errorf("Object %v has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}, 3: {0}, 4: {1, 3}, 5: {4}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[Version]struct{}{2: struct{}{}, 5: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %v has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[Version]uint64{0: 0, 1: 1}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %v graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be a conflict between v2 and v5 with v1 as ancestor.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(isConflict && newHead == 5 && oldHead == 2 && ancestor == 1 && errConflict == nil) {
- t.Errorf("Object %v wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if logrec, e := dag.getLogrec(oid, oldHead); e != nil || logrec != "logrec-02" {
- t.Errorf("Invalid logrec for oldhead object %v:%d in DAG file %s: %v", oid, oldHead, dagfile, logrec)
- }
- if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-05" {
- t.Errorf("Invalid logrec for newhead object %v:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
- }
- if logrec, e := dag.getLogrec(oid, ancestor); e != nil || logrec != "logrec-01" {
- t.Errorf("Invalid logrec for ancestor object %v:%d in DAG file %s: %v", oid, ancestor, dagfile, logrec)
- }
-
- checkDAGStats(t, "remote-conf2-pre", 1, 6, 0, 0)
-
- // Resolve the conflict by adding a new local v6 derived from v2 and v5 (this replay moves the head).
- if err = dagReplayCommands(dag, "local-resolve-00.sync"); err != nil {
- t.Fatal(err)
- }
-
- // Verify that the head moved to v6 and the parent map shows the resolution.
- if head, e := dag.getHead(oid); e != nil || head != 6 {
- t.Errorf("Object %v has wrong head after conflict resolution in DAG file %s: %d", oid, dagfile, head)
- }
-
- exp[6] = []Version{2, 5}
- pmap = dag.getParentMap(oid)
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map after conflict resolution in DAG file %s: (%v) instead of (%v)",
- oid, dagfile, pmap, exp)
- }
-
- checkDAGStats(t, "remote-conf2-post", 1, 7, 0, 0)
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestAncestorIterator checks that the iterator goes over the correct set
-// of ancestor nodes for an object given a starting node. It should traverse
-// reconvergent DAG branches only visiting each ancestor once:
-// v0 -> v1 -> v2 -> v4 -> v5 -> v7 -> v8
-// |--> v3 ---| |
-// +--> v6 ---------------+
-// - Starting at v0 it should only cover v0.
-// - Starting at v2 it should only cover v0-v2.
-// - Starting at v5 it should only cover v0-v5.
-// - Starting at v8 it should cover all nodes (v0-v8).
-func TestAncestorIterator(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-01.sync"); err != nil {
- t.Fatal(err)
- }
-
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- // Loop checking the iteration behavior for different starting nodes.
- for _, start := range []Version{0, 2, 5, 8} {
- visitCount := make(map[Version]int)
- err = dag.ancestorIter(oid, []Version{start},
- func(oid ObjId, v Version, node *dagNode) error {
- visitCount[v]++
- return nil
- })
-
- // Check that all prior nodes are visited only once.
- for i := Version(0); i < (start + 1); i++ {
- if visitCount[i] != 1 {
- t.Errorf("wrong visit count for iter on object %v node %d starting from node %d: %d instead of 1",
- oid, i, start, visitCount[i])
- }
- }
- }
-
- // Make sure an error in the callback is returned through the iterator.
- cbErr := errors.New("callback error")
- err = dag.ancestorIter(oid, []Version{8}, func(oid ObjId, v Version, node *dagNode) error {
- if v == 0 {
- return cbErr
- }
- return nil
- })
- if err != cbErr {
- t.Errorf("wrong error returned from callback: %v instead of %v", err, cbErr)
- }
-
- checkDAGStats(t, "ancestor-iter", 1, 9, 0, 0)
-
- if err = checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestPruning tests sync pruning of the DAG for an object with 3 concurrent
-// updates (i.e. 2 conflict resolution convergent points). The pruning must
-// get rid of the DAG branches across the reconvergence points:
-// v0 -> v1 -> v2 -> v4 -> v5 -> v7 -> v8
-// |--> v3 ---| |
-// +--> v6 ---------------+
-// By pruning at v0, nothing is deleted.
-// Then by pruning at v1, only v0 is deleted.
-// Then by pruning at v5, v1-v4 are deleted leaving v5 and "v6 -> v7 -> v8".
-// Then by pruning at v7, v5-v6 are deleted leaving "v7 -> v8".
-// Then by pruning at v8, v7 is deleted leaving v8 as the head.
-// Then by pruning again at v8 nothing changes.
-func TestPruning(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-01.sync"); err != nil {
- t.Fatal(err)
- }
-
- checkDAGStats(t, "prune-init", 1, 9, 0, 0)
-
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}, 3: {1}, 4: {2, 3}, 5: {4}, 6: {1}, 7: {5, 6}, 8: {7}}
-
- // Loop pruning at an invalid version (333) then at v0, v5, v8 and again at v8.
- testVersions := []Version{333, 0, 1, 5, 7, 8, 8}
- delCounts := []int{0, 0, 1, 4, 2, 1, 0}
- which := "prune-snip-"
- remain := 9
-
- for i, version := range testVersions {
- del := 0
- err = dag.prune(oid, version, func(lr string) error {
- del++
- return nil
- })
-
- if i == 0 && err == nil {
- t.Errorf("pruning non-existent object %v:%d did not fail in DAG file %s", oid, version, dagfile)
- } else if i > 0 && err != nil {
- t.Errorf("pruning object %v:%d failed in DAG file %s: %v", oid, version, dagfile, err)
- }
-
- if del != delCounts[i] {
- t.Errorf("pruning object %v:%d deleted %d log records instead of %d", oid, version, del, delCounts[i])
- }
-
- which += "*"
- remain -= del
- checkDAGStats(t, which, 1, int64(remain), 0, 0)
-
- if head, err := dag.getHead(oid); err != nil || head != 8 {
- t.Errorf("Object %v has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- err = dag.pruneDone()
- if err != nil {
- t.Errorf("pruneDone() failed in DAG file %s: %v", dagfile, err)
- }
-
- // Remove pruned nodes from the expected parent map used to validate
- // and set the parents of the pruned node to nil.
- if version < 10 {
- for j := Version(0); j < version; j++ {
- delete(exp, j)
- }
- exp[version] = nil
- }
-
- pmap := dag.getParentMap(oid)
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
- }
-
- checkDAGStats(t, "prune-end", 1, 1, 0, 0)
-
- err = dag.pruneAll(oid, func(lr string) error {
- return nil
- })
- if err != nil {
- t.Errorf("pruneAll() for object %v failed in DAG file %s: %v", oid, dagfile, err)
- }
-
- if err = checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestPruningCallbackError tests sync pruning of the DAG when the callback
-// function returns an error. The pruning must try to delete as many nodes
-// and log records as possible and properly adjust the parent pointers of
-// the pruning node. The object DAG is:
-// v0 -> v1 -> v2 -> v4 -> v5 -> v7 -> v8
-// |--> v3 ---| |
-// +--> v6 ---------------+
-// By pruning at v8 and having the callback function fail for v3, all other
-// nodes must be deleted and only v8 remains as the head.
-func TestPruningCallbackError(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-01.sync"); err != nil {
- t.Fatal(err)
- }
-
- checkDAGStats(t, "prune-cb-init", 1, 9, 0, 0)
-
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- exp := map[Version][]Version{8: nil}
-
- // Prune at v8 with a callback function that fails for v3.
- del, expDel := 0, 8
- version := Version(8)
- err = dag.prune(oid, version, func(lr string) error {
- del++
- if lr == "logrec-03" {
- return fmt.Errorf("refuse to delete %s", lr)
- }
- return nil
- })
-
- if err == nil {
- t.Errorf("pruning object %v:%d did not fail in DAG file %s", oid, version, dagfile)
- }
- if del != expDel {
- t.Errorf("pruning object %v:%d deleted %d log records instead of %d", oid, version, del, expDel)
- }
-
- err = dag.pruneDone()
- if err != nil {
- t.Errorf("pruneDone() failed in DAG file %s: %v", dagfile, err)
- }
-
- if head, err := dag.getHead(oid); err != nil || head != 8 {
- t.Errorf("Object %v has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- checkDAGStats(t, "prune-cb-end", 1, 1, 0, 0)
-
- if err = checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteLinkedNoConflictSameHead tests sync of remote updates that contain
-// linked nodes (conflict resolution by selecting an existing version) on top of
-// a local initial state without conflict. An object is created locally and
-// updated twice (v1 -> v2 -> v3). Another device has learned about v1, created
-// (v1 -> v4), then learned about (v1 -> v2) and resolved that conflict by selecting
-// v2 over v4. Now it sends that new info (v4 and the v2/v4 link) back to the
-// original (local) device. Instead of a v3/v4 conflict, the device sees that
-// v2 was chosen over v4 and resolves it as a no-conflict case.
-func TestRemoteLinkedNoConflictSameHead(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-noconf-link-00.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- checkDAGStats(t, "linked-noconf", 1, 4, 0, 0)
-
- // The head must not have moved (i.e. still at v3) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 3 {
- t.Errorf("Object %v has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[Version][]Version{1: nil, 2: {1, 4}, 3: {2}, 4: {1}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[Version]struct{}{3: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %v has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[Version]uint64{1: 0, 4: 1}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %v graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be no conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(!isConflict && newHead == 3 && oldHead == 3 && ancestor == NoVersion && errConflict == nil) {
- t.Errorf("Object %v wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- // Clear the grafting data and verify that hasConflict() fails without it.
- dag.clearGraft()
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if errConflict == nil {
- t.Errorf("hasConflict() on %v did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteLinkedConflict tests sync of remote updates that contain linked
-// nodes (conflict resolution by selecting an existing version) on top of a local
-// initial state triggering a local conflict. An object is created locally and
-// updated twice (v1 -> v2 -> v3). Another device has along the way learned about v1,
-// created (v1 -> v4), then learned about (v1 -> v2) and resolved that conflict by
-// selecting v4 over v2. Now it sends that new info (v4 and the v4/v2 link) back
-// to the original (local) device. The device sees a v3/v4 conflict.
-func TestRemoteLinkedConflict(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-conf-link.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- checkDAGStats(t, "linked-conf", 1, 4, 0, 0)
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 3 {
- t.Errorf("Object %v has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[Version][]Version{1: nil, 2: {1}, 3: {2}, 4: {1, 2}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[Version]struct{}{3: struct{}{}, 4: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %v has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[Version]uint64{1: 0, 2: 1}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %v graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be a conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(isConflict && newHead == 4 && oldHead == 3 && ancestor == 2 && errConflict == nil) {
- t.Errorf("Object %v wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- // Clear the grafting data and verify that hasConflict() fails without it.
- dag.clearGraft()
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if errConflict == nil {
- t.Errorf("hasConflict() on %v did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteLinkedNoConflictNewHead tests sync of remote updates that contain
-// linked nodes (conflict resolution by selecting an existing version) on top of
-// a local initial state without conflict, but moves the head node to a new one.
-// An object is created locally and updated twice (v1 -> v2 -> v3). Another device
-// has along the way learned about v1, created (v1 -> v4), then learned about
-// (v1 -> v2 -> v3) and resolved that conflict by selecting v4 over v3. Now it
-// sends that new info (v4 and the v4/v3 link) back to the original (local) device.
-// The device sees that the new head v4 is "derived" from v3 thus no conflict.
-func TestRemoteLinkedConflictNewHead(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-noconf-link-01.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- checkDAGStats(t, "linked-conf2", 1, 4, 0, 0)
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 3 {
- t.Errorf("Object %v has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[Version][]Version{1: nil, 2: {1}, 3: {2}, 4: {1, 3}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[Version]struct{}{4: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %v has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[Version]uint64{1: 0, 3: 2}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %v graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be no conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(!isConflict && newHead == 4 && oldHead == 3 && ancestor == NoVersion && errConflict == nil) {
- t.Errorf("Object %v wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- // Clear the grafting data and verify that hasConflict() fails without it.
- dag.clearGraft()
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if errConflict == nil {
- t.Errorf("hasConflict() on %v did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestRemoteLinkedNoConflictNewHeadOvertake tests sync of remote updates that
-// contain linked nodes (conflict resolution by selecting an existing version)
-// on top of a local initial state without conflict, but moves the head node
-// to a new one that overtook the linked node.
-// An object is created locally and updated twice (v1 -> v2 -> v3). Another
-// device has along the way learned about v1, created (v1 -> v4), then learned
-// about (v1 -> v2 -> v3) and resolved that conflict by selecting v3 over v4.
-// Then it creates a new update v5 from v3 (v3 -> v5). Now it sends that new
-// info (v4, the v3/v4 link, and v5) back to the original (local) device.
-// The device sees that the new head v5 is "derived" from v3 thus no conflict.
-func TestRemoteLinkedConflictNewHeadOvertake(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
- t.Fatal(err)
- }
- if err = dagReplayCommands(dag, "remote-noconf-link-02.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- checkDAGStats(t, "linked-conf3-pre", 1, 5, 0, 0)
-
- // The head must not have moved (i.e. still at v2) and the parent map
- // shows the newly grafted DAG fragment on top of the prior DAG.
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 3 {
- t.Errorf("Object %v has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- pmap := dag.getParentMap(oid)
-
- exp := map[Version][]Version{1: nil, 2: {1}, 3: {2, 4}, 4: {1}, 5: {3}}
-
- if !reflect.DeepEqual(pmap, exp) {
- t.Errorf("Invalid object %v parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
- }
-
- // Verify the grafting of remote nodes.
- newHeads, grafts := dag.getGraftNodes(oid)
-
- expNewHeads := map[Version]struct{}{5: struct{}{}}
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %v has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts := map[Version]uint64{1: 0, 3: 2, 4: 1}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %v graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- // There should be no conflict.
- isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
- if !(!isConflict && newHead == 5 && oldHead == 3 && ancestor == NoVersion && errConflict == nil) {
- t.Errorf("Object %v wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- // Then we can move the head and clear the grafting data.
- if err = dag.moveHead(oid, newHead); err != nil {
- t.Errorf("Object %v cannot move head to %d in DAG file %s: %v", oid, newHead, dagfile, err)
- }
-
- // Clear the grafting data and verify that hasConflict() fails without it.
- dag.clearGraft()
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if errConflict == nil {
- t.Errorf("hasConflict() on %v did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- // Now new info comes from another device repeating the v2/v3 link.
- // Verify that it is a NOP (no changes).
- if err = dagReplayCommands(dag, "remote-noconf-link-repeat.log.sync"); err != nil {
- t.Fatal(err)
- }
-
- if head, e := dag.getHead(oid); e != nil || head != 5 {
- t.Errorf("Object %v has wrong head in DAG file %s: %d", oid, dagfile, head)
- }
-
- newHeads, grafts = dag.getGraftNodes(oid)
- if !reflect.DeepEqual(newHeads, expNewHeads) {
- t.Errorf("Object %v has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
- }
-
- expgrafts = map[Version]uint64{}
- if !reflect.DeepEqual(grafts, expgrafts) {
- t.Errorf("Invalid object %v graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
- }
-
- isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
- if !(!isConflict && newHead == 5 && oldHead == 5 && ancestor == NoVersion && errConflict == nil) {
- t.Errorf("Object %v wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
- oid, isConflict, newHead, oldHead, ancestor, errConflict)
- }
-
- checkDAGStats(t, "linked-conf3-post", 1, 5, 0, 0)
-
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
-}
-
-// TestAddNodeTransactional tests adding multiple DAG nodes grouped within a transaction.
-func TestAddNodeTransactional(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-02.sync"); err != nil {
- t.Fatal(err)
- }
-
- checkDAGStats(t, "add-tx-init", 3, 5, 0, 0)
-
- oid_a, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
- oid_b, err := strToObjId("6789")
- if err != nil {
- t.Fatal(err)
- }
- oid_c, err := strToObjId("2222")
- if err != nil {
- t.Fatal(err)
- }
-
- // Verify NoTxId is reported as an error.
- if err := dag.addNodeTxEnd(NoTxId, 0); err == nil {
- t.Errorf("addNodeTxEnd() did not fail for invalid 'NoTxId' value")
- }
- if _, err := dag.getTransaction(NoTxId); err == nil {
- t.Errorf("getTransaction() did not fail for invalid 'NoTxId' value")
- }
- if err := dag.setTransaction(NoTxId, nil); err == nil {
- t.Errorf("setTransaction() did not fail for invalid 'NoTxId' value")
- }
- if err := dag.delTransaction(NoTxId); err == nil {
- t.Errorf("delTransaction() did not fail for invalid 'NoTxId' value")
- }
-
- // Mutate 2 objects within a transaction.
- tid_1 := dag.addNodeTxStart(NoTxId)
- if tid_1 == NoTxId {
- t.Fatal("Cannot start 1st DAG addNode() transaction")
- }
- if err := dag.addNodeTxEnd(tid_1, 0); err == nil {
- t.Errorf("addNodeTxEnd() did not fail for a zero-count transaction")
- }
-
- txSt, ok := dag.txSet[tid_1]
- if !ok {
- t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_1, dagfile)
- }
- if n := len(txSt.TxMap); n != 0 {
- t.Errorf("Transactions map for Tx ID %v has length %d instead of 0 in DAG file %s", tid_1, n, dagfile)
- }
-
- if err := dag.addNode(oid_a, 3, false, false, []Version{2}, "logrec-a-03", tid_1); err != nil {
- t.Errorf("Cannot addNode() on object %v and Tx ID %v in DAG file %s: %v", oid_a, tid_1, dagfile, err)
- }
-
- if tTmp := dag.addNodeTxStart(tid_1); tTmp != tid_1 {
- t.Fatal("restarting transaction failed")
- }
-
- if err := dag.addNode(oid_b, 3, false, false, []Version{2}, "logrec-b-03", tid_1); err != nil {
- t.Errorf("Cannot addNode() on object %v and Tx ID %v in DAG file %s: %v", oid_b, tid_1, dagfile, err)
- }
-
- // At the same time mutate the 3rd object in another transaction.
- tid_2 := dag.addNodeTxStart(NoTxId)
- if tid_2 == NoTxId {
- t.Fatal("Cannot start 2nd DAG addNode() transaction")
- }
-
- txSt, ok = dag.txSet[tid_2]
- if !ok {
- t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_2, dagfile)
- }
- if n := len(txSt.TxMap); n != 0 {
- t.Errorf("Transactions map for Tx ID %v has length %d instead of 0 in DAG file %s", tid_2, n, dagfile)
- }
-
- if err := dag.addNode(oid_c, 2, false, false, []Version{1}, "logrec-c-02", tid_2); err != nil {
- t.Errorf("Cannot addNode() on object %v and Tx ID %v in DAG file %s: %v", oid_c, tid_2, dagfile, err)
- }
-
- // Verify the in-memory transaction sets constructed.
- txSt, ok = dag.txSet[tid_1]
- if !ok {
- t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_1, dagfile)
- }
- expTxSt := &dagTxState{dagTxMap{oid_a: 3, oid_b: 3}, 0}
- if !reflect.DeepEqual(txSt, expTxSt) {
- t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_1, dagfile, txSt, expTxSt)
- }
-
- txSt, ok = dag.txSet[tid_2]
- if !ok {
- t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_2, dagfile)
- }
- expTxSt = &dagTxState{dagTxMap{oid_c: 2}, 0}
- if !reflect.DeepEqual(txSt, expTxSt) {
- t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txSt, expTxSt)
- }
-
- // Verify failing to use a Tx ID not returned by addNodeTxStart().
- bad_tid := tid_1 + 1
- for bad_tid == NoTxId || bad_tid == tid_2 {
- bad_tid++
- }
-
- if err := dag.addNode(oid_c, 3, false, false, []Version{2}, "logrec-c-03", bad_tid); err == nil {
- t.Errorf("addNode() did not fail on object %v for a bad Tx ID %v in DAG file %s", oid_c, bad_tid, dagfile)
- }
- if err := dag.addNodeTxEnd(bad_tid, 1); err == nil {
- t.Errorf("addNodeTxEnd() did not fail for a bad Tx ID %v in DAG file %s", bad_tid, dagfile)
- }
-
- // End the 1st transaction and verify the in-memory and in-DAG data.
- if err := dag.addNodeTxEnd(tid_1, 2); err != nil {
- t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
- }
-
- checkDAGStats(t, "add-tx-1", 3, 8, 1, 0)
-
- if _, ok = dag.txSet[tid_1]; ok {
- t.Errorf("Transactions state for Tx ID %v still exists in DAG file %s", tid_1, dagfile)
- }
-
- txSt, err = dag.getTransaction(tid_1)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
- }
-
- expTxSt = &dagTxState{dagTxMap{oid_a: 3, oid_b: 3}, 2}
- if !reflect.DeepEqual(txSt, expTxSt) {
- t.Errorf("Invalid transaction state from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
- tid_1, dagfile, txSt, expTxSt)
- }
-
- txSt, ok = dag.txSet[tid_2]
- if !ok {
- t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_2, dagfile)
- }
-
- expTxSt = &dagTxState{dagTxMap{oid_c: 2}, 0}
- if !reflect.DeepEqual(txSt, expTxSt) {
- t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txSt, expTxSt)
- }
-
- // End the 2nd transaction and re-verify the in-memory and in-DAG data.
- if err := dag.addNodeTxEnd(tid_2, 1); err != nil {
- t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
- }
-
- checkDAGStats(t, "add-tx-2", 3, 8, 2, 0)
-
- if _, ok = dag.txSet[tid_2]; ok {
- t.Errorf("Transactions state for Tx ID %v still exists in DAG file %s", tid_2, dagfile)
- }
-
- txSt, err = dag.getTransaction(tid_2)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
- }
-
- expTxSt = &dagTxState{dagTxMap{oid_c: 2}, 1}
- if !reflect.DeepEqual(txSt, expTxSt) {
- t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txSt, expTxSt)
- }
-
- if n := len(dag.txSet); n != 0 {
- t.Errorf("Transaction sets in-memory: %d entries found, should be empty in DAG file %s", n, dagfile)
- }
-
- // Test incrementally filling up a transaction.
- tid_3 := TxId(100)
- if _, ok = dag.txSet[tid_3]; ok {
- t.Errorf("Transactions state for Tx ID %v found in DAG file %s", tid_3, dagfile)
- }
-
- if tTmp := dag.addNodeTxStart(tid_3); tTmp != tid_3 {
- t.Fatalf("Cannot start transaction %v", tid_3)
- }
-
- txSt, ok = dag.txSet[tid_3]
- if !ok {
- t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_3, dagfile)
- }
- if n := len(txSt.TxMap); n != 0 {
- t.Errorf("Transactions map for Tx ID %v has length %d instead of 0 in DAG file %s", tid_3, n, dagfile)
- }
-
- if err := dag.addNode(oid_a, 4, false, false, []Version{3}, "logrec-a-04", tid_3); err != nil {
- t.Errorf("Cannot addNode() on object %v and Tx ID %v in DAG file %s: %v", oid_a, tid_3, dagfile, err)
- }
-
- if err := dag.addNodeTxEnd(tid_3, 2); err != nil {
- t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_3, dagfile, err)
- }
-
- checkDAGStats(t, "add-tx-3", 3, 9, 3, 0)
-
- if _, ok = dag.txSet[tid_3]; ok {
- t.Errorf("Transactions state for Tx ID %v still exists in DAG file %s", tid_3, dagfile)
- }
-
- txSt, err = dag.getTransaction(tid_3)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_3, dagfile, err)
- }
-
- expTxSt = &dagTxState{dagTxMap{oid_a: 4}, 2}
- if !reflect.DeepEqual(txSt, expTxSt) {
- t.Errorf("Invalid transaction state from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
- tid_3, dagfile, txSt, expTxSt)
- }
-
- if tTmp := dag.addNodeTxStart(tid_3); tTmp != tid_3 {
- t.Fatalf("Cannot start transaction %v", tid_3)
- }
-
- txSt, ok = dag.txSet[tid_3]
- if !ok {
- t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_3, dagfile)
- }
-
- if !reflect.DeepEqual(txSt, expTxSt) {
- t.Errorf("Invalid transaction state from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
- tid_3, dagfile, txSt, expTxSt)
- }
-
- if err := dag.addNode(oid_b, 4, false, false, []Version{3}, "logrec-b-04", tid_3); err != nil {
- t.Errorf("Cannot addNode() on object %v and Tx ID %v in DAG file %s: %v", oid_b, tid_3, dagfile, err)
- }
-
- if err := dag.addNodeTxEnd(tid_3, 3); err == nil {
- t.Errorf("addNodeTxEnd() didn't fail for Tx ID %v in DAG file %s: %v", tid_3, dagfile, err)
- }
-
- if err := dag.addNodeTxEnd(tid_3, 2); err != nil {
- t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_3, dagfile, err)
- }
-
- checkDAGStats(t, "add-tx-4", 3, 10, 3, 0)
-
- txSt, err = dag.getTransaction(tid_3)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_3, dagfile, err)
- }
-
- expTxSt = &dagTxState{dagTxMap{oid_a: 4, oid_b: 4}, 2}
- if !reflect.DeepEqual(txSt, expTxSt) {
- t.Errorf("Invalid transaction state from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
- tid_3, dagfile, txSt, expTxSt)
- }
-
- // Get the 3 new nodes from the DAG and verify their Tx IDs.
- node, err := dag.getNode(oid_a, 3)
- if err != nil {
- t.Errorf("Cannot find object %v:3 in DAG file %s: %v", oid_a, dagfile, err)
- }
- if node.TxId != tid_1 {
- t.Errorf("Invalid TxId for object %v:3 in DAG file %s: %v instead of %v", oid_a, dagfile, node.TxId, tid_1)
- }
- node, err = dag.getNode(oid_a, 4)
- if err != nil {
- t.Errorf("Cannot find object %v:4 in DAG file %s: %v", oid_a, dagfile, err)
- }
- if node.TxId != tid_3 {
- t.Errorf("Invalid TxId for object %v:4 in DAG file %s: %v instead of %v", oid_a, dagfile, node.TxId, tid_3)
- }
- node, err = dag.getNode(oid_b, 3)
- if err != nil {
- t.Errorf("Cannot find object %v:3 in DAG file %s: %v", oid_b, dagfile, err)
- }
- if node.TxId != tid_1 {
- t.Errorf("Invalid TxId for object %v:3 in DAG file %s: %v instead of %v", oid_b, dagfile, node.TxId, tid_1)
- }
- node, err = dag.getNode(oid_b, 4)
- if err != nil {
- t.Errorf("Cannot find object %v:4 in DAG file %s: %v", oid_b, dagfile, err)
- }
- if node.TxId != tid_3 {
- t.Errorf("Invalid TxId for object %v:4 in DAG file %s: %v instead of %v", oid_b, dagfile, node.TxId, tid_3)
- }
- node, err = dag.getNode(oid_c, 2)
- if err != nil {
- t.Errorf("Cannot find object %v:2 in DAG file %s: %v", oid_c, dagfile, err)
- }
- if node.TxId != tid_2 {
- t.Errorf("Invalid TxId for object %v:2 in DAG file %s: %v instead of %v", oid_c, dagfile, node.TxId, tid_2)
- }
-
- for _, oid := range []ObjId{oid_a, oid_b, oid_c} {
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
- }
-}
-
-// TestPruningTransactions tests pruning DAG nodes grouped within transactions.
-func TestPruningTransactions(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-02.sync"); err != nil {
- t.Fatal(err)
- }
-
- checkDAGStats(t, "prune-tx-init", 3, 5, 0, 0)
-
- oid_a, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
- oid_b, err := strToObjId("6789")
- if err != nil {
- t.Fatal(err)
- }
- oid_c, err := strToObjId("2222")
- if err != nil {
- t.Fatal(err)
- }
-
- // Mutate objects in 2 transactions then add non-transactional mutations
- // to act as the pruning points. Before pruning the DAG is:
- // a1 -- a2 -- (a3) --- a4
- // b1 -- b2 -- (b3) -- (b4) -- b5
- // c1 ---------------- (c2)
- // Now by pruning at (a4, b5, c2), the new DAG should be:
- // a4
- // b5
- // (c2)
- // Transaction 1 (a3, b3) gets deleted, but transaction 2 (b4, c2) still
- // has (c2) dangling waiting for a future pruning.
- tid_1 := dag.addNodeTxStart(NoTxId)
- if tid_1 == NoTxId {
- t.Fatal("Cannot start 1st DAG addNode() transaction")
- }
- if err := dag.addNode(oid_a, 3, false, false, []Version{2}, "logrec-a-03", tid_1); err != nil {
- t.Errorf("Cannot addNode() on object %v and Tx ID %v in DAG file %s: %v", oid_a, tid_1, dagfile, err)
- }
- if err := dag.addNode(oid_b, 3, false, false, []Version{2}, "logrec-b-03", tid_1); err != nil {
- t.Errorf("Cannot addNode() on object %v and Tx ID %v in DAG file %s: %v", oid_b, tid_1, dagfile, err)
- }
- if err := dag.addNodeTxEnd(tid_1, 2); err != nil {
- t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
- }
-
- checkDAGStats(t, "prune-tx-1", 3, 7, 1, 0)
-
- tid_2 := dag.addNodeTxStart(NoTxId)
- if tid_2 == NoTxId {
- t.Fatal("Cannot start 2nd DAG addNode() transaction")
- }
- if err := dag.addNode(oid_b, 4, false, false, []Version{3}, "logrec-b-04", tid_2); err != nil {
- t.Errorf("Cannot addNode() on object %v and Tx ID %v in DAG file %s: %v", oid_b, tid_2, dagfile, err)
- }
- if err := dag.addNode(oid_c, 2, false, false, []Version{1}, "logrec-c-02", tid_2); err != nil {
- t.Errorf("Cannot addNode() on object %v and Tx ID %v in DAG file %s: %v", oid_c, tid_2, dagfile, err)
- }
- if err := dag.addNodeTxEnd(tid_2, 2); err != nil {
- t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
- }
-
- checkDAGStats(t, "prune-tx-2", 3, 9, 2, 0)
-
- if err := dag.addNode(oid_a, 4, false, false, []Version{3}, "logrec-a-04", NoTxId); err != nil {
- t.Errorf("Cannot addNode() on object %v and Tx ID %v in DAG file %s: %v", oid_a, tid_1, dagfile, err)
- }
- if err := dag.addNode(oid_b, 5, false, false, []Version{4}, "logrec-b-05", NoTxId); err != nil {
- t.Errorf("Cannot addNode() on object %v and Tx ID %v in DAG file %s: %v", oid_b, tid_2, dagfile, err)
- }
-
- if err = dag.moveHead(oid_a, 4); err != nil {
- t.Errorf("Object %v cannot move head in DAG file %s: %v", oid_a, dagfile, err)
- }
- if err = dag.moveHead(oid_b, 5); err != nil {
- t.Errorf("Object %v cannot move head in DAG file %s: %v", oid_b, dagfile, err)
- }
- if err = dag.moveHead(oid_c, 2); err != nil {
- t.Errorf("Object %v cannot move head in DAG file %s: %v", oid_c, dagfile, err)
- }
-
- checkDAGStats(t, "prune-tx-3", 3, 11, 2, 0)
-
- // Verify the transaction sets.
- txSt, err := dag.getTransaction(tid_1)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
- }
-
- expTxSt := &dagTxState{dagTxMap{oid_a: 3, oid_b: 3}, 2}
- if !reflect.DeepEqual(txSt, expTxSt) {
- t.Errorf("Invalid transaction state from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
- tid_1, dagfile, txSt, expTxSt)
- }
-
- txSt, err = dag.getTransaction(tid_2)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
- }
-
- expTxSt = &dagTxState{dagTxMap{oid_b: 4, oid_c: 2}, 2}
- if !reflect.DeepEqual(txSt, expTxSt) {
- t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txSt, expTxSt)
- }
-
- // Prune the 3 objects at their head nodes.
- for _, oid := range []ObjId{oid_a, oid_b, oid_c} {
- head, err := dag.getHead(oid)
- if err != nil {
- t.Errorf("Cannot getHead() on object %v in DAG file %s: %v", oid, dagfile, err)
- }
- err = dag.prune(oid, head, func(lr string) error {
- return nil
- })
- if err != nil {
- t.Errorf("Cannot prune() on object %v in DAG file %s: %v", oid, dagfile, err)
- }
- }
-
- if err = dag.pruneDone(); err != nil {
- t.Errorf("pruneDone() failed in DAG file %s: %v", dagfile, err)
- }
-
- if n := len(dag.txGC); n != 0 {
- t.Errorf("Transaction GC map not empty after pruneDone() in DAG file %s: %d", dagfile, n)
- }
-
- // Verify that Tx-1 was deleted and Tx-2 still has c2 in it.
- checkDAGStats(t, "prune-tx-4", 3, 3, 1, 0)
-
- txSt, err = dag.getTransaction(tid_1)
- if err == nil {
- t.Errorf("getTransaction() did not fail for Tx ID %v in DAG file %s: %v", tid_1, dagfile, txSt)
- }
-
- txSt, err = dag.getTransaction(tid_2)
- if err != nil {
- t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
- }
-
- expTxSt = &dagTxState{dagTxMap{oid_c: 2}, 2}
- if !reflect.DeepEqual(txSt, expTxSt) {
- t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txSt, expTxSt)
- }
-
- // Add c3 as a new head and prune at that point. This should GC Tx-2.
- if err := dag.addNode(oid_c, 3, false, false, []Version{2}, "logrec-c-03", NoTxId); err != nil {
- t.Errorf("Cannot addNode() on object %v in DAG file %s: %v", oid_c, dagfile, err)
- }
- if err = dag.moveHead(oid_c, 3); err != nil {
- t.Errorf("Object %v cannot move head in DAG file %s: %v", oid_c, dagfile, err)
- }
-
- checkDAGStats(t, "prune-tx-5", 3, 4, 1, 0)
-
- err = dag.prune(oid_c, 3, func(lr string) error {
- return nil
- })
- if err != nil {
- t.Errorf("Cannot prune() on object %v in DAG file %s: %v", oid_c, dagfile, err)
- }
- if err = dag.pruneDone(); err != nil {
- t.Errorf("pruneDone() #2 failed in DAG file %s: %v", dagfile, err)
- }
- if n := len(dag.txGC); n != 0 {
- t.Errorf("Transaction GC map not empty after pruneDone() in DAG file %s: %d", dagfile, n)
- }
-
- checkDAGStats(t, "prune-tx-6", 3, 3, 0, 0)
-
- txSt, err = dag.getTransaction(tid_2)
- if err == nil {
- t.Errorf("getTransaction() did not fail for Tx ID %v in DAG file %s: %v", tid_2, dagfile, txSt)
- }
-
- for _, oid := range []ObjId{oid_a, oid_b, oid_c} {
- if err := checkEndOfSync(dag, oid); err != nil {
- t.Fatal(err)
- }
- }
-}
-
-// TestHasDeletedDescendant tests lookup of DAG deleted nodes descending from a given node.
-func TestHasDeletedDescendant(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- if err = dagReplayCommands(dag, "local-init-03.sync"); err != nil {
- t.Fatal(err)
- }
-
- oid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- // Delete node v3 to create a dangling parent link from v7 (increase code coverage).
- if err = dag.delNode(oid, 3); err != nil {
- t.Errorf("cannot delete node %v:3 in DAG file %s: %v", oid, dagfile, err)
- }
-
- type hasDelDescTest struct {
- node Version
- result bool
- }
- tests := []hasDelDescTest{
- {NoVersion, false},
- {999, false},
- {1, true},
- {2, true},
- {3, false},
- {4, false},
- {5, false},
- {6, false},
- {7, false},
- {8, false},
- }
-
- for _, test := range tests {
- result := dag.hasDeletedDescendant(oid, test.node)
- if result != test.result {
- t.Errorf("hasDeletedDescendant() for node %d in DAG file %s: %v instead of %v",
- test.node, dagfile, result, test.result)
- }
- }
-
- dag.close()
-}
-
-// TestPrivNode tests access to the private nodes table in a DAG.
-func TestPrivNode(t *testing.T) {
- dagfile := dagFilename()
- defer os.Remove(dagfile)
-
- dag, err := openDAG(dagfile)
- if err != nil {
- t.Fatalf("Cannot open new DAG file %s", dagfile)
- }
-
- oid, err := strToObjId("2222")
- if err != nil {
- t.Fatal(err)
- }
-
- priv, err := dag.getPrivNode(oid)
- if err == nil || priv != nil {
- t.Errorf("Found non-existing private object %v in DAG file %s: %v, err %v", oid, dagfile, priv, err)
- }
-
- priv = &privNode{
- //Mutation: &raw.Mutation{ID: oid, PriorVersion: 0x0, Version: 0x55104dc76695721d, Value: "value-foobar"},
- PathIDs: []ObjId{oid, ObjId("haha"), ObjId("foobar")},
- TxId: 56789,
- }
-
- if err = dag.setPrivNode(oid, priv); err != nil {
- t.Fatalf("Cannot set private object %v (%v) in DAG file %s: %v", oid, priv, dagfile, err)
- }
-
- checkDAGStats(t, "priv-1", 0, 0, 0, 1)
-
- priv2, err := dag.getPrivNode(oid)
- if err != nil {
- t.Fatalf("Cannot get private object %v from DAG file %s: %v", oid, dagfile, err)
- }
- if !reflect.DeepEqual(priv2, priv) {
- t.Errorf("Private object %v has wrong data in DAG file %s: %v instead of %v", oid, dagfile, priv2, priv)
- }
-
- //priv.Mutation.PriorVersion = priv.Mutation.Version
- //priv.Mutation.Version = 0x55555ddddd345abc
- //priv.Mutation.Value = "value-new"
- priv.TxId = 98765
-
- if err = dag.setPrivNode(oid, priv); err != nil {
- t.Fatalf("Cannot overwrite private object %v (%v) in DAG file %s: %v", oid, priv, dagfile, err)
- }
-
- checkDAGStats(t, "priv-1", 0, 0, 0, 1)
-
- priv2, err = dag.getPrivNode(oid)
- if err != nil {
- t.Fatalf("Cannot get updated private object %v from DAG file %s: %v", oid, dagfile, err)
- }
- if !reflect.DeepEqual(priv2, priv) {
- t.Errorf("Private object %v has wrong data post-update in DAG file %s: %v instead of %v", oid, dagfile, priv2, priv)
- }
-
- err = dag.delPrivNode(oid)
- if err != nil {
- t.Fatalf("Cannot delete private object %v in DAG file %s: %v", oid, dagfile, err)
- }
-
- checkDAGStats(t, "priv-1", 0, 0, 0, 0)
-
- priv3, err := dag.getPrivNode(oid)
- if err == nil || priv3 != nil {
- t.Errorf("Found deleted private object %v in DAG file %s: %v, err %v", oid, dagfile, priv3, err)
- }
-
- dag.close()
-}
diff --git a/services/syncbase/sync/devtable.go b/services/syncbase/sync/devtable.go
deleted file mode 100644
index 9a463a1..0000000
--- a/services/syncbase/sync/devtable.go
+++ /dev/null
@@ -1,546 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Package vsync provides veyron sync DevTable utility functions.
-// DevTable is indexed by the device id and stores device level
-// information needed by sync. Main component of a device's info are
-// a set of generation vectors: one per SyncRoot. Generation vector
-// is the version vector for a device's view of a SyncRoot,
-// representing all the different generations (from different devices)
-// seen by that device for that SyncRoot. A generation represents a
-// collection of updates that originated on a device during an
-// interval of time. It serves as a checkpoint when communicating with
-// other devices. Generations do not overlap and all updates belong to
-// a generation.
-//
-// Synchronization for a given SyncRoot between two devices A and B
-// uses generation vectors as follows:
-// A B
-// <== B's generation vector
-// diff(A's generation vector, B's generation vector)
-// log records of missing generations ==>
-// cache B's generation vector (for space reclamation)
-//
-// Implementation notes: DevTable is stored in a persistent K/V
-// database in the current implementation. Generation vector is
-// implemented as a map of (Device ID -> Generation ID), one entry for
-// every known device. If the generation vector contains an entry
-// (Device ID -> Generation ID), it implies that the device has
-// learned of all the generations until and including Generation
-// ID. Generation IDs start from 1. A generation ID of 0 is a
-// reserved bootstrap value, and indicates the device has no updates.
-import (
- "errors"
- "fmt"
- "sort"
- "time"
-
- "v.io/x/lib/vlog"
-)
-
-var (
- errInvalidDTab = errors.New("invalid devtable db")
-)
-
-// devInfo is the information stored per device.
-type devInfo struct {
- Vectors map[ObjId]GenVector // device generation vectors.
- Ts time.Time // last communication time stamp.
-}
-
-// devTableHeader contains the header metadata.
-type devTableHeader struct {
- Resmark []byte // resume marker for watch.
- // Generation vector for space reclamation. All generations
- // less than this generation vector are deleted from storage.
- ReclaimVec GenVector
-}
-
-// devTable contains the metadata for the device table db.
-type devTable struct {
- fname string // file pathname.
- db *kvdb // underlying K/V DB.
- devices *kvtable // pointer to the "devices" table in the kvdb. Contains device info.
-
- // Key:"Head" Value:devTableHeader
- header *kvtable // pointer to the "header" table in the kvdb. Contains device table header.
- head *devTableHeader // devTable head cached in memory.
-
- s *syncd // pointer to the sync daemon object.
-}
-
-// genOrder represents a generation along with its position in the log.
-type genOrder struct {
- devID DeviceId
- srID ObjId
- genID GenId
- order uint32
-}
-
-// byOrder is used to sort the genOrder array.
-type byOrder []*genOrder
-
-func (a byOrder) Len() int {
- return len(a)
-}
-
-func (a byOrder) Swap(i, j int) {
- a[i], a[j] = a[j], a[i]
-}
-
-func (a byOrder) Less(i, j int) bool {
- return a[i].order < a[j].order
-}
-
-// openDevTable opens or creates a devTable for the given filename.
-func openDevTable(filename string, sin *syncd) (*devTable, error) {
- dtab := &devTable{
- fname: filename,
- s: sin,
- }
- // Open the file and create it if it does not exist.
- // Also initialize the kvdb and its collection.
- db, tbls, err := kvdbOpen(filename, []string{"devices", "header"})
- if err != nil {
- return nil, err
- }
-
- dtab.db = db
- dtab.devices = tbls[0]
- dtab.header = tbls[1]
-
- // Initialize the devTable header.
- dtab.head = &devTableHeader{
- ReclaimVec: GenVector{
- dtab.s.id: 0,
- },
- }
- // If header already exists in db, read it back from db.
- if dtab.hasHead() {
- if err := dtab.getHead(); err != nil {
- dtab.db.close() // this also closes the tables.
- return nil, err
- }
- }
-
- return dtab, nil
-}
-
-// close closes the devTable and invalidates its struct.
-func (dt *devTable) close() error {
- if dt.db == nil {
- return errInvalidDTab
- }
- // Flush the dirty data.
- if err := dt.flush(); err != nil {
- return err
- }
- dt.db.close() // this also closes the tables.
-
- *dt = devTable{} // zero out the devTable struct.
- return nil
-}
-
-// flush flushes the devTable db to storage.
-func (dt *devTable) flush() error {
- if dt.db == nil {
- return errInvalidDTab
- }
- // Set the head from memory before flushing.
- if err := dt.putHead(); err != nil {
- return err
- }
- dt.db.flush()
- return nil
-}
-
-// initSyncRoot initializes the local generation vector for this SyncRoot.
-func (dt *devTable) initSyncRoot(srid ObjId) error {
- if dt.db == nil {
- return errInvalidDTab
- }
- if _, err := dt.getGenVec(dt.s.id, srid); err == nil {
- return fmt.Errorf("syncroot already exists %v", srid)
- }
- return dt.putGenVec(dt.s.id, srid, GenVector{dt.s.id: 0})
-}
-
-// delSyncRoot deletes the generation vector for this SyncRoot.
-func (dt *devTable) delSyncRoot(srid ObjId) error {
- if dt.db == nil {
- return errInvalidDTab
- }
- info, err := dt.getDevInfo(dt.s.id)
- if err != nil {
- return fmt.Errorf("dev doesn't exists %v", dt.s.id)
- }
- if _, ok := info.Vectors[srid]; !ok {
- return fmt.Errorf("syncroot doesn't exist %v", srid)
- }
- delete(info.Vectors, srid)
- if len(info.Vectors) == 0 {
- return dt.delDevInfo(dt.s.id)
- }
- return dt.putDevInfo(dt.s.id, info)
-}
-
-// putHead puts the devTable head into the devTable db.
-func (dt *devTable) putHead() error {
- return dt.header.set("Head", dt.head)
-}
-
-// getHead gets the devTable head from the devTable db.
-func (dt *devTable) getHead() error {
- if dt.head == nil {
- return errors.New("nil devTable header")
- }
- return dt.header.get("Head", dt.head)
-}
-
-// hasHead returns true if the devTable db has a devTable head.
-func (dt *devTable) hasHead() bool {
- return dt.header.hasKey("Head")
-}
-
-// putDevInfo puts a devInfo struct in the devTable db.
-func (dt *devTable) putDevInfo(devid DeviceId, info *devInfo) error {
- if dt.db == nil {
- return errInvalidDTab
- }
- return dt.devices.set(string(devid), info)
-}
-
-// getDevInfo gets a devInfo struct from the devTable db.
-func (dt *devTable) getDevInfo(devid DeviceId) (*devInfo, error) {
- if dt.db == nil {
- return nil, errInvalidDTab
- }
- var info devInfo
- if err := dt.devices.get(string(devid), &info); err != nil {
- return nil, err
- }
- if info.Vectors == nil {
- return nil, errors.New("nil genvectors")
- }
- return &info, nil
-}
-
-// hasDevInfo returns true if the device (devid) has any devInfo in the devTable db.
-func (dt *devTable) hasDevInfo(devid DeviceId) bool {
- if dt.db == nil {
- return false
- }
- return dt.devices.hasKey(string(devid))
-}
-
-// delDevInfo deletes devInfo struct in the devTable db.
-func (dt *devTable) delDevInfo(devid DeviceId) error {
- if dt.db == nil {
- return errInvalidDTab
- }
- return dt.devices.del(string(devid))
-}
-
-// putGenVec puts a generation vector in the devTable db.
-func (dt *devTable) putGenVec(devid DeviceId, srid ObjId, v GenVector) error {
- if dt.db == nil {
- return errInvalidDTab
- }
- var info *devInfo
- if dt.hasDevInfo(devid) {
- var err error
- if info, err = dt.getDevInfo(devid); err != nil {
- return err
- }
- } else {
- info = &devInfo{
- Vectors: make(map[ObjId]GenVector),
- }
- }
- info.Vectors[srid] = v
- info.Ts = time.Now().UTC()
- return dt.putDevInfo(devid, info)
-}
-
-// getGenVec gets a generation vector from the devTable db.
-func (dt *devTable) getGenVec(devid DeviceId, srid ObjId) (GenVector, error) {
- if dt.db == nil {
- return nil, errInvalidDTab
- }
- info, err := dt.getDevInfo(devid)
- if err != nil {
- return nil, err
- }
- v, ok := info.Vectors[srid]
- if !ok {
- return nil, fmt.Errorf("srid %s doesn't exist", srid.String())
- }
- return v, nil
-}
-
-// populateGenOrderEntry populates a genOrder entry.
-func (dt *devTable) populateGenOrderEntry(e *genOrder, id DeviceId, srid ObjId, gnum GenId) error {
- e.devID = id
- e.srID = srid
- e.genID = gnum
-
- o, err := dt.s.log.getGenMetadata(id, srid, gnum)
- if err != nil {
- return err
- }
- e.order = o.Pos
- return nil
-}
-
-// updateGeneration updates a single generation (upID, upGen) in a device's generation vector for SyncRoot srID.
-func (dt *devTable) updateGeneration(key DeviceId, srID ObjId, upID DeviceId, upGen GenId) error {
- if dt.db == nil {
- return errInvalidDTab
- }
- info, err := dt.getDevInfo(key)
- if err != nil {
- return err
- }
-
- v, ok := info.Vectors[srID]
- if !ok {
- return fmt.Errorf("srid %s doesn't exist", srID.String())
- }
- v[upID] = upGen
- return dt.putDevInfo(key, info)
-}
-
-// updateLocalGenVector updates local generation vector based on the remote generation vector.
-func (dt *devTable) updateLocalGenVector(local, remote GenVector) error {
- if dt.db == nil {
- return errInvalidDTab
- }
- if local == nil || remote == nil {
- return errors.New("invalid input args to function")
- }
- for rid, rgen := range remote {
- lgen, ok := local[rid]
- if !ok || lgen < rgen {
- local[rid] = rgen
- }
- }
- return nil
-}
-
-// diffGenVectors diffs generation vectors for a given SyncRoot belonging
-// to src and dest and returns the generations known to src and not known to
-// dest. In addition, sync needs to maintain the order in which device
-// generations are created/received. Hence, when two generation
-// vectors are diffed, the differing generations are returned in a
-// sorted order based on their position in the src's log. genOrder
-// array consists of every generation that is missing between src and
-// dest sorted using its position in the src's log.
-// Example: Generation vector for device A (src) AVec = {A:10, B:5, C:1}
-// Generation vector for device B (dest) BVec = {A:5, B:10, D:2}
-// Missing generations in unsorted order: {A:6, A:7, A:8, A:9, A:10, C:1}
-//
-// TODO(hpucha): Revisit for the case of a lot of generations to
-// send back (say during bootstrap).
-func (dt *devTable) diffGenVectors(srcVec, destVec GenVector, srid ObjId) ([]*genOrder, error) {
- if dt.db == nil {
- return nil, errInvalidDTab
- }
-
- // Create an array for the generations that need to be returned.
- var gens []*genOrder
-
- // Compute missing generations for devices that are in destination and source vector.
- for devid, genid := range destVec {
- srcGenId, ok := srcVec[devid]
- // Skip since src doesn't know of this device.
- if !ok {
- continue
- }
- // Need to include all generations in the interval [genid+1, srcGenId],
- // genid+1 and srcGenId inclusive.
- // Check against reclaimVec to see if required generations are already GCed.
- // Starting gen is then max(oldGen, genid+1)
- startGen := genid + 1
- oldGen := dt.getOldestGen(devid) + 1
- if startGen < oldGen {
- vlog.VI(1).Infof("diffGenVectors:: Adjusting starting generations from %d to %d",
- startGen, oldGen)
- startGen = oldGen
- }
- for i := startGen; i <= srcGenId; i++ {
- // Populate the genorder entry.
- var entry genOrder
- if err := dt.populateGenOrderEntry(&entry, devid, srid, i); err != nil {
- return nil, err
- }
- gens = append(gens, &entry)
- }
- }
- // Compute missing generations for devices not in destination vector but in source vector.
- for devid, genid := range srcVec {
- // Add devices destination does not know about.
- if _, ok := destVec[devid]; !ok {
- // Bootstrap generation to oldest available.
- destGenId := dt.getOldestGen(devid) + 1
- // Need to include all generations in the interval [destGenId, genid],
- // destGenId and genid inclusive.
- for i := destGenId; i <= genid; i++ {
- // Populate the genorder entry.
- var entry genOrder
- if err := dt.populateGenOrderEntry(&entry, devid, srid, i); err != nil {
- return nil, err
- }
- gens = append(gens, &entry)
- }
- }
- }
-
- // Sort generations in log order.
- sort.Sort(byOrder(gens))
- return gens, nil
-}
-
-// getOldestGen returns the most recent gc'ed generation for the device "dev".
-func (dt *devTable) getOldestGen(dev DeviceId) GenId {
- return dt.head.ReclaimVec[dev]
-}
-
-// // computeReclaimVector computes a generation vector such that the
-// // generations less than or equal to those in the vector can be
-// // garbage collected. Caller holds a lock on s.lock.
-// //
-// // Approach: For each device in the system, we compute its maximum
-// // generation known to all the other devices in the system. This is a
-// // O(N^2) algorithm where N is the number of devices in the system. N
-// // is assumed to be small, of the order of hundreds of devices.
-// func (dt *devTable) computeReclaimVector() (GenVector, error) {
-// // Get local generation vector to create the set of devices in
-// // the system. Local generation vector is a good bootstrap
-// // device set since it contains all the devices whose log
-// // records were ever stored locally.
-// devSet, err := dt.getGenVec(dt.s.id)
-// if err != nil {
-// return nil, err
-// }
-//
-// newReclaimVec := GenVector{}
-// for devid := range devSet {
-// if !dt.hasDevInfo(devid) {
-// // This node knows of devid, but hasn't yet
-// // contacted the device. Do not garbage
-// // collect any further. For instance, when
-// // node A learns of node C's generations from
-// // node B, node A may not have an entry for
-// // node C yet, but node C will be part of its
-// // devSet.
-// for dev := range devSet {
-// newReclaimVec[dev] = dt.getOldestGen(dev)
-// }
-// return newReclaimVec, nil
-// }
-//
-// vec, err := dt.getGenVec(devid)
-// if err != nil {
-// return nil, err
-// }
-// for dev := range devSet {
-// gen1, ok := vec[dev]
-// // Device "devid" does not know about device "dev".
-// if !ok {
-// newReclaimVec[dev] = dt.getOldestGen(dev)
-// continue
-// }
-// gen2, ok := newReclaimVec[dev]
-// if !ok || (gen1 < gen2) {
-// newReclaimVec[dev] = gen1
-// }
-// }
-// }
-// return newReclaimVec, nil
-// }
-//
-// // addDevice adds a newly learned device to the devTable state.
-// func (dt *devTable) addDevice(newDev DeviceId) error {
-// // Create an entry in the device table for the new device.
-// vector := GenVector{
-// newDev: 0,
-// }
-// if err := dt.putGenVec(newDev, vector); err != nil {
-// return err
-// }
-//
-// // Update local generation vector with the new device.
-// local, err := dt.getDevInfo(dt.s.id)
-// if err != nil {
-// return err
-// }
-// if err := dt.updateLocalGenVector(local.Vector, vector); err != nil {
-// return err
-// }
-// if err := dt.putDevInfo(dt.s.id, local); err != nil {
-// return err
-// }
-// return nil
-// }
-//
-// // updateReclaimVec updates the reclaim vector to track gc'ed generations.
-// func (dt *devTable) updateReclaimVec(minGens GenVector) error {
-// for dev, min := range minGens {
-// gen, ok := dt.head.ReclaimVec[dev]
-// if !ok {
-// if min < 1 {
-// vlog.Errorf("updateReclaimVec:: Received bad generation %s %d",
-// dev, min)
-// dt.head.ReclaimVec[dev] = 0
-// } else {
-// dt.head.ReclaimVec[dev] = min - 1
-// }
-// continue
-// }
-//
-// // We obtained a generation that is already reclaimed.
-// if min <= gen {
-// return errors.New("requested gen smaller than GC'ed gen")
-// }
-// }
-// return nil
-// }
-
-// getDeviceIds returns the IDs of all devices that are involved in synchronization.
-func (dt *devTable) getDeviceIds() ([]DeviceId, error) {
- if dt.db == nil {
- return nil, errInvalidDTab
- }
-
- devIDs := make([]DeviceId, 0)
- dt.devices.keyIter(func(devStr string) {
- devIDs = append(devIDs, DeviceId(devStr))
- })
-
- return devIDs, nil
-}
-
-// dump writes to the log file information on all device table entries.
-func (dt *devTable) dump() {
- if dt.db == nil {
- return
- }
-
- vlog.VI(1).Infof("DUMP: Dev: self %v: resmark %v, reclaim %v",
- dt.s.id, dt.head.Resmark, dt.head.ReclaimVec)
-
- dt.devices.keyIter(func(devStr string) {
- info, err := dt.getDevInfo(DeviceId(devStr))
- if err != nil {
- return
- }
-
- vlog.VI(1).Infof("DUMP: Dev: %s: #SR %d, time %v", devStr, len(info.Vectors), info.Ts)
- for sr, vec := range info.Vectors {
- vlog.VI(1).Infof("DUMP: Dev: %s: SR %v, vec %v", devStr, sr, vec)
- }
- })
-}
diff --git a/services/syncbase/sync/devtable_test.go b/services/syncbase/sync/devtable_test.go
deleted file mode 100644
index 8cf5307..0000000
--- a/services/syncbase/sync/devtable_test.go
+++ /dev/null
@@ -1,1227 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Tests for the Veyron Sync devTable component.
-import (
- "os"
- "reflect"
- "runtime"
- "testing"
- "time"
-)
-
-// TestDevTabStore tests creating a backing file for devTable.
-func TestDevTabStore(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- fsize := getFileSize(devfile)
- if fsize < 0 {
- //t.Errorf("DevTable file %s not created", devfile)
- }
-
- if err := dtab.flush(); err != nil {
- t.Errorf("Cannot flush devTable file %s, err %v", devfile, err)
- }
-
- oldfsize := fsize
- fsize = getFileSize(devfile)
- if fsize <= oldfsize {
- //t.Errorf("DevTable file %s not flushed", devfile)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-
- oldfsize = getFileSize(devfile)
-
- dtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot re-open existing devTable file %s, err %v", devfile, err)
- }
-
- fsize = getFileSize(devfile)
- if fsize != oldfsize {
- t.Errorf("DevTable file %s size changed across re-open (%d %d)", devfile, fsize, oldfsize)
- }
-
- if err := dtab.flush(); err != nil {
- t.Errorf("Cannot flush devTable file %s, err %v", devfile, err)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestInvalidDTab tests devTable methods on an invalid (closed) devTable ptr.
-func TestInvalidDTab(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-
- validateError := func(err error, funcName string) {
- _, file, line, _ := runtime.Caller(1)
- if err == nil || err != errInvalidDTab {
- t.Errorf("%s:%d %s() did not fail on a closed devTable: %v", file, line, funcName, err)
- }
- }
-
- err = dtab.close()
- validateError(err, "close")
-
- err = dtab.flush()
- validateError(err, "flush")
-
- srid := ObjId("foo")
-
- err = dtab.initSyncRoot(srid)
- validateError(err, "initSyncRoot")
-
- err = dtab.putDevInfo(s.id, &devInfo{})
- validateError(err, "putDevInfo")
-
- _, err = dtab.getDevInfo(s.id)
- validateError(err, "getDevInfo")
-
- err = dtab.delDevInfo(s.id)
- validateError(err, "delDevInfo")
-
- if dtab.hasDevInfo(s.id) {
- t.Errorf("hasDevInfo() did not fail on a closed devTable: %v", err)
- }
-
- err = dtab.putGenVec(s.id, srid, GenVector{})
- validateError(err, "putGenVec")
-
- _, err = dtab.getGenVec(s.id, srid)
- validateError(err, "getGenVec")
-
- err = dtab.updateGeneration(s.id, srid, s.id, 0)
- validateError(err, "updateGeneration")
-
- err = dtab.updateLocalGenVector(GenVector{}, GenVector{})
- validateError(err, "updateLocalGenVector")
-
- _, err = dtab.diffGenVectors(GenVector{}, GenVector{}, srid)
- validateError(err, "diffGenVectors")
-
- _, err = dtab.getDeviceIds()
- validateError(err, "getDeviceIds")
-
- // Harmless NOP.
- dtab.dump()
-}
-
-// TestPutGetDevTableHeader tests setting and getting devTable header across devTable open/close/reopen.
-func TestPutGetDevTableHeader(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- // In memory head should be initialized.
- if dtab.head.Resmark != nil {
- t.Errorf("First time log create should reset header: %v", dtab.head.Resmark)
- }
- expVec := GenVector{dtab.s.id: 0}
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
- devfile, dtab.head.ReclaimVec, expVec)
- }
-
- // No head should be there in db.
- if err = dtab.getHead(); err == nil {
- t.Errorf("getHead() found non-existent head in devTable file %s, err %v", devfile, err)
- }
-
- if dtab.hasHead() {
- t.Errorf("hasHead() found non-existent head in devTable file %s", devfile)
- }
-
- expMark := []byte{1, 2, 3}
- expVec = GenVector{
- "VeyronTab": 30,
- "VeyronPhone": 10,
- }
- dtab.head = &devTableHeader{
- Resmark: expMark,
- ReclaimVec: expVec,
- }
-
- if err := dtab.putHead(); err != nil {
- t.Errorf("Cannot put head %v in devTable file %s, err %v", dtab.head, devfile, err)
- }
-
- // Reset values.
- dtab.head.Resmark = nil
- dtab.head.ReclaimVec = GenVector{}
-
- for i := 0; i < 2; i++ {
- if err := dtab.getHead(); err != nil {
- t.Fatalf("getHead() can not find head (i=%d) in devTable file %s, err %v", i, devfile, err)
- }
-
- if !dtab.hasHead() {
- t.Errorf("hasHead() can not find head (i=%d) in devTable file %s", i, devfile)
- }
-
- if !reflect.DeepEqual(dtab.head.Resmark, expMark) {
- t.Errorf("Data mismatch for resmark (i=%d) in devTable file %s: %v instead of %v",
- i, devfile, dtab.head.Resmark, expMark)
- }
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec (i=%d) in devTable file %s: %v instead of %v",
- i, devfile, dtab.head.ReclaimVec, expVec)
- }
-
- if i == 0 {
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
- dtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot re-open devTable file %s, err %v", devfile, err)
- }
- }
- }
-
- dtab.dump()
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestPersistDevTableHeader tests that devTable header is
-// automatically persisted across devTable open/close/reopen.
-func TestPersistDevTableHeader(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- // In memory head should be initialized.
- if dtab.head.Resmark != nil {
- t.Errorf("First time log create should reset header: %v", dtab.head.Resmark)
- }
- expVec := GenVector{dtab.s.id: 0}
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
- devfile, dtab.head.ReclaimVec, expVec)
- }
-
- expMark := []byte{0, 2, 255}
- expVec = GenVector{
- "VeyronTab": 100,
- "VeyronPhone": 10000,
- }
- dtab.head = &devTableHeader{
- Resmark: expMark,
- ReclaimVec: expVec,
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-
- dtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- // In memory head should be initialized from db.
- if !reflect.DeepEqual(dtab.head.Resmark, expMark) {
- t.Errorf("Data mismatch for resmark in devTable file %s: %v instead of %v",
- devfile, dtab.head.Resmark, expMark)
- }
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
- devfile, dtab.head.ReclaimVec, expVec)
- }
-
- expMark = []byte{60, 180, 7}
- expVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 1987,
- }
- dtab.head = &devTableHeader{
- Resmark: expMark,
- ReclaimVec: expVec,
- }
-
- if err := dtab.flush(); err != nil {
- t.Errorf("Cannot flush devTable file %s, err %v", devfile, err)
- }
-
- // Reset values.
- dtab.head.Resmark = nil
- dtab.head.ReclaimVec = GenVector{}
-
- if err := dtab.getHead(); err != nil {
- t.Fatalf("getHead() can not find head in devTable file %s, err %v", devfile, err)
- }
-
- // In memory head should be initialized from db.
- if !reflect.DeepEqual(dtab.head.Resmark, expMark) {
- t.Errorf("Data mismatch for resmark in devTable file %s: %v instead of %v",
- devfile, dtab.head.Resmark, expMark)
- }
- if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
- t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
- devfile, dtab.head.ReclaimVec, expVec)
- }
-
- dtab.dump()
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestDTabInitDelSyncRoot tests initing and deleting a new SyncRoot.
-func TestDTabInitDelSyncRoot(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
- srid1 := ObjId("foo")
- if err := dtab.initSyncRoot(srid1); err != nil {
- t.Fatalf("Cannot create new SyncRoot %s, err %v", srid1.String(), err)
- }
- srid2 := ObjId("bar")
- if err := dtab.initSyncRoot(srid2); err != nil {
- t.Fatalf("Cannot create new SyncRoot %s, err %v", srid2.String(), err)
- }
- if err := dtab.initSyncRoot(srid2); err == nil {
- t.Fatalf("Creating existing SyncRoot didn't fail %s", srid2.String())
- }
-
- info, err := dtab.getDevInfo(s.id)
- if err != nil {
- t.Errorf("GetDevInfo() can not find device %s in devTable file %s err %v",
- s.id, devfile, err)
- }
- expVec := map[ObjId]GenVector{
- srid1: {s.id: 0},
- srid2: {s.id: 0},
- }
- if !reflect.DeepEqual(info.Vectors, expVec) {
- t.Errorf("Data mismatch for device %s %v instead of %v",
- s.id, info.Vectors, expVec)
- }
- if err := dtab.delSyncRoot(srid1); err != nil {
- t.Fatalf("Cannot delete SyncRoot %s, err %v", srid1.String(), err)
- }
- if err := dtab.delSyncRoot(srid1); err == nil {
- t.Fatalf("Deleting non-existent SyncRoot didn't fail %s", srid1.String())
- }
- if err := dtab.delSyncRoot(srid2); err != nil {
- t.Fatalf("Cannot delete SyncRoot %s, err %v", srid2.String(), err)
- }
-
- if _, err = dtab.getDevInfo(s.id); err == nil {
- t.Errorf("GetDevInfo() found device %s in devTable file %s err %v",
- s.id, devfile, err)
- }
-}
-
-// TestPutGetDevInfo tests setting and getting devInfo across devTable open/close/reopen.
-func TestPutGetDevInfo(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- var devid DeviceId = "VeyronTab"
-
- info, err := dtab.getDevInfo(devid)
- if err == nil || info != nil {
- t.Errorf("GetDevInfo() found non-existent device %s in devTable file %s: %v, err %v",
- devid, devfile, info, err)
- }
-
- if dtab.hasDevInfo(devid) {
- t.Errorf("HasDevInfo() found non-existent device %s in devTable file %s",
- devid, devfile)
- }
- info = &devInfo{
- Vectors: map[ObjId]GenVector{ObjId("haha"): GenVector{"VeyronTab": 0, "VeyronPhone": 10},
- ObjId("hello"): GenVector{"VeyronLaptop": 20, "VeyronPhone": 30, "VeyronTab": 80}},
- Ts: time.Now().UTC(),
- }
-
- if err := dtab.putDevInfo(devid, info); err != nil {
- t.Errorf("Cannot put device %s (%v) in devTable file %s, err %v", devid, info, devfile, err)
- }
-
- for i := 0; i < 2; i++ {
- curInfo, err := dtab.getDevInfo(devid)
- if err != nil || curInfo == nil {
- t.Fatalf("GetDevInfo() can not find device %s (i=%d) in devTable file %s: %v, err: %v",
- devid, i, devfile, curInfo, err)
- }
-
- if !dtab.hasDevInfo(devid) {
- t.Errorf("HasDevInfo() can not find device %s (i=%d) in devTable file %s",
- devid, i, devfile)
- }
-
- if !reflect.DeepEqual(curInfo, info) {
- t.Errorf("Data mismatch for device %s (i=%d) in devTable file %s: %v instead of %v",
- devid, i, devfile, curInfo, info)
- }
-
- if i == 0 {
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
- dtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot re-open devTable file %s, err %v", devfile, err)
- }
- }
- }
-
- dtab.dump()
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestPutGetGenVec tests setting and getting generation vector across dtab open/close/reopen.
-func TestPutGetGenVec(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- var devid DeviceId = "VeyronTab"
- srids := []ObjId{"foo", "bar"}
- vecs := []GenVector{GenVector{"VeyronTab": 0, "VeyronPhone": 10, "VeyronDesktop": 20, "VeyronLaptop": 2},
- GenVector{"VeyronTab": 400, "VeyronPhone": 100, "VeyronDesktop": 200, "VeyronLaptop": 20}}
-
- for i, sr := range srids {
- vec, err := dtab.getGenVec(devid, sr)
- if err == nil || vec != nil {
- t.Errorf("GetGenVec() found non-existent device %s in devTable file %s: %v, err %v",
- devid, devfile, vec, err)
- }
-
- if err := dtab.putGenVec(devid, sr, vecs[i]); err != nil {
- t.Errorf("Cannot put device %s (%s %v) in devTable file %s, err %v", devid, sr.String(),
- vecs[i], devfile, err)
- }
- }
-
- for k, sr := range srids {
- for i := 0; i < 2; i++ {
- // Check for devid.
- curVec, err := dtab.getGenVec(devid, sr)
- if err != nil || curVec == nil {
- t.Fatalf("GetGenVec() can not find device %s (i=%d) in devTable file %s, err %v",
- devid, i, devfile, err)
- }
-
- if !reflect.DeepEqual(curVec, vecs[k]) {
- t.Errorf("Data mismatch for device %s srid %s (i=%d) in devTable file %s: %v instead of %v",
- devid, sr.String(), i, devfile, curVec, vecs[k])
- }
-
- if i == 0 {
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
- dtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot re-open devTable file %s, err %v", devfile, err)
- }
- }
- }
- }
-
- dtab.dump()
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestUpdateGeneration tests updating a generation.
-func TestUpdateGeneration(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- var devid DeviceId = "VeyronTab"
- srid := ObjId("foo")
-
- err = dtab.updateGeneration(devid, srid, devid, 10)
- if err == nil {
- t.Errorf("UpdateGeneration() found non-existent device %s in devTable file %s, err %v",
- devid, devfile, err)
- }
- v := GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
-
- if err := dtab.putGenVec(devid, srid, v); err != nil {
- t.Errorf("Cannot put device %s (%s %v) in devTable file %s, err %v", devid,
- srid.String(), v, devfile, err)
- }
-
- // Try a non-existent SyncRoot ID.
- if err = dtab.updateGeneration(devid, ObjId("haha"), devid, 10); err == nil {
- t.Errorf("UpdateGeneration() did not fail on a wrong SyncRoot for %s in devTable file %s", devid, devfile)
- }
-
- err = dtab.updateGeneration(devid, srid, devid, 10)
- if err != nil {
- t.Errorf("UpdateGeneration() failed for %s in devTable file %s with error %v",
- devid, devfile, err)
- }
- err = dtab.updateGeneration(devid, srid, "VeyronLaptop", 18)
- if err != nil {
- t.Errorf("UpdateGeneration() failed for %s in devTable file %s with error %v",
- devid, devfile, err)
- }
- curVec, err := dtab.getGenVec(devid, srid)
- if err != nil || curVec == nil {
- t.Fatalf("GetGenVec() can not find device %s in devTable file %s, err %v",
- devid, devfile, err)
- }
- vExp := GenVector{
- "VeyronTab": 10,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 18,
- }
-
- if !reflect.DeepEqual(curVec, vExp) {
- t.Errorf("Data mismatch for device %s srid %s in devTable file %s: %v instead of %v",
- devid, srid.String(), devfile, v, vExp)
- }
-
- dtab.dump()
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestUpdateLocalGenVector tests updating a gen vector.
-func TestUpdateLocalGenVector(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- s := &syncd{id: "VeyronPhone"}
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- // Test nil args.
- if err := dtab.updateLocalGenVector(nil, nil); err == nil {
- t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
- devfile, err)
- }
-
- // Nothing to update.
- local := GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 1,
- }
- remote := GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 1,
- }
- if err := dtab.updateLocalGenVector(local, remote); err != nil {
- t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
- devfile, err)
- }
-
- if !reflect.DeepEqual(local, remote) {
- t.Errorf("Data mismatch for object %v instead of %v",
- local, remote)
- }
-
- // local is missing a generation.
- local = GenVector{
- "VeyronPhone": 1,
- }
- if err := dtab.updateLocalGenVector(local, remote); err != nil {
- t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
- devfile, err)
- }
- if !reflect.DeepEqual(local, remote) {
- t.Errorf("Data mismatch for object %v instead of %v",
- local, remote)
- }
-
- // local is stale compared to remote.
- local = GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 0,
- }
- remote = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 0,
- "VeyronLaptop": 2,
- }
- if err := dtab.updateLocalGenVector(local, remote); err != nil {
- t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
- devfile, err)
- }
- if !reflect.DeepEqual(local, remote) {
- t.Errorf("Data mismatch for object %v instead of %v",
- local, remote)
- }
-
- // local is partially stale.
- local = GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 0,
- "VeyronDesktop": 20,
- }
- remote = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronLaptop": 2,
- }
- localExp := GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
- if err := dtab.updateLocalGenVector(local, remote); err != nil {
- t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
- devfile, err)
- }
- if !reflect.DeepEqual(local, localExp) {
- t.Errorf("Data mismatch for object %v instead of %v",
- local, localExp)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestDiffGenVectors tests diffing gen vectors.
-func TestDiffGenVectors(t *testing.T) {
- logOrder := []DeviceId{"VeyronTab", "VeyronPhone", "VeyronDesktop", "VeyronLaptop"}
- var expGens []*genOrder
- srid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- // set reclaimVec such that it doesn't affect diffs.
- reclaimVec := GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 0,
- "VeyronDesktop": 0,
- "VeyronLaptop": 0,
- }
-
- // src and dest are identical vectors.
- vec := GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
- setupAndTestDiff(t, vec, vec, reclaimVec, logOrder, expGens)
-
- // src has no updates.
- srcVec := GenVector{
- "VeyronTab": 0,
- }
- remoteVec := GenVector{
- "VeyronTab": 5,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 8,
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, []DeviceId{}, expGens)
-
- // src and remote have no updates.
- srcVec = GenVector{
- "VeyronTab": 0,
- }
- remoteVec = GenVector{
- "VeyronTab": 0,
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, []DeviceId{}, expGens)
-
- // set reclaimVec such that it doesn't affect diffs.
- reclaimVec = GenVector{
- "VeyronTab": 0,
- }
-
- // src is staler than remote.
- srcVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
- remoteVec = GenVector{
- "VeyronTab": 5,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 8,
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
-
- // src is fresher than remote.
- srcVec = GenVector{
- "VeyronTab": 5,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
- remoteVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- }
- expGens = make([]*genOrder, 4)
- for i := 0; i < 4; i++ {
- expGens[i] = &genOrder{
- devID: "VeyronTab",
- srID: srid,
- genID: GenId(i + 2),
- order: uint32(i + 1),
- }
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
-
- // src is fresher than remote in all but one device.
- srcVec = GenVector{
- "VeyronTab": 5,
- "VeyronPhone": 10,
- "VeyronDesktop": 22,
- "VeyronLaptop": 2,
- }
- remoteVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 10,
- "VeyronDesktop": 20,
- "VeyronLaptop": 2,
- "VeyronCloud": 40,
- }
- expGens = make([]*genOrder, 6)
- for i := 0; i < 6; i++ {
- switch {
- case i < 4:
- expGens[i] = &genOrder{
- devID: "VeyronTab",
- srID: srid,
- genID: GenId(i + 2),
- order: uint32(i + 1),
- }
- default:
- expGens[i] = &genOrder{
- devID: "VeyronDesktop",
- srID: srid,
- genID: GenId(i - 4 + 21),
- order: uint32(i - 4 + 35),
- }
- }
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
-
- // src is fresher than dest, scramble log order.
- o := []DeviceId{"VeyronTab", "VeyronLaptop", "VeyronPhone", "VeyronDesktop"}
- srcVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 2,
- "VeyronDesktop": 3,
- "VeyronLaptop": 4,
- }
- remoteVec = GenVector{
- "VeyronTab": 0,
- "VeyronPhone": 2,
- "VeyronDesktop": 0,
- }
- expGens = make([]*genOrder, 8)
- for i := 0; i < 8; i++ {
- switch {
- case i < 1:
- expGens[i] = &genOrder{
- devID: "VeyronTab",
- srID: srid,
- genID: GenId(i + 1),
- order: uint32(i),
- }
- case i >= 1 && i < 5:
- expGens[i] = &genOrder{
- devID: "VeyronLaptop",
- srID: srid,
- genID: GenId(i),
- order: uint32(i),
- }
- default:
- expGens[i] = &genOrder{
- devID: "VeyronDesktop",
- srID: srid,
- genID: GenId(i - 4),
- order: uint32(i - 5 + 7),
- }
- }
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, o, expGens)
-
- // remote has no updates.
- srcVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 2,
- "VeyronDesktop": 3,
- "VeyronLaptop": 4,
- }
- remoteVec = GenVector{
- "VeyronPhone": 0,
- }
- expGens = make([]*genOrder, 10)
- for i := 0; i < 10; i++ {
- switch {
- case i < 1:
- expGens[i] = &genOrder{
- devID: "VeyronTab",
- srID: srid,
- genID: GenId(i + 1),
- order: uint32(i),
- }
- case i >= 1 && i < 3:
- expGens[i] = &genOrder{
- devID: "VeyronPhone",
- srID: srid,
- genID: GenId(i),
- order: uint32(i),
- }
- case i >= 3 && i < 6:
- expGens[i] = &genOrder{
- devID: "VeyronDesktop",
- srID: srid,
- genID: GenId(i - 2),
- order: uint32(i),
- }
- default:
- expGens[i] = &genOrder{
- devID: "VeyronLaptop",
- srID: srid,
- genID: GenId(i - 5),
- order: uint32(i),
- }
- }
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
-
- // Test with reclaimVec fast-fwded.
- reclaimVec = GenVector{
- "VeyronPhone": 1,
- "VeyronLaptop": 2,
- }
- srcVec = GenVector{
- "VeyronTab": 1,
- "VeyronPhone": 2,
- "VeyronDesktop": 3,
- "VeyronLaptop": 4,
- }
- remoteVec = GenVector{
- "VeyronPhone": 0,
- }
- expGens = make([]*genOrder, 7)
- for i := 0; i < 7; i++ {
- switch {
- case i < 1:
- expGens[i] = &genOrder{
- devID: "VeyronTab",
- srID: srid,
- genID: GenId(i + 1),
- order: uint32(i),
- }
- case i == 1:
- expGens[i] = &genOrder{
- devID: "VeyronPhone",
- srID: srid,
- genID: GenId(i + 1),
- order: uint32(i + 1),
- }
- case i >= 2 && i < 5:
- expGens[i] = &genOrder{
- devID: "VeyronDesktop",
- srID: srid,
- genID: GenId(i - 1),
- order: uint32(i + 1),
- }
- default:
- expGens[i] = &genOrder{
- devID: "VeyronLaptop",
- srID: srid,
- genID: GenId(i - 2),
- order: uint32(i + 3),
- }
- }
- }
- setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
-}
-
-// setupAndTestDiff is an utility function to test diffing generation vectors.
-func setupAndTestDiff(t *testing.T, srcVec, remoteVec, reclaimVec GenVector, logOrder []DeviceId, expGens []*genOrder) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- logfile := getFileName()
- defer os.Remove(logfile)
-
- var srcid DeviceId = "VeyronTab"
- var destid DeviceId = "VeyronPhone"
-
- var err error
- s := &syncd{id: srcid}
- s.log, err = openILog(logfile, s)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
- dtab, err := openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
- dtab.head.ReclaimVec = reclaimVec
-
- srid, err := strToObjId("1234")
- if err != nil {
- t.Fatal(err)
- }
- // Populate generations in log order.
- var order uint32
- for _, k := range logOrder {
- v, ok := (srcVec)[k]
- if !ok {
- t.Errorf("Cannot find key %s in srcVec %v", k, srcVec)
- }
- for i := GenId(1); i <= v; i++ {
- val := &genMetadata{Pos: order}
- if err := s.log.putGenMetadata(k, srid, i, val); err != nil {
- t.Errorf("Cannot put object %s:%d in log file %s, err %v", k, v, logfile, err)
- }
- order++
- }
- }
- gens, err := dtab.diffGenVectors(srcVec, remoteVec, srid)
- if err != nil {
- t.Fatalf("DiffGenVectors() failed src: %s %v dest: %s %v in devTable file %s, err %v",
- srcid, srcVec, destid, remoteVec, devfile, err)
- }
-
- if !reflect.DeepEqual(gens, expGens) {
- t.Fatalf("Data mismatch for genorder %v instead of %v, src %v dest %v reclaim %v",
- gens, expGens, srcVec, remoteVec, reclaimVec)
- }
-
- if err := dtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// TestGetOldestGen tests obtaining generations from reclaimVec.
-func TestGetOldestGen(t *testing.T) {
- devfile := getFileName()
- defer os.Remove(devfile)
-
- var srcid DeviceId = "VeyronTab"
- s := &syncd{id: srcid}
- var err error
- s.devtab, err = openDevTable(devfile, s)
- if err != nil {
- t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
- }
-
- if s.devtab.getOldestGen(srcid) != 0 {
- t.Errorf("Cannot get generation for device %s in devTable file %s",
- srcid, devfile)
- }
-
- var destid DeviceId = "VeyronPhone"
- if s.devtab.getOldestGen(destid) != 0 {
- t.Errorf("Cannot get generation for device %s in devTable file %s",
- destid, devfile)
- }
-
- s.devtab.head.ReclaimVec[srcid] = 10
- if s.devtab.getOldestGen(srcid) != 10 {
- t.Errorf("Cannot get generation for device %s in devTable file %s",
- srcid, devfile)
- }
- if s.devtab.getOldestGen(destid) != 0 {
- t.Errorf("Cannot get generation for device %s in devTable file %s",
- destid, devfile)
- }
-
- if err := s.devtab.close(); err != nil {
- t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
- }
-}
-
-// // TestComputeReclaimVector tests reclaim vector computation.
-// func TestComputeReclaimVector(t *testing.T) {
-// devArr := []DeviceId{"VeyronTab", "VeyronPhone", "VeyronDesktop", "VeyronLaptop"}
-// genVecArr := make([]GenVector, 4)
-//
-// // All devices are up-to-date.
-// genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// setupAndTestReclaimVector(t, devArr, genVecArr, nil, genVecArr[0])
-//
-// // Every device is missing at least one other device. Not possible to gc.
-// genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronLaptop": 4}
-// genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3}
-// genVecArr[3] = GenVector{"VeyronDesktop": 3, "VeyronLaptop": 4}
-// expReclaimVec := GenVector{"VeyronTab": 0, "VeyronPhone": 0, "VeyronDesktop": 0, "VeyronLaptop": 0}
-// setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
-//
-// // All devices know at least one generation from other devices.
-// genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 2, "VeyronLaptop": 2}
-// genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 3, "VeyronLaptop": 1}
-// genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 1, "VeyronLaptop": 4}
-// expReclaimVec = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 1, "VeyronLaptop": 1}
-// setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
-//
-// // One device is missing from one other device.
-// genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 2}
-// genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 3, "VeyronLaptop": 1}
-// genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 1, "VeyronLaptop": 4}
-// expReclaimVec = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 1, "VeyronLaptop": 0}
-// setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
-//
-// // All devices know at least "n" generations from other devices.
-// var n GenId = 10
-// genVecArr[0] = GenVector{"VeyronTab": n + 10, "VeyronPhone": n,
-// "VeyronDesktop": n + 8, "VeyronLaptop": n + 4}
-// genVecArr[1] = GenVector{"VeyronTab": n + 6, "VeyronPhone": n + 10,
-// "VeyronDesktop": n, "VeyronLaptop": n + 3}
-// genVecArr[2] = GenVector{"VeyronTab": n, "VeyronPhone": n + 2,
-// "VeyronDesktop": n + 10, "VeyronLaptop": n}
-// genVecArr[3] = GenVector{"VeyronTab": n + 7, "VeyronPhone": n + 1,
-// "VeyronDesktop": n + 5, "VeyronLaptop": n + 10}
-// expReclaimVec = GenVector{"VeyronTab": n, "VeyronPhone": n, "VeyronDesktop": n, "VeyronLaptop": n}
-// setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
-//
-// // Never contacted a device.
-// devArr = []DeviceId{"VeyronTab", "VeyronDesktop", "VeyronLaptop"}
-// genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// expReclaimVec = GenVector{"VeyronTab": 0, "VeyronPhone": 0, "VeyronDesktop": 0, "VeyronLaptop": 0}
-// setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
-//
-// // Start from existing reclaim vector.
-// devArr = []DeviceId{"VeyronTab", "VeyronPhone", "VeyronDesktop", "VeyronLaptop"}
-// reclaimVec := GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// genVecArr[0] = GenVector{"VeyronTab": 6, "VeyronPhone": 6, "VeyronDesktop": 6, "VeyronLaptop": 6}
-// genVecArr[1] = GenVector{"VeyronTab": 6, "VeyronPhone": 6, "VeyronDesktop": 3, "VeyronLaptop": 6}
-// genVecArr[2] = GenVector{"VeyronTab": 6, "VeyronPhone": 6, "VeyronDesktop": 6, "VeyronLaptop": 4}
-// genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 6, "VeyronLaptop": 6}
-//
-// setupAndTestReclaimVector(t, devArr, genVecArr, reclaimVec, reclaimVec)
-// }
-//
-// // setupAndTestReclaimVector is an utility function to test reclaim vector computation.
-// func setupAndTestReclaimVector(t *testing.T, devArr []DeviceId, genVecArr []GenVector, reclaimStart, expReclaimVec GenVector) {
-// devfile := getFileName()
-// defer os.Remove(devfile)
-//
-// s := &syncd{id: "VeyronTab"}
-// dtab, err := openDevTable(devfile, s)
-// if err != nil {
-// t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
-// }
-// if reclaimStart != nil {
-// dtab.head.ReclaimVec = reclaimStart
-// }
-//
-// for i := range devArr {
-// if err := dtab.putGenVec(devArr[i], genVecArr[i]); err != nil {
-// t.Errorf("Cannot put object %s (%v) in devTable file %s, err %v",
-// devArr[i], genVecArr[i], devfile, err)
-// }
-// }
-//
-// reclaimVec, err := dtab.computeReclaimVector()
-// if err != nil {
-// t.Fatalf("computeReclaimVector() failed devices: %v, vectors: %v in devTable file %s, err %v",
-// devArr, genVecArr, devfile, err)
-// }
-//
-// if !reflect.DeepEqual(reclaimVec, expReclaimVec) {
-// t.Fatalf("Data mismatch for reclaimVec %v instead of %v",
-// reclaimVec, expReclaimVec)
-// }
-//
-// if err := dtab.close(); err != nil {
-// t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
-// }
-// }
-//
-// // TestAddDevice tests adding a device to the devTable.
-// func TestAddDevice(t *testing.T) {
-// devfile := getFileName()
-// defer os.Remove(devfile)
-//
-// s := &syncd{id: "VeyronPhone"}
-// dtab, err := openDevTable(devfile, s)
-// if err != nil {
-// t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
-// }
-//
-// var dev DeviceId = "VeyronLaptop"
-// if err := dtab.addDevice(dev); err != nil {
-// t.Fatalf("Cannot add new device in devTable file %s, err %v", devfile, err)
-// }
-//
-// vec, err := dtab.getGenVec(dev)
-// if err != nil || vec == nil {
-// t.Fatalf("GetGenVec() can not find object %s in devTable file %s, err %v",
-// dev, devfile, err)
-// }
-// expVec := GenVector{dev: 0}
-// if !reflect.DeepEqual(vec, expVec) {
-// t.Errorf("Data mismatch for object %s in devTable file %s: %v instead of %v",
-// dev, devfile, vec, expVec)
-// }
-//
-// vec, err = dtab.getGenVec(dtab.s.id)
-// if err != nil || vec == nil {
-// t.Fatalf("GetGenVec() can not find object %s in devTable file %s, err %v",
-// dtab.s.id, devfile, err)
-// }
-// expVec = GenVector{dtab.s.id: 0, dev: 0}
-// if !reflect.DeepEqual(vec, expVec) {
-// t.Errorf("Data mismatch for object %s in devTable file %s: %v instead of %v",
-// dtab.s.id, devfile, vec, expVec)
-// }
-//
-// expVec = GenVector{dtab.s.id: 10, "VeyronDesktop": 40, dev: 80}
-// if err := dtab.putGenVec(dtab.s.id, expVec); err != nil {
-// t.Fatalf("PutGenVec() can not put object %s in devTable file %s, err %v",
-// dtab.s.id, devfile, err)
-// }
-// dev = "VeyronTab"
-// if err := dtab.addDevice(dev); err != nil {
-// t.Fatalf("Cannot add new device in devTable file %s, err %v", devfile, err)
-// }
-// expVec[dev] = 0
-//
-// vec, err = dtab.getGenVec(dtab.s.id)
-// if err != nil || vec == nil {
-// t.Fatalf("GetGenVec() can not find object %s in devTable file %s, err %v",
-// dtab.s.id, devfile, err)
-// }
-// if !reflect.DeepEqual(vec, expVec) {
-// t.Errorf("Data mismatch for object %s in devTable file %s: %v instead of %v",
-// dtab.s.id, devfile, vec, expVec)
-// }
-//
-// if err := dtab.close(); err != nil {
-// t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
-// }
-// }
-//
-// // TestUpdateReclaimVec tests updating the reclaim vector.
-// func TestUpdateReclaimVec(t *testing.T) {
-// devfile := getFileName()
-// defer os.Remove(devfile)
-//
-// s := &syncd{id: "VeyronPhone"}
-// dtab, err := openDevTable(devfile, s)
-// if err != nil {
-// t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
-// }
-//
-// minGens := GenVector{"VeyronTab": 1, "VeyronDesktop": 3, "VeyronLaptop": 4}
-// if err := dtab.updateReclaimVec(minGens); err != nil {
-// t.Fatalf("Cannot update reclaimvec in devTable file %s, err %v", devfile, err)
-// }
-// expVec := GenVector{dtab.s.id: 0, "VeyronTab": 0, "VeyronDesktop": 2, "VeyronLaptop": 3}
-// if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
-// t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
-// devfile, dtab.head.ReclaimVec, expVec)
-// }
-//
-// dtab.head.ReclaimVec[DeviceId("VeyronTab")] = 4
-// minGens = GenVector{"VeyronTab": 1}
-// if err := dtab.updateReclaimVec(minGens); err == nil {
-// t.Fatalf("Update reclaimvec didn't fail in devTable file %s", devfile)
-// }
-//
-// minGens = GenVector{"VeyronTab": 5}
-// if err := dtab.updateReclaimVec(minGens); err != nil {
-// t.Fatalf("Cannot update reclaimvec in devTable file %s, err %v", devfile, err)
-// }
-// expVec = GenVector{dtab.s.id: 0, "VeyronTab": 4, "VeyronDesktop": 2, "VeyronLaptop": 3}
-// if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
-// t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
-// devfile, dtab.head.ReclaimVec, expVec)
-// }
-//
-// if err := dtab.close(); err != nil {
-// t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
-// }
-// }
diff --git a/services/syncbase/sync/ilog.go b/services/syncbase/sync/ilog.go
deleted file mode 100644
index 9e35c24..0000000
--- a/services/syncbase/sync/ilog.go
+++ /dev/null
@@ -1,627 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Package vsync provides veyron sync ILog utility functions. ILog
-// (Indexed Log) provides log functionality with indexing support.
-// ILog stores log records that are locally generated or obtained over
-// the network. Indexing is needed since sync needs to selectively
-// retrieve log records that belong to a particular device, syncroot
-// and generation during synchronization.
-//
-// When a device receives a request to send log records, it first
-// computes the missing generations between itself and the incoming
-// request on a per-syncroot basis. It then sends all the log records
-// belonging to each missing generation. A device that receives log
-// records over the network replays all the records received from
-// another device in a single batch. Each replayed log record adds a
-// new version to the dag of the object contained in the log
-// record. At the end of replaying all the log records, conflict
-// detection and resolution is carried out for all the objects learned
-// during this iteration. Conflict detection and resolution is carried
-// out after a batch of log records are replayed, instead of
-// incrementally after each record is replayed, to avoid repeating
-// conflict resolution already performed by other devices.
-//
-// New log records are created when objects in the local store are
-// created/updated. Local log records are also replayed to keep the
-// per-object dags consistent with the local store state.
-//
-// Note that syncgroups are mainly tracked between syncd/store and the
-// client. Sync-related metadata (log records, generations and
-// generation vectors) is unaware of syncgroups, and uses syncroots
-// instead. Each syncgroup contains a root object and a unique root
-// object ID. In case of peer syncgroups, all the peer syncgroups
-// contain the same root object ID. Sync translates any given
-// syncgroup to a syncroot rooted at this root object ID and syncs all
-// the syncroots the device is part of. Thus, although a device may
-// be part of more than 1 peer syncgroup, at the sync level, a single
-// syncroot is synced on behalf of all the peer syncgroups.
-//
-// Implementation notes: ILog records are stored in a persistent K/V
-// database in the current implementation. ILog db consists of 3
-// tables:
-// ** records: table consists of all the log records indexed
-// by deviceid:srid:genid:lsn referring to
-// the device that creates the log record
-// the root oid of the syncroot to which the log record belongs
-// the generation on the syncroot the log record is part of
-// the sequence number in that generation
-// Note that lsn in each generation starts from 0 and genid starts from 1.
-// ** gens: table consists of the generation metadata for each
-// generation, and is indexed by deviceid:srid:genid.
-// ** head: table consists of the log header.
-import (
- "errors"
- "fmt"
- "strconv"
- "strings"
-
- "v.io/x/lib/vlog"
-)
-
-var (
- errNoUpdates = errors.New("no new local updates")
- errInvalidLog = errors.New("invalid log db")
-)
-
-// curLocalGen contains metadata re. the current local generation for a SyncRoot.
-type curLocalGen struct {
- CurGenNum GenId // generation id for a SyncRoot's current generation.
- CurSeqNum SeqNum // log sequence number for a SyncRoot's current generation.
-}
-
-// iLogHeader contains the log header metadata.
-type iLogHeader struct {
- CurSRGens map[ObjId]*curLocalGen // local generation info per SyncRoot.
- Curorder uint32 // position in log for the next generation.
-}
-
-// genMetadata contains the metadata for a generation.
-type genMetadata struct {
- // All generations stored in the log are ordered wrt each
- // other and this order needs to be preserved.
- // Position of this generation in the log.
- Pos uint32
-
- // Number of log records in this generation still stored in the log.
- // This count is used during garbage collection.
- Count uint64
-
- // Maximum SeqNum that was part of this generation.
- // This is useful during garbage collection to track any unclaimed log records.
- MaxSeqNum SeqNum
-}
-
-// iLog contains the metadata for the ILog db.
-type iLog struct {
- fname string // file pathname.
- db *kvdb // underlying k/v db.
-
- // Key:deviceid:srid:genid:lsn Value:LogRecord. Pointer to
- // the "records" table in the kvdb. Contains log records.
- records *kvtable
-
- // Key:deviceid:srid:genid Value:genMetadata. Pointer to the
- // "gens" table in the kvdb. Contains generation metadata for
- // each generation.
- gens *kvtable
-
- // Key:"Head" Value:iLogHeader. Pointer to the "header" table
- // in the kvdb. Contains logheader.
- header *kvtable
-
- head *iLogHeader // log head cached in memory.
-
- s *syncd // pointer to the sync daemon object.
-}
-
-// openILog opens or creates a ILog for the given filename.
-func openILog(filename string, sin *syncd) (*iLog, error) {
- ilog := &iLog{
- fname: filename,
- head: &iLogHeader{},
- s: sin,
- }
- // Open the file and create it if it does not exist.
- // Also initialize the kvdb and its three collections.
- db, tbls, err := kvdbOpen(filename, []string{"records", "gens", "header"})
- if err != nil {
- return nil, err
- }
-
- ilog.db = db
- ilog.records = tbls[0]
- ilog.gens = tbls[1]
- ilog.header = tbls[2]
-
- // If header already exists in db, read it back from db.
- if ilog.hasHead() {
- if err := ilog.getHead(); err != nil {
- ilog.db.close() // this also closes the tables.
- return nil, err
- }
- } else {
- ilog.head.CurSRGens = make(map[ObjId]*curLocalGen)
- }
-
- return ilog, nil
-}
-
-// close closes the ILog and invalidate its struct.
-func (l *iLog) close() error {
- if l.db == nil {
- return errInvalidLog
- }
- // Flush the dirty data.
- if err := l.flush(); err != nil {
- return err
- }
-
- l.db.close() // this also closes the tables.
-
- *l = iLog{} // zero out the ILog struct.
- return nil
-}
-
-// flush flushes the ILog db to disk.
-func (l *iLog) flush() error {
- if l.db == nil {
- return errInvalidLog
- }
- // Set the head from memory before flushing.
- if err := l.putHead(); err != nil {
- return err
- }
-
- l.db.flush()
-
- return nil
-}
-
-// initSyncRoot initializes the log header for this SyncRoot.
-func (l *iLog) initSyncRoot(srid ObjId) error {
- if l.db == nil {
- return errInvalidLog
- }
- if _, ok := l.head.CurSRGens[srid]; ok {
- return fmt.Errorf("syncroot already exists %v", srid)
- }
- // First generation to be created is generation 1. A
- // generation of 0 represents no updates on the device.
- l.head.CurSRGens[srid] = &curLocalGen{CurGenNum: 1}
- return nil
-}
-
-// delSyncRoot deletes the log header for this SyncRoot.
-func (l *iLog) delSyncRoot(srid ObjId) error {
- if l.db == nil {
- return errInvalidLog
- }
- if _, ok := l.head.CurSRGens[srid]; !ok {
- return fmt.Errorf("syncroot doesn't exist %v", srid)
- }
-
- delete(l.head.CurSRGens, srid)
- return nil
-}
-
-// putHead puts the log head into the ILog db.
-func (l *iLog) putHead() error {
- return l.header.set("Head", l.head)
-}
-
-// getHead gets the log head from the ILog db.
-func (l *iLog) getHead() error {
- if l.head == nil {
- return errors.New("nil log header")
- }
- if err := l.header.get("Head", l.head); err != nil {
- return err
- }
- // When we put an empty map in kvdb, we get back a "nil" map
- // (due to VOM encoding/decoding). To keep putHead/getHead
- // symmetrical, we add this additional initialization.
- if l.head.CurSRGens == nil {
- l.head.CurSRGens = make(map[ObjId]*curLocalGen)
- }
- return nil
-}
-
-// hasHead returns true if the ILog db has a log head.
-func (l *iLog) hasHead() bool {
- return l.header.hasKey("Head")
-}
-
-// logRecKey creates a key for a log record.
-func logRecKey(devid DeviceId, srid ObjId, gnum GenId, lsn SeqNum) string {
- return fmt.Sprintf("%s:%s:%d:%d", devid, srid.String(), uint64(gnum), uint64(lsn))
-}
-
-// strToGenId converts a string to GenId.
-func strToGenId(genIDStr string) (GenId, error) {
- id, err := strconv.ParseUint(genIDStr, 10, 64)
- if err != nil {
- return 0, err
- }
- return GenId(id), nil
-}
-
-// strToObjId converts a string to ObjId.
-func strToObjId(objIDStr string) (ObjId, error) {
- return ObjId(objIDStr), nil
-}
-
-// splitLogRecKey splits a : separated logrec key into its components.
-func splitLogRecKey(key string) (DeviceId, ObjId, GenId, SeqNum, error) {
- args := strings.Split(key, ":")
- if len(args) != 4 {
- return "", NoObjId, 0, 0, fmt.Errorf("bad logrec key %s", key)
- }
- srid, err := strToObjId(args[1])
- if err != nil {
- return "", NoObjId, 0, 0, err
- }
- gnum, err := strToGenId(args[2])
- if err != nil {
- return "", NoObjId, 0, 0, err
- }
- lsn, err := strconv.ParseUint(args[3], 10, 64)
- if err != nil {
- return "", NoObjId, 0, 0, err
- }
- return DeviceId(args[0]), srid, gnum, SeqNum(lsn), nil
-}
-
-// putLogRec puts the log record into the ILog db.
-func (l *iLog) putLogRec(rec *LogRec) (string, error) {
- if l.db == nil {
- return "", errInvalidLog
- }
- key := logRecKey(rec.DevId, rec.SyncRootId, rec.GenNum, rec.SeqNum)
- return key, l.records.set(key, rec)
-}
-
-// getLogRec gets the log record from the ILog db.
-func (l *iLog) getLogRec(devid DeviceId, srid ObjId, gnum GenId, lsn SeqNum) (*LogRec, error) {
- if l.db == nil {
- return nil, errInvalidLog
- }
- key := logRecKey(devid, srid, gnum, lsn)
- var rec LogRec
- if err := l.records.get(key, &rec); err != nil {
- return nil, err
- }
- return &rec, nil
-}
-
-// hasLogRec returns true if the ILog db has a log record matching (devid, srid, gnum, lsn).
-func (l *iLog) hasLogRec(devid DeviceId, srid ObjId, gnum GenId, lsn SeqNum) bool {
- if l.db == nil {
- return false
- }
- key := logRecKey(devid, srid, gnum, lsn)
- return l.records.hasKey(key)
-}
-
-// delLogRec deletes the log record matching (devid, srid, gnum, lsn) from the ILog db.
-func (l *iLog) delLogRec(devid DeviceId, srid ObjId, gnum GenId, lsn SeqNum) error {
- if l.db == nil {
- return errInvalidLog
- }
- key := logRecKey(devid, srid, gnum, lsn)
- return l.records.del(key)
-}
-
-// generationKey creates a key for a generation.
-func generationKey(devid DeviceId, srid ObjId, gnum GenId) string {
- return fmt.Sprintf("%s:%s:%d", devid, srid.String(), gnum)
-}
-
-// splitGenerationKey splits a : separated generation key into its components.
-func splitGenerationKey(key string) (DeviceId, ObjId, GenId, error) {
- args := strings.Split(key, ":")
- if len(args) != 3 {
- return "", NoObjId, 0, fmt.Errorf("bad generation key %s", key)
- }
- srid, err := strToObjId(args[1])
- if err != nil {
- return "", NoObjId, 0, err
- }
- gnum, err := strToGenId(args[2])
- if err != nil {
- return "", NoObjId, 0, err
- }
- return DeviceId(args[0]), srid, gnum, nil
-}
-
-// putGenMetadata puts the metadata of the generation (devid, srid, gnum) into the ILog db.
-func (l *iLog) putGenMetadata(devid DeviceId, srid ObjId, gnum GenId, val *genMetadata) error {
- key := generationKey(devid, srid, gnum)
- return l.gens.set(key, val)
-}
-
-// getGenMetadata gets the metadata of the generation (devid, srid, gnum) from the ILog db.
-func (l *iLog) getGenMetadata(devid DeviceId, srid ObjId, gnum GenId) (*genMetadata, error) {
- if l.db == nil {
- return nil, errInvalidLog
- }
- key := generationKey(devid, srid, gnum)
- var val genMetadata
- if err := l.gens.get(key, &val); err != nil {
- return nil, err
- }
- return &val, nil
-}
-
-// hasGenMetadata returns true if the ILog db has the generation (devid, srid, gnum).
-func (l *iLog) hasGenMetadata(devid DeviceId, srid ObjId, gnum GenId) bool {
- key := generationKey(devid, srid, gnum)
- return l.gens.hasKey(key)
-}
-
-// delGenMetadata deletes the generation (devid, srid, gnum) metadata from the ILog db.
-func (l *iLog) delGenMetadata(devid DeviceId, srid ObjId, gnum GenId) error {
- if l.db == nil {
- return errInvalidLog
- }
- key := generationKey(devid, srid, gnum)
- return l.gens.del(key)
-}
-
-// getLocalGenInfo gets the local generation info for the given syncroot.
-func (l *iLog) getLocalGenInfo(srid ObjId) (*curLocalGen, error) {
- gen, ok := l.head.CurSRGens[srid]
- if !ok {
- return nil, fmt.Errorf("no gen info found for srid %s", srid.String())
- }
- return gen, nil
-}
-
-// createLocalLogRec creates a new local log record of type NodeRec.
-func (l *iLog) createLocalLogRec(obj ObjId, vers Version,
- par []Version, val *LogValue, srid ObjId) (*LogRec, error) {
- gen, err := l.getLocalGenInfo(srid)
- if err != nil {
- return nil, err
- }
- rec := &LogRec{
- DevId: l.s.id,
- SyncRootId: srid,
- GenNum: gen.CurGenNum,
- SeqNum: gen.CurSeqNum,
- RecType: NodeRec,
-
- ObjId: obj,
- CurVers: vers,
- Parents: par,
- Value: *val,
- }
-
- // Increment the SeqNum for the local log.
- gen.CurSeqNum++
-
- return rec, nil
-}
-
-// createLocalLinkLogRec creates a new local log record of type LinkRec.
-func (l *iLog) createLocalLinkLogRec(obj ObjId, vers, par Version, srid ObjId) (*LogRec, error) {
- gen, err := l.getLocalGenInfo(srid)
- if err != nil {
- return nil, err
- }
- rec := &LogRec{
- DevId: l.s.id,
- SyncRootId: srid,
- GenNum: gen.CurGenNum,
- SeqNum: gen.CurSeqNum,
- RecType: LinkRec,
-
- ObjId: obj,
- CurVers: vers,
- Parents: []Version{par},
- Value: LogValue{},
- }
-
- // Increment the SeqNum for the local log.
- gen.CurSeqNum++
-
- return rec, nil
-}
-
-// createRemoteGeneration adds a new remote generation.
-func (l *iLog) createRemoteGeneration(dev DeviceId, srid ObjId, gnum GenId, gen *genMetadata) error {
- if l.db == nil {
- return errInvalidLog
- }
-
- if gen.Count != uint64(gen.MaxSeqNum+1) {
- return errors.New("mismatch in count and lsn")
- }
-
- vlog.VI(2).Infof("createRemoteGeneration:: dev %s srid %s gen %d %v", dev, srid.String(), gnum, gen)
-
- gen.Pos = l.head.Curorder
- l.head.Curorder++
-
- return l.putGenMetadata(dev, srid, gnum, gen)
-}
-
-// createLocalGeneration creates a new local generation.
-func (l *iLog) createLocalGeneration(srid ObjId) (GenId, error) {
- if l.db == nil {
- return 0, errInvalidLog
- }
-
- g, err := l.getLocalGenInfo(srid)
- if err != nil {
- return 0, err
- }
-
- gnum := g.CurGenNum
-
- // If there are no updates, there will be no new generation.
- if g.CurSeqNum == 0 {
- return gnum - 1, errNoUpdates
- }
-
- // Add the current generation to the db.
- val := &genMetadata{
- Pos: l.head.Curorder,
- Count: uint64(g.CurSeqNum),
- MaxSeqNum: g.CurSeqNum - 1,
- }
-
- err = l.putGenMetadata(l.s.id, srid, gnum, val)
-
- vlog.VI(2).Infof("createLocalGeneration:: created srid %s gen %d %v", srid.String(), gnum, val)
- // Move to the next generation irrespective of err.
- l.head.Curorder++
- g.CurGenNum++
- g.CurSeqNum = 0
-
- return gnum, err
-}
-
-// processWatchRecord processes new object versions obtained from the local store.
-func (l *iLog) processWatchRecord(objID ObjId, vers, parent Version, val *LogValue, srid ObjId) error {
- if l.db == nil {
- return errInvalidLog
- }
-
- vlog.VI(2).Infof("processWatchRecord:: adding for object %v %v (srid %v)", objID, vers, srid)
-
- if !srid.IsValid() {
- return errors.New("invalid syncroot id")
- }
-
- // Filter out the echo from the watcher. When syncd puts mutations into store,
- // it hears back these mutations once again on the watch stream. We need to
- // filter out these echoes and only process brand new watch changes.
- if vers != NoVersion {
- // Check if the object's vers already exists in the DAG.
- if l.s.dag.hasNode(objID, vers) {
- return nil
- }
-
- // When we successfully join a SyncGroup, Store
- // creates an empty directory with object ID as the
- // rootObjId of the SyncGroup joined. We do not care
- // about this version of the directory. We will start
- // accepting local mutations on this object only after
- // we hear about it remotely first.
- //
- // NOTE: Since this new directory is protected from
- // any local updates via an Permissions until after the first
- // sync, waiting to hear remotely first works without
- // dropping any updates. However, we cache in the
- // "priv" table its version in the Store so that we
- // can correctly specify prior version when we put
- // mutations into the store.
- if _, err := l.s.dag.getHead(objID); err != nil && objID == srid {
- priv := &privNode{ /*Mutation: &val.Mutation, */ SyncTime: val.SyncTime, TxId: val.TxId, TxCount: val.TxCount}
- return l.s.dag.setPrivNode(objID, priv)
- }
- } else {
- // Check if the parent version has a deleted
- // descendant already in the DAG.
- if l.s.dag.hasDeletedDescendant(objID, parent) {
- return nil
- }
- }
-
- var pars []Version
- if parent != NoVersion {
- pars = []Version{parent}
- }
-
- // If the current version is a deletion, generate a new version number.
- if val.Delete {
- if vers != NoVersion {
- return fmt.Errorf("deleted vers is %v", vers)
- }
- vers = NewVersion()
- //val.Mutation.Version = vers
- }
-
- // Create a log record from Watch's Change Record.
- rec, err := l.createLocalLogRec(objID, vers, pars, val, srid)
- if err != nil {
- return err
- }
-
- // Insert the new log record into the log.
- logKey, err := l.putLogRec(rec)
- if err != nil {
- return err
- }
-
- // Insert the new log record into dag.
- if err = l.s.dag.addNode(rec.ObjId, rec.CurVers, false, val.Delete, rec.Parents, logKey, val.TxId); err != nil {
- return err
- }
-
- // Move the head.
- return l.s.dag.moveHead(rec.ObjId, rec.CurVers)
-}
-
-// dump writes to the log file information on ILog internals.
-func (l *iLog) dump() {
- if l.db == nil {
- return
- }
-
- vlog.VI(1).Infof("DUMP: ILog: #SR %d, cur %d", len(l.head.CurSRGens), l.head.Curorder)
- for sr, gen := range l.head.CurSRGens {
- vlog.VI(1).Infof("DUMP: ILog: SR %v: gen %v, lsn %v", sr, gen.CurGenNum, gen.CurSeqNum)
- }
-
- // Find for each (devid, srid) pair its lowest and highest generation numbers.
- type genInfo struct{ min, max GenId }
- devs := make(map[DeviceId]map[ObjId]*genInfo)
-
- l.gens.keyIter(func(genKey string) {
- devid, srid, gnum, err := splitGenerationKey(genKey)
- if err != nil {
- return
- }
-
- srids, ok := devs[devid]
- if !ok {
- srids = make(map[ObjId]*genInfo)
- devs[devid] = srids
- }
-
- info, ok := srids[srid]
- if ok {
- if gnum > info.max {
- info.max = gnum
- }
- if gnum < info.min {
- info.min = gnum
- }
- } else {
- srids[srid] = &genInfo{min: gnum, max: gnum}
- }
- })
-
- // For each (devid, srid) pair dump its generation info in order from
- // min to max generation numbers, inclusive.
- for devid, srids := range devs {
- for srid, info := range srids {
- for gnum := info.min; gnum <= info.max; gnum++ {
- meta, err := l.getGenMetadata(devid, srid, gnum)
- if err == nil {
- vlog.VI(1).Infof(
- "DUMP: ILog: gen (%v, %v, %v): pos %v, count %v, maxlsn %v",
- devid, srid, gnum, meta.Pos, meta.Count, meta.MaxSeqNum)
- } else {
- vlog.VI(1).Infof("DUMP: ILog: gen (%v, %v, %v): missing generation",
- devid, srid, gnum)
- }
- }
- }
- }
-}
diff --git a/services/syncbase/sync/ilog_test.go b/services/syncbase/sync/ilog_test.go
deleted file mode 100644
index ba12bf0..0000000
--- a/services/syncbase/sync/ilog_test.go
+++ /dev/null
@@ -1,1024 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Tests for the Veyron Sync ILog component.
-import (
- "fmt"
- "math/rand"
- "os"
- "reflect"
- "runtime"
- "testing"
-)
-
-// TestILogStore tests creating a backing file for ILog.
-func TestILogStore(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new ILog file %s, err %v", logfile, err)
- }
-
- fsize := getFileSize(logfile)
- if fsize < 0 {
- //t.Errorf("Log file %s not created", logfile)
- }
-
- if err := log.flush(); err != nil {
- t.Errorf("Cannot flush ILog file %s, err %v", logfile, err)
- }
-
- oldfsize := fsize
- fsize = getFileSize(logfile)
- if fsize <= oldfsize {
- //t.Errorf("Log file %s not flushed", logfile)
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close ILog file %s, err %v", logfile, err)
- }
-
- oldfsize = getFileSize(logfile)
-
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open existing log file %s, err %v", logfile, err)
- }
-
- fsize = getFileSize(logfile)
- if fsize != oldfsize {
- t.Errorf("Log file %s size changed across re-open (%d %d)", logfile, fsize, oldfsize)
- }
-
- if err := log.flush(); err != nil {
- t.Errorf("Cannot flush ILog file %s, err %v", logfile, err)
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close ILog file %s, err %v", logfile, err)
- }
-}
-
-// TestInvalidLog tests log methods on an invalid (closed) log ptr.
-func TestInvalidLog(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new ILog file %s, err %v", logfile, err)
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close ILog file %s, err %v", logfile, err)
- }
-
- validateError := func(err error, funcName string) {
- _, file, line, _ := runtime.Caller(1)
- if err == nil || err != errInvalidLog {
- t.Errorf("%s:%d %s() did not fail on a closed log: %v", file, line, funcName, err)
- }
- }
-
- err = log.close()
- validateError(err, "close")
-
- err = log.flush()
- validateError(err, "flush")
-
- srid := ObjId("haha")
-
- err = log.initSyncRoot(srid)
- validateError(err, "initSyncRoot")
-
- _, err = log.putLogRec(&LogRec{})
- validateError(err, "putLogRec")
-
- var devid DeviceId = "VeyronPhone"
- var gnum GenId = 1
- var lsn SeqNum
-
- _, err = log.getLogRec(devid, srid, gnum, lsn)
- validateError(err, "getLogRec")
-
- if log.hasLogRec(devid, srid, gnum, lsn) {
- t.Errorf("hasLogRec() did not fail on a closed log: %v", err)
- }
-
- err = log.delLogRec(devid, srid, gnum, lsn)
- validateError(err, "delLogRec")
-
- _, err = log.getGenMetadata(devid, srid, gnum)
- validateError(err, "getGenMetadata")
-
- err = log.delGenMetadata(devid, srid, gnum)
- validateError(err, "delGenMetadata")
-
- err = log.createRemoteGeneration(devid, srid, gnum, &genMetadata{})
- validateError(err, "createRemoteGeneration")
-
- _, err = log.createLocalGeneration(srid)
- validateError(err, "createLocalGeneration")
-
- err = log.processWatchRecord(ObjId("foobar"), 2, Version(999), &LogValue{}, srid)
- validateError(err, "processWatchRecord")
-
- // Harmless NOP.
- log.dump()
-}
-
-// TestPutGetLogHeader tests setting and getting log header across log open/close/reopen.
-func TestPutGetLogHeader(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- // In memory head should be initialized.
- if len(log.head.CurSRGens) != 0 || log.head.Curorder != 0 {
- t.Errorf("First time log create should reset header")
- }
-
- // No head should be there in db.
- if err = log.getHead(); err == nil {
- t.Errorf("getHead() found non-existent head in log file %s, err %v", logfile, err)
- }
-
- if log.hasHead() {
- t.Errorf("hasHead() found non-existent head in log file %s", logfile)
- }
-
- expVal := map[ObjId]*curLocalGen{ObjId("bla"): &curLocalGen{CurGenNum: 10, CurSeqNum: 100},
- ObjId("qwerty"): &curLocalGen{CurGenNum: 40, CurSeqNum: 80}}
-
- log.head = &iLogHeader{
- CurSRGens: expVal,
- Curorder: 1000,
- }
-
- if err := log.putHead(); err != nil {
- t.Errorf("Cannot put head %v in log file %s, err %v", log.head, logfile, err)
- }
-
- // Reset values.
- log.head.CurSRGens = nil
- log.head.Curorder = 0
-
- for i := 0; i < 2; i++ {
- if err := log.getHead(); err != nil {
- t.Fatalf("getHead() can not find head (i=%d) in log file %s, err %v", i, logfile, err)
- }
-
- if !log.hasHead() {
- t.Errorf("hasHead() can not find head (i=%d) in log file %s", i, logfile)
- }
-
- if !reflect.DeepEqual(log.head.CurSRGens, expVal) || log.head.Curorder != 1000 {
- t.Errorf("Data mismatch for head (i=%d) in log file %s: %v",
- i, logfile, log.head)
- }
-
- if i == 0 {
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- }
- }
-
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestPersistLogHeader tests that log header is automatically persisted across log open/close/reopen.
-func TestPersistLogHeader(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- // In memory head should be initialized.
- if len(log.head.CurSRGens) != 0 || log.head.Curorder != 0 {
- t.Errorf("First time log create should reset header")
- }
-
- expVal := map[ObjId]*curLocalGen{ObjId("blabla"): &curLocalGen{CurGenNum: 10, CurSeqNum: 100},
- ObjId("xyz"): &curLocalGen{CurGenNum: 40, CurSeqNum: 80}}
- log.head = &iLogHeader{
- CurSRGens: expVal,
- Curorder: 1000,
- }
-
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- // In memory head should be initialized from db.
- if !reflect.DeepEqual(log.head.CurSRGens, expVal) || log.head.Curorder != 1000 {
- t.Errorf("Data mismatch for head in log file %s: %v", logfile, log.head)
- }
-
- for sr := range expVal {
- expVal[sr] = &curLocalGen{CurGenNum: 1000, CurSeqNum: 10}
- break
- }
- log.head = &iLogHeader{
- CurSRGens: expVal,
- Curorder: 100,
- }
-
- if err := log.flush(); err != nil {
- t.Errorf("Cannot flush ILog file %s, err %v", logfile, err)
- }
-
- // Reset values.
- log.head.CurSRGens = nil
- log.head.Curorder = 0
-
- if err := log.getHead(); err != nil {
- t.Fatalf("getHead() can not find head in log file %s, err %v", logfile, err)
- }
-
- // In memory head should be initialized from db.
- if !reflect.DeepEqual(log.head.CurSRGens, expVal) || log.head.Curorder != 100 {
- t.Errorf("Data mismatch for head in log file %s: %v", logfile, log.head)
- }
-
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestLogInitDelSyncRoot tests initing and deleting a new SyncRoot.
-func TestLogInitDelSyncRoot(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- srid1 := ObjId("haha")
- if err := log.initSyncRoot(srid1); err != nil {
- t.Fatalf("Cannot create new SyncRoot %s, err %v", srid1.String(), err)
- }
- srid2 := ObjId("asdf")
- if err := log.initSyncRoot(srid2); err != nil {
- t.Fatalf("Cannot create new SyncRoot %s, err %v", srid2.String(), err)
- }
- if err := log.initSyncRoot(srid2); err == nil {
- t.Fatalf("Creating existing SyncRoot didn't fail %s, err %v", srid2.String(), err)
- }
- expGen := &curLocalGen{CurGenNum: 1, CurSeqNum: 0}
- expSRGens := map[ObjId]*curLocalGen{
- srid1: expGen,
- srid2: expGen,
- }
- if !reflect.DeepEqual(log.head.CurSRGens, expSRGens) {
- t.Errorf("Data mismatch for head in log file %s: %v instead of %v",
- logfile, log.head.CurSRGens, expSRGens)
- }
-
- if err := log.delSyncRoot(srid1); err != nil {
- t.Fatalf("Cannot delete SyncRoot %s, err %v", srid1.String(), err)
- }
- if err := log.delSyncRoot(srid2); err != nil {
- t.Fatalf("Cannot delete SyncRoot %s, err %v", srid1.String(), err)
- }
- if err := log.delSyncRoot(srid1); err == nil {
- t.Fatalf("Deleting non-existent SyncRoot didn't fail %s", srid1.String())
- }
- expSRGens = map[ObjId]*curLocalGen{}
- if !reflect.DeepEqual(log.head.CurSRGens, expSRGens) {
- t.Errorf("Data mismatch for head in log file %s: %v instead of %v",
- logfile, log.head.CurSRGens, expSRGens)
- }
-
- if err := log.putHead(); err != nil {
- t.Errorf("Cannot put head %v in log file %s, err %v", log.head, logfile, err)
- }
-
- // Reset values.
- log.head.CurSRGens = nil
- log.head.Curorder = 0
-
- if err := log.getHead(); err != nil {
- t.Fatalf("getHead() can not find head in log file %s, err %v", logfile, err)
- }
-
- if !reflect.DeepEqual(log.head.CurSRGens, expSRGens) {
- t.Errorf("Data mismatch for head in log file %s: %v instead of %v",
- logfile, log.head.CurSRGens, expSRGens)
- }
-}
-
-// TestStringAndParseKeys tests conversion to/from string for log record and generation keys.
-func TestStringAndParseKeys(t *testing.T) {
- var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
- randString := func(n int) string {
- b := make([]rune, n)
- for i := range b {
- b[i] = letters[rand.Intn(len(letters))]
- }
- return string(b)
- }
-
- for i := 0; i < 10; i++ {
- devid := DeviceId(randString(rand.Intn(20) + 1))
- srid := ObjId(fmt.Sprintf("haha-%d", i))
- gnum := GenId(rand.Int63())
- lsn := SeqNum(rand.Int63())
-
- devid1, srid1, gnum1, lsn1, err := splitLogRecKey(logRecKey(devid, srid, gnum, lsn))
- if err != nil || devid1 != devid || srid1 != srid || gnum1 != gnum || lsn1 != lsn {
- t.Fatalf("LogRec conversion failed in iter %d: got %s %v %v %v want %s %v %v %v, err %v",
- i, devid1, srid1, gnum1, lsn1, devid, srid, gnum, lsn, err)
- }
-
- devid2, srid2, gnum2, err := splitGenerationKey(generationKey(devid, srid, gnum))
- if err != nil || devid2 != devid || srid2 != srid || gnum2 != gnum {
- t.Fatalf("Gen conversion failed in iter %d: got %s %v %v want %s %v %v, err %v",
- i, devid2, srid2, gnum2, devid, srid, gnum, err)
- }
- }
-
- //badStrs := []string{"abc:3:4", "abc:123:4:5", "abc:1234:-1:10", "ijk:7890:1000:-1", "abc:3:4:5:6", "abc::3:4:5"}
- badStrs := []string{"abc:3:4", "abc:1234:-1:10", "ijk:7890:1000:-1", "abc:3:4:5:6", "abc::3:4:5"}
- for _, s := range badStrs {
- if _, _, _, _, err := splitLogRecKey(s); err == nil {
- t.Fatalf("LogRec conversion didn't fail for str %s", s)
- }
- }
-
- //badStrs = []string{"abc:3", "abc:123:4", "abc:1234:-1", "abc:3:4:5", "abc:4::5"}
- badStrs = []string{"abc:3", "abc:1234:-1", "abc:3:4:5", "abc:4::5"}
- for _, s := range badStrs {
- if _, _, _, err := splitGenerationKey(s); err == nil {
- t.Fatalf("Gen conversion didn't fail for str %s", s)
- }
- }
-}
-
-// TestPutGetLogRec tests setting and getting a log record across log open/close/reopen.
-func TestPutGetLogRec(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- var devid DeviceId = "VeyronTab"
- srid := ObjId("haha")
- var gnum GenId = 100
- var lsn SeqNum = 1000
-
- rec, err := log.getLogRec(devid, srid, gnum, lsn)
- if err == nil || rec != nil {
- t.Errorf("GetLogRec() found non-existent object %s:%s:%d:%d in log file %s: %v, err %v",
- devid, srid.String(), gnum, lsn, logfile, rec, err)
- }
-
- if log.hasLogRec(devid, srid, gnum, lsn) {
- t.Errorf("HasLogRec() found non-existent object %s:%s:%d:%d in log file %s",
- devid, srid.String(), gnum, lsn, logfile)
- }
-
- rec = &LogRec{
- DevId: devid,
- SyncRootId: srid,
- GenNum: gnum,
- SeqNum: lsn,
- ObjId: ObjId("bla"),
- CurVers: 2,
- Parents: []Version{0, 1},
- Value: LogValue{},
- }
-
- if _, err := log.putLogRec(rec); err != nil {
- t.Errorf("Cannot put object %s:%s:%d:%d (%v) in log file %s, err %v", devid, srid.String(), gnum, lsn, rec, logfile, err)
- }
-
- for i := 0; i < 2; i++ {
- curRec, err := log.getLogRec(devid, srid, gnum, lsn)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:%s:%d:%d (i=%d) in log file %s, err %v",
- devid, srid.String(), gnum, lsn, i, logfile, err)
- }
-
- if !log.hasLogRec(devid, srid, gnum, lsn) {
- t.Errorf("HasLogRec() can not find object %s:%s:%d:%d (i=%d) in log file %s",
- devid, srid.String(), gnum, lsn, i, logfile)
- }
-
- if !reflect.DeepEqual(curRec, rec) {
- t.Errorf("Data mismatch for object %s:%d:%d (i=%d) in log file %s: %v instead of %v",
- devid, gnum, lsn, i, logfile, curRec, rec)
- }
-
- if i == 0 {
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- }
- }
-
- log.dump()
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestDelLogRec tests deleting a log record across log open/close/reopen.
-func TestDelLogRec(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- var devid DeviceId = "VeyronTab"
- srid := ObjId("haha")
- var gnum GenId = 100
- var lsn SeqNum = 1000
-
- rec := &LogRec{
- DevId: devid,
- SyncRootId: srid,
- GenNum: gnum,
- SeqNum: lsn,
- ObjId: ObjId("bla"),
- CurVers: 2,
- Parents: []Version{0, 1},
- Value: LogValue{},
- }
-
- if _, err := log.putLogRec(rec); err != nil {
- t.Errorf("Cannot put object %s:%s:%d:%d (%v) in log file %s, err %v", devid, srid.String(), gnum, lsn, rec, logfile, err)
- }
-
- curRec, err := log.getLogRec(devid, srid, gnum, lsn)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:%s:%d:%d in log file %s, err %v",
- devid, srid.String(), gnum, lsn, logfile, err)
- }
-
- if err := log.delLogRec(devid, srid, gnum, lsn); err != nil {
- t.Fatalf("DelLogRec() can not delete object %s:%s:%d:%d in log file %s, err %v",
- devid, srid.String(), gnum, lsn, logfile, err)
- }
-
- for i := 0; i < 2; i++ {
- curRec, err = log.getLogRec(devid, srid, gnum, lsn)
- if err == nil || curRec != nil {
- t.Fatalf("GetLogRec() finds deleted object %s:%s:%d:%d (i=%d) in log file %s, err %v",
- devid, srid.String(), gnum, lsn, i, logfile, err)
- }
-
- if log.hasLogRec(devid, srid, gnum, lsn) {
- t.Errorf("HasLogRec() finds deleted object %s:%s:%d:%d (i=%d) in log file %s",
- devid, srid.String(), gnum, lsn, i, logfile)
- }
-
- if i == 0 {
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- }
- }
-
- log.dump()
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-
-}
-
-// TestPutGetGenMetadata tests setting and getting generation metadata across log open/close/reopen.
-func TestPutGetGenMetadata(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- var devid DeviceId = "VeyronTab"
- srid := ObjId("haha")
- var gnum GenId = 100
-
- val, err := log.getGenMetadata(devid, srid, gnum)
- if err == nil || val != nil {
- t.Errorf("GetGenMetadata() found non-existent object %s:%s:%d in log file %s: %v, err %v",
- devid, srid.String(), gnum, logfile, val, err)
- }
-
- if log.hasGenMetadata(devid, srid, gnum) {
- t.Errorf("hasGenMetadata() found non-existent object %s:%s:%d in log file %s",
- devid, srid.String(), gnum, logfile)
- }
-
- val = &genMetadata{Pos: 40, Count: 100, MaxSeqNum: 99}
- if err := log.putGenMetadata(devid, srid, gnum, val); err != nil {
- t.Errorf("Cannot put object %s:%s:%d in log file %s, err %v", devid, srid.String(), gnum, logfile, err)
- }
-
- for i := 0; i < 2; i++ {
- curVal, err := log.getGenMetadata(devid, srid, gnum)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%s:%d (i=%d) in log file %s, err %v",
- devid, srid.String(), gnum, i, logfile, err)
- }
-
- if !log.hasGenMetadata(devid, srid, gnum) {
- t.Errorf("hasGenMetadata() can not find object %s:%s:%d (i=%d) in log file %s",
- devid, srid.String(), gnum, i, logfile)
- }
-
- if !reflect.DeepEqual(curVal, val) {
- t.Errorf("Data mismatch for object %s:%s:%d (i=%d) in log file %s: %v instead of %v",
- devid, srid.String(), gnum, i, logfile, curVal, val)
- }
-
- if i == 0 {
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- }
- }
-
- log.dump()
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestDelGenMetadata tests deleting generation metadata across log open/close/reopen.
-func TestDelGenMetadata(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- var devid DeviceId = "VeyronTab"
- srid := ObjId("haha")
- var gnum GenId = 100
-
- val := &genMetadata{Pos: 40, Count: 100, MaxSeqNum: 99}
- if err := log.putGenMetadata(devid, srid, gnum, val); err != nil {
- t.Errorf("Cannot put object %s:%s:%d in log file %s, err %v", devid, srid.String(), gnum, logfile, err)
- }
-
- curVal, err := log.getGenMetadata(devid, srid, gnum)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%s:%d in log file %s, err %v",
- devid, srid.String(), gnum, logfile, err)
- }
-
- if err := log.delGenMetadata(devid, srid, gnum); err != nil {
- t.Fatalf("DelGenMetadata() can not delete object %s:%s:%d in log file %s, err %v",
- devid, srid.String(), gnum, logfile, err)
- }
-
- for i := 0; i < 2; i++ {
- curVal, err := log.getGenMetadata(devid, srid, gnum)
- if err == nil || curVal != nil {
- t.Fatalf("GetGenMetadata() finds deleted object %s:%s:%d (i=%d) in log file %s, err %v",
- devid, srid.String(), gnum, i, logfile, err)
- }
-
- if log.hasGenMetadata(devid, srid, gnum) {
- t.Errorf("hasGenMetadata() finds deleted object %s:%s:%d (i=%d) in log file %s",
- devid, srid.String(), gnum, i, logfile)
- }
-
- if i == 0 {
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- }
- }
-
- log.dump()
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestPersistLogState tests that generation metadata and record state
-// is persisted across log open/close/reopen.
-func TestPersistLogState(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- log, err := openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- var devid DeviceId = "VeyronTab"
- srid := ObjId("haha")
-
- // Add several generations.
- for i := uint32(0); i < 10; i++ {
- val := &genMetadata{Pos: i}
- if err := log.putGenMetadata(devid, srid, GenId(i+10), val); err != nil {
- t.Errorf("Cannot put object %s:%s:%d in log file %s, err %v", devid, srid.String(),
- i, logfile, err)
- }
- }
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
- log, err = openILog(logfile, nil)
- if err != nil {
- t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
- }
- for i := uint32(0); i < 10; i++ {
- curVal, err := log.getGenMetadata(devid, srid, GenId(i+10))
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%s:%d in log file %s, err %v",
- devid, srid.String(), i, logfile, err)
- }
- if curVal.Pos != i {
- t.Errorf("Data mismatch for object %s:%s:%d in log file %s: %v",
- devid, srid.String(), i, logfile, curVal)
- }
- // Should safely overwrite the same keys.
- curVal.Pos = i + 10
- if err := log.putGenMetadata(devid, srid, GenId(i+10), curVal); err != nil {
- t.Errorf("Cannot put object %s:%s:%d in log file %s, err %v", devid, srid.String(),
- i, logfile, err)
- }
- }
- for i := uint32(0); i < 10; i++ {
- curVal, err := log.getGenMetadata(devid, srid, GenId(i+10))
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%s:%d in log file %s, err %v",
- devid, srid.String(), i, logfile, err)
- }
- if curVal.Pos != (i + 10) {
- t.Errorf("Data mismatch for object %s:%s:%d in log file %s: %v, err %v",
- devid, srid.String(), i, logfile, curVal, err)
- }
- }
-
- log.dump()
- if err := log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// fillFakeLogRecords fills fake log records for testing purposes.
-func (l *iLog) fillFakeLogRecords(srid ObjId) {
- const num = 10
- var parvers []Version
- id := ObjId("haha")
- for i := int(0); i < num; i++ {
- // Create a local log record.
- curvers := Version(i)
- rec, err := l.createLocalLogRec(id, curvers, parvers, &LogValue{}, srid)
- if err != nil {
- return
- }
- // Insert the new log record into the log.
- _, err = l.putLogRec(rec)
- if err != nil {
- return
- }
- parvers = []Version{curvers}
- }
-}
-
-// TestCreateGeneration tests that local log records and local
-// generations are created uniquely and remote generations are
-// correctly inserted in log order.
-func TestCreateGeneration(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- s := &syncd{id: "VeyronTab"}
- log, err := openILog(logfile, s)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- // Try using a wrong SyncRoot ID.
- bad_srid := ObjId("xyz")
- if _, err := log.createLocalLogRec(ObjId("asdf"), NoVersion, nil, &LogValue{}, bad_srid); err == nil {
- t.Errorf("createLocalLogRec did not fail when using an invalid SyncRoot ID %v", bad_srid)
- }
- if _, err := log.createLocalLinkLogRec(ObjId("qwerty"), NoVersion, NoVersion, bad_srid); err == nil {
- t.Errorf("createLocalLinkLogRec did not fail when using an invalid SyncRoot ID %v", bad_srid)
- }
- if _, err := log.createLocalGeneration(bad_srid); err == nil {
- t.Errorf("createLocalGeneration did not fail when using an invalid SyncRoot ID %v", bad_srid)
- }
-
- srids := []ObjId{ObjId("foo"), ObjId("bar")}
- num := []uint64{10, 20}
- for pos, sr := range srids {
- if err := log.initSyncRoot(sr); err != nil {
- t.Fatalf("Cannot create new SyncRoot %s, err %v", sr.String(), err)
- }
- if g, err := log.createLocalGeneration(sr); err != errNoUpdates {
- t.Errorf("Should not find local updates gen %d with error %v", g, err)
- }
-
- var parvers []Version
- id := ObjId(fmt.Sprintf("haha-%d", pos))
- for i := uint64(0); i < num[pos]; i++ {
- // Create a local log record.
- curvers := Version(i)
- rec, err := log.createLocalLogRec(id, curvers, parvers, &LogValue{}, sr)
- if err != nil {
- t.Fatalf("Cannot create local log rec ObjId: %v Current: %v Parents: %v Error: %v",
- id, curvers, parvers, err)
- }
-
- temprec := &LogRec{
- DevId: log.s.id,
- SyncRootId: sr,
- GenNum: GenId(1),
- SeqNum: SeqNum(i),
- ObjId: id,
- CurVers: curvers,
- Parents: parvers,
- Value: LogValue{},
- }
- // Verify that the log record has the right values.
- if !reflect.DeepEqual(rec, temprec) {
- t.Errorf("Data mismtach in log record %v instead of %v", rec, temprec)
- }
-
- // Insert the new log record into the log.
- _, err = log.putLogRec(rec)
- if err != nil {
- t.Errorf("Cannot put log record:: failed with err %v", err)
- }
-
- parvers = []Version{curvers}
- }
- }
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-
- log, err = openILog(logfile, s)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- for pos, sr := range srids {
- if log.head.CurSRGens[sr].CurGenNum != 1 || log.head.CurSRGens[sr].CurSeqNum != SeqNum(num[pos]) {
- t.Errorf("Data mismatch in log header %v (pos=%d)", log.head, pos)
- }
-
- g, err := log.createLocalGeneration(sr)
- if g != 1 || err != nil {
- t.Errorf("Could not create local generation srid %s gen %d (pos=%d) with error %v",
- sr.String(), g, pos, err)
- }
- curVal, err := log.getGenMetadata(log.s.id, sr, g)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%s:%d (pos=%d) in log file %s, err %v",
- log.s.id, sr.String(), g, pos, logfile, err)
- }
- expVal := &genMetadata{Pos: uint32(pos), Count: num[pos], MaxSeqNum: SeqNum(num[pos] - 1)}
- if !reflect.DeepEqual(curVal, expVal) {
- t.Errorf("Data mismatch for object %s:%s:%d (pos=%d) in log file %s: %v instead of %v",
- log.s.id, sr.String(), g, pos, logfile, curVal, expVal)
- }
- if log.head.CurSRGens[sr].CurGenNum != 2 || log.head.CurSRGens[sr].CurSeqNum != 0 ||
- log.head.Curorder != uint32(pos+1) {
- t.Errorf("Data mismatch in log header (pos=%d) %v %v",
- pos, log.head.CurSRGens[sr], log.head.Curorder)
- }
-
- if g, err := log.createLocalGeneration(sr); err != errNoUpdates {
- t.Errorf("Should not find local updates sr %s gen %d (pos=%d) with error %v",
- sr.String(), g, pos, err)
- }
- }
-
- // Populate one more generation for only one syncroot.
- log.fillFakeLogRecords(srids[0])
-
- if g, err := log.createLocalGeneration(srids[0]); g != 2 || err != nil {
- t.Errorf("Could not create local generation sgid %s gen %d with error %v",
- srids[0].String(), g, err)
- }
-
- if g, err := log.createLocalGeneration(srids[1]); err != errNoUpdates {
- t.Errorf("Should not find local updates sr %s gen %d with error %v", srids[1].String(), g, err)
- }
-
- // Create a remote generation.
- expGen := &genMetadata{Count: 1, MaxSeqNum: 50}
- if err = log.createRemoteGeneration("VeyronPhone", srids[0], 1, expGen); err == nil {
- t.Errorf("Remote generation create should have failed")
- }
- expGen.MaxSeqNum = 0
- if err = log.createRemoteGeneration("VeyronPhone", srids[0], 1, expGen); err != nil {
- t.Errorf("createRemoteGeneration failed with err %v", err)
- }
- if expGen.Pos != 3 {
- t.Errorf("createRemoteGeneration created incorrect log order %d", expGen.Pos)
- }
-
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-
- // Reopen the log and ensure that all log records for generations exist.
- // Also ensure that generation metadata is accurate.
- log, err = openILog(logfile, s)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- gens := []GenId{2, 1}
- order := map[ObjId][]uint32{srids[0]: []uint32{0, 2},
- srids[1]: []uint32{1}}
- for pos, sr := range srids {
- for g := GenId(1); g <= gens[pos]; g++ {
- // Check all log records.
- for i := SeqNum(0); i < SeqNum(num[pos]); i++ {
- curRec, err := log.getLogRec(log.s.id, sr, g, i)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:%s:%d:%d in log file %s, err %v",
- log.s.id, sr.String(), g, i, logfile, err)
- }
- }
- // Check generation metadata.
- curVal, err := log.getGenMetadata(log.s.id, sr, g)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object %s:%s:%d in log file %s, err %v",
- log.s.id, sr.String(), g, logfile, err)
- }
- expVal := &genMetadata{Count: num[pos], MaxSeqNum: SeqNum(num[pos] - 1), Pos: order[sr][g-1]}
- if !reflect.DeepEqual(curVal, expVal) {
- t.Errorf("Data mismatch for object %s:%s:%d in log file %s: %v instead of %v",
- log.s.id, sr.String(), g, logfile, curVal, expVal)
- }
- }
- }
-
- // Check remote generation metadata.
- curVal, err := log.getGenMetadata("VeyronPhone", srids[0], 1)
- if err != nil || curVal == nil {
- t.Fatalf("GetGenMetadata() can not find object in log file %s, err %v",
- logfile, err)
- }
- if !reflect.DeepEqual(curVal, expGen) {
- t.Errorf("Data mismatch for object in log file %s: %v instead of %v",
- logfile, curVal, expGen)
- }
-
- expSRGens := map[ObjId]*curLocalGen{srids[0]: &curLocalGen{3, 0},
- srids[1]: &curLocalGen{2, 0}}
-
- if !reflect.DeepEqual(log.head.CurSRGens, expSRGens) || log.head.Curorder != 4 {
- t.Errorf("Data mismatch in log header %v %v", log.head.CurSRGens, log.head.Curorder)
- }
-
- log.dump()
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-
-// TestProcessWatchRecord tests that local updates are correctly handled.
-// Commands are in file testdata/<local-init-00.log.sync, local-init-02.log.sync>.
-/*
-func TestProcessWatchRecord(t *testing.T) {
- logfile := getFileName()
- defer os.Remove(logfile)
-
- dagfile := getFileName()
- defer os.Remove(dagfile)
-
- var err error
- s := &syncd{id: "VeyronTab"}
-
- if s.dag, err = openDAG(dagfile); err != nil {
- t.Fatalf("Cannot open new dag file %s, err %v", logfile, err)
- }
- log, err := openILog(logfile, s)
- if err != nil {
- t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
- }
-
- srids := []ObjId{"foo", "bar"}
- fnames := []string{"local-init-00.log.sync", "local-init-02.log.sync"}
- num := []uint32{3, 5}
-
- // Try using an invalid SyncRoot ID.
- if err = log.processWatchRecord(ObjId("haha"), 2, Version(999), &LogValue{}, NoObjId); err == nil {
- t.Errorf("processWatchRecord did not fail when using an invalid SyncRoot ID")
- }
-
- // Test echo suppression on JoinSyncGroup.
- if err := log.processWatchRecord(srids[0], 5, NoVersion, &LogValue{}, srids[0]); err != nil {
- t.Fatalf("Echo suppression on JoinSyncGroup failed with err %v", err)
- }
-
- for pos, sr := range srids {
- if err := log.initSyncRoot(sr); err != nil {
- t.Fatalf("Cannot create new SyncRoot %s, err %v", sr.String(), err)
- }
-
- if err := logReplayCommands(log, fnames[pos], sr); err != nil {
- t.Error(err)
- }
- }
-
- checkObject := func(obj string, expHead Version) {
- _, file, line, _ := runtime.Caller(1)
- objid, err := strToObjId(obj)
- if err != nil {
- t.Errorf("%s:%d Could not create objid %v", file, line, err)
- }
-
- // Verify DAG state.
- if head, err := log.s.dag.getHead(objid); err != nil || head != expHead {
- t.Errorf("%s:%d Invalid object %d head in DAG %v want %v, err %v", file, line,
- objid, head, expHead, err)
- }
- }
- for pos, sr := range srids {
- if log.head.CurSRGens[sr].CurGenNum != 1 || log.head.CurSRGens[sr].CurSeqNum != SeqNum(num[pos]) ||
- log.head.Curorder != 0 {
- t.Errorf("Data mismatch in log header %v", log.head)
- }
-
- // Check all log records.
- for i := SeqNum(0); i < SeqNum(num[pos]); i++ {
- curRec, err := log.getLogRec(log.s.id, sr, GenId(1), i)
- if err != nil || curRec == nil {
- t.Fatalf("GetLogRec() can not find object %s:%s:1:%d in log file %s, err %v",
- log.s.id, sr.String(), i, logfile, err)
- }
- }
-
- if pos == 0 {
- checkObject("1234", 3)
- } else if pos == 1 {
- checkObject("12", 3)
- checkObject("45", 20)
- }
- }
-
- s.dag.flush()
- s.dag.close()
-
- log.dump()
- if err = log.close(); err != nil {
- t.Errorf("Cannot close log file %s, err %v", logfile, err)
- }
-}
-*/
diff --git a/services/syncbase/sync/initiator.go b/services/syncbase/sync/initiator.go
deleted file mode 100644
index 8d23976..0000000
--- a/services/syncbase/sync/initiator.go
+++ /dev/null
@@ -1,943 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-import (
- "errors"
- "fmt"
- "math/rand"
- "os"
- "time"
-
- "v.io/v23/context"
- "v.io/v23/naming"
- "v.io/v23/vtrace"
- "v.io/x/lib/vlog"
-)
-
-// Policies to pick a peer to sync with.
-const (
- // Picks a peer at random from the available set.
- selectRandom = iota
-
- // TODO(hpucha): implement other policies.
- // Picks a peer with most differing generations.
- selectMostDiff
-
- // Picks a peer that was synced with the furthest in the past.
- selectOldest
-)
-
-// Policies for conflict resolution.
-const (
- // Resolves conflicts by picking the mutation with the most recent timestamp.
- useTime = iota
-
- // TODO(hpucha): implement other policies.
- // Resolves conflicts by using the app conflict resolver callbacks via store.
- useCallback
-)
-
-var (
- // peerSyncInterval is the duration between two consecutive
- // sync events. In every sync event, the initiator contacts
- // one of its peers to obtain any pending updates.
- peerSyncInterval = 50 * time.Millisecond
-
- // peerSelectionPolicy is the policy used to select a peer when
- // the initiator gets a chance to sync.
- peerSelectionPolicy = selectRandom
-
- // conflictResolutionPolicy is the policy used to resolve conflicts.
- conflictResolutionPolicy = useTime
-
- errNoUsefulPeer = errors.New("no useful peer to contact")
-)
-
-// syncInitiator contains the metadata and state for the initiator thread.
-type syncInitiator struct {
- syncd *syncd
-
- // State accumulated during an initiation round.
- iState *initiationState
-}
-
-// initiationState accumulated during an initiation round.
-type initiationState struct {
-
- // Veyron name of peer being synced with.
- peer string
-
- // Local generation vector.
- local map[ObjId]GenVector
-
- // SyncGroups being requested in the initiation round.
- syncGroups map[ObjId]GroupIdSet
-
- // Map to track new generations received in the RPC reply.
- newGens map[string]*genMetadata
-
- // Array to track order of arrival for the generations.
- // We need to preserve this order for the replay.
- orderGens []string
-
- // Generation vector to track the oldest generation received
- // in the RPC reply per device, for garbage collection.
- minGens map[ObjId]GenVector
-
- // Generation vector from the remote peer.
- remote map[ObjId]GenVector
-
- // Tmp kvdb state.
- tmpFile string
- tmpDB *kvdb
- tmpTbl *kvtable
-
- // State to track updated objects during a log replay.
- updObjects map[ObjId]*objConflictState
-
- // State to delete objects from the "priv" table.
- delPrivObjs map[ObjId]struct{}
-}
-
-// objConflictState contains the conflict state for objects that are
-// updated during an initiator run.
-type objConflictState struct {
- isConflict bool
- newHead Version
- oldHead Version
- ancestor Version
- resolvVal *LogValue
- srID ObjId
-}
-
-// newInitiator creates a new initiator instance attached to the given syncd instance.
-func newInitiator(syncd *syncd, syncTick time.Duration) *syncInitiator {
- i := &syncInitiator{syncd: syncd}
-
- // Override the default peerSyncInterval value if syncTick is specified.
- if syncTick > 0 {
- peerSyncInterval = syncTick
- }
-
- vlog.VI(1).Infof("newInitiator: My device ID: %s", i.syncd.id)
- vlog.VI(1).Infof("newInitiator: Sync interval: %v", peerSyncInterval)
-
- return i
-}
-
-// contactPeers wakes up every peerSyncInterval to contact peers and get deltas from them.
-func (i *syncInitiator) contactPeers() {
- ticker := time.NewTicker(peerSyncInterval)
- for {
- select {
- case <-i.syncd.closed:
- ticker.Stop()
- i.syncd.pending.Done()
- return
- case <-ticker.C:
- }
-
- peerRelName, err := i.pickPeer()
- if err != nil {
- continue
- }
-
- i.getDeltasFromPeer(peerRelName)
- }
-}
-
-// pickPeer picks a sync endpoint to sync with.
-func (i *syncInitiator) pickPeer() (string, error) {
- myID := string(i.syncd.relName)
-
- switch peerSelectionPolicy {
- case selectRandom:
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.RLock()
- // neighbors, err := i.syncd.sgtab.getMembers()
- neighbors := make(map[string]uint32)
- var err error
- i.syncd.lock.RUnlock()
-
- if err != nil {
- return "", err
- }
-
- // Remove myself from the set so only neighbors are counted.
- delete(neighbors, myID)
-
- if len(neighbors) == 0 {
- return "", errNoUsefulPeer
- }
-
- // Pick a neighbor at random.
- ind := rand.Intn(len(neighbors))
- for k := range neighbors {
- if ind == 0 {
- return k, nil
- }
- ind--
- }
- return "", fmt.Errorf("random selection didn't succeed")
- default:
- return "", fmt.Errorf("unknown peer selection policy")
- }
-}
-
-// getDeltasFromPeer performs an initiation round to the specified
-// peer. An initiation round consists of:
-// * Creating local generation for syncroots that are going to be requested in this round.
-// * Contacting the peer to receive all the deltas based on the local gen vector.
-// * Processing those deltas to discover objects which have been updated.
-// * Processing updated objects to detect and resolve any conflicts if needed.
-// * Communicating relevant object updates to the store.
-func (i *syncInitiator) getDeltasFromPeer(peerRelName string) {
-
- vlog.VI(2).Infof("getDeltasFromPeer:: %s", peerRelName)
- // Initialize initiation state.
- i.newInitiationState()
- defer i.clearInitiationState()
-
- // Freeze the most recent batch of local changes
- // before fetching remote changes from a peer.
- //
- // We only allow an initiator to create new local
- // generations (not responders/watcher) in order to
- // maintain a static baseline for the duration of a
- // sync. This addresses the following race condition:
- // If we allow responders to create new local
- // generations while the initiator is in progress,
- // they may beat the initiator and send these new
- // generations to remote devices. These remote
- // devices in turn can send these generations back to
- // the initiator in progress which was started with
- // older generation information.
- err := i.updateLocalGeneration(peerRelName)
- if err != nil {
- vlog.Fatalf("getDeltasFromPeer:: error updating local generation: err %v", err)
- }
-
- // Obtain deltas from the peer over the network. These deltas
- // are stored in a tmp kvdb.
- if err := i.getDeltas(); err != nil {
- vlog.Errorf("getDeltasFromPeer:: error getting deltas: err %v", err)
- return
- }
-
- i.syncd.sgOp.Lock()
- defer i.syncd.sgOp.Unlock()
-
- if err := i.processDeltas(); err != nil {
- vlog.Fatalf("getDeltasFromPeer:: error processing logs: err %v", err)
- }
-
- if err := i.processUpdatedObjects(); err != nil {
- vlog.Fatalf("getDeltasFromPeer:: error processing objects: err %v", err)
- }
-}
-
-// newInitiationState creates new initiation state.
-func (i *syncInitiator) newInitiationState() {
- st := &initiationState{}
- st.local = make(map[ObjId]GenVector)
- st.syncGroups = make(map[ObjId]GroupIdSet)
- st.newGens = make(map[string]*genMetadata)
- st.minGens = make(map[ObjId]GenVector)
- st.remote = make(map[ObjId]GenVector)
- st.updObjects = make(map[ObjId]*objConflictState)
- st.delPrivObjs = make(map[ObjId]struct{})
-
- i.iState = st
-}
-
-// clearInitiationState cleans up the state from the current initiation round.
-func (i *syncInitiator) clearInitiationState() {
- if i.iState.tmpDB != nil {
- i.iState.tmpDB.close()
- }
- if i.iState.tmpFile != "" {
- os.Remove(i.iState.tmpFile)
- }
-
- for o := range i.iState.delPrivObjs {
- i.syncd.dag.delPrivNode(o)
- }
-
- i.syncd.dag.clearGraft()
-
- i.iState = nil
-}
-
-// updateLocalGeneration creates a new local generation if needed and
-// populates the newest local generation vector.
-func (i *syncInitiator) updateLocalGeneration(peerRelName string) error {
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.Lock()
- defer i.syncd.lock.Unlock()
-
- // peerInfo, err := i.syncd.sgtab.getMemberInfo(peerRelName)
- // if err != nil {
- // return err
- // }
-
- // Re-construct all mount table possibilities.
- mtTables := make(map[string]struct{})
-
- // for sg := range peerInfo.gids {
- // sgData, err := i.syncd.sgtab.getSyncGroupByID(sg)
- // if err != nil {
- // return err
- // }
- // sr := ObjId(sgData.SrvInfo.RootObjId)
- // for _, mt := range sgData.SrvInfo.Config.MountTables {
- // mtTables[mt] = struct{}{}
- // }
- // i.iState.syncGroups[sr] = append(i.iState.syncGroups[sr], sg)
- // }
-
- // Create a new local generation if there are any local updates
- // for every syncroot that is common with the "peer" to be
- // contacted.
- for sr := range i.iState.syncGroups {
- gen, err := i.syncd.log.createLocalGeneration(sr)
- if err != nil && err != errNoUpdates {
- return err
- }
-
- if err == nil {
- vlog.VI(2).Infof("updateLocalGeneration:: created gen for sr %s at %d", sr.String(), gen)
-
- // Update local generation vector in devTable.
- if err = i.syncd.devtab.updateGeneration(i.syncd.id, sr, i.syncd.id, gen); err != nil {
- return err
- }
- }
-
- v, err := i.syncd.devtab.getGenVec(i.syncd.id, sr)
- if err != nil {
- return err
- }
-
- i.iState.local[sr] = v
- }
-
- // Check if the name is absolute, and if so, use the name as-is.
- if naming.Rooted(peerRelName) {
- i.iState.peer = peerRelName
- return nil
- }
-
- // Pick any global name to contact the peer.
- for mt := range mtTables {
- i.iState.peer = naming.Join(mt, peerRelName)
- vlog.VI(2).Infof("updateLocalGeneration:: contacting peer %s", i.iState.peer)
- return nil
- }
-
- return fmt.Errorf("no mounttables found")
-}
-
-// getDeltas obtains the deltas from the peer and stores them in a tmp kvdb.
-func (i *syncInitiator) getDeltas() error {
- // As log records are streamed, they are saved
- // in a tmp kvdb so that they can be processed in one batch.
- if err := i.initTmpKVDB(); err != nil {
- return err
- }
-
- ctx, _ := vtrace.WithNewTrace(i.syncd.ctx)
- ctx, cancel := context.WithTimeout(ctx, time.Minute)
- defer cancel()
-
- vlog.VI(1).Infof("getDeltas:: From peer with DeviceId %s at %v", i.iState.peer, time.Now().UTC())
-
- // Construct a new stub that binds to peer endpoint.
- c := SyncClient(naming.JoinAddressName(i.iState.peer, SyncSuffix))
-
- vlog.VI(1).Infof("GetDeltasFromPeer:: Sending local information: %v", i.iState.local)
-
- // Issue a GetDeltas() rpc.
- stream, err := c.GetDeltas(ctx, i.iState.local, i.iState.syncGroups, i.syncd.id)
- if err != nil {
- return err
- }
-
- return i.recvLogStream(stream)
-}
-
-// initTmpKVDB initializes the tmp kvdb to store the log record stream.
-func (i *syncInitiator) initTmpKVDB() error {
- i.iState.tmpFile = fmt.Sprintf("%s/tmp_%d", i.syncd.kvdbPath, time.Now().UnixNano())
- tmpDB, tbls, err := kvdbOpen(i.iState.tmpFile, []string{"tmprec"})
- if err != nil {
- return err
- }
- i.iState.tmpDB = tmpDB
- i.iState.tmpTbl = tbls[0]
- return nil
-}
-
-// recvLogStream receives the log records from the GetDeltas stream
-// and puts them in tmp kvdb for later processing.
-func (i *syncInitiator) recvLogStream(stream SyncGetDeltasClientCall) error {
- rStream := stream.RecvStream()
- for rStream.Advance() {
- rec := rStream.Value()
-
- // Insert log record in tmpTbl.
- if err := i.iState.tmpTbl.set(logRecKey(rec.DevId, rec.SyncRootId, rec.GenNum, rec.SeqNum), &rec); err != nil {
- // TODO(hpucha): do we need to cancel stream?
- return err
- }
-
- // Populate the generation metadata.
- genKey := generationKey(rec.DevId, rec.SyncRootId, rec.GenNum)
- if gen, ok := i.iState.newGens[genKey]; !ok {
- // New generation in the stream.
- i.iState.orderGens = append(i.iState.orderGens, genKey)
- i.iState.newGens[genKey] = &genMetadata{
- Count: 1,
- MaxSeqNum: rec.SeqNum,
- }
- if _, ok := i.iState.minGens[rec.SyncRootId]; !ok {
- i.iState.minGens[rec.SyncRootId] = GenVector{}
- }
- g, ok := i.iState.minGens[rec.SyncRootId][rec.DevId]
- if !ok || g > rec.GenNum {
- i.iState.minGens[rec.SyncRootId][rec.DevId] = rec.GenNum
- }
- } else {
- gen.Count++
- if rec.SeqNum > gen.MaxSeqNum {
- gen.MaxSeqNum = rec.SeqNum
- }
- }
- }
-
- if err := rStream.Err(); err != nil {
- return err
- }
-
- var err error
- if i.iState.remote, err = stream.Finish(); err != nil {
- return err
- }
- vlog.VI(1).Infof("recvLogStream:: Local vector %v", i.iState.local)
- vlog.VI(1).Infof("recvLogStream:: Remote vector %v", i.iState.remote)
- vlog.VI(2).Infof("recvLogStream:: orderGens %v", i.iState.orderGens)
- return nil
-}
-
-// processDeltas replays an entire log stream spanning multiple
-// generations from different devices received from a single GetDeltas
-// call. It does not perform any conflict resolution during replay.
-// This avoids resolving conflicts that have already been resolved by
-// other devices.
-func (i *syncInitiator) processDeltas() error {
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.Lock()
- defer i.syncd.lock.Unlock()
-
- // Track received transactions.
- txMap := make(map[TxId]uint32)
-
- // Loop through each received generation in order.
- for _, key := range i.iState.orderGens {
- gen := i.iState.newGens[key]
- dev, sr, gnum, err := splitGenerationKey(key)
- if err != nil {
- return err
- }
-
- // If "sr" has been left since getDeltas, skip processing.
- // if !i.syncd.sgtab.isSyncRoot(sr) {
- // continue
- // }
-
- for l := SeqNum(0); l <= gen.MaxSeqNum; l++ {
- var rec LogRec
- if err := i.iState.tmpTbl.get(logRecKey(dev, sr, gnum, l), &rec); err != nil {
- return err
- }
-
- // Begin a new transaction if needed.
- curTx := rec.Value.TxId
- if curTx != NoTxId {
- if cnt, ok := txMap[curTx]; !ok {
- if i.syncd.dag.addNodeTxStart(curTx) != curTx {
- return fmt.Errorf("failed to create transaction")
- }
- txMap[curTx] = rec.Value.TxCount
- vlog.VI(2).Infof("processDeltas:: Begin Tx %v, count %d", curTx, rec.Value.TxCount)
- } else if cnt != rec.Value.TxCount {
- return fmt.Errorf("inconsistent counts for tid %v %d, %d", curTx, cnt, rec.Value.TxCount)
- }
- }
-
- if err := i.insertRecInLogAndDag(&rec, curTx); err != nil {
- return err
- }
-
- // Mark object dirty.
- i.iState.updObjects[rec.ObjId] = &objConflictState{
- srID: rec.SyncRootId,
- }
- }
- // Insert the generation metadata.
- if err := i.syncd.log.createRemoteGeneration(dev, sr, gnum, gen); err != nil {
- return err
- }
- }
-
- // End the started transactions if any.
- for t, cnt := range txMap {
- if err := i.syncd.dag.addNodeTxEnd(t, cnt); err != nil {
- return err
- }
- vlog.VI(2).Infof("processLogStream:: End Tx %v %v", t, cnt)
- }
-
- return nil
-}
-
-// insertRecInLogAndDag adds a new log record to log and dag data structures.
-func (i *syncInitiator) insertRecInLogAndDag(rec *LogRec, txID TxId) error {
- logKey, err := i.syncd.log.putLogRec(rec)
- if err != nil {
- return err
- }
-
- vlog.VI(2).Infof("insertRecInLogAndDag:: Adding log record %v, Tx %v", rec, txID)
- switch rec.RecType {
- case NodeRec:
- return i.syncd.dag.addNode(rec.ObjId, rec.CurVers, true, rec.Value.Delete, rec.Parents, logKey, txID)
- case LinkRec:
- return i.syncd.dag.addParent(rec.ObjId, rec.CurVers, rec.Parents[0], true)
- default:
- return fmt.Errorf("unknown log record type")
- }
-}
-
-// processUpdatedObjects processes all the updates received by the
-// initiator, one object at a time. For each updated object, we first
-// check if the object has any conflicts, resulting in three
-// possibilities:
-//
-// * There is no conflict, and no updates are needed to the store
-// (isConflict=false, newHead == oldHead). All changes received convey
-// information that still keeps the local head as the most recent
-// version. This occurs when conflicts are resolved by blessings.
-//
-// * There is no conflict, but a remote version is discovered that
-// builds on the local head (isConflict=false, newHead != oldHead). In
-// this case, we generate a store mutation to simply update the store
-// to the latest value.
-//
-// * There is a conflict and we call into the app or the system to
-// resolve the conflict, resulting in three possibilties: (a) conflict
-// was resolved by blessing the local version. In this case, store
-// need not be updated, but a link is added to record the
-// blessing. (b) conflict was resolved by blessing the remote
-// version. In this case, store is updated with the remote version and
-// a link is added as well. (c) conflict was resolved by generating a
-// new store mutation. In this case, store is updated with the new
-// version.
-//
-// We then put all these mutations in the store. If the put succeeds,
-// we update the log and dag state suitably (move the head ptr of the
-// object in the dag to the latest version, and create a new log
-// record reflecting conflict resolution if any). Puts to store can
-// fail since preconditions on the objects may have been violated. In
-// this case, we wait to get the latest versions of objects from the
-// store, and recheck if the object has any conflicts and repeat the
-// above steps, until put to store succeeds.
-func (i *syncInitiator) processUpdatedObjects() error {
- for {
- if err := i.detectConflicts(); err != nil {
- return err
- }
-
- if err := i.resolveConflicts(); err != nil {
- return err
- }
-
- err := i.updateStoreAndSync()
- if err == nil {
- break
- }
-
- vlog.Errorf("PutMutations failed %v. Will retry", err)
- // TODO(hpucha): Sleeping and retrying is a temporary
- // solution. Next iteration will have coordination
- // with watch thread to intelligently retry. Hence
- // this value is not a config param.
- time.Sleep(1 * time.Second)
- }
-
- return nil
-}
-
-// detectConflicts iterates through all the updated objects to detect
-// conflicts.
-func (i *syncInitiator) detectConflicts() error {
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.RLock()
- defer i.syncd.lock.RUnlock()
-
- for obj, st := range i.iState.updObjects {
- // Check if object has a conflict.
- var err error
- st.isConflict, st.newHead, st.oldHead, st.ancestor, err = i.syncd.dag.hasConflict(obj)
- vlog.VI(2).Infof("detectConflicts:: object %v state %v err %v",
- obj, st, err)
- if err != nil {
- return err
- }
- }
- return nil
-}
-
-// resolveConflicts resolves conflicts for updated objects. Conflicts
-// may be resolved by adding new versions or blessing either the local
-// or the remote version.
-func (i *syncInitiator) resolveConflicts() error {
- for obj, st := range i.iState.updObjects {
- if !st.isConflict {
- continue
- }
-
- res, err := i.resolveObjConflict(obj, st.oldHead, st.newHead, st.ancestor)
- if err != nil {
- return err
- }
-
- st.resolvVal = res
- }
-
- return nil
-}
-
-// resolveObjConflict resolves a conflict for an object given its ID and
-// the 3 versions that express the conflict: the object's local version, its
-// remote version (from the device contacted), and the common ancestor from
-// which both versions branched away. The function returns the new object
-// value according to the conflict resolution policy.
-func (i *syncInitiator) resolveObjConflict(oid ObjId,
- local, remote, ancestor Version) (*LogValue, error) {
-
- // Fetch the log records of the 3 object versions.
- versions := []Version{local, remote, ancestor}
- lrecs, err := i.getLogRecsBatch(oid, versions)
- if err != nil {
- return nil, err
- }
-
- // Resolve the conflict according to the resolution policy.
- var res *LogValue
-
- switch conflictResolutionPolicy {
- case useTime:
- res, err = i.resolveObjConflictByTime(oid, lrecs[0], lrecs[1], lrecs[2])
- default:
- err = fmt.Errorf("unknown conflict resolution policy: %v", conflictResolutionPolicy)
- }
-
- if err != nil {
- return nil, err
- }
-
- resCopy := *res
- // resCopy.Mutation.Version = NewVersion()
- // resCopy.Mutation.Dir = resDir
- resCopy.SyncTime = time.Now().UnixNano()
- resCopy.TxId = NoTxId
- resCopy.TxCount = 0
- return &resCopy, nil
-}
-
-// resolveObjConflictByTime resolves conflicts using the timestamps
-// of the conflicting mutations. It picks a mutation with the larger
-// timestamp, i.e. the most recent update. If the timestamps are equal,
-// it uses the mutation version numbers as a tie-breaker, picking the
-// mutation with the larger version.
-// Instead of creating a new version that resolves the conflict, we are
-// blessing an existing version as the conflict resolution.
-func (i *syncInitiator) resolveObjConflictByTime(oid ObjId,
- local, remote, ancestor *LogRec) (*LogValue, error) {
-
- var res *LogValue
- switch {
- case local.Value.SyncTime > remote.Value.SyncTime:
- res = &local.Value
- case local.Value.SyncTime < remote.Value.SyncTime:
- res = &remote.Value
- // case local.Value.Mutation.Version > remote.Value.Mutation.Version:
- // res = &local.Value
- // case local.Value.Mutation.Version < remote.Value.Mutation.Version:
- // res = &remote.Value
- }
-
- return res, nil
-}
-
-// getLogRecsBatch gets the log records for an array of versions.
-func (i *syncInitiator) getLogRecsBatch(obj ObjId, versions []Version) ([]*LogRec, error) {
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.RLock()
- defer i.syncd.lock.RUnlock()
-
- lrecs := make([]*LogRec, len(versions))
- var err error
- for p, v := range versions {
- lrecs[p], err = i.getLogRec(obj, v)
- if err != nil {
- return nil, err
- }
- }
- return lrecs, nil
-}
-
-// updateStoreAndSync updates the store, and if that is successful,
-// updates log and dag data structures.
-func (i *syncInitiator) updateStoreAndSync() error {
-
- // TODO(hpucha): Eliminate reaching into syncd's lock.
- i.syncd.lock.Lock()
- defer i.syncd.lock.Unlock()
-
- // var m []raw.Mutation
- // for obj, st := range i.iState.updObjects {
- // if !st.isConflict {
- // rec, err := i.getLogRec(obj, st.newHead)
- // if err != nil {
- // return err
- // }
- // st.resolvVal = &rec.Value
- // // Sanity check.
- // if st.resolvVal.Mutation.Version != st.newHead {
- // return fmt.Errorf("bad mutation %d %d",
- // st.resolvVal.Mutation.Version, st.newHead)
- // }
- // }
-
- // // If the local version is picked, no further updates
- // // to the store are needed. If the remote version is
- // // picked, we put it in the store.
- // if st.resolvVal.Mutation.Version != st.oldHead {
- // st.resolvVal.Mutation.PriorVersion = st.oldHead
-
- // // Convert resolvVal.Mutation into a mutation for the Store.
- // stMutation, err := i.storeMutation(obj, st.resolvVal)
- // if err != nil {
- // return err
- // }
-
- // vlog.VI(2).Infof("updateStoreAndSync:: Try to append mutation %v (%v) for obj %v (nh %v, oh %v)",
- // st.resolvVal.Mutation, stMutation, obj, st.newHead, st.oldHead)
-
- // // Append to mutations, skipping a delete following a delete mutation.
- // if stMutation.Version != NoVersion ||
- // stMutation.PriorVersion != NoVersion {
- // vlog.VI(2).Infof("updateStoreAndSync:: appending mutation %v for obj %v",
- // stMutation, obj)
- // m = append(m, stMutation)
- // }
- // }
- // }
-
- // TODO(hpucha): We will hold the lock across PutMutations rpc
- // to prevent a race with watcher. The next iteration will
- // clean up this coordination.
- // if store := i.syncd.store; store != nil && len(m) > 0 {
- // ctx, _ := vtrace.SetNewTrace(i.syncd.ctx)
- // ctx, cancel := context.WithTimeout(ctx, time.Minute)
- // defer cancel()
-
- // stream, err := store.PutMutations(ctx)
- // if err != nil {
- // vlog.Errorf("updateStoreAndSync:: putmutations err %v", err)
- // return err
- // }
- // sender := stream.SendStream()
- // for i := range m {
- // if err := sender.Send(m[i]); err != nil {
- // vlog.Errorf("updateStoreAndSync:: send err %v", err)
- // return err
- // }
- // }
- // if err := sender.Close(); err != nil {
- // vlog.Errorf("updateStoreAndSync:: closesend err %v", err)
- // return err
- // }
- // if err := stream.Finish(); err != nil {
- // vlog.Errorf("updateStoreAndSync:: finish err %v", err)
- // return err
- // }
- // }
-
- vlog.VI(2).Infof("updateStoreAndSync:: putmutations succeeded")
- if err := i.updateLogAndDag(); err != nil {
- return err
- }
-
- if err := i.updateGenVecs(); err != nil {
- return err
- }
-
- return nil
-}
-
-// storeMutation converts a resolved mutation generated by syncd to
-// one that can be sent to the store. To send to the store, it
-// converts the version numbers corresponding to object deletions to
-// NoVersion when required. It also converts the version number
-// appropriately to handle SyncGroup join.
-// func (i *syncInitiator) storeMutation(obj ObjId, resolvVal *LogValue) (raw.Mutation, error) {
-// curDelete := resolvVal.Delete
-// priorDelete := false
-// if resolvVal.Mutation.PriorVersion != raw.NoVersion {
-// oldRec, err := i.getLogRec(obj, resolvVal.Mutation.PriorVersion)
-// if err != nil {
-// return raw.Mutation{}, err
-// }
-// priorDelete = oldRec.Value.Delete
-// }
-
-// // Handle the versioning of a SyncGroup's root ObjId during join.
-// if resolvVal.Mutation.PriorVersion == raw.NoVersion {
-// if i.syncd.sgtab.isSyncRoot(obj) {
-// node, err := i.syncd.dag.getPrivNode(obj)
-// if err != nil {
-// return raw.Mutation{}, err
-// }
-// resolvVal.Mutation.PriorVersion = node.Mutation.Version
-// i.iState.delPrivObjs[obj] = struct{}{}
-// }
-// }
-
-// // Current version and prior versions are not deletes.
-// if !curDelete && !priorDelete {
-// return resolvVal.Mutation, nil
-// }
-
-// // Creating a new copy of the mutation to adjust version
-// // numbers when handling deletions.
-// stMutation := resolvVal.Mutation
-// // Adjust the current version if this a deletion.
-// if curDelete {
-// stMutation.Version = NoVersion
-// }
-// // Adjust the prior version if it is a deletion.
-// if priorDelete {
-// stMutation.PriorVersion = NoVersion
-// }
-
-// return stMutation, nil
-// }
-
-// getLogRec returns the log record corresponding to a given object and its version.
-func (i *syncInitiator) getLogRec(obj ObjId, vers Version) (*LogRec, error) {
- logKey, err := i.syncd.dag.getLogrec(obj, vers)
- vlog.VI(2).Infof("getLogRec:: logkey from dag is %s", logKey)
- if err != nil {
- return nil, err
- }
- dev, sg, gen, lsn, err := splitLogRecKey(logKey)
- if err != nil {
- return nil, err
- }
- vlog.VI(2).Infof("getLogRec:: splitting logkey %s to %s %v %v %v", logKey, dev, sg, gen, lsn)
- rec, err := i.syncd.log.getLogRec(dev, sg, gen, lsn)
- if err != nil {
- return nil, err
- }
- return rec, nil
-}
-
-// updateLogAndDag updates the log and dag data structures on a successful store put.
-func (i *syncInitiator) updateLogAndDag() error {
- for obj, st := range i.iState.updObjects {
- if st.isConflict {
- // Object had a conflict, which was resolved successfully.
- // Put is successful, create a log record.
- var err error
- var rec *LogRec
-
- // switch {
- // case st.resolvVal.Mutation.Version == st.oldHead:
- // // Local version was blessed as the conflict resolution.
- // rec, err = i.syncd.log.createLocalLinkLogRec(obj, st.oldHead, st.newHead, st.srID)
- // case st.resolvVal.Mutation.Version == st.newHead:
- // // Remote version was blessed as the conflict resolution.
- // rec, err = i.syncd.log.createLocalLinkLogRec(obj, st.newHead, st.oldHead, st.srID)
- // default:
- // // New version was created to resolve the conflict.
- // parents := []Version{st.newHead, st.oldHead}
- // rec, err = i.syncd.log.createLocalLogRec(obj, st.resolvVal.Mutation.Version, parents, st.resolvVal, st.srID)
- // }
- if err != nil {
- return err
- }
- logKey, err := i.syncd.log.putLogRec(rec)
- if err != nil {
- return err
- }
- // Add a new DAG node.
- switch rec.RecType {
- case NodeRec:
- // TODO(hpucha): addNode operations arising out of conflict resolution
- // may need to be part of a transaction when app-driven resolution
- // is introduced.
- err = i.syncd.dag.addNode(obj, rec.CurVers, false, rec.Value.Delete, rec.Parents, logKey, NoTxId)
- case LinkRec:
- err = i.syncd.dag.addParent(obj, rec.CurVers, rec.Parents[0], false)
- default:
- return fmt.Errorf("unknown log record type")
- }
- if err != nil {
- return err
- }
- }
-
- // Move the head. This should be idempotent. We may
- // move head to the local head in some cases.
- // if err := i.syncd.dag.moveHead(obj, st.resolvVal.Mutation.Version); err != nil {
- // return err
- // }
- }
- return nil
-}
-
-// updateGenVecs updates local, reclaim and remote vectors at the end of an initiator cycle.
-func (i *syncInitiator) updateGenVecs() error {
- // Update the local gen vector and put it in kvdb only if we have new updates.
- if len(i.iState.updObjects) > 0 {
- // remote can be a subset of local.
- for sr, rVec := range i.iState.remote {
- lVec := i.iState.local[sr]
-
- if err := i.syncd.devtab.updateLocalGenVector(lVec, rVec); err != nil {
- return err
- }
-
- if err := i.syncd.devtab.putGenVec(i.syncd.id, sr, lVec); err != nil {
- return err
- }
-
- // if err := i.syncd.devtab.updateReclaimVec(minGens); err != nil {
- // return err
- // }
- }
- }
-
- for sr, rVec := range i.iState.remote {
- // Cache the remote generation vector for space reclamation.
- if err := i.syncd.devtab.putGenVec(i.syncd.nameToDevId(i.iState.peer), sr, rVec); err != nil {
- return err
- }
- }
- return nil
-}
diff --git a/services/syncbase/sync/kvdb.go b/services/syncbase/sync/kvdb.go
deleted file mode 100644
index 79b1101..0000000
--- a/services/syncbase/sync/kvdb.go
+++ /dev/null
@@ -1,129 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Helpful wrappers to a persistent key/value (K/V) DB used by Veyron Sync.
-// The current underlying DB is an in-memory map.
-
-import (
- "bytes"
- "fmt"
- "sort"
-
- "v.io/v23/vom"
-)
-
-var memStore map[string]*kvdb
-
-type kvdb struct {
- tables map[string]*kvtable
-}
-
-type kvtable struct {
- data map[string][]byte
-}
-
-// kvdbOpen opens or creates a K/V DB for the given filename and table names
-// within the DB. It returns the DB handler and handlers for each table.
-func kvdbOpen(filename string, tables []string) (*kvdb, []*kvtable, error) {
- if memStore == nil {
- memStore = make(map[string]*kvdb)
- }
-
- db := memStore[filename]
- if db == nil {
- db = &kvdb{tables: make(map[string]*kvtable)}
- memStore[filename] = db
- }
-
- tbls := make([]*kvtable, len(tables))
-
- for i, table := range tables {
- t := db.tables[table]
- if t == nil {
- t = &kvtable{data: make(map[string][]byte)}
- db.tables[table] = t
- }
- tbls[i] = t
- }
-
- return db, tbls, nil
-}
-
-// close closes the given K/V DB.
-func (db *kvdb) close() {
-}
-
-// flush flushes the given K/V DB to disk.
-func (db *kvdb) flush() {
-}
-
-// set stores (or overwrites) the given key/value pair in the DB table.
-func (t *kvtable) set(key string, value interface{}) error {
- var val bytes.Buffer
- if err := vom.NewEncoder(&val).Encode(value); err != nil {
- return err
- }
- t.data[key] = val.Bytes()
- return nil
-}
-
-// create stores the given key/value pair in the DB table only if
-// the key does not already exist. Otherwise it returns an error.
-func (t *kvtable) create(key string, value interface{}) error {
- if t.hasKey(key) {
- return fmt.Errorf("key %s exists", key)
- }
- return t.set(key, value)
-}
-
-// update stores the given key/value pair in the DB table only if
-// the key already exists. Otherwise it returns an error.
-func (t *kvtable) update(key string, value interface{}) error {
- if !t.hasKey(key) {
- return fmt.Errorf("key %s does not exist", key)
- }
- return t.set(key, value)
-}
-
-// get retrieves the value of a key from the DB table.
-func (t *kvtable) get(key string, value interface{}) error {
- val := t.data[key]
- if val == nil {
- return fmt.Errorf("entry %s not found in the K/V DB table", key)
- }
- return vom.NewDecoder(bytes.NewBuffer(val)).Decode(value)
-}
-
-// del deletes the entry in the DB table given its key.
-func (t *kvtable) del(key string) error {
- delete(t.data, key)
- return nil
-}
-
-// hasKey returns true if the given key exists in the DB table.
-func (t *kvtable) hasKey(key string) bool {
- val, ok := t.data[key]
- return ok && val != nil
-}
-
-// keyIter iterates over all keys in a DB table invoking the given callback
-// function for each one. The key iterator callback is passed the item key.
-func (t *kvtable) keyIter(keyIterCB func(key string)) error {
- keys := make([]string, 0, len(t.data))
- for k := range t.data {
- keys = append(keys, k)
- }
- sort.Strings(keys)
- for _, k := range keys {
- keyIterCB(k)
- }
- return nil
-}
-
-// getNumKeys returns the number of keys (entries) in the DB table.
-func (t *kvtable) getNumKeys() uint64 {
- return uint64(len(t.data))
-}
diff --git a/services/syncbase/sync/kvdb_test.go b/services/syncbase/sync/kvdb_test.go
deleted file mode 100644
index 69116e3..0000000
--- a/services/syncbase/sync/kvdb_test.go
+++ /dev/null
@@ -1,407 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Tests for the Veyron Sync K/V DB component.
-
-import (
- "fmt"
- "os"
- "reflect"
- "testing"
- "time"
-)
-
-// A user structure stores info in the "users" table.
-type user struct {
- Username string
- Drinks []string
-}
-
-// A drink structure stores info in the "drinks" table.
-type drink struct {
- Name string
- Alcohol bool
-}
-
-var (
- users = []user{
- {Username: "lancelot", Drinks: []string{"beer", "coffee"}},
- {Username: "arthur", Drinks: []string{"coke", "beer", "coffee"}},
- {Username: "robin", Drinks: []string{"pepsi"}},
- {Username: "galahad"},
- }
- drinks = []drink{
- {Name: "coke", Alcohol: false},
- {Name: "pepsi", Alcohol: false},
- {Name: "beer", Alcohol: true},
- {Name: "coffee", Alcohol: false},
- }
-)
-
-// createTestDB creates a K/V DB with 2 tables.
-func createTestDB(t *testing.T) (fname string, db *kvdb, usersTbl, drinksTbl *kvtable) {
- fname = fmt.Sprintf("%s/sync_kvdb_test_%d_%d", os.TempDir(), os.Getpid(), time.Now().UnixNano())
- db, tbls, err := kvdbOpen(fname, []string{"users", "drinks"})
- if err != nil {
- os.Remove(fname)
- t.Fatalf("cannot create new K/V DB file %s: %v", fname, err)
- }
-
- usersTbl, drinksTbl = tbls[0], tbls[1]
- return
-}
-
-// initTestTables initializes the K/V tables used by the tests.
-func initTestTables(t *testing.T, usersTbl, drinksTbl *kvtable, useCreate bool) {
- userPut, drinkPut, funcName := usersTbl.set, drinksTbl.set, "set()"
- if useCreate {
- userPut, drinkPut, funcName = usersTbl.create, drinksTbl.create, "create()"
- }
-
- for _, uu := range users {
- if err := userPut(uu.Username, &uu); err != nil {
- t.Fatalf("%s failed for user %s", funcName, uu.Username)
- }
- }
-
- for _, dd := range drinks {
- if err := drinkPut(dd.Name, &dd); err != nil {
- t.Fatalf("%s failed for drink %s", funcName, dd.Name)
- }
- }
-
- return
-}
-
-// checkTestTables verifies the contents of the K/V tables.
-func checkTestTables(t *testing.T, usersTbl, drinksTbl *kvtable) {
- for _, uu := range users {
- var u2 user
- if err := usersTbl.get(uu.Username, &u2); err != nil {
- t.Fatalf("get() failed for user %s", uu.Username)
- }
- if !reflect.DeepEqual(u2, uu) {
- t.Fatalf("got wrong data for user %s: %#v instead of %#v", uu.Username, u2, uu)
- }
- if !usersTbl.hasKey(uu.Username) {
- t.Fatalf("hasKey() did not find user %s", uu.Username)
- }
- }
- for _, dd := range drinks {
- var d2 drink
- if err := drinksTbl.get(dd.Name, &d2); err != nil {
- t.Fatalf("get() failed for drink %s", dd.Name)
- }
- if !reflect.DeepEqual(d2, dd) {
- t.Fatalf("got wrong data for drink %s: %#v instead of %#v", dd.Name, d2, dd)
- }
- if !drinksTbl.hasKey(dd.Name) {
- t.Fatalf("hasKey() did not find drink %s", dd.Name)
- }
- }
-
- if num := usersTbl.getNumKeys(); num != uint64(len(users)) {
- t.Fatalf("getNumKeys(): wrong user count: got %v instead of %v", num, len(users))
- }
- if num := drinksTbl.getNumKeys(); num != uint64(len(drinks)) {
- t.Fatalf("getNumKeys(): wrong drink count: got %v instead of %v", num, len(drinks))
- }
-}
-
-func TestKVDBSet(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
-
- db.flush()
-}
-
-func TestKVDBCreate(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, true)
-
- db.flush()
-}
-
-func TestKVDBBadGet(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- // The DB is empty, all gets must fail.
- for _, uu := range users {
- var u2 user
- if err := usersTbl.get(uu.Username, &u2); err == nil {
- t.Fatalf("get() found non-existent user %s in file %s: %v", uu.Username, kvdbfile, u2)
- }
- }
- for _, dd := range drinks {
- var d2 drink
- if err := drinksTbl.get(dd.Name, &d2); err == nil {
- t.Fatalf("get() found non-existent drink %s in file %s: %v", dd.Name, kvdbfile, d2)
- }
- }
-}
-
-func TestKVDBBadUpdate(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- // The DB is empty, all updates must fail.
- for _, uu := range users {
- u2 := user{Username: uu.Username}
- if err := usersTbl.update(uu.Username, &u2); err == nil {
- t.Fatalf("update() worked for a non-existent user %s in file %s", uu.Username, kvdbfile)
- }
- }
- for _, dd := range drinks {
- d2 := drink{Name: dd.Name}
- if err := drinksTbl.update(dd.Name, &d2); err == nil {
- t.Fatalf("update() worked for a non-existent drink %s in file %s", dd.Name, kvdbfile)
- }
- }
-}
-
-func TestKVDBBadHasKey(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- // The DB is empty, all key-checks must fail.
- for _, uu := range users {
- if usersTbl.hasKey(uu.Username) {
- t.Fatalf("hasKey() found non-existent user %s in file %s", uu.Username, kvdbfile)
- }
- }
- for _, dd := range drinks {
- if drinksTbl.hasKey(dd.Name) {
- t.Fatalf("hasKey() found non-existent drink %s in file %s", dd.Name, kvdbfile)
- }
- }
-}
-
-func TestKVDBGet(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
- checkTestTables(t, usersTbl, drinksTbl)
-
- db.flush()
- checkTestTables(t, usersTbl, drinksTbl)
-}
-
-func TestKVDBBadCreate(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
-
- // Must not be able to re-create the same entries.
- for _, uu := range users {
- u2 := user{Username: uu.Username}
- if err := usersTbl.create(uu.Username, &u2); err == nil {
- t.Fatalf("create() worked for an existing user %s in file %s", uu.Username, kvdbfile)
- }
- }
- for _, dd := range drinks {
- d2 := drink{Name: dd.Name}
- if err := drinksTbl.create(dd.Name, &d2); err == nil {
- t.Fatalf("create() worked for an existing drink %s in file %s", dd.Name, kvdbfile)
- }
- }
-
- db.flush()
-}
-
-func TestKVDBReopen(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
-
- initTestTables(t, usersTbl, drinksTbl, true)
-
- // Close the re-open the file.
- db.flush()
- db.close()
-
- db, tbls, err := kvdbOpen(kvdbfile, []string{"users", "drinks"})
- if err != nil {
- t.Fatalf("Cannot re-open existing K/V DB file %s", kvdbfile)
- }
- defer db.close()
-
- usersTbl, drinksTbl = tbls[0], tbls[1]
- checkTestTables(t, usersTbl, drinksTbl)
-}
-
-func TestKVDBKeyIter(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
-
- // Get the list of all entry keys in each table.
- keylist := ""
- err := usersTbl.keyIter(func(key string) {
- keylist += key + ","
- })
- if err != nil || keylist != "arthur,galahad,lancelot,robin," {
- t.Fatalf("keyIter() failed in file %s: err %v, user names: %v", kvdbfile, err, keylist)
- }
- keylist = ""
- err = drinksTbl.keyIter(func(key string) {
- keylist += key + ","
- })
- if err != nil || keylist != "beer,coffee,coke,pepsi," {
- t.Fatalf("keyIter() failed in file %s: err %v, drink names: %v", kvdbfile, err, keylist)
- }
-
- db.flush()
-}
-
-func TestKVDBUpdate(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
-
- initTestTables(t, usersTbl, drinksTbl, false)
- db.flush()
- db.close()
-
- db, tbls, err := kvdbOpen(kvdbfile, []string{"users", "drinks"})
- if err != nil {
- t.Fatalf("Cannot re-open existing K/V DB file %s", kvdbfile)
- }
- defer db.close()
-
- usersTbl, drinksTbl = tbls[0], tbls[1]
-
- for _, uu := range users {
- key := uu.Username
- u2 := uu
- u2.Username += "-New"
-
- if err = usersTbl.update(key, &u2); err != nil {
- t.Fatalf("update() failed for user %s in file %s", key, kvdbfile)
- }
-
- var u3 user
- if err = usersTbl.get(key, &u3); err != nil {
- t.Fatalf("get() failed for user %s in file %s", key, kvdbfile)
- }
- if !reflect.DeepEqual(u3, u2) {
- t.Fatalf("got wrong new data for user %s in file %s: %#v instead of %#v", key, kvdbfile, u3, u2)
- }
- }
-
- for _, dd := range drinks {
- key := dd.Name
- d2 := dd
- d2.Alcohol = !d2.Alcohol
-
- if err = drinksTbl.update(key, &d2); err != nil {
- t.Fatalf("update() failed for drink %s in file %s", key, kvdbfile)
- }
-
- var d3 drink
- if err = drinksTbl.get(key, &d3); err != nil {
- t.Fatalf("get() failed for drink %s in file %s", key, kvdbfile)
- }
- if !reflect.DeepEqual(d3, d2) {
- t.Fatalf("got wrong new data for drink %s in file %s: %#v instead of %#v", key, kvdbfile, d3, d2)
- }
- }
-
- db.flush()
-}
-
-func TestKVDBSetAgain(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
-
- for _, uu := range users {
- key := uu.Username
- u2 := uu
- u2.Username += "-New"
-
- if err := usersTbl.set(key, &u2); err != nil {
- t.Fatalf("set() again failed for user %s in file %s", key, kvdbfile)
- }
-
- var u3 user
- if err := usersTbl.get(key, &u3); err != nil {
- t.Fatalf("get() failed for user %s in file %s", key, kvdbfile)
- }
- if !reflect.DeepEqual(u3, u2) {
- t.Fatalf("got wrong new data for user %s in file %s: %#v instead of %#v", key, kvdbfile, u3, u2)
- }
- }
-
- for _, dd := range drinks {
- key := dd.Name
- d2 := dd
- d2.Alcohol = !d2.Alcohol
-
- if err := drinksTbl.update(key, &d2); err != nil {
- t.Fatalf("set() again failed for drink %s in file %s", key, kvdbfile)
- }
-
- var d3 drink
- if err := drinksTbl.get(key, &d3); err != nil {
- t.Fatalf("get() failed for drink %s in file %s", key, kvdbfile)
- }
- if !reflect.DeepEqual(d3, d2) {
- t.Fatalf("got wrong new data for drink %s in file %s: %#v instead of %#v", key, kvdbfile, d3, d2)
- }
- }
-
- db.flush()
-}
-
-func TestKVDBDelete(t *testing.T) {
- kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
- defer os.Remove(kvdbfile)
- defer db.close()
-
- initTestTables(t, usersTbl, drinksTbl, false)
-
- db.flush()
-
- // Delete entries and verify that they no longer exist.
-
- for _, uu := range users {
- key := uu.Username
- if err := usersTbl.del(key); err != nil {
- t.Errorf("del() failed for user %s in file %s", key, kvdbfile)
- }
- if usersTbl.hasKey(key) {
- t.Errorf("hasKey() still finds deleted user %s in file %s", key, kvdbfile)
- }
- }
-
- for _, dd := range drinks {
- key := dd.Name
- if err := drinksTbl.del(key); err != nil {
- t.Errorf("del() failed for drink %s in file %s", key, kvdbfile)
- }
- if drinksTbl.hasKey(key) {
- t.Errorf("hasKey() still finds deleted drink %s in file %s", key, kvdbfile)
- }
- }
-
- db.flush()
-}
diff --git a/services/syncbase/sync/replay_test.go b/services/syncbase/sync/replay_test.go
deleted file mode 100644
index 218abae..0000000
--- a/services/syncbase/sync/replay_test.go
+++ /dev/null
@@ -1,226 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Used to ease the setup of Veyron Sync test scenarios.
-// Parses a sync command file and returns a vector of commands to execute.
-//
-// Used by different test replay engines:
-// - dagReplayCommands() executes the parsed commands at the DAG API level.
-// - logReplayCommands() executes the parsed commands at the Log API level.
-
-import (
- "bufio"
- "fmt"
- "os"
- "strconv"
- "strings"
-)
-
-const (
- addLocal = iota
- addRemote
- setDevTable
- linkLocal
- linkRemote
-)
-
-type syncCommand struct {
- cmd int
- objID ObjId
- version Version
- parents []Version
- logrec string
- devID DeviceId
- genVec GenVector
- txID TxId
- txCount uint32
- deleted bool
-}
-
-func strToVersion(verStr string) (Version, error) {
- ver, err := strconv.ParseUint(verStr, 10, 64)
- if err != nil {
- return 0, err
- }
- return Version(ver), nil
-}
-
-func parseSyncCommands(file string) ([]syncCommand, error) {
- cmds := []syncCommand{}
- sf, err := os.Open("testdata/" + file)
- if err != nil {
- return nil, err
- }
- defer sf.Close()
-
- scanner := bufio.NewScanner(sf)
- lineno := 0
- for scanner.Scan() {
- lineno++
- line := strings.TrimSpace(scanner.Text())
- if line == "" || line[0] == '#' {
- continue
- }
-
- args := strings.Split(line, "|")
- nargs := len(args)
-
- switch args[0] {
- case "addl", "addr":
- expNargs := 9
- if nargs != expNargs {
- return nil, fmt.Errorf("%s:%d: need %d args instead of %d", file, lineno, expNargs, nargs)
- }
- version, err := strToVersion(args[2])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid version: %s", file, lineno, args[2])
- }
- var parents []Version
- for i := 3; i <= 4; i++ {
- if args[i] != "" {
- pver, err := strToVersion(args[i])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid parent: %s", file, lineno, args[i])
- }
- parents = append(parents, pver)
- }
- }
-
- txID, err := strToTxId(args[6])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid TxId: %s", file, lineno, args[6])
- }
- txCount, err := strconv.ParseUint(args[7], 10, 32)
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid tx count: %s", file, lineno, args[7])
- }
- del, err := strconv.ParseBool(args[8])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid deleted bit: %s", file, lineno, args[8])
- }
- cmd := syncCommand{
- version: version,
- parents: parents,
- logrec: args[5],
- txID: txID,
- txCount: uint32(txCount),
- deleted: del,
- }
- if args[0] == "addl" {
- cmd.cmd = addLocal
- } else {
- cmd.cmd = addRemote
- }
- if cmd.objID, err = strToObjId(args[1]); err != nil {
- return nil, fmt.Errorf("%s:%d: invalid object ID: %s", file, lineno, args[1])
- }
- cmds = append(cmds, cmd)
-
- case "setdev":
- expNargs := 3
- if nargs != expNargs {
- return nil, fmt.Errorf("%s:%d: need %d args instead of %d", file, lineno, expNargs, nargs)
- }
-
- genVec := make(GenVector)
- for _, elem := range strings.Split(args[2], ",") {
- kv := strings.Split(elem, ":")
- if len(kv) != 2 {
- return nil, fmt.Errorf("%s:%d: invalid gen vector key/val: %s", file, lineno, elem)
- }
- genID, err := strToGenId(kv[1])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid gen ID: %s", file, lineno, kv[1])
- }
- genVec[DeviceId(kv[0])] = genID
- }
-
- cmd := syncCommand{cmd: setDevTable, devID: DeviceId(args[1]), genVec: genVec}
- cmds = append(cmds, cmd)
-
- case "linkl", "linkr":
- expNargs := 6
- if nargs != expNargs {
- return nil, fmt.Errorf("%s:%d: need %d args instead of %d", file, lineno, expNargs, nargs)
- }
-
- version, err := strToVersion(args[2])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid version: %s", file, lineno, args[2])
- }
- if args[3] == "" {
- return nil, fmt.Errorf("%s:%d: parent (to-node) version not specified", file, lineno)
- }
- if args[4] != "" {
- return nil, fmt.Errorf("%s:%d: cannot specify a 2nd parent (to-node): %s", file, lineno, args[4])
- }
- parent, err := strToVersion(args[3])
- if err != nil {
- return nil, fmt.Errorf("%s:%d: invalid parent (to-node) version: %s", file, lineno, args[3])
- }
-
- cmd := syncCommand{version: version, parents: []Version{parent}, logrec: args[5]}
- if args[0] == "linkl" {
- cmd.cmd = linkLocal
- } else {
- cmd.cmd = linkRemote
- }
- if cmd.objID, err = strToObjId(args[1]); err != nil {
- return nil, fmt.Errorf("%s:%d: invalid object ID: %s", file, lineno, args[1])
- }
- cmds = append(cmds, cmd)
-
- default:
- return nil, fmt.Errorf("%s:%d: invalid operation: %s", file, lineno, args[0])
- }
- }
-
- err = scanner.Err()
- return cmds, err
-}
-
-func dagReplayCommands(dag *dag, syncfile string) error {
- cmds, err := parseSyncCommands(syncfile)
- if err != nil {
- return err
- }
-
- for _, cmd := range cmds {
- switch cmd.cmd {
- case addLocal:
- err = dag.addNode(cmd.objID, cmd.version, false, cmd.deleted, cmd.parents, cmd.logrec, NoTxId)
- if err != nil {
- return fmt.Errorf("cannot add local node %v:%d to DAG: %v", cmd.objID, cmd.version, err)
- }
- if err := dag.moveHead(cmd.objID, cmd.version); err != nil {
- return fmt.Errorf("cannot move head to %v:%d in DAG: %v", cmd.objID, cmd.version, err)
- }
- dag.flush()
-
- case addRemote:
- err = dag.addNode(cmd.objID, cmd.version, true, cmd.deleted, cmd.parents, cmd.logrec, NoTxId)
- if err != nil {
- return fmt.Errorf("cannot add remote node %v:%d to DAG: %v", cmd.objID, cmd.version, err)
- }
- dag.flush()
-
- case linkLocal:
- if err = dag.addParent(cmd.objID, cmd.version, cmd.parents[0], false); err != nil {
- return fmt.Errorf("cannot add local parent %d to DAG node %v:%d: %v",
- cmd.parents[0], cmd.objID, cmd.version, err)
- }
- dag.flush()
-
- case linkRemote:
- if err = dag.addParent(cmd.objID, cmd.version, cmd.parents[0], true); err != nil {
- return fmt.Errorf("cannot add remote parent %d to DAG node %v:%d: %v",
- cmd.parents[0], cmd.objID, cmd.version, err)
- }
- dag.flush()
- }
- }
- return nil
-}
diff --git a/services/syncbase/sync/sgtable.go b/services/syncbase/sync/sgtable.go
deleted file mode 100644
index 41818c1..0000000
--- a/services/syncbase/sync/sgtable.go
+++ /dev/null
@@ -1,500 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// The SyncGroup Table stores the group information in a K/V DB. It also
-// maintains an index to provide access by SyncGroup ID or name.
-//
-// The SyncGroup info is fetched from the SyncGroup server by the create or
-// join operations, and is regularly updated after that.
-//
-// The DB contains two tables persisted to disk (data, names) and one
-// in-memory (ephemeral) map (members):
-// * data: one entry per SyncGroup ID containing the SyncGroup data
-// * names: one entry per SyncGroup name pointing to its SyncGroup ID
-// * members: an inverted index of SyncGroup members to SyncGroup IDs
-// built from the list of SyncGroup joiners
-
-import (
- "errors"
- "fmt"
- "path"
- "strconv"
-
- "v.io/x/lib/vlog"
- "v.io/x/ref/lib/stats"
-)
-
-var (
- errBadSGTable = errors.New("invalid SyncGroup Table")
-)
-
-type syncGroupTable struct {
- fname string // file pathname
- store *kvdb // underlying K/V store
- sgData *kvtable // pointer to "data" table in the kvdb
- sgNames *kvtable // pointer to "names" table in the kvdb
- members map[string]*memberInfo // in-memory tracking of SyncGroup member info
-
- // SyncGroup Table stats
- numSGs *stats.Integer // number of SyncGroups
- numMembers *stats.Integer // number of Sync members
-}
-
-type syncGroupData struct {
- SrvInfo SyncGroupInfo // SyncGroup info from SyncGroupServer
- LocalPath string // local path of the SyncGroup in the Store
-}
-
-type memberInfo struct {
- gids map[GroupId]*memberMetaData // map of SyncGroup IDs joined and their metadata
-}
-
-type memberMetaData struct {
- metaData JoinerMetaData // joiner metadata at the SyncGroup server
-}
-
-type sgSet map[GroupId]struct{} // a set of SyncGroups
-
-// strToGroupId converts a SyncGroup ID in string format to an GroupId.
-func strToGroupId(str string) (GroupId, error) {
- id, err := strconv.ParseUint(str, 10, 64)
- if err != nil {
- return NoGroupId, err
- }
- return GroupId(id), nil
-}
-
-// openSyncGroupTable opens or creates a syncGroupTable for the given filename.
-func openSyncGroupTable(filename string) (*syncGroupTable, error) {
- // Open the file and create it if it does not exist.
- // Also initialize the store and its tables.
- db, tbls, err := kvdbOpen(filename, []string{"data", "names"})
- if err != nil {
- return nil, err
- }
-
- s := &syncGroupTable{
- fname: filename,
- store: db,
- sgData: tbls[0],
- sgNames: tbls[1],
- members: make(map[string]*memberInfo),
- numSGs: stats.NewInteger(statsNumSyncGroup),
- numMembers: stats.NewInteger(statsNumMember),
- }
-
- // Reconstruct the in-memory tracking maps by iterating over the SyncGroups.
- // This is needed when an existing SyncGroup Table file is re-opened.
- s.sgData.keyIter(func(gidStr string) {
- // Get the SyncGroup data given the group ID in string format (as the data table key).
- gid, err := strToGroupId(gidStr)
- if err != nil {
- return
- }
-
- data, err := s.getSyncGroupByID(gid)
- if err != nil {
- return
- }
-
- s.numSGs.Incr(1)
-
- // Add all SyncGroup members to the members inverted index.
- s.addAllMembers(data)
- })
-
- return s, nil
-}
-
-// close closes the syncGroupTable and invalidates its structure.
-func (s *syncGroupTable) close() {
- if s.store != nil {
- s.store.close() // this also closes the tables
- stats.Delete(statsNumSyncGroup)
- stats.Delete(statsNumMember)
- }
- *s = syncGroupTable{} // zero out the structure
-}
-
-// flush flushes the syncGroupTable store to disk.
-func (s *syncGroupTable) flush() {
- if s.store != nil {
- s.store.flush()
- }
-}
-
-// addSyncGroup adds a new SyncGroup given its information.
-func (s *syncGroupTable) addSyncGroup(sgData *syncGroupData) error {
- if s.store == nil {
- return errBadSGTable
- }
- if sgData == nil {
- return errors.New("group information not specified")
- }
- gid, name := sgData.SrvInfo.Id, path.Join(sgData.SrvInfo.ServerName, sgData.SrvInfo.GroupName)
- if name == "" {
- return errors.New("group name not specified")
- }
- if sgData.LocalPath == "" {
- return errors.New("group local path not specified")
- }
- if len(sgData.SrvInfo.Joiners) == 0 {
- return errors.New("group has no joiners")
- }
-
- if s.hasSGDataEntry(gid) {
- return fmt.Errorf("group %d already exists", gid)
- }
- if s.hasSGNameEntry(name) {
- return fmt.Errorf("group name %s already exists", name)
- }
-
- // Add the group name and data entries.
- if err := s.setSGNameEntry(name, gid); err != nil {
- return err
- }
-
- if err := s.setSGDataEntry(gid, sgData); err != nil {
- s.delSGNameEntry(name)
- return err
- }
-
- s.numSGs.Incr(1)
- s.addAllMembers(sgData)
- return nil
-}
-
-// getSyncGroupID retrieves the SyncGroup ID given its name.
-func (s *syncGroupTable) getSyncGroupID(name string) (GroupId, error) {
- return s.getSGNameEntry(name)
-}
-
-// getSyncGroupName retrieves the SyncGroup name given its ID.
-func (s *syncGroupTable) getSyncGroupName(gid GroupId) (string, error) {
- data, err := s.getSyncGroupByID(gid)
- if err != nil {
- return "", err
- }
-
- return path.Join(data.SrvInfo.ServerName, data.SrvInfo.GroupName), nil
-}
-
-// getSyncGroupByID retrieves the SyncGroup given its ID.
-func (s *syncGroupTable) getSyncGroupByID(gid GroupId) (*syncGroupData, error) {
- return s.getSGDataEntry(gid)
-}
-
-// getSyncGroupByName retrieves the SyncGroup given its name.
-func (s *syncGroupTable) getSyncGroupByName(name string) (*syncGroupData, error) {
- gid, err := s.getSyncGroupID(name)
- if err != nil {
- return nil, err
- }
- return s.getSyncGroupByID(gid)
-}
-
-// updateSyncGroup updates the SyncGroup data.
-func (s *syncGroupTable) updateSyncGroup(data *syncGroupData) error {
- if s.store == nil {
- return errBadSGTable
- }
- if data == nil {
- return errors.New("SyncGroup data not specified")
- }
- if data.SrvInfo.GroupName == "" {
- return errors.New("group name not specified")
- }
- if len(data.SrvInfo.Joiners) == 0 {
- return errors.New("group has no joiners")
- }
-
- fullGroupName := path.Join(data.SrvInfo.ServerName, data.SrvInfo.GroupName)
- oldData, err := s.getSyncGroupByName(fullGroupName)
- if err != nil {
- return err
- }
-
- if data.SrvInfo.Id != oldData.SrvInfo.Id {
- return fmt.Errorf("cannot change ID of SyncGroup name %s", fullGroupName)
- }
- if data.LocalPath == "" {
- data.LocalPath = oldData.LocalPath
- } else if data.LocalPath != oldData.LocalPath {
- return fmt.Errorf("cannot change local path of SyncGroup name %s", fullGroupName)
- }
-
- // Get the old set of SyncGroup joiners and diff it with the new set.
- // Add all the current members because this inserts the new members and
- // updates the metadata of the existing ones (addMember() is like a "put").
- // Delete the members that are no longer part of the SyncGroup.
- gid := oldData.SrvInfo.Id
- newJoiners, oldJoiners := data.SrvInfo.Joiners, oldData.SrvInfo.Joiners
-
- for member, memberData := range newJoiners {
- s.addMember(member, gid, memberData)
- }
-
- for member := range oldJoiners {
- if _, ok := newJoiners[member]; !ok {
- s.delMember(member, gid)
- }
- }
-
- return s.setSGDataEntry(gid, data)
-}
-
-// delSyncGroupByID deletes the SyncGroup given its ID.
-func (s *syncGroupTable) delSyncGroupByID(gid GroupId) error {
- data, err := s.getSyncGroupByID(gid)
- if err != nil {
- return err
- }
- if err = s.delSGNameEntry(path.Join(data.SrvInfo.ServerName, data.SrvInfo.GroupName)); err != nil {
- return err
- }
-
- s.numSGs.Incr(-1)
- s.delAllMembers(data)
- return s.delSGDataEntry(gid)
-}
-
-// delSyncGroupByName deletes the SyncGroup given its name.
-func (s *syncGroupTable) delSyncGroupByName(name string) error {
- gid, err := s.getSyncGroupID(name)
- if err != nil {
- return err
- }
-
- return s.delSyncGroupByID(gid)
-}
-
-// getAllSyncGroupNames returns the names of all SyncGroups.
-func (s *syncGroupTable) getAllSyncGroupNames() ([]string, error) {
- if s.store == nil {
- return nil, errBadSGTable
- }
-
- names := make([]string, 0)
-
- err := s.sgNames.keyIter(func(name string) {
- names = append(names, name)
- })
-
- if err != nil {
- return nil, err
- }
- return names, nil
-}
-
-// getMembers returns all SyncGroup members and the count of SyncGroups each one joined.
-func (s *syncGroupTable) getMembers() (map[string]uint32, error) {
- if s.store == nil {
- return nil, errBadSGTable
- }
-
- members := make(map[string]uint32)
- for member, info := range s.members {
- members[member] = uint32(len(info.gids))
- }
-
- return members, nil
-}
-
-// getMemberInfo returns SyncGroup information for a given member.
-func (s *syncGroupTable) getMemberInfo(member string) (*memberInfo, error) {
- if s.store == nil {
- return nil, errBadSGTable
- }
-
- info, ok := s.members[member]
- if !ok {
- return nil, fmt.Errorf("unknown member: %s", member)
- }
-
- return info, nil
-}
-
-// addMember inserts or updates a (member, group ID) entry in the in-memory
-// structure that indexes SyncGroup memberships based on member names and stores
-// in it the member's joiner metadata.
-func (s *syncGroupTable) addMember(member string, gid GroupId, metadata JoinerMetaData) {
- if s.store == nil {
- return
- }
-
- info, ok := s.members[member]
- if !ok {
- info = &memberInfo{gids: make(map[GroupId]*memberMetaData)}
- s.members[member] = info
- s.numMembers.Incr(1)
- }
-
- info.gids[gid] = &memberMetaData{metaData: metadata}
-}
-
-// delMember removes a (member, group ID) entry from the in-memory structure
-// that indexes SyncGroup memberships based on member names.
-func (s *syncGroupTable) delMember(member string, gid GroupId) {
- if s.store == nil {
- return
- }
-
- info, ok := s.members[member]
- if !ok {
- return
- }
-
- delete(info.gids, gid)
- if len(info.gids) == 0 {
- delete(s.members, member)
- s.numMembers.Incr(-1)
- }
-}
-
-// addAllMembers inserts all members of a SyncGroup in the in-memory structure
-// that indexes SyncGroup memberships based on member names.
-func (s *syncGroupTable) addAllMembers(data *syncGroupData) {
- if s.store == nil || data == nil {
- return
- }
-
- gid := data.SrvInfo.Id
- for member, memberData := range data.SrvInfo.Joiners {
- s.addMember(member, gid, memberData)
- }
-}
-
-// delAllMembers removes all members of a SyncGroup from the in-memory structure
-// that indexes SyncGroup memberships based on member names.
-func (s *syncGroupTable) delAllMembers(data *syncGroupData) {
- if s.store == nil || data == nil {
- return
- }
-
- gid := data.SrvInfo.Id
- for member := range data.SrvInfo.Joiners {
- s.delMember(member, gid)
- }
-}
-
-// Low-level functions to access the tables in the K/V DB.
-// They directly access the table entries without tracking their relationships.
-
-// sgDataKey returns the key used to access the SyncGroup data in the DB.
-func sgDataKey(gid GroupId) string {
- return fmt.Sprintf("%d", gid)
-}
-
-// hasSGDataEntry returns true if the SyncGroup data entry exists in the DB.
-func (s *syncGroupTable) hasSGDataEntry(gid GroupId) bool {
- if s.store == nil {
- return false
- }
- key := sgDataKey(gid)
- return s.sgData.hasKey(key)
-}
-
-// setSGDataEntry stores the SyncGroup data in the DB.
-func (s *syncGroupTable) setSGDataEntry(gid GroupId, data *syncGroupData) error {
- if s.store == nil {
- return errBadSGTable
- }
- key := sgDataKey(gid)
- return s.sgData.set(key, data)
-}
-
-// getSGDataEntry retrieves from the DB the SyncGroup data for a given group ID.
-func (s *syncGroupTable) getSGDataEntry(gid GroupId) (*syncGroupData, error) {
- if s.store == nil {
- return nil, errBadSGTable
- }
- var data syncGroupData
- key := sgDataKey(gid)
- if err := s.sgData.get(key, &data); err != nil {
- return nil, err
- }
- return &data, nil
-}
-
-// delSGDataEntry deletes the SyncGroup data from the DB.
-func (s *syncGroupTable) delSGDataEntry(gid GroupId) error {
- if s.store == nil {
- return errBadSGTable
- }
- key := sgDataKey(gid)
- return s.sgData.del(key)
-}
-
-// sgNameKey returns the key used to access the SyncGroup name in the DB.
-func sgNameKey(name string) string {
- return name
-}
-
-// hasSGNameEntry returns true if the SyncGroup name entry exists in the DB.
-func (s *syncGroupTable) hasSGNameEntry(name string) bool {
- if s.store == nil {
- return false
- }
- key := sgNameKey(name)
- return s.sgNames.hasKey(key)
-}
-
-// setSGNameEntry stores the SyncGroup name to ID mapping in the DB.
-func (s *syncGroupTable) setSGNameEntry(name string, gid GroupId) error {
- if s.store == nil {
- return errBadSGTable
- }
- key := sgNameKey(name)
- return s.sgNames.set(key, gid)
-}
-
-// getSGNameEntry retrieves the SyncGroup name to ID mapping from the DB.
-func (s *syncGroupTable) getSGNameEntry(name string) (GroupId, error) {
- var gid GroupId
- if s.store == nil {
- return gid, errBadSGTable
- }
- key := sgNameKey(name)
- err := s.sgNames.get(key, &gid)
- return gid, err
-}
-
-// delSGNameEntry deletes the SyncGroup name to ID mapping from the DB.
-func (s *syncGroupTable) delSGNameEntry(name string) error {
- if s.store == nil {
- return errBadSGTable
- }
- key := sgNameKey(name)
- return s.sgNames.del(key)
-}
-
-// dump writes to the log file information on all SyncGroups.
-func (s *syncGroupTable) dump() {
- if s.store == nil {
- return
- }
-
- s.sgData.keyIter(func(gidStr string) {
- // Get the SyncGroup data given the group ID in string format (as the data table key).
- gid, err := strToGroupId(gidStr)
- if err != nil {
- return
- }
-
- data, err := s.getSyncGroupByID(gid)
- if err != nil {
- return
- }
-
- members := make([]string, 0, len(data.SrvInfo.Joiners))
- for joiner := range data.SrvInfo.Joiners {
- members = append(members, joiner)
- }
- vlog.VI(1).Infof("DUMP: SyncGroup %s: id %v, path %s, members: %s",
- path.Join(data.SrvInfo.ServerName, data.SrvInfo.GroupName),
- gid, data.LocalPath, members)
- })
-}
diff --git a/services/syncbase/sync/sgtable_test.go b/services/syncbase/sync/sgtable_test.go
deleted file mode 100644
index c80a2a9..0000000
--- a/services/syncbase/sync/sgtable_test.go
+++ /dev/null
@@ -1,769 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Tests for the Veyron SyncGroup Table.
-
-import (
- "os"
- "reflect"
- "testing"
-
- "v.io/x/ref/lib/stats"
-)
-
-// TestSyncGroupTableOpen tests the creation of a SyncGroup Table, closing and re-opening it.
-// It also verifies that its backing file is created and that a 2nd close is safe.
-func TestSyncGroupTableOpen(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- fsize := getFileSize(sgfile)
- if fsize < 0 {
- //t.Fatalf("SyncGroup Table file %s not created", sgfile)
- }
-
- sg.flush()
- oldfsize := fsize
- fsize = getFileSize(sgfile)
- if fsize <= oldfsize {
- //t.Fatalf("SyncGroup Table file %s not flushed", sgfile)
- }
-
- sg.close()
-
- sg, err = openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot re-open existing SyncGroup Table file %s", sgfile)
- }
-
- oldfsize = fsize
- fsize = getFileSize(sgfile)
- if fsize != oldfsize {
- t.Fatalf("SyncGroup Table file %s size changed across re-open", sgfile)
- }
-
- sg.close()
- sg.close() // multiple closes should be a safe NOP
-
- fsize = getFileSize(sgfile)
- if fsize != oldfsize {
- t.Fatalf("SyncGroup Table file %s size changed across close", sgfile)
- }
-
- // Fail opening a SyncGroup Table in a non-existent directory.
- _, err = openSyncGroupTable("/not/really/there/junk.sg")
- if err == nil {
- //t.Fatalf("openSyncGroupTable() did not fail when using a bad pathname")
- }
-}
-
-// TestInvalidSyncGroupTable tests using methods on an invalid (closed) SyncGroup Table.
-func TestInvalidSyncGroupTable(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- sg.close()
-
- sgid, err := strToGroupId("1234")
- if err != nil {
- t.Error(err)
- }
-
- validateError := func(t *testing.T, err error, funcName string) {
- if err == nil || err.Error() != "invalid SyncGroup Table" {
- t.Errorf("%s() did not fail on a closed SyncGroup Table: %v", funcName, err)
- }
- }
-
- err = sg.addSyncGroup(&syncGroupData{})
- validateError(t, err, "addSyncGroup")
-
- _, err = sg.getSyncGroupID("foobar")
- validateError(t, err, "getSyncGroupID")
-
- _, err = sg.getSyncGroupName(sgid)
- validateError(t, err, "getSyncGroupName")
-
- _, err = sg.getSyncGroupByID(sgid)
- validateError(t, err, "getSyncGroupByID")
-
- _, err = sg.getSyncGroupByName("foobar")
- validateError(t, err, "getSyncGroupByName")
-
- err = sg.updateSyncGroup(&syncGroupData{})
- validateError(t, err, "updateSyncGroup")
-
- err = sg.delSyncGroupByID(sgid)
- validateError(t, err, "delSyncGroupByID")
-
- err = sg.delSyncGroupByName("foobar")
- validateError(t, err, "delSyncGroupByName")
-
- _, err = sg.getAllSyncGroupNames()
- validateError(t, err, "getAllSyncGroupNames")
-
- _, err = sg.getMembers()
- validateError(t, err, "getMembers")
-
- _, err = sg.getMemberInfo("foobar")
- validateError(t, err, "getMemberInfo")
-
- err = sg.setSGDataEntry(sgid, &syncGroupData{})
- validateError(t, err, "setSGDataEntry")
-
- _, err = sg.getSGDataEntry(sgid)
- validateError(t, err, "getSGDataEntry")
-
- err = sg.delSGDataEntry(sgid)
- validateError(t, err, "delSGDataEntry")
-
- err = sg.setSGNameEntry("foobar", sgid)
- validateError(t, err, "setSGNameEntry")
-
- _, err = sg.getSGNameEntry("foobar")
- validateError(t, err, "getSGNameEntry")
-
- err = sg.delSGNameEntry("foobar")
- validateError(t, err, "delSGNameEntry")
-
- // These calls should be harmless NOPs.
- sg.dump()
- sg.flush()
- sg.close()
- sg.addMember("foobar", sgid, JoinerMetaData{})
- sg.delMember("foobar", sgid)
- sg.addAllMembers(&syncGroupData{})
- sg.delAllMembers(&syncGroupData{})
-
- if sg.hasSGDataEntry(sgid) {
- t.Errorf("hasSGDataEntry() found an entry on a closed SyncGroup Table")
- }
- if sg.hasSGNameEntry("foobar") {
- t.Errorf("hasSGNameEntry() found an entry on a closed SyncGroup Table")
- }
-}
-
-// checkSGStats verifies the SyncGroup Table stats counters.
-func checkSGStats(t *testing.T, which string, numSG, numMembers int64) {
- if num, err := stats.Value(statsNumSyncGroup); err != nil || num != numSG {
- t.Errorf("num-syncgroups (%s): got %v (err: %v) instead of %v", which, num, err, numSG)
- }
- if num, err := stats.Value(statsNumMember); err != nil || num != numMembers {
- t.Errorf("num-members (%s): got %v (err: %v) instead of %v", which, num, err, numMembers)
- }
-}
-
-// TestAddSyncGroup tests adding SyncGroups.
-func TestAddSyncGroup(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- checkSGStats(t, "add-1", 0, 0)
-
- sgname := "foobar"
- sgid, err := strToGroupId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- sgData := &syncGroupData{
- SrvInfo: SyncGroupInfo{
- Id: sgid,
- GroupName: sgname,
- Joiners: map[string]JoinerMetaData{
- "phone": JoinerMetaData{SyncPriority: 10},
- "tablet": JoinerMetaData{SyncPriority: 25},
- "cloud": JoinerMetaData{SyncPriority: 1},
- },
- },
- LocalPath: "/foo/bar",
- }
-
- err = sg.addSyncGroup(sgData)
- if err != nil {
- t.Errorf("adding SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- // Verify SyncGroup ID, name, and data.
- if id, err := sg.getSyncGroupID(sgname); err != nil || id != sgid {
- t.Errorf("cannot get back ID of SyncGroup %s: got ID %d instead of %d; err: %v", sgname, id, sgid, err)
- }
- if name, err := sg.getSyncGroupName(sgid); err != nil || name != sgname {
- t.Errorf("cannot get back name of SyncGroup ID %d: got %s instead of %s; err: %v", sgid, name, sgname, err)
- }
-
- data, err := sg.getSyncGroupByID(sgid)
- if err != nil {
- t.Errorf("cannot get SyncGroup by ID %d: %v", sgid, err)
- }
- if !reflect.DeepEqual(data, sgData) {
- t.Errorf("invalid SyncGroup data for group ID %d: got %v instead of %v", sgid, data, sgData)
- }
-
- data, err = sg.getSyncGroupByName(sgname)
- if err != nil {
- t.Errorf("cannot get SyncGroup by Name %s: %v", sgname, err)
- }
- if !reflect.DeepEqual(data, sgData) {
- t.Errorf("invalid SyncGroup data for group name %s: got %v instead of %v", sgname, data, sgData)
- }
-
- // Verify membership data.
- members, err := sg.getMembers()
- if err != nil {
- t.Errorf("cannot get all SyncGroup members: %v", err)
- }
- expMembers := map[string]uint32{"phone": 1, "tablet": 1, "cloud": 1}
- if !reflect.DeepEqual(members, expMembers) {
- t.Errorf("invalid SyncGroup members: got %v instead of %v", members, expMembers)
- }
-
- expMetaData := map[string]*memberMetaData{
- "phone": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 10}},
- "tablet": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 25}},
- "cloud": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 1}},
- }
- for mm := range members {
- info, err := sg.getMemberInfo(mm)
- if err != nil || info == nil {
- t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
- }
- if len(info.gids) != 1 {
- t.Errorf("invalid info for SyncGroup member %s: %v", mm, info)
- }
- expJoinerMetaData := expMetaData[mm]
- joinerMetaData := info.gids[sgid]
- if !reflect.DeepEqual(joinerMetaData, expJoinerMetaData) {
- t.Errorf("invalid joiner Data for SyncGroup member %s under group ID %d: got %v instead of %v",
- mm, sgid, joinerMetaData, expJoinerMetaData)
- }
- }
-
- checkSGStats(t, "add-2", 1, 3)
-
- // Use a non-existent member.
- if info, err := sg.getMemberInfo("should-not-be-there"); err == nil {
- t.Errorf("found info for invalid SyncGroup member: %v", info)
- }
-
- // Adding a SyncGroup for a pre-existing group ID or name should fail.
- err = sg.addSyncGroup(sgData)
- if err == nil {
- t.Errorf("re-adding SyncGroup %d did not fail", sgid)
- }
-
- sgData.SrvInfo.Id, err = strToGroupId("5555")
- if err != nil {
- t.Fatal(err)
- }
- err = sg.addSyncGroup(sgData)
- if err == nil {
- t.Errorf("adding SyncGroup %s with a different ID did not fail", sgname)
- }
-
- checkSGStats(t, "add-3", 1, 3)
-
- sg.dump()
- sg.close()
-}
-
-// TestInvalidAddSyncGroup tests adding SyncGroups.
-func TestInvalidAddSyncGroup(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- sgname := "foobar"
- sgid, err := strToGroupId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- err = sg.addSyncGroup(nil)
- if err == nil {
- t.Errorf("adding a nil SyncGroup did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData := &syncGroupData{}
- sgData.SrvInfo.Id = sgid
-
- err = sg.addSyncGroup(sgData)
- if err == nil {
- t.Errorf("adding a SyncGroup with an empty name did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData.SrvInfo.GroupName = sgname
-
- err = sg.addSyncGroup(sgData)
- if err == nil {
- t.Errorf("adding a SyncGroup with no local path did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData.LocalPath = "/foo/bar"
-
- err = sg.addSyncGroup(sgData)
- if err == nil {
- t.Errorf("adding a SyncGroup with no joiners did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sg.dump()
- sg.close()
-}
-
-// TestUpdateSyncGroup tests updating a SyncGroup.
-func TestUpdateSyncGroup(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- err = sg.updateSyncGroup(nil)
- if err == nil {
- t.Errorf("updating a nil SyncGroup did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData := &syncGroupData{}
- err = sg.updateSyncGroup(sgData)
- if err == nil {
- t.Errorf("updating a SyncGroup with an empty name did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData.SrvInfo.GroupName = "blabla"
- err = sg.updateSyncGroup(sgData)
- if err == nil {
- t.Errorf("updating a SyncGroup with no joiners did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData.SrvInfo.Joiners = map[string]JoinerMetaData{
- "phone": JoinerMetaData{SyncPriority: 10},
- }
- err = sg.updateSyncGroup(sgData)
- if err == nil {
- t.Errorf("updating a SyncGroup with a non-existing name did not fail in SyncGroup Table file %s", sgfile)
- }
-
- // Create the SyncGroup to update later.
- sgname := "foobar"
- sgid, err := strToGroupId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- sgData = &syncGroupData{
- SrvInfo: SyncGroupInfo{
- Id: sgid,
- GroupName: sgname,
- Joiners: map[string]JoinerMetaData{
- "phone": JoinerMetaData{SyncPriority: 10},
- "tablet": JoinerMetaData{SyncPriority: 25},
- "cloud": JoinerMetaData{SyncPriority: 1},
- },
- },
- LocalPath: "/foo/bar",
- }
-
- err = sg.addSyncGroup(sgData)
- if err != nil {
- t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- checkSGStats(t, "up-1", 1, 3)
-
- // Update it using different group or root IDs, which is not allowed.
- xid, err := strToGroupId("9999")
- if err != nil {
- t.Fatal(err)
- }
-
- sgData.SrvInfo.Id = xid
-
- err = sg.updateSyncGroup(sgData)
- if err == nil {
- t.Errorf("updating a SyncGroup with an ID mismatch did not fail in SyncGroup Table file %s", sgfile)
- }
-
- sgData.SrvInfo.Id = sgid
- sgData.LocalPath = "hahahaha"
- err = sg.updateSyncGroup(sgData)
- if err == nil {
- t.Errorf("updating a SyncGroup with a local path mismatch did not fail in SyncGroup Table file %s", sgfile)
- }
-
- checkSGStats(t, "up-2", 1, 3)
-
- // Update it using a modified set of joiners.
- // An empty string indicates no change to the local path.
- sgData.LocalPath = ""
- sgData.SrvInfo.Joiners["universe"] = JoinerMetaData{SyncPriority: 0}
- delete(sgData.SrvInfo.Joiners, "cloud")
-
- err = sg.updateSyncGroup(sgData)
- if err != nil {
- t.Errorf("updating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- // Do some NOP member deletions (bad member, bad group ID).
- // SyncGroup verification (below) should see the expected info asserting these were NOPs.
- sg.delMember("blablablablabla", sgid)
- sg.delMember("phone", xid)
-
- checkSGStats(t, "up-3", 1, 3)
-
- // Verify updated SyncGroup.
- if id, err := sg.getSyncGroupID(sgname); err != nil || id != sgid {
- t.Errorf("cannot get back ID of updated SyncGroup %s: got ID %d instead of %d; err: %v", sgname, id, sgid, err)
- }
- if name, err := sg.getSyncGroupName(sgid); err != nil || name != sgname {
- t.Errorf("cannot get back name of updated SyncGroup ID %d: got %s instead of %s; err: %v", sgid, name, sgname, err)
- }
-
- expData := &syncGroupData{
- SrvInfo: SyncGroupInfo{
- Id: sgid,
- GroupName: sgname,
- Joiners: map[string]JoinerMetaData{
- "phone": JoinerMetaData{SyncPriority: 10},
- "tablet": JoinerMetaData{SyncPriority: 25},
- "universe": JoinerMetaData{SyncPriority: 0},
- },
- },
- LocalPath: "/foo/bar",
- }
-
- data, err := sg.getSyncGroupByID(sgid)
- if err != nil {
- t.Errorf("cannot get updated SyncGroup by ID %d: %v", sgid, err)
- }
- if !reflect.DeepEqual(data, expData) {
- t.Errorf("invalid SyncGroup data for updated group ID %d: got %v instead of %v", sgid, data, expData)
- }
-
- data, err = sg.getSyncGroupByName(sgname)
- if err != nil {
- t.Errorf("cannot get updated SyncGroup by Name %s: %v", sgname, err)
- }
- if !reflect.DeepEqual(data, expData) {
- t.Errorf("invalid SyncGroup data for updated group name %s: got %v instead of %v", sgname, data, expData)
- }
-
- // Verify membership data.
- members, err := sg.getMembers()
- if err != nil {
- t.Errorf("cannot get all SyncGroup members after update: %v", err)
- }
- expMembers := map[string]uint32{"phone": 1, "tablet": 1, "universe": 1}
- if !reflect.DeepEqual(members, expMembers) {
- t.Errorf("invalid SyncGroup members after update: got %v instead of %v", members, expMembers)
- }
-
- expMetaData := map[string]*memberMetaData{
- "phone": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 10}},
- "tablet": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 25}},
- "universe": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 0}},
- }
- for mm := range members {
- info, err := sg.getMemberInfo(mm)
- if err != nil || info == nil {
- t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
- }
- if len(info.gids) != 1 {
- t.Errorf("invalid info for SyncGroup member %s: %v", mm, info)
- }
- expJoinerMetaData := expMetaData[mm]
- joinerMetaData := info.gids[sgid]
- if !reflect.DeepEqual(joinerMetaData, expJoinerMetaData) {
- t.Errorf("invalid joiner Data for SyncGroup member %s under group ID %d: got %v instead of %v",
- mm, sgid, joinerMetaData, expJoinerMetaData)
- }
- }
-
- sg.dump()
- sg.close()
-}
-
-// TestDeleteSyncGroup tests deleting a SyncGroup.
-func TestDeleteSyncGroup(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- sgname := "foobar"
- sgid, err := strToGroupId("1234")
- if err != nil {
- t.Fatal(err)
- }
-
- // Delete non-existing SyncGroups.
- err = sg.delSyncGroupByID(sgid)
- if err == nil {
- t.Errorf("deleting a non-existing SyncGroup ID did not fail in SyncGroup Table file %s", sgfile)
- }
-
- err = sg.delSyncGroupByName(sgname)
- if err == nil {
- t.Errorf("deleting a non-existing SyncGroup name did not fail in SyncGroup Table file %s", sgfile)
- }
-
- checkSGStats(t, "del-1", 0, 0)
-
- // Create the SyncGroup to delete later.
- sgData := &syncGroupData{
- SrvInfo: SyncGroupInfo{
- Id: sgid,
- GroupName: sgname,
- Joiners: map[string]JoinerMetaData{
- "phone": JoinerMetaData{SyncPriority: 10},
- "tablet": JoinerMetaData{SyncPriority: 25},
- "cloud": JoinerMetaData{SyncPriority: 1},
- },
- },
- LocalPath: "/foo/bar",
- }
-
- err = sg.addSyncGroup(sgData)
- if err != nil {
- t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- checkSGStats(t, "del-2", 1, 3)
-
- // Delete it by ID.
- err = sg.delSyncGroupByID(sgid)
- if err != nil {
- t.Errorf("deleting SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- checkSGStats(t, "del-3", 0, 0)
-
- // Create it again then delete it by name.
- err = sg.addSyncGroup(sgData)
- if err != nil {
- t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
- }
-
- checkSGStats(t, "del-4", 1, 3)
-
- err = sg.delSyncGroupByName(sgname)
- if err != nil {
- t.Errorf("deleting SyncGroup name %s failed in SyncGroup Table file %s: %v", sgname, sgfile, err)
- }
-
- checkSGStats(t, "del-5", 0, 0)
-
- sg.dump()
- sg.close()
-}
-
-// TestMultiSyncGroups tests creating multiple SyncGroups.
-func TestMultiSyncGroups(t *testing.T) {
- sgfile := getFileName()
- defer os.Remove(sgfile)
-
- sg, err := openSyncGroupTable(sgfile)
- if err != nil {
- t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
- }
-
- sgname1, sgname2 := "foo", "bar"
- sgid1, err := strToGroupId("1234")
- if err != nil {
- t.Fatal(err)
- }
- sgid2, err := strToGroupId("8888")
- if err != nil {
- t.Fatal(err)
- }
-
- // Add two SyncGroups.
- sgData1 := &syncGroupData{
- SrvInfo: SyncGroupInfo{
- Id: sgid1,
- GroupName: sgname1,
- Joiners: map[string]JoinerMetaData{
- "phone": JoinerMetaData{SyncPriority: 10},
- "tablet": JoinerMetaData{SyncPriority: 25},
- "cloud": JoinerMetaData{SyncPriority: 1},
- },
- },
- LocalPath: "/foo/bar",
- }
-
- sgData2 := &syncGroupData{
- SrvInfo: SyncGroupInfo{
- Id: sgid2,
- GroupName: sgname2,
- Joiners: map[string]JoinerMetaData{
- "tablet": JoinerMetaData{SyncPriority: 111},
- "door": JoinerMetaData{SyncPriority: 33},
- "lamp": JoinerMetaData{SyncPriority: 9},
- },
- },
- LocalPath: "/foo/bar",
- }
-
- err = sg.addSyncGroup(sgData1)
- if err != nil {
- t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid1, sgfile, err)
- }
-
- checkSGStats(t, "multi-1", 1, 3)
-
- err = sg.addSyncGroup(sgData2)
- if err != nil {
- t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid2, sgfile, err)
- }
-
- checkSGStats(t, "multi-2", 2, 5)
-
- // Verify SyncGroup names.
- sgNames, err := sg.getAllSyncGroupNames()
- if err != nil {
- t.Errorf("cannot get all SyncGroup names: %v", err)
- }
- if len(sgNames) != 2 {
- t.Errorf("wrong number of SyncGroup names: %d instead of 2", len(sgNames))
- }
- expNames := map[string]struct{}{sgname1: struct{}{}, sgname2: struct{}{}}
- for _, name := range sgNames {
- if _, ok := expNames[name]; !ok {
- t.Errorf("unknown SyncGroup name returned: %s", name)
- } else {
- delete(expNames, name)
- }
- }
-
- if len(expNames) > 0 {
- t.Errorf("SyncGroup names missing, not returned: %v", expNames)
- }
-
- // Verify SyncGroup membership data.
- members, err := sg.getMembers()
- if err != nil {
- t.Errorf("cannot get all SyncGroup members: %v", err)
- }
-
- expMembers := map[string]uint32{"phone": 1, "tablet": 2, "cloud": 1, "door": 1, "lamp": 1}
- if !reflect.DeepEqual(members, expMembers) {
- t.Errorf("invalid SyncGroup members: got %v instead of %v", members, expMembers)
- }
-
- expMemberInfo := map[string]*memberInfo{
- "phone": &memberInfo{
- gids: map[GroupId]*memberMetaData{
- sgid1: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 10}},
- },
- },
- "tablet": &memberInfo{
- gids: map[GroupId]*memberMetaData{
- sgid1: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 25}},
- sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 111}},
- },
- },
- "cloud": &memberInfo{
- gids: map[GroupId]*memberMetaData{
- sgid1: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 1}},
- },
- },
- "door": &memberInfo{
- gids: map[GroupId]*memberMetaData{
- sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 33}},
- },
- },
- "lamp": &memberInfo{
- gids: map[GroupId]*memberMetaData{
- sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 9}},
- },
- },
- }
-
- for mm := range members {
- info, err := sg.getMemberInfo(mm)
- if err != nil || info == nil {
- t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
- }
- expInfo := expMemberInfo[mm]
- if !reflect.DeepEqual(info, expInfo) {
- t.Errorf("invalid info for SyncGroup member %s: got %v instead of %v", mm, info, expInfo)
- }
- }
-
- // Delete the 1st SyncGroup.
- err = sg.delSyncGroupByID(sgid1)
- if err != nil {
- t.Errorf("deleting SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid1, sgfile, err)
- }
-
- checkSGStats(t, "multi-3", 1, 3)
-
- // Verify SyncGroup membership data.
- members, err = sg.getMembers()
- if err != nil {
- t.Errorf("cannot get all SyncGroup members: %v", err)
- }
-
- expMembers = map[string]uint32{"tablet": 1, "door": 1, "lamp": 1}
- if !reflect.DeepEqual(members, expMembers) {
- t.Errorf("invalid SyncGroup members: got %v instead of %v", members, expMembers)
- }
-
- expMemberInfo = map[string]*memberInfo{
- "tablet": &memberInfo{
- gids: map[GroupId]*memberMetaData{
- sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 111}},
- },
- },
- "door": &memberInfo{
- gids: map[GroupId]*memberMetaData{
- sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 33}},
- },
- },
- "lamp": &memberInfo{
- gids: map[GroupId]*memberMetaData{
- sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 9}},
- },
- },
- }
-
- for mm := range members {
- info, err := sg.getMemberInfo(mm)
- if err != nil || info == nil {
- t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
- }
- expInfo := expMemberInfo[mm]
- if !reflect.DeepEqual(info, expInfo) {
- t.Errorf("invalid info for SyncGroup member %s: got %v instead of %v", mm, info, expInfo)
- }
- }
-
- sg.dump()
- sg.close()
-}
diff --git a/services/syncbase/sync/testdata/local-init-00.log.sync b/services/syncbase/sync/testdata/local-init-00.log.sync
deleted file mode 100644
index 46b3502..0000000
--- a/services/syncbase/sync/testdata/local-init-00.log.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object locally and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addl|1234|1|||logrec-00|0|1|false
-addl|1234|2|1||logrec-01|0|1|false
-addl|1234|3|2||logrec-02|0|1|false
diff --git a/services/syncbase/sync/testdata/local-init-00.sync b/services/syncbase/sync/testdata/local-init-00.sync
deleted file mode 100644
index 9ab99dc..0000000
--- a/services/syncbase/sync/testdata/local-init-00.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object locally and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addl|1234|0|||logrec-00|0|1|false
-addl|1234|1|0||logrec-01|0|1|false
-addl|1234|2|1||logrec-02|0|1|false
diff --git a/services/syncbase/sync/testdata/local-init-01.log.sync b/services/syncbase/sync/testdata/local-init-01.log.sync
deleted file mode 100644
index 2f5930c..0000000
--- a/services/syncbase/sync/testdata/local-init-01.log.sync
+++ /dev/null
@@ -1,9 +0,0 @@
-# Create objects locally and update one and delete another.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addl|12|1|||logrec-00|0|1|false
-addl|12|2|1||logrec-01|0|1|false
-addl|12|3|2||logrec-02|0|1|false
-
-addl|45|1|||logrec-00|0|1|false
-addl|45|0|1||logrec-00|0|1|true
\ No newline at end of file
diff --git a/services/syncbase/sync/testdata/local-init-01.sync b/services/syncbase/sync/testdata/local-init-01.sync
deleted file mode 100644
index 1178c62..0000000
--- a/services/syncbase/sync/testdata/local-init-01.sync
+++ /dev/null
@@ -1,12 +0,0 @@
-# Create an object DAG locally with branches and resolved conflicts.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addl|1234|0|||logrec-00|0|1|false
-addl|1234|1|0||logrec-01|0|1|false
-addl|1234|2|1||logrec-02|0|1|false
-addl|1234|3|1||logrec-03|0|1|false
-addl|1234|4|2|3|logrec-04|0|1|false
-addl|1234|5|4||logrec-05|0|1|false
-addl|1234|6|1||logrec-06|0|1|false
-addl|1234|7|5|6|logrec-07|0|1|false
-addl|1234|8|7||logrec-08|0|1|false
diff --git a/services/syncbase/sync/testdata/local-init-02.log.sync b/services/syncbase/sync/testdata/local-init-02.log.sync
deleted file mode 100644
index 7f9f6a7..0000000
--- a/services/syncbase/sync/testdata/local-init-02.log.sync
+++ /dev/null
@@ -1,9 +0,0 @@
-# Create objects locally and update one and delete another.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addl|12|1|||logrec-00|0|1|false
-addl|12|2|1||logrec-01|0|1|false
-addl|12|3|2||logrec-02|0|1|false
-
-addl|45|10|||logrec-00|0|1|false
-addl|45|20|10||logrec-00|0|1|false
\ No newline at end of file
diff --git a/services/syncbase/sync/testdata/local-init-02.sync b/services/syncbase/sync/testdata/local-init-02.sync
deleted file mode 100644
index cb60a79..0000000
--- a/services/syncbase/sync/testdata/local-init-02.sync
+++ /dev/null
@@ -1,10 +0,0 @@
-# Create DAGs for 3 objects locally.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addl|1234|1|||logrec-a-01|0|1|false
-addl|1234|2|1||logrec-a-02|0|1|false
-
-addl|6789|1|||logrec-b-01|0|1|false
-addl|6789|2|1||logrec-b-02|0|1|false
-
-addl|2222|1|||logrec-c-01|0|1|false
diff --git a/services/syncbase/sync/testdata/local-init-03.sync b/services/syncbase/sync/testdata/local-init-03.sync
deleted file mode 100644
index 202a752..0000000
--- a/services/syncbase/sync/testdata/local-init-03.sync
+++ /dev/null
@@ -1,10 +0,0 @@
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addl|1234|1|||logrec-01|0|1|false
-addl|1234|2|1||logrec-02|0|1|false
-addl|1234|3|1||logrec-03|0|1|false
-addl|1234|4|2||logrec-04|0|1|false
-addl|1234|5|2||logrec-05|0|1|true
-addl|1234|6|4|5|logrec-06|0|1|false
-addl|1234|7|3|5|logrec-07|0|1|false
-addl|1234|8|6|7|logrec-08|0|1|false
diff --git a/services/syncbase/sync/testdata/local-init-watch.log.sync b/services/syncbase/sync/testdata/local-init-watch.log.sync
deleted file mode 100644
index 3e895c8..0000000
--- a/services/syncbase/sync/testdata/local-init-watch.log.sync
+++ /dev/null
@@ -1,3 +0,0 @@
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addl|082bc42e15af4fcf611d7f19a8d7831f|4|||logrec-00|0|1|false
diff --git a/services/syncbase/sync/testdata/local-resolve-00.sync b/services/syncbase/sync/testdata/local-resolve-00.sync
deleted file mode 100644
index 7026060..0000000
--- a/services/syncbase/sync/testdata/local-resolve-00.sync
+++ /dev/null
@@ -1,4 +0,0 @@
-# Create an object locally and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addl|1234|6|2|5|logrec-06|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-2obj-del.log.sync b/services/syncbase/sync/testdata/remote-2obj-del.log.sync
deleted file mode 100644
index 5274ecf..0000000
--- a/services/syncbase/sync/testdata/remote-2obj-del.log.sync
+++ /dev/null
@@ -1,7 +0,0 @@
-# Update one object and delete another object remotely.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|12|4|3||VeyronPhone:10:1:0|0|1|false
-addr|12|5|4||VeyronPhone:10:1:1|0|1|false
-addr|12|6|5||VeyronPhone:10:1:2|0|1|true
-addr|45|2|1||VeyronPhone:10:1:3|0|1|false
\ No newline at end of file
diff --git a/services/syncbase/sync/testdata/remote-conf-00.log.sync b/services/syncbase/sync/testdata/remote-conf-00.log.sync
deleted file mode 100644
index 1f9bb5b..0000000
--- a/services/syncbase/sync/testdata/remote-conf-00.log.sync
+++ /dev/null
@@ -1,8 +0,0 @@
-# Update an object remotely three times triggering one conflict after
-# it was created locally up to v3 (i.e. assume the remote sync received
-# it from the local sync at v2, then updated separately).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|4|2||VeyronPhone:10:1:0|0|1|false
-addr|1234|5|4||VeyronPhone:10:1:1|0|1|false
-addr|1234|6|5||VeyronPhone:10:1:2|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-conf-00.sync b/services/syncbase/sync/testdata/remote-conf-00.sync
deleted file mode 100644
index 8fae794..0000000
--- a/services/syncbase/sync/testdata/remote-conf-00.sync
+++ /dev/null
@@ -1,8 +0,0 @@
-# Update an object remotely three times triggering one conflict after
-# it was created locally up to v2 (i.e. assume the remote sync received
-# it from the local sync at v1, then updated separately).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|3|1||logrec-03|0|1|false
-addr|1234|4|3||logrec-04|0|1|false
-addr|1234|5|4||logrec-05|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-conf-01.log.sync b/services/syncbase/sync/testdata/remote-conf-01.log.sync
deleted file mode 100644
index 9581f69..0000000
--- a/services/syncbase/sync/testdata/remote-conf-01.log.sync
+++ /dev/null
@@ -1,10 +0,0 @@
-# Update an object remotely three times triggering a conflict with
-# 2 graft points: v1 and v4. This assumes that the remote sync got
-# v1, made its own conflicting v4 that it resolved into v5 (against v2)
-# then made a v6 change. When the local sync gets back this info it
-# sees 2 graft points: v1-v4 and v2-v5.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|4|1||VeyronLaptop:10:1:0|0|1|false
-addr|1234|5|2|4|VeyronPhone:10:1:0|0|1|false
-addr|1234|6|5||VeyronPhone:10:1:1|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-conf-01.sync b/services/syncbase/sync/testdata/remote-conf-01.sync
deleted file mode 100644
index 7485aeb..0000000
--- a/services/syncbase/sync/testdata/remote-conf-01.sync
+++ /dev/null
@@ -1,10 +0,0 @@
-# Update an object remotely three times triggering a conflict with
-# 2 graft points: v0 and v2. This assumes that the remote sync got
-# v0, made its own conflicting v3 that it resolved into v4 (against v1)
-# then made a v5 change. When the local sync gets back this info it
-# sees 2 graft points: v0-v3 and v1-v4.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|3|0||logrec-03|0|1|false
-addr|1234|4|1|3|logrec-04|0|1|false
-addr|1234|5|4||logrec-05|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-conf-02.log.sync b/services/syncbase/sync/testdata/remote-conf-02.log.sync
deleted file mode 100644
index 3c36277..0000000
--- a/services/syncbase/sync/testdata/remote-conf-02.log.sync
+++ /dev/null
@@ -1,15 +0,0 @@
-# Update an object remotely three times triggering one conflict after
-# it was created locally up to v3 (i.e. assume the remote sync received
-# it from the local sync at v2, then updated separately).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|4|2||VeyronPhone:10:1:0|0|1|false
-addr|1234|5|4||VeyronPhone:10:1:1|0|1|false
-addr|1234|6|5||VeyronPhone:10:1:2|0|1|false
-
-addr|12|4|2||VeyronPhone:99:1:0|0|1|false
-addr|12|5|4||VeyronPhone:99:1:1|0|1|false
-addr|12|6|5||VeyronPhone:99:1:2|0|1|false
-
-addr|45|30|20||VeyronPhone:99:2:0|0|1|false
-addr|45|40|30||VeyronPhone:99:2:1|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-conf-link.log.sync b/services/syncbase/sync/testdata/remote-conf-link.log.sync
deleted file mode 100644
index a324e4f..0000000
--- a/services/syncbase/sync/testdata/remote-conf-link.log.sync
+++ /dev/null
@@ -1,5 +0,0 @@
-# Update an object remotely, detect conflict, and bless the local version.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|4|1||VeyronPhone:10:1:0|0|1|false
-linkr|1234|4|2||VeyronPhone:10:1:1
diff --git a/services/syncbase/sync/testdata/remote-init-00.log.sync b/services/syncbase/sync/testdata/remote-init-00.log.sync
deleted file mode 100644
index 9795d53..0000000
--- a/services/syncbase/sync/testdata/remote-init-00.log.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object remotely and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|1|||VeyronPhone:10:1:0|0|1|false
-addr|1234|2|1||VeyronPhone:10:1:1|0|1|false
-addr|1234|3|2||VeyronPhone:10:1:2|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-init-00.sync b/services/syncbase/sync/testdata/remote-init-00.sync
deleted file mode 100644
index 18d288e..0000000
--- a/services/syncbase/sync/testdata/remote-init-00.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object remotely and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|0|||logrec-00|0|1|false
-addr|1234|1|0||logrec-01|0|1|false
-addr|1234|2|1||logrec-02|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-init-01.log.sync b/services/syncbase/sync/testdata/remote-init-01.log.sync
deleted file mode 100644
index 1db4c50..0000000
--- a/services/syncbase/sync/testdata/remote-init-01.log.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object remotely and update it twice (linked-list).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|1|||VeyronPhone:10:5:0|0|1|false
-addr|1234|2|1||VeyronPhone:10:5:1|0|1|false
-addr|1234|3|2||VeyronPhone:10:5:2|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-init-02.log.sync b/services/syncbase/sync/testdata/remote-init-02.log.sync
deleted file mode 100644
index 1274b7d..0000000
--- a/services/syncbase/sync/testdata/remote-init-02.log.sync
+++ /dev/null
@@ -1,17 +0,0 @@
-# Create objects and transactions remotely.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|12|1|||VeyronPhone:10:1:0|0|1|false
-
-addr|12|2|1||VeyronPhone:10:1:1|100|3|false
-addr|45|1|||VeyronPhone:10:1:2|100|3|false
-addr|78|1|||VeyronPhone:10:1:3|100|3|false
-
-addr|78|2|1||VeyronPhone:10:1:4|0|1|false
-
-addr|78|3|1||VeyronLaptop:10:1:0|0|1|false
-
-addr|78|4|2|3|VeyronPhone:10:2:0|0|1|false
-
-addr|12|3|2||VeyronPhone:10:2:1|101|2|false
-addr|45|2|1||VeyronPhone:10:2:2|101|2|false
diff --git a/services/syncbase/sync/testdata/remote-init-03.log.sync b/services/syncbase/sync/testdata/remote-init-03.log.sync
deleted file mode 100644
index 1ccac35..0000000
--- a/services/syncbase/sync/testdata/remote-init-03.log.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Create an object remotely and delete it.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|1|||VeyronPhone:10:1:0|0|1|false
-addr|1234|2|1||VeyronPhone:10:1:1|0|1|false
-addr|1234|3|2||VeyronPhone:10:1:2|0|1|true
diff --git a/services/syncbase/sync/testdata/remote-noconf-00.log.sync b/services/syncbase/sync/testdata/remote-noconf-00.log.sync
deleted file mode 100644
index e2e2afa..0000000
--- a/services/syncbase/sync/testdata/remote-noconf-00.log.sync
+++ /dev/null
@@ -1,8 +0,0 @@
-# Update an object remotely three times without triggering a conflict
-# after it was created locally up to v3 (i.e. assume the remote sync
-# received it from the local sync first, then updated it).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|4|3||VeyronPhone:10:1:0|0|1|false
-addr|1234|5|4||VeyronPhone:10:1:1|0|1|false
-addr|1234|6|5||VeyronPhone:10:1:2|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-noconf-00.sync b/services/syncbase/sync/testdata/remote-noconf-00.sync
deleted file mode 100644
index d44b6ac..0000000
--- a/services/syncbase/sync/testdata/remote-noconf-00.sync
+++ /dev/null
@@ -1,8 +0,0 @@
-# Update an object remotely three times without triggering a conflict
-# after it was created locally up to v2 (i.e. assume the remote sync
-# received it from the local sync first, then updated it).
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|3|2||logrec-03|0|1|false
-addr|1234|4|3||logrec-04|0|1|false
-addr|1234|5|4||logrec-05|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-noconf-link-00.log.sync b/services/syncbase/sync/testdata/remote-noconf-link-00.log.sync
deleted file mode 100644
index 6945bb2..0000000
--- a/services/syncbase/sync/testdata/remote-noconf-link-00.log.sync
+++ /dev/null
@@ -1,5 +0,0 @@
-# Update an object remotely, detect conflict, and bless the remote version.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|4|1||VeyronPhone:10:1:0|0|1|false
-linkr|1234|2|4||VeyronPhone:10:1:1
diff --git a/services/syncbase/sync/testdata/remote-noconf-link-01.log.sync b/services/syncbase/sync/testdata/remote-noconf-link-01.log.sync
deleted file mode 100644
index 0c6969e..0000000
--- a/services/syncbase/sync/testdata/remote-noconf-link-01.log.sync
+++ /dev/null
@@ -1,5 +0,0 @@
-# Update an object remotely, detect conflict, and bless the local version.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|4|1||VeyronPhone:10:1:0|0|1|false
-linkr|1234|4|3||VeyronPhone:10:1:1
diff --git a/services/syncbase/sync/testdata/remote-noconf-link-02.log.sync b/services/syncbase/sync/testdata/remote-noconf-link-02.log.sync
deleted file mode 100644
index df9e128..0000000
--- a/services/syncbase/sync/testdata/remote-noconf-link-02.log.sync
+++ /dev/null
@@ -1,6 +0,0 @@
-# Update an object remotely, detect conflict, and bless the remote version, and continue updating.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-addr|1234|4|1||VeyronPhone:10:1:0|0|1|false
-linkr|1234|3|4||VeyronPhone:10:1:1
-addr|1234|5|3||VeyronPhone:10:2:0|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-noconf-link-repeat.log.sync b/services/syncbase/sync/testdata/remote-noconf-link-repeat.log.sync
deleted file mode 100644
index 82e11c6..0000000
--- a/services/syncbase/sync/testdata/remote-noconf-link-repeat.log.sync
+++ /dev/null
@@ -1,4 +0,0 @@
-# Resolve the same conflict on two different devices.
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-
-linkr|1234|3|4||VeyronLaptop:10:1:0
diff --git a/services/syncbase/sync/testdata/test-1obj.gc.sync b/services/syncbase/sync/testdata/test-1obj.gc.sync
deleted file mode 100644
index 4d1a8d0..0000000
--- a/services/syncbase/sync/testdata/test-1obj.gc.sync
+++ /dev/null
@@ -1,14 +0,0 @@
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-# Local node is A. Remote nodes are B and C.
-### NOT UP-TO-DATE
-addr|12345|0|||C:1:0|false|false
-addr|12345|1|0||B:1:0|false|false
-addl|12345|2|0||A:1:0|false|false
-addl|12345|3|1|2|A:2:0|false|false
-addr|12345|4|3||C:2:0|false|false
-addr|12345|5|3||B:2:0|false|false
-addr|12345|6|4|5|B:3:0|false|false
-# Devtable state
-setdev|A|A:2,B:3,C:2
-setdev|B|A:2,B:3,C:2
-setdev|C|A:2,B:1,C:2
diff --git a/services/syncbase/sync/testdata/test-3obj.gc.sync b/services/syncbase/sync/testdata/test-3obj.gc.sync
deleted file mode 100644
index ce0656c..0000000
--- a/services/syncbase/sync/testdata/test-3obj.gc.sync
+++ /dev/null
@@ -1,45 +0,0 @@
-# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
-# Local node is A. Remote nodes are B and C.
-### NOT UP-TO-DATE
-addl|123|1|||A:1:0|false|false
-
-addr|456|1|||B:1:0|false|false
-
-addr|456|2|1||B:2:0|false|false
-addr|123|2|1||B:2:1|false|false
-
-addl|456|3|2||A:2:0|false|false
-addl|123|4|2||A:2:1|false|false
-
-addr|789|1|||C:1:0|false|false
-
-addr|789|2|1||C:2:0|false|false
-
-addr|123|3|1||C:3:0|false|false
-addr|789|3|2||C:3:1|false|false
-
-addr|123|5|3|2|C:4:0|false|false
-
-addl|123|6|4|5|A:3:0|false|false
-addl|456|4|3||A:3:1|false|false
-addl|789|4|3||A:3:2|false|false
-
-addr|456|5|2||B:3:0|false|false
-
-addl|456|7|4|5|A:4:0|false|false
-
-addr|456|6|2||C:5:0|false|false
-addr|123|7|5||C:5:1|false|false
-addr|123|8|7||C:5:2|false|false
-addr|789|5|3||C:5:3|false|false
-
-addl|123|9|6|8|A:5:0|false|false
-addl|456|8|6|7|A:5:1|false|false
-addl|789|6|4|5|A:5:2|false|false
-
-addl|123|10|9||A:6:0|false|false
-
-# Devtable state
-setdev|A|A:6,B:3,C:5
-setdev|B|A:4,B:3,C:4
-setdev|C|A:4,B:3,C:4
diff --git a/services/syncbase/sync/util_test.go b/services/syncbase/sync/util_test.go
deleted file mode 100644
index 54ed906..0000000
--- a/services/syncbase/sync/util_test.go
+++ /dev/null
@@ -1,250 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Utilities for testing.
-import (
- "container/list"
- "fmt"
- "os"
- "time"
-)
-
-// getFileName generates a filename for a temporary (per unit test) kvdb file.
-func getFileName() string {
- return fmt.Sprintf("%s/sync_test_%d_%d", os.TempDir(), os.Getpid(), time.Now().UnixNano())
-}
-
-// createTempDir creates a unique temporary directory to store kvdb files.
-func createTempDir() (string, error) {
- dir := fmt.Sprintf("%s/sync_test_%d_%d/", os.TempDir(), os.Getpid(), time.Now().UnixNano())
- if err := os.MkdirAll(dir, 0700); err != nil {
- return "", err
- }
- return dir, nil
-}
-
-// getFileSize returns the size of a file.
-func getFileSize(fname string) int64 {
- finfo, err := os.Stat(fname)
- if err != nil {
- return -1
- }
- return finfo.Size()
-}
-
-// dummyStream struct emulates stream of log records received from RPC.
-type dummyStream struct {
- l *list.List
- value LogRec
-}
-
-func newStream() *dummyStream {
- ds := &dummyStream{
- l: list.New(),
- }
- return ds
-}
-
-func (ds *dummyStream) Advance() bool {
- if ds.l.Len() > 0 {
- ds.value = ds.l.Remove(ds.l.Front()).(LogRec)
- return true
- }
- return false
-}
-
-func (ds *dummyStream) Value() LogRec {
- return ds.value
-}
-
-func (ds *dummyStream) RecvStream() interface {
- Advance() bool
- Value() LogRec
- Err() error
-} {
- return ds
-}
-
-func (*dummyStream) Err() error { return nil }
-
-func (ds *dummyStream) Finish() (map[ObjId]GenVector, error) {
- return nil, nil
-}
-
-func (ds *dummyStream) Cancel() {
-}
-
-func (ds *dummyStream) add(rec LogRec) {
- ds.l.PushBack(rec)
-}
-
-// logReplayCommands replays local log records parsed from the input file.
-func logReplayCommands(log *iLog, syncfile string, srid ObjId) error {
- cmds, err := parseSyncCommands(syncfile)
- if err != nil {
- return err
- }
-
- for _, cmd := range cmds {
- switch cmd.cmd {
- case addLocal:
- parent := NoVersion
- if cmd.parents != nil {
- parent = cmd.parents[0]
- }
-
- val := &LogValue{
- //Mutation: raw.Mutation{Version: cmd.version},
- Delete: cmd.deleted,
- TxId: cmd.txID,
- TxCount: cmd.txCount,
- }
- err = log.processWatchRecord(cmd.objID, cmd.version, parent, val, srid)
- if err != nil {
- return fmt.Errorf("cannot replay local log records %v:%v err %v",
- cmd.objID, cmd.version, err)
- }
- default:
- return fmt.Errorf("unknown cmd %v", cmd.cmd)
- }
- }
-
- return nil
-}
-
-// createReplayStream creates a dummy stream of log records parsed from the input file.
-func createReplayStream(syncfile string) (*dummyStream, error) {
- cmds, err := parseSyncCommands(syncfile)
- if err != nil {
- return nil, err
- }
-
- stream := newStream()
- for _, cmd := range cmds {
- id, srid, gnum, lsn, err := splitLogRecKey(cmd.logrec)
- if err != nil {
- return nil, err
- }
- rec := LogRec{
- DevId: id,
- SyncRootId: srid,
- GenNum: gnum,
- SeqNum: lsn,
- ObjId: cmd.objID,
- CurVers: cmd.version,
- Parents: cmd.parents,
- Value: LogValue{
- //Mutation: raw.Mutation{Version: cmd.version},
- Delete: cmd.deleted,
- TxId: cmd.txID,
- TxCount: cmd.txCount,
- },
- }
-
- switch cmd.cmd {
- case addRemote:
- rec.RecType = NodeRec
- case linkRemote:
- rec.RecType = LinkRec
- default:
- return nil, err
- }
- stream.add(rec)
- }
-
- return stream, nil
-}
-
-//
-// // populates the log and dag state as part of state initialization.
-// func populateLogAndDAG(s *syncd, rec *LogRec) error {
-// logKey, err := s.log.putLogRec(rec)
-// if err != nil {
-// return err
-// }
-//
-// if err := s.dag.addNode(rec.ObjId, rec.CurVers, false, rec.Value.Delete, rec.Parents, logKey, NoTxId); err != nil {
-// return err
-// }
-// if err := s.dag.moveHead(rec.ObjId, rec.CurVers); err != nil {
-// return err
-// }
-// return nil
-// }
-//
-//
-// // vsyncInitState initializes log, dag and devtable state obtained from an input trace-like file.
-// func vsyncInitState(s *syncd, syncfile string) error {
-// cmds, err := parseSyncCommands(syncfile)
-// if err != nil {
-// return err
-// }
-//
-// var curGen GenId
-// genMap := make(map[string]*genMetadata)
-//
-// for _, cmd := range cmds {
-// switch cmd.cmd {
-// case addLocal, addRemote:
-// id, sgid, gnum, lsn, err := splitLogRecKey(cmd.logrec)
-// if err != nil {
-// return err
-// }
-// rec := &LogRec{
-// DevId: id,
-// SGrpID: sgid,
-// GenNum: gnum,
-// SeqNum: lsn,
-// ObjId: cmd.objID,
-// CurVers: cmd.version,
-// Parents: cmd.parents,
-// Value: LogValue{Continued: cmd.continued, Delete: cmd.deleted},
-// }
-// if err := populateLogAndDAG(s, rec); err != nil {
-// return err
-// }
-// key := generationKey(id, sgid, gnum)
-// if m, ok := genMap[key]; !ok {
-// genMap[key] = &genMetadata{
-// Pos: s.log.head.Curorder,
-// Count: 1,
-// MaxSeqNum: rec.SeqNum,
-// }
-// s.log.head.Curorder++
-// } else {
-// m.Count++
-// if rec.SeqNum > m.MaxSeqNum {
-// m.MaxSeqNum = rec.SeqNum
-// }
-// }
-// if cmd.cmd == addLocal {
-// curGen = gnum
-// }
-//
-// case setDevTable:
-// if err := s.devtab.putGenVec(cmd.devID, cmd.genVec); err != nil {
-// return err
-// }
-// }
-// }
-//
-// // Initializing genMetadata.
-// for key, gen := range genMap {
-// dev, gnum, err := splitGenerationKey(key)
-// if err != nil {
-// return err
-// }
-// if err := s.log.putGenMetadata(dev, gnum, gen); err != nil {
-// return err
-// }
-// }
-//
-// // Initializing generation in log header.
-// s.log.head.Curgen = curGen + 1
-//
-// return nil
-// }
-//
diff --git a/services/syncbase/sync/vsync.vdl b/services/syncbase/sync/vsync.vdl
deleted file mode 100644
index 8ad9d30..0000000
--- a/services/syncbase/sync/vsync.vdl
+++ /dev/null
@@ -1,182 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-import (
- "v.io/v23/security/access"
-)
-
-// temporary types
-type ObjId string
-type Version uint64
-type GroupId uint64
-
-// DeviceId is the globally unique Id of a device.
-type DeviceId string
-// GenId is the unique Id per generation per device.
-type GenId uint64
-// SeqNum is the log sequence number.
-type SeqNum uint64
-// GenVector is the generation vector.
-type GenVector map[DeviceId]GenId
-// TxId is the unique Id per transaction.
-type TxId uint64
-// GroupIdSet is the list of SyncGroup Ids.
-type GroupIdSet []GroupId
-
-const (
- // NodeRec type log record adds a new node in the dag.
- NodeRec = byte(0)
- // LinkRec type log record adds a new link in the dag.
- LinkRec = byte(1)
-
- // Sync interface has Object name "global/vsync/<devid>/sync".
- SyncSuffix = "sync"
-
- // temporary nil values
- NoObjId = ObjId("")
- NoVersion = Version(0)
- NoGroupId = GroupId(0)
-)
-
-// LogRec represents a single log record that is exchanged between two
-// peers.
-//
-// It contains log related metadata: DevId is the id of the device
-// that created the log record, SyncRootId is the id of a SyncRoot this log
-// record was created under, GNum is the Id of the generation that the
-// log record is part of, SeqNum is the log sequence number of the log
-// record in the generation GNum, and RecType is the type of log
-// record.
-//
-// It also contains information relevant to the updates to an object
-// in the store: ObjId is the id of the object that was
-// updated. CurVers is the current version number of the
-// object. Parents can contain 0, 1 or 2 parent versions that the
-// current version is derived from, and Value is the actual value of
-// the object mutation.
-type LogRec struct {
- // Log related information.
- DevId DeviceId
- SyncRootId ObjId
- GenNum GenId
- SeqNum SeqNum
- RecType byte
-
- // Object related information.
- ObjId ObjId
- CurVers Version
- Parents []Version
- Value LogValue
-}
-
-// LogValue represents an object mutation within a transaction.
-type LogValue struct {
- // Mutation is the store mutation representing the change in the object.
- //Mutation raw.Mutation
- // SyncTime is the timestamp of the mutation when it arrives at the Sync server.
- SyncTime int64
- // Delete indicates whether the mutation resulted in the object being
- // deleted from the store.
- Delete bool
- // TxId is the unique Id of the transaction this mutation belongs to.
- TxId TxId
- // TxCount is the number of mutations in the transaction TxId.
- TxCount uint32
-}
-
-// DeviceStats contains high-level information on a device participating in
-// peer-to-peer synchronization.
-type DeviceStats struct {
- DevId DeviceId // Device Id.
- LastSync int64 // Timestamp of last sync from the device.
- GenVectors map[ObjId]GenVector // Generation vectors per SyncRoot.
- IsSelf bool // True if the responder is on this device.
-}
-
-// SyncGroupStats contains high-level information on a SyncGroup.
-type SyncGroupStats struct {
- Name string // Global name of the SyncGroup.
- Id GroupId // Global Id of the SyncGroup.
- Path string // Local store path for the root of the SyncGroup.
- RootObjId ObjId // Id of the store object at the root path.
- NumJoiners uint32 // Number of members currently in the SyncGroup.
-}
-
-// SyncGroupMember contains information on a SyncGroup member.
-type SyncGroupMember struct {
- Name string // Name of SyncGroup member.
- Id GroupId // Global Id of the SyncGroup.
- Metadata JoinerMetaData // Extra member metadata.
-}
-
-// A SyncGroupInfo is the conceptual state of a SyncGroup object.
-type SyncGroupInfo struct {
- Id GroupId // Globally unique SyncGroup Id.
- ServerName string // Global Vanadium name of SyncGroupServer.
- GroupName string // Relative name of group; global name is ServerName/GroupName.
- Config SyncGroupConfig // Configuration parameters of this SyncGroup.
- ETag string // Version Id for concurrency control.
-
- // A map from joiner names to the associated metaData for devices that
- // have called Join() or Create() and not subsequently called Leave()
- // or had Eject() called on them. The map returned by the calls below
- // may contain only a subset of joiners if the number is large.
- Joiners map[string]JoinerMetaData
-
- // Blessings for joiners of this SyncGroup will be self-signed by the
- // SyncGroupServer, and will have names matching
- // JoinerBlessingPrefix/Name/...
- JoinerBlessingPrefix string
-}
-
-// A SyncGroupConfig contains some fields of SyncGroupInfo that
-// are passed at create time, but which can be changed later.
-type SyncGroupConfig struct {
- Desc string // Human readable description.
- Options map[string]any // Options for future evolution.
- Permissions access.Permissions // The object's Permissions.
-
- // Mount tables used to advertise for synchronization.
- // Typically, we will have only one entry. However, an array allows
- // mount tables to be changed over time.
- MountTables []string
-
- BlessingsDurationNanos int64 // Duration of blessings, in nanoseconds. 0 => use server default.
-}
-
-// A JoinerMetaData contains the non-name information stored per joiner.
-type JoinerMetaData struct {
- // SyncPriority is a hint to bias the choice of syncing partners.
- // Members of the SyncGroup should choose to synchronize more often
- // with partners with lower values.
- SyncPriority int32
-}
-
-// Sync allows a device to GetDeltas from another device.
-type Sync interface {
- // GetDeltas returns a device's current generation vector and all
- // the missing log records when compared to the incoming generation vector.
- GetDeltas(in map[ObjId]GenVector, sgs map[ObjId]GroupIdSet, clientId DeviceId) stream<_, LogRec> (map[ObjId]GenVector | error) {access.Write}
-
- // GetObjectHistory returns the mutation history of a store object.
- GetObjectHistory(oid ObjId) stream<_, LogRec> (Version | error)
-
- // GetDeviceStats returns information on devices participating in
- // peer-to-peer synchronization.
- GetDeviceStats() stream<_, DeviceStats> error {access.Admin}
-
- // GetSyncGroupMembers returns information on SyncGroup members.
- // If SyncGroup names are specified, only members of these SyncGroups
- // are returned. Otherwise, if the slice of names is nil or empty,
- // members of all SyncGroups are returned.
- GetSyncGroupMembers(sgNames []string) stream<_, SyncGroupMember> error {access.Admin}
-
- // GetSyncGroupStats returns high-level information on all SyncGroups.
- GetSyncGroupStats() stream<_, SyncGroupStats> error {access.Admin}
-
- // Dump writes to the Sync log internal information used for debugging.
- Dump() error {access.Admin}
-}
diff --git a/services/syncbase/sync/vsync.vdl.go b/services/syncbase/sync/vsync.vdl.go
deleted file mode 100644
index 8c7a6ae..0000000
--- a/services/syncbase/sync/vsync.vdl.go
+++ /dev/null
@@ -1,1091 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This file was auto-generated by the vanadium vdl tool.
-// Source: vsync.vdl
-
-package vsync
-
-import (
- // VDL system imports
- "io"
- "v.io/v23"
- "v.io/v23/context"
- "v.io/v23/rpc"
- "v.io/v23/vdl"
-
- // VDL user imports
- "v.io/v23/security/access"
-)
-
-// temporary types
-type ObjId string
-
-func (ObjId) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.ObjId"`
-}) {
-}
-
-type Version uint64
-
-func (Version) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.Version"`
-}) {
-}
-
-type GroupId uint64
-
-func (GroupId) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.GroupId"`
-}) {
-}
-
-// DeviceId is the globally unique Id of a device.
-type DeviceId string
-
-func (DeviceId) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.DeviceId"`
-}) {
-}
-
-// GenId is the unique Id per generation per device.
-type GenId uint64
-
-func (GenId) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.GenId"`
-}) {
-}
-
-// SeqNum is the log sequence number.
-type SeqNum uint64
-
-func (SeqNum) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.SeqNum"`
-}) {
-}
-
-// GenVector is the generation vector.
-type GenVector map[DeviceId]GenId
-
-func (GenVector) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.GenVector"`
-}) {
-}
-
-// TxId is the unique Id per transaction.
-type TxId uint64
-
-func (TxId) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.TxId"`
-}) {
-}
-
-// GroupIdSet is the list of SyncGroup Ids.
-type GroupIdSet []GroupId
-
-func (GroupIdSet) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.GroupIdSet"`
-}) {
-}
-
-// LogRec represents a single log record that is exchanged between two
-// peers.
-//
-// It contains log related metadata: DevId is the id of the device
-// that created the log record, SyncRootId is the id of a SyncRoot this log
-// record was created under, GNum is the Id of the generation that the
-// log record is part of, SeqNum is the log sequence number of the log
-// record in the generation GNum, and RecType is the type of log
-// record.
-//
-// It also contains information relevant to the updates to an object
-// in the store: ObjId is the id of the object that was
-// updated. CurVers is the current version number of the
-// object. Parents can contain 0, 1 or 2 parent versions that the
-// current version is derived from, and Value is the actual value of
-// the object mutation.
-type LogRec struct {
- // Log related information.
- DevId DeviceId
- SyncRootId ObjId
- GenNum GenId
- SeqNum SeqNum
- RecType byte
- // Object related information.
- ObjId ObjId
- CurVers Version
- Parents []Version
- Value LogValue
-}
-
-func (LogRec) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.LogRec"`
-}) {
-}
-
-// LogValue represents an object mutation within a transaction.
-type LogValue struct {
- // Mutation is the store mutation representing the change in the object.
- //Mutation raw.Mutation
- // SyncTime is the timestamp of the mutation when it arrives at the Sync server.
- SyncTime int64
- // Delete indicates whether the mutation resulted in the object being
- // deleted from the store.
- Delete bool
- // TxId is the unique Id of the transaction this mutation belongs to.
- TxId TxId
- // TxCount is the number of mutations in the transaction TxId.
- TxCount uint32
-}
-
-func (LogValue) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.LogValue"`
-}) {
-}
-
-// DeviceStats contains high-level information on a device participating in
-// peer-to-peer synchronization.
-type DeviceStats struct {
- DevId DeviceId // Device Id.
- LastSync int64 // Timestamp of last sync from the device.
- GenVectors map[ObjId]GenVector // Generation vectors per SyncRoot.
- IsSelf bool // True if the responder is on this device.
-}
-
-func (DeviceStats) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.DeviceStats"`
-}) {
-}
-
-// SyncGroupStats contains high-level information on a SyncGroup.
-type SyncGroupStats struct {
- Name string // Global name of the SyncGroup.
- Id GroupId // Global Id of the SyncGroup.
- Path string // Local store path for the root of the SyncGroup.
- RootObjId ObjId // Id of the store object at the root path.
- NumJoiners uint32 // Number of members currently in the SyncGroup.
-}
-
-func (SyncGroupStats) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.SyncGroupStats"`
-}) {
-}
-
-// SyncGroupMember contains information on a SyncGroup member.
-type SyncGroupMember struct {
- Name string // Name of SyncGroup member.
- Id GroupId // Global Id of the SyncGroup.
- Metadata JoinerMetaData // Extra member metadata.
-}
-
-func (SyncGroupMember) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.SyncGroupMember"`
-}) {
-}
-
-// A SyncGroupInfo is the conceptual state of a SyncGroup object.
-type SyncGroupInfo struct {
- Id GroupId // Globally unique SyncGroup Id.
- ServerName string // Global Vanadium name of SyncGroupServer.
- GroupName string // Relative name of group; global name is ServerName/GroupName.
- Config SyncGroupConfig // Configuration parameters of this SyncGroup.
- ETag string // Version Id for concurrency control.
- // A map from joiner names to the associated metaData for devices that
- // have called Join() or Create() and not subsequently called Leave()
- // or had Eject() called on them. The map returned by the calls below
- // may contain only a subset of joiners if the number is large.
- Joiners map[string]JoinerMetaData
- // Blessings for joiners of this SyncGroup will be self-signed by the
- // SyncGroupServer, and will have names matching
- // JoinerBlessingPrefix/Name/...
- JoinerBlessingPrefix string
-}
-
-func (SyncGroupInfo) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.SyncGroupInfo"`
-}) {
-}
-
-// A SyncGroupConfig contains some fields of SyncGroupInfo that
-// are passed at create time, but which can be changed later.
-type SyncGroupConfig struct {
- Desc string // Human readable description.
- Options map[string]*vdl.Value // Options for future evolution.
- Permissions access.Permissions // The object's Permissions.
- // Mount tables used to advertise for synchronization.
- // Typically, we will have only one entry. However, an array allows
- // mount tables to be changed over time.
- MountTables []string
- BlessingsDurationNanos int64 // Duration of blessings, in nanoseconds. 0 => use server default.
-}
-
-func (SyncGroupConfig) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.SyncGroupConfig"`
-}) {
-}
-
-// A JoinerMetaData contains the non-name information stored per joiner.
-type JoinerMetaData struct {
- // SyncPriority is a hint to bias the choice of syncing partners.
- // Members of the SyncGroup should choose to synchronize more often
- // with partners with lower values.
- SyncPriority int32
-}
-
-func (JoinerMetaData) __VDLReflect(struct {
- Name string `vdl:"v.io/syncbase/x/ref/services/syncbase/sync.JoinerMetaData"`
-}) {
-}
-
-func init() {
- vdl.Register((*ObjId)(nil))
- vdl.Register((*Version)(nil))
- vdl.Register((*GroupId)(nil))
- vdl.Register((*DeviceId)(nil))
- vdl.Register((*GenId)(nil))
- vdl.Register((*SeqNum)(nil))
- vdl.Register((*GenVector)(nil))
- vdl.Register((*TxId)(nil))
- vdl.Register((*GroupIdSet)(nil))
- vdl.Register((*LogRec)(nil))
- vdl.Register((*LogValue)(nil))
- vdl.Register((*DeviceStats)(nil))
- vdl.Register((*SyncGroupStats)(nil))
- vdl.Register((*SyncGroupMember)(nil))
- vdl.Register((*SyncGroupInfo)(nil))
- vdl.Register((*SyncGroupConfig)(nil))
- vdl.Register((*JoinerMetaData)(nil))
-}
-
-// NodeRec type log record adds a new node in the dag.
-const NodeRec = byte(0)
-
-// LinkRec type log record adds a new link in the dag.
-const LinkRec = byte(1)
-
-// Sync interface has Object name "global/vsync/<devid>/sync".
-const SyncSuffix = "sync"
-
-// temporary nil values
-const NoObjId = ObjId("")
-
-const NoVersion = Version(0)
-
-const NoGroupId = GroupId(0)
-
-// SyncClientMethods is the client interface
-// containing Sync methods.
-//
-// Sync allows a device to GetDeltas from another device.
-type SyncClientMethods interface {
- // GetDeltas returns a device's current generation vector and all
- // the missing log records when compared to the incoming generation vector.
- GetDeltas(ctx *context.T, in map[ObjId]GenVector, sgs map[ObjId]GroupIdSet, clientId DeviceId, opts ...rpc.CallOpt) (SyncGetDeltasClientCall, error)
- // GetObjectHistory returns the mutation history of a store object.
- GetObjectHistory(ctx *context.T, oid ObjId, opts ...rpc.CallOpt) (SyncGetObjectHistoryClientCall, error)
- // GetDeviceStats returns information on devices participating in
- // peer-to-peer synchronization.
- GetDeviceStats(*context.T, ...rpc.CallOpt) (SyncGetDeviceStatsClientCall, error)
- // GetSyncGroupMembers returns information on SyncGroup members.
- // If SyncGroup names are specified, only members of these SyncGroups
- // are returned. Otherwise, if the slice of names is nil or empty,
- // members of all SyncGroups are returned.
- GetSyncGroupMembers(ctx *context.T, sgNames []string, opts ...rpc.CallOpt) (SyncGetSyncGroupMembersClientCall, error)
- // GetSyncGroupStats returns high-level information on all SyncGroups.
- GetSyncGroupStats(*context.T, ...rpc.CallOpt) (SyncGetSyncGroupStatsClientCall, error)
- // Dump writes to the Sync log internal information used for debugging.
- Dump(*context.T, ...rpc.CallOpt) error
-}
-
-// SyncClientStub adds universal methods to SyncClientMethods.
-type SyncClientStub interface {
- SyncClientMethods
- rpc.UniversalServiceMethods
-}
-
-// SyncClient returns a client stub for Sync.
-func SyncClient(name string) SyncClientStub {
- return implSyncClientStub{name}
-}
-
-type implSyncClientStub struct {
- name string
-}
-
-func (c implSyncClientStub) GetDeltas(ctx *context.T, i0 map[ObjId]GenVector, i1 map[ObjId]GroupIdSet, i2 DeviceId, opts ...rpc.CallOpt) (ocall SyncGetDeltasClientCall, err error) {
- var call rpc.ClientCall
- if call, err = v23.GetClient(ctx).StartCall(ctx, c.name, "GetDeltas", []interface{}{i0, i1, i2}, opts...); err != nil {
- return
- }
- ocall = &implSyncGetDeltasClientCall{ClientCall: call}
- return
-}
-
-func (c implSyncClientStub) GetObjectHistory(ctx *context.T, i0 ObjId, opts ...rpc.CallOpt) (ocall SyncGetObjectHistoryClientCall, err error) {
- var call rpc.ClientCall
- if call, err = v23.GetClient(ctx).StartCall(ctx, c.name, "GetObjectHistory", []interface{}{i0}, opts...); err != nil {
- return
- }
- ocall = &implSyncGetObjectHistoryClientCall{ClientCall: call}
- return
-}
-
-func (c implSyncClientStub) GetDeviceStats(ctx *context.T, opts ...rpc.CallOpt) (ocall SyncGetDeviceStatsClientCall, err error) {
- var call rpc.ClientCall
- if call, err = v23.GetClient(ctx).StartCall(ctx, c.name, "GetDeviceStats", nil, opts...); err != nil {
- return
- }
- ocall = &implSyncGetDeviceStatsClientCall{ClientCall: call}
- return
-}
-
-func (c implSyncClientStub) GetSyncGroupMembers(ctx *context.T, i0 []string, opts ...rpc.CallOpt) (ocall SyncGetSyncGroupMembersClientCall, err error) {
- var call rpc.ClientCall
- if call, err = v23.GetClient(ctx).StartCall(ctx, c.name, "GetSyncGroupMembers", []interface{}{i0}, opts...); err != nil {
- return
- }
- ocall = &implSyncGetSyncGroupMembersClientCall{ClientCall: call}
- return
-}
-
-func (c implSyncClientStub) GetSyncGroupStats(ctx *context.T, opts ...rpc.CallOpt) (ocall SyncGetSyncGroupStatsClientCall, err error) {
- var call rpc.ClientCall
- if call, err = v23.GetClient(ctx).StartCall(ctx, c.name, "GetSyncGroupStats", nil, opts...); err != nil {
- return
- }
- ocall = &implSyncGetSyncGroupStatsClientCall{ClientCall: call}
- return
-}
-
-func (c implSyncClientStub) Dump(ctx *context.T, opts ...rpc.CallOpt) (err error) {
- err = v23.GetClient(ctx).Call(ctx, c.name, "Dump", nil, nil, opts...)
- return
-}
-
-// SyncGetDeltasClientStream is the client stream for Sync.GetDeltas.
-type SyncGetDeltasClientStream interface {
- // RecvStream returns the receiver side of the Sync.GetDeltas client stream.
- RecvStream() interface {
- // Advance stages an item so that it may be retrieved via Value. Returns
- // true iff there is an item to retrieve. Advance must be called before
- // Value is called. May block if an item is not available.
- Advance() bool
- // Value returns the item that was staged by Advance. May panic if Advance
- // returned false or was not called. Never blocks.
- Value() LogRec
- // Err returns any error encountered by Advance. Never blocks.
- Err() error
- }
-}
-
-// SyncGetDeltasClientCall represents the call returned from Sync.GetDeltas.
-type SyncGetDeltasClientCall interface {
- SyncGetDeltasClientStream
- // Finish blocks until the server is done, and returns the positional return
- // values for call.
- //
- // Finish returns immediately if the call has been canceled; depending on the
- // timing the output could either be an error signaling cancelation, or the
- // valid positional return values from the server.
- //
- // Calling Finish is mandatory for releasing stream resources, unless the call
- // has been canceled or any of the other methods return an error. Finish should
- // be called at most once.
- Finish() (map[ObjId]GenVector, error)
-}
-
-type implSyncGetDeltasClientCall struct {
- rpc.ClientCall
- valRecv LogRec
- errRecv error
-}
-
-func (c *implSyncGetDeltasClientCall) RecvStream() interface {
- Advance() bool
- Value() LogRec
- Err() error
-} {
- return implSyncGetDeltasClientCallRecv{c}
-}
-
-type implSyncGetDeltasClientCallRecv struct {
- c *implSyncGetDeltasClientCall
-}
-
-func (c implSyncGetDeltasClientCallRecv) Advance() bool {
- c.c.valRecv = LogRec{}
- c.c.errRecv = c.c.Recv(&c.c.valRecv)
- return c.c.errRecv == nil
-}
-func (c implSyncGetDeltasClientCallRecv) Value() LogRec {
- return c.c.valRecv
-}
-func (c implSyncGetDeltasClientCallRecv) Err() error {
- if c.c.errRecv == io.EOF {
- return nil
- }
- return c.c.errRecv
-}
-func (c *implSyncGetDeltasClientCall) Finish() (o0 map[ObjId]GenVector, err error) {
- err = c.ClientCall.Finish(&o0)
- return
-}
-
-// SyncGetObjectHistoryClientStream is the client stream for Sync.GetObjectHistory.
-type SyncGetObjectHistoryClientStream interface {
- // RecvStream returns the receiver side of the Sync.GetObjectHistory client stream.
- RecvStream() interface {
- // Advance stages an item so that it may be retrieved via Value. Returns
- // true iff there is an item to retrieve. Advance must be called before
- // Value is called. May block if an item is not available.
- Advance() bool
- // Value returns the item that was staged by Advance. May panic if Advance
- // returned false or was not called. Never blocks.
- Value() LogRec
- // Err returns any error encountered by Advance. Never blocks.
- Err() error
- }
-}
-
-// SyncGetObjectHistoryClientCall represents the call returned from Sync.GetObjectHistory.
-type SyncGetObjectHistoryClientCall interface {
- SyncGetObjectHistoryClientStream
- // Finish blocks until the server is done, and returns the positional return
- // values for call.
- //
- // Finish returns immediately if the call has been canceled; depending on the
- // timing the output could either be an error signaling cancelation, or the
- // valid positional return values from the server.
- //
- // Calling Finish is mandatory for releasing stream resources, unless the call
- // has been canceled or any of the other methods return an error. Finish should
- // be called at most once.
- Finish() (Version, error)
-}
-
-type implSyncGetObjectHistoryClientCall struct {
- rpc.ClientCall
- valRecv LogRec
- errRecv error
-}
-
-func (c *implSyncGetObjectHistoryClientCall) RecvStream() interface {
- Advance() bool
- Value() LogRec
- Err() error
-} {
- return implSyncGetObjectHistoryClientCallRecv{c}
-}
-
-type implSyncGetObjectHistoryClientCallRecv struct {
- c *implSyncGetObjectHistoryClientCall
-}
-
-func (c implSyncGetObjectHistoryClientCallRecv) Advance() bool {
- c.c.valRecv = LogRec{}
- c.c.errRecv = c.c.Recv(&c.c.valRecv)
- return c.c.errRecv == nil
-}
-func (c implSyncGetObjectHistoryClientCallRecv) Value() LogRec {
- return c.c.valRecv
-}
-func (c implSyncGetObjectHistoryClientCallRecv) Err() error {
- if c.c.errRecv == io.EOF {
- return nil
- }
- return c.c.errRecv
-}
-func (c *implSyncGetObjectHistoryClientCall) Finish() (o0 Version, err error) {
- err = c.ClientCall.Finish(&o0)
- return
-}
-
-// SyncGetDeviceStatsClientStream is the client stream for Sync.GetDeviceStats.
-type SyncGetDeviceStatsClientStream interface {
- // RecvStream returns the receiver side of the Sync.GetDeviceStats client stream.
- RecvStream() interface {
- // Advance stages an item so that it may be retrieved via Value. Returns
- // true iff there is an item to retrieve. Advance must be called before
- // Value is called. May block if an item is not available.
- Advance() bool
- // Value returns the item that was staged by Advance. May panic if Advance
- // returned false or was not called. Never blocks.
- Value() DeviceStats
- // Err returns any error encountered by Advance. Never blocks.
- Err() error
- }
-}
-
-// SyncGetDeviceStatsClientCall represents the call returned from Sync.GetDeviceStats.
-type SyncGetDeviceStatsClientCall interface {
- SyncGetDeviceStatsClientStream
- // Finish blocks until the server is done, and returns the positional return
- // values for call.
- //
- // Finish returns immediately if the call has been canceled; depending on the
- // timing the output could either be an error signaling cancelation, or the
- // valid positional return values from the server.
- //
- // Calling Finish is mandatory for releasing stream resources, unless the call
- // has been canceled or any of the other methods return an error. Finish should
- // be called at most once.
- Finish() error
-}
-
-type implSyncGetDeviceStatsClientCall struct {
- rpc.ClientCall
- valRecv DeviceStats
- errRecv error
-}
-
-func (c *implSyncGetDeviceStatsClientCall) RecvStream() interface {
- Advance() bool
- Value() DeviceStats
- Err() error
-} {
- return implSyncGetDeviceStatsClientCallRecv{c}
-}
-
-type implSyncGetDeviceStatsClientCallRecv struct {
- c *implSyncGetDeviceStatsClientCall
-}
-
-func (c implSyncGetDeviceStatsClientCallRecv) Advance() bool {
- c.c.valRecv = DeviceStats{}
- c.c.errRecv = c.c.Recv(&c.c.valRecv)
- return c.c.errRecv == nil
-}
-func (c implSyncGetDeviceStatsClientCallRecv) Value() DeviceStats {
- return c.c.valRecv
-}
-func (c implSyncGetDeviceStatsClientCallRecv) Err() error {
- if c.c.errRecv == io.EOF {
- return nil
- }
- return c.c.errRecv
-}
-func (c *implSyncGetDeviceStatsClientCall) Finish() (err error) {
- err = c.ClientCall.Finish()
- return
-}
-
-// SyncGetSyncGroupMembersClientStream is the client stream for Sync.GetSyncGroupMembers.
-type SyncGetSyncGroupMembersClientStream interface {
- // RecvStream returns the receiver side of the Sync.GetSyncGroupMembers client stream.
- RecvStream() interface {
- // Advance stages an item so that it may be retrieved via Value. Returns
- // true iff there is an item to retrieve. Advance must be called before
- // Value is called. May block if an item is not available.
- Advance() bool
- // Value returns the item that was staged by Advance. May panic if Advance
- // returned false or was not called. Never blocks.
- Value() SyncGroupMember
- // Err returns any error encountered by Advance. Never blocks.
- Err() error
- }
-}
-
-// SyncGetSyncGroupMembersClientCall represents the call returned from Sync.GetSyncGroupMembers.
-type SyncGetSyncGroupMembersClientCall interface {
- SyncGetSyncGroupMembersClientStream
- // Finish blocks until the server is done, and returns the positional return
- // values for call.
- //
- // Finish returns immediately if the call has been canceled; depending on the
- // timing the output could either be an error signaling cancelation, or the
- // valid positional return values from the server.
- //
- // Calling Finish is mandatory for releasing stream resources, unless the call
- // has been canceled or any of the other methods return an error. Finish should
- // be called at most once.
- Finish() error
-}
-
-type implSyncGetSyncGroupMembersClientCall struct {
- rpc.ClientCall
- valRecv SyncGroupMember
- errRecv error
-}
-
-func (c *implSyncGetSyncGroupMembersClientCall) RecvStream() interface {
- Advance() bool
- Value() SyncGroupMember
- Err() error
-} {
- return implSyncGetSyncGroupMembersClientCallRecv{c}
-}
-
-type implSyncGetSyncGroupMembersClientCallRecv struct {
- c *implSyncGetSyncGroupMembersClientCall
-}
-
-func (c implSyncGetSyncGroupMembersClientCallRecv) Advance() bool {
- c.c.valRecv = SyncGroupMember{}
- c.c.errRecv = c.c.Recv(&c.c.valRecv)
- return c.c.errRecv == nil
-}
-func (c implSyncGetSyncGroupMembersClientCallRecv) Value() SyncGroupMember {
- return c.c.valRecv
-}
-func (c implSyncGetSyncGroupMembersClientCallRecv) Err() error {
- if c.c.errRecv == io.EOF {
- return nil
- }
- return c.c.errRecv
-}
-func (c *implSyncGetSyncGroupMembersClientCall) Finish() (err error) {
- err = c.ClientCall.Finish()
- return
-}
-
-// SyncGetSyncGroupStatsClientStream is the client stream for Sync.GetSyncGroupStats.
-type SyncGetSyncGroupStatsClientStream interface {
- // RecvStream returns the receiver side of the Sync.GetSyncGroupStats client stream.
- RecvStream() interface {
- // Advance stages an item so that it may be retrieved via Value. Returns
- // true iff there is an item to retrieve. Advance must be called before
- // Value is called. May block if an item is not available.
- Advance() bool
- // Value returns the item that was staged by Advance. May panic if Advance
- // returned false or was not called. Never blocks.
- Value() SyncGroupStats
- // Err returns any error encountered by Advance. Never blocks.
- Err() error
- }
-}
-
-// SyncGetSyncGroupStatsClientCall represents the call returned from Sync.GetSyncGroupStats.
-type SyncGetSyncGroupStatsClientCall interface {
- SyncGetSyncGroupStatsClientStream
- // Finish blocks until the server is done, and returns the positional return
- // values for call.
- //
- // Finish returns immediately if the call has been canceled; depending on the
- // timing the output could either be an error signaling cancelation, or the
- // valid positional return values from the server.
- //
- // Calling Finish is mandatory for releasing stream resources, unless the call
- // has been canceled or any of the other methods return an error. Finish should
- // be called at most once.
- Finish() error
-}
-
-type implSyncGetSyncGroupStatsClientCall struct {
- rpc.ClientCall
- valRecv SyncGroupStats
- errRecv error
-}
-
-func (c *implSyncGetSyncGroupStatsClientCall) RecvStream() interface {
- Advance() bool
- Value() SyncGroupStats
- Err() error
-} {
- return implSyncGetSyncGroupStatsClientCallRecv{c}
-}
-
-type implSyncGetSyncGroupStatsClientCallRecv struct {
- c *implSyncGetSyncGroupStatsClientCall
-}
-
-func (c implSyncGetSyncGroupStatsClientCallRecv) Advance() bool {
- c.c.valRecv = SyncGroupStats{}
- c.c.errRecv = c.c.Recv(&c.c.valRecv)
- return c.c.errRecv == nil
-}
-func (c implSyncGetSyncGroupStatsClientCallRecv) Value() SyncGroupStats {
- return c.c.valRecv
-}
-func (c implSyncGetSyncGroupStatsClientCallRecv) Err() error {
- if c.c.errRecv == io.EOF {
- return nil
- }
- return c.c.errRecv
-}
-func (c *implSyncGetSyncGroupStatsClientCall) Finish() (err error) {
- err = c.ClientCall.Finish()
- return
-}
-
-// SyncServerMethods is the interface a server writer
-// implements for Sync.
-//
-// Sync allows a device to GetDeltas from another device.
-type SyncServerMethods interface {
- // GetDeltas returns a device's current generation vector and all
- // the missing log records when compared to the incoming generation vector.
- GetDeltas(ctx *context.T, call SyncGetDeltasServerCall, in map[ObjId]GenVector, sgs map[ObjId]GroupIdSet, clientId DeviceId) (map[ObjId]GenVector, error)
- // GetObjectHistory returns the mutation history of a store object.
- GetObjectHistory(ctx *context.T, call SyncGetObjectHistoryServerCall, oid ObjId) (Version, error)
- // GetDeviceStats returns information on devices participating in
- // peer-to-peer synchronization.
- GetDeviceStats(*context.T, SyncGetDeviceStatsServerCall) error
- // GetSyncGroupMembers returns information on SyncGroup members.
- // If SyncGroup names are specified, only members of these SyncGroups
- // are returned. Otherwise, if the slice of names is nil or empty,
- // members of all SyncGroups are returned.
- GetSyncGroupMembers(ctx *context.T, call SyncGetSyncGroupMembersServerCall, sgNames []string) error
- // GetSyncGroupStats returns high-level information on all SyncGroups.
- GetSyncGroupStats(*context.T, SyncGetSyncGroupStatsServerCall) error
- // Dump writes to the Sync log internal information used for debugging.
- Dump(*context.T, rpc.ServerCall) error
-}
-
-// SyncServerStubMethods is the server interface containing
-// Sync methods, as expected by rpc.Server.
-// The only difference between this interface and SyncServerMethods
-// is the streaming methods.
-type SyncServerStubMethods interface {
- // GetDeltas returns a device's current generation vector and all
- // the missing log records when compared to the incoming generation vector.
- GetDeltas(ctx *context.T, call *SyncGetDeltasServerCallStub, in map[ObjId]GenVector, sgs map[ObjId]GroupIdSet, clientId DeviceId) (map[ObjId]GenVector, error)
- // GetObjectHistory returns the mutation history of a store object.
- GetObjectHistory(ctx *context.T, call *SyncGetObjectHistoryServerCallStub, oid ObjId) (Version, error)
- // GetDeviceStats returns information on devices participating in
- // peer-to-peer synchronization.
- GetDeviceStats(*context.T, *SyncGetDeviceStatsServerCallStub) error
- // GetSyncGroupMembers returns information on SyncGroup members.
- // If SyncGroup names are specified, only members of these SyncGroups
- // are returned. Otherwise, if the slice of names is nil or empty,
- // members of all SyncGroups are returned.
- GetSyncGroupMembers(ctx *context.T, call *SyncGetSyncGroupMembersServerCallStub, sgNames []string) error
- // GetSyncGroupStats returns high-level information on all SyncGroups.
- GetSyncGroupStats(*context.T, *SyncGetSyncGroupStatsServerCallStub) error
- // Dump writes to the Sync log internal information used for debugging.
- Dump(*context.T, rpc.ServerCall) error
-}
-
-// SyncServerStub adds universal methods to SyncServerStubMethods.
-type SyncServerStub interface {
- SyncServerStubMethods
- // Describe the Sync interfaces.
- Describe__() []rpc.InterfaceDesc
-}
-
-// SyncServer returns a server stub for Sync.
-// It converts an implementation of SyncServerMethods into
-// an object that may be used by rpc.Server.
-func SyncServer(impl SyncServerMethods) SyncServerStub {
- stub := implSyncServerStub{
- impl: impl,
- }
- // Initialize GlobState; always check the stub itself first, to handle the
- // case where the user has the Glob method defined in their VDL source.
- if gs := rpc.NewGlobState(stub); gs != nil {
- stub.gs = gs
- } else if gs := rpc.NewGlobState(impl); gs != nil {
- stub.gs = gs
- }
- return stub
-}
-
-type implSyncServerStub struct {
- impl SyncServerMethods
- gs *rpc.GlobState
-}
-
-func (s implSyncServerStub) GetDeltas(ctx *context.T, call *SyncGetDeltasServerCallStub, i0 map[ObjId]GenVector, i1 map[ObjId]GroupIdSet, i2 DeviceId) (map[ObjId]GenVector, error) {
- return s.impl.GetDeltas(ctx, call, i0, i1, i2)
-}
-
-func (s implSyncServerStub) GetObjectHistory(ctx *context.T, call *SyncGetObjectHistoryServerCallStub, i0 ObjId) (Version, error) {
- return s.impl.GetObjectHistory(ctx, call, i0)
-}
-
-func (s implSyncServerStub) GetDeviceStats(ctx *context.T, call *SyncGetDeviceStatsServerCallStub) error {
- return s.impl.GetDeviceStats(ctx, call)
-}
-
-func (s implSyncServerStub) GetSyncGroupMembers(ctx *context.T, call *SyncGetSyncGroupMembersServerCallStub, i0 []string) error {
- return s.impl.GetSyncGroupMembers(ctx, call, i0)
-}
-
-func (s implSyncServerStub) GetSyncGroupStats(ctx *context.T, call *SyncGetSyncGroupStatsServerCallStub) error {
- return s.impl.GetSyncGroupStats(ctx, call)
-}
-
-func (s implSyncServerStub) Dump(ctx *context.T, call rpc.ServerCall) error {
- return s.impl.Dump(ctx, call)
-}
-
-func (s implSyncServerStub) Globber() *rpc.GlobState {
- return s.gs
-}
-
-func (s implSyncServerStub) Describe__() []rpc.InterfaceDesc {
- return []rpc.InterfaceDesc{SyncDesc}
-}
-
-// SyncDesc describes the Sync interface.
-var SyncDesc rpc.InterfaceDesc = descSync
-
-// descSync hides the desc to keep godoc clean.
-var descSync = rpc.InterfaceDesc{
- Name: "Sync",
- PkgPath: "v.io/syncbase/x/ref/services/syncbase/sync",
- Doc: "// Sync allows a device to GetDeltas from another device.",
- Methods: []rpc.MethodDesc{
- {
- Name: "GetDeltas",
- Doc: "// GetDeltas returns a device's current generation vector and all\n// the missing log records when compared to the incoming generation vector.",
- InArgs: []rpc.ArgDesc{
- {"in", ``}, // map[ObjId]GenVector
- {"sgs", ``}, // map[ObjId]GroupIdSet
- {"clientId", ``}, // DeviceId
- },
- OutArgs: []rpc.ArgDesc{
- {"", ``}, // map[ObjId]GenVector
- },
- Tags: []*vdl.Value{vdl.ValueOf(access.Tag("Write"))},
- },
- {
- Name: "GetObjectHistory",
- Doc: "// GetObjectHistory returns the mutation history of a store object.",
- InArgs: []rpc.ArgDesc{
- {"oid", ``}, // ObjId
- },
- OutArgs: []rpc.ArgDesc{
- {"", ``}, // Version
- },
- },
- {
- Name: "GetDeviceStats",
- Doc: "// GetDeviceStats returns information on devices participating in\n// peer-to-peer synchronization.",
- Tags: []*vdl.Value{vdl.ValueOf(access.Tag("Admin"))},
- },
- {
- Name: "GetSyncGroupMembers",
- Doc: "// GetSyncGroupMembers returns information on SyncGroup members.\n// If SyncGroup names are specified, only members of these SyncGroups\n// are returned. Otherwise, if the slice of names is nil or empty,\n// members of all SyncGroups are returned.",
- InArgs: []rpc.ArgDesc{
- {"sgNames", ``}, // []string
- },
- Tags: []*vdl.Value{vdl.ValueOf(access.Tag("Admin"))},
- },
- {
- Name: "GetSyncGroupStats",
- Doc: "// GetSyncGroupStats returns high-level information on all SyncGroups.",
- Tags: []*vdl.Value{vdl.ValueOf(access.Tag("Admin"))},
- },
- {
- Name: "Dump",
- Doc: "// Dump writes to the Sync log internal information used for debugging.",
- Tags: []*vdl.Value{vdl.ValueOf(access.Tag("Admin"))},
- },
- },
-}
-
-// SyncGetDeltasServerStream is the server stream for Sync.GetDeltas.
-type SyncGetDeltasServerStream interface {
- // SendStream returns the send side of the Sync.GetDeltas server stream.
- SendStream() interface {
- // Send places the item onto the output stream. Returns errors encountered
- // while sending. Blocks if there is no buffer space; will unblock when
- // buffer space is available.
- Send(item LogRec) error
- }
-}
-
-// SyncGetDeltasServerCall represents the context passed to Sync.GetDeltas.
-type SyncGetDeltasServerCall interface {
- rpc.ServerCall
- SyncGetDeltasServerStream
-}
-
-// SyncGetDeltasServerCallStub is a wrapper that converts rpc.StreamServerCall into
-// a typesafe stub that implements SyncGetDeltasServerCall.
-type SyncGetDeltasServerCallStub struct {
- rpc.StreamServerCall
-}
-
-// Init initializes SyncGetDeltasServerCallStub from rpc.StreamServerCall.
-func (s *SyncGetDeltasServerCallStub) Init(call rpc.StreamServerCall) {
- s.StreamServerCall = call
-}
-
-// SendStream returns the send side of the Sync.GetDeltas server stream.
-func (s *SyncGetDeltasServerCallStub) SendStream() interface {
- Send(item LogRec) error
-} {
- return implSyncGetDeltasServerCallSend{s}
-}
-
-type implSyncGetDeltasServerCallSend struct {
- s *SyncGetDeltasServerCallStub
-}
-
-func (s implSyncGetDeltasServerCallSend) Send(item LogRec) error {
- return s.s.Send(item)
-}
-
-// SyncGetObjectHistoryServerStream is the server stream for Sync.GetObjectHistory.
-type SyncGetObjectHistoryServerStream interface {
- // SendStream returns the send side of the Sync.GetObjectHistory server stream.
- SendStream() interface {
- // Send places the item onto the output stream. Returns errors encountered
- // while sending. Blocks if there is no buffer space; will unblock when
- // buffer space is available.
- Send(item LogRec) error
- }
-}
-
-// SyncGetObjectHistoryServerCall represents the context passed to Sync.GetObjectHistory.
-type SyncGetObjectHistoryServerCall interface {
- rpc.ServerCall
- SyncGetObjectHistoryServerStream
-}
-
-// SyncGetObjectHistoryServerCallStub is a wrapper that converts rpc.StreamServerCall into
-// a typesafe stub that implements SyncGetObjectHistoryServerCall.
-type SyncGetObjectHistoryServerCallStub struct {
- rpc.StreamServerCall
-}
-
-// Init initializes SyncGetObjectHistoryServerCallStub from rpc.StreamServerCall.
-func (s *SyncGetObjectHistoryServerCallStub) Init(call rpc.StreamServerCall) {
- s.StreamServerCall = call
-}
-
-// SendStream returns the send side of the Sync.GetObjectHistory server stream.
-func (s *SyncGetObjectHistoryServerCallStub) SendStream() interface {
- Send(item LogRec) error
-} {
- return implSyncGetObjectHistoryServerCallSend{s}
-}
-
-type implSyncGetObjectHistoryServerCallSend struct {
- s *SyncGetObjectHistoryServerCallStub
-}
-
-func (s implSyncGetObjectHistoryServerCallSend) Send(item LogRec) error {
- return s.s.Send(item)
-}
-
-// SyncGetDeviceStatsServerStream is the server stream for Sync.GetDeviceStats.
-type SyncGetDeviceStatsServerStream interface {
- // SendStream returns the send side of the Sync.GetDeviceStats server stream.
- SendStream() interface {
- // Send places the item onto the output stream. Returns errors encountered
- // while sending. Blocks if there is no buffer space; will unblock when
- // buffer space is available.
- Send(item DeviceStats) error
- }
-}
-
-// SyncGetDeviceStatsServerCall represents the context passed to Sync.GetDeviceStats.
-type SyncGetDeviceStatsServerCall interface {
- rpc.ServerCall
- SyncGetDeviceStatsServerStream
-}
-
-// SyncGetDeviceStatsServerCallStub is a wrapper that converts rpc.StreamServerCall into
-// a typesafe stub that implements SyncGetDeviceStatsServerCall.
-type SyncGetDeviceStatsServerCallStub struct {
- rpc.StreamServerCall
-}
-
-// Init initializes SyncGetDeviceStatsServerCallStub from rpc.StreamServerCall.
-func (s *SyncGetDeviceStatsServerCallStub) Init(call rpc.StreamServerCall) {
- s.StreamServerCall = call
-}
-
-// SendStream returns the send side of the Sync.GetDeviceStats server stream.
-func (s *SyncGetDeviceStatsServerCallStub) SendStream() interface {
- Send(item DeviceStats) error
-} {
- return implSyncGetDeviceStatsServerCallSend{s}
-}
-
-type implSyncGetDeviceStatsServerCallSend struct {
- s *SyncGetDeviceStatsServerCallStub
-}
-
-func (s implSyncGetDeviceStatsServerCallSend) Send(item DeviceStats) error {
- return s.s.Send(item)
-}
-
-// SyncGetSyncGroupMembersServerStream is the server stream for Sync.GetSyncGroupMembers.
-type SyncGetSyncGroupMembersServerStream interface {
- // SendStream returns the send side of the Sync.GetSyncGroupMembers server stream.
- SendStream() interface {
- // Send places the item onto the output stream. Returns errors encountered
- // while sending. Blocks if there is no buffer space; will unblock when
- // buffer space is available.
- Send(item SyncGroupMember) error
- }
-}
-
-// SyncGetSyncGroupMembersServerCall represents the context passed to Sync.GetSyncGroupMembers.
-type SyncGetSyncGroupMembersServerCall interface {
- rpc.ServerCall
- SyncGetSyncGroupMembersServerStream
-}
-
-// SyncGetSyncGroupMembersServerCallStub is a wrapper that converts rpc.StreamServerCall into
-// a typesafe stub that implements SyncGetSyncGroupMembersServerCall.
-type SyncGetSyncGroupMembersServerCallStub struct {
- rpc.StreamServerCall
-}
-
-// Init initializes SyncGetSyncGroupMembersServerCallStub from rpc.StreamServerCall.
-func (s *SyncGetSyncGroupMembersServerCallStub) Init(call rpc.StreamServerCall) {
- s.StreamServerCall = call
-}
-
-// SendStream returns the send side of the Sync.GetSyncGroupMembers server stream.
-func (s *SyncGetSyncGroupMembersServerCallStub) SendStream() interface {
- Send(item SyncGroupMember) error
-} {
- return implSyncGetSyncGroupMembersServerCallSend{s}
-}
-
-type implSyncGetSyncGroupMembersServerCallSend struct {
- s *SyncGetSyncGroupMembersServerCallStub
-}
-
-func (s implSyncGetSyncGroupMembersServerCallSend) Send(item SyncGroupMember) error {
- return s.s.Send(item)
-}
-
-// SyncGetSyncGroupStatsServerStream is the server stream for Sync.GetSyncGroupStats.
-type SyncGetSyncGroupStatsServerStream interface {
- // SendStream returns the send side of the Sync.GetSyncGroupStats server stream.
- SendStream() interface {
- // Send places the item onto the output stream. Returns errors encountered
- // while sending. Blocks if there is no buffer space; will unblock when
- // buffer space is available.
- Send(item SyncGroupStats) error
- }
-}
-
-// SyncGetSyncGroupStatsServerCall represents the context passed to Sync.GetSyncGroupStats.
-type SyncGetSyncGroupStatsServerCall interface {
- rpc.ServerCall
- SyncGetSyncGroupStatsServerStream
-}
-
-// SyncGetSyncGroupStatsServerCallStub is a wrapper that converts rpc.StreamServerCall into
-// a typesafe stub that implements SyncGetSyncGroupStatsServerCall.
-type SyncGetSyncGroupStatsServerCallStub struct {
- rpc.StreamServerCall
-}
-
-// Init initializes SyncGetSyncGroupStatsServerCallStub from rpc.StreamServerCall.
-func (s *SyncGetSyncGroupStatsServerCallStub) Init(call rpc.StreamServerCall) {
- s.StreamServerCall = call
-}
-
-// SendStream returns the send side of the Sync.GetSyncGroupStats server stream.
-func (s *SyncGetSyncGroupStatsServerCallStub) SendStream() interface {
- Send(item SyncGroupStats) error
-} {
- return implSyncGetSyncGroupStatsServerCallSend{s}
-}
-
-type implSyncGetSyncGroupStatsServerCallSend struct {
- s *SyncGetSyncGroupStatsServerCallStub
-}
-
-func (s implSyncGetSyncGroupStatsServerCallSend) Send(item SyncGroupStats) error {
- return s.s.Send(item)
-}
diff --git a/services/syncbase/sync/vsyncd.go b/services/syncbase/sync/vsyncd.go
deleted file mode 100644
index ac25c29..0000000
--- a/services/syncbase/sync/vsyncd.go
+++ /dev/null
@@ -1,647 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Package vsync provides veyron sync daemon utility functions. Sync
-// daemon serves incoming GetDeltas requests and contacts other peers
-// to get deltas from them. When it receives a GetDeltas request, the
-// incoming generation vector is diffed with the local generation
-// vector, and missing generations are sent back. When it receives
-// log records in response to a GetDeltas request, it replays those
-// log records to get in sync with the sender.
-import (
- "fmt"
- "math/rand"
- "strings"
- "sync"
- "time"
-
- "v.io/v23/context"
- "v.io/v23/naming"
- "v.io/v23/rpc"
- "v.io/v23/security"
- "v.io/x/lib/vlog"
- "v.io/x/ref/lib/stats"
-)
-
-var (
- rng *rand.Rand
-)
-
-// Names of Sync stats entries.
-const (
- statsKvdbPath = "vsync/kvdb-path"
- statsDevId = "vsync/dev-id"
- statsNumDagObj = "vsync/num-dag-objects"
- statsNumDagNode = "vsync/num-dag-nodes"
- statsNumDagTx = "vsync/num-dag-tx"
- statsNumDagPrivNode = "vsync/num-dag-privnodes"
- statsNumSyncGroup = "vsync/num-syncgroups"
- statsNumMember = "vsync/num-members"
-)
-
-// syncd contains the metadata for the sync daemon.
-type syncd struct {
- // Pointers to metadata structures.
- log *iLog
- devtab *devTable
- dag *dag
- sgtab *syncGroupTable
-
- // Filesystem path to store metadata.
- kvdbPath string
-
- // Local device id.
- id DeviceId
- // Handle to the server to publish names to mount tables during syncgroup create/join.
- server rpc.Server
- // Map to track the set of names already published for this sync service.
- names map[string]struct{}
- // Relative name of the sync service to be advertised to the syncgroup server.
- // This is built off the local device id.
- relName string
-
- // Authorizer for sync service.
- auth security.Authorizer
-
- // RWlock to concurrently access log and device table data structures.
- lock sync.RWMutex
-
- // Mutex to serialize syncgroup operations and an going
- // initiation round. If both "lock" and "sgOp" need to be
- // acquired, sgOp must be acquired first.
- sgOp sync.Mutex
-
- // Channel used by Sync responders to notify the SyncGroup refresher
- // to fetch from SyncGroupServer updated info for some SyncGroups.
- // sgRefresh chan *refreshRequest
-
- // State to coordinate shutting down all spawned goroutines.
- pending sync.WaitGroup
- closed chan struct{}
-
- // Handlers for goroutine procedures.
- // hdlGC *syncGC
- hdlWatcher *syncWatcher
- hdlInitiator *syncInitiator
-
- // The context used for background activities.
- ctx *context.T
-}
-
-func (o ObjId) String() string {
- return string(o)
-}
-
-func (o ObjId) IsValid() bool {
- return o != NoObjId
-}
-
-func init() {
- rng = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
-}
-
-func NewVersion() Version {
- for {
- if v := Version(rng.Int63()); v != NoVersion {
- return v
- }
- }
-}
-
-// NewSyncd creates a new syncd instance.
-//
-// Syncd concurrency: syncd initializes three goroutines at
-// startup. The "watcher" thread is responsible for watching the store
-// for changes to its objects. The "initiator" thread is responsible
-// for periodically checking the neighborhood and contacting a peer to
-// obtain changes from that peer. The "gc" thread is responsible for
-// periodically checking if any log records and dag state can be
-// pruned. All these 3 threads perform write operations to the data
-// structures, and synchronize by acquiring a write lock on s.lock. In
-// addition, when syncd receives an incoming RPC, it responds to the
-// request by acquiring a read lock on s.lock. Thus, at any instant in
-// time, either one of the watcher, initiator or gc threads is active,
-// or any number of responders can be active, serving incoming
-// requests. Fairness between these threads follows from
-// sync.RWMutex. The spec says that the writers cannot be starved by
-// the readers but it does not guarantee FIFO. We may have to revisit
-// this in the future.
-func NewSyncd(devid string, syncTick time.Duration, server rpc.Server, ctx *context.T) *syncd {
- return newSyncdCore(devid, syncTick, server, ctx)
-}
-
-// newSyncdCore is the internal function that creates the Syncd
-// structure and initilizes its thread (goroutines). It takes a
-// Veyron Store parameter to separate the core of Syncd setup from the
-// external dependency on Veyron Store.
-func newSyncdCore(devid string, syncTick time.Duration,
- server rpc.Server, ctx *context.T) *syncd {
- // TODO(kash): Get this to compile
- return nil
- // s := &syncd{ctx: ctx}
-
- // storePath := "/tmp"
-
- // // If no authorizer is specified, allow access to all Principals.
- // if s.auth = vflag.NewAuthorizerOrDie(); s.auth == nil {
- // s.auth = allowEveryone{}
- // }
-
- // // Bootstrap my own DeviceId.
- // s.id = DeviceId(devid)
-
- // // Cache the server to publish names if required.
- // s.server = server
- // s.relName = devIDToName(s.id)
- // s.names = make(map[string]struct{})
-
- // var err error
- // // Log init.
- // if s.log, err = openILog(storePath+"/vsync_ilog.db", s); err != nil {
- // vlog.Fatalf("newSyncd: openILog failed: err %v", err)
- // }
-
- // // DevTable init.
- // if s.devtab, err = openDevTable(storePath+"/vsync_dtab.db", s); err != nil {
- // vlog.Fatalf("newSyncd: openDevTable failed: err %v", err)
- // }
-
- // // Dag init.
- // if s.dag, err = openDAG(storePath + "/vsync_dag.db"); err != nil {
- // vlog.Fatalf("newSyncd: OpenDag failed: err %v", err)
- // }
-
- // // SyncGroupTable init.
- // if s.sgtab, err = openSyncGroupTable(storePath + "/vsync_sgtab.db"); err != nil {
- // vlog.Fatalf("newSyncd: openSyncGroupTable failed: err %v", err)
- // }
-
- // // Channel for responders to notify the SyncGroup refresher.
- // // Give the channel some buffering to ease a burst of responders
- // // while the refresher is still finishing a previous request.
- // // s.sgRefresh = make(chan *refreshRequest, 4096)
-
- // // Channel to propagate close event to all threads.
- // s.closed = make(chan struct{})
-
- // s.pending.Add(2)
-
- // // Get deltas every peerSyncInterval.
- // s.hdlInitiator = newInitiator(s, syncTick)
- // go s.hdlInitiator.contactPeers()
-
- // // // Garbage collect every garbageCollectInterval.
- // // s.hdlGC = newGC(s)
- // // go s.hdlGC.garbageCollect()
-
- // // Start a watcher thread that will get updates from local store.
- // s.hdlWatcher = newWatcher(s)
- // go s.hdlWatcher.watchStore()
-
- // // Start a thread to refresh SyncGroup info from SyncGroupServer.
- // // go s.syncGroupRefresher()
-
- // // Server stats.
- // stats.NewString(statsKvdbPath).Set(s.kvdbPath)
- // stats.NewString(statsDevId).Set(string(s.id))
-
- // return s
-}
-
-// devIDToName converts a device ID to its relative name.
-func devIDToName(id DeviceId) string {
- return naming.Join([]string{"global", "vsync", string(id)}...)
-}
-
-// nameToDevId extracts the DeviceId from a name.
-func (s *syncd) nameToDevId(name string) DeviceId {
- parts := strings.Split(name, "/")
- return DeviceId(parts[len(parts)-1])
-}
-
-// Close cleans up syncd state.
-func (s *syncd) Close() {
- close(s.closed)
- s.pending.Wait()
-
- stats.Delete(statsKvdbPath)
- stats.Delete(statsDevId)
-
- // TODO(hpucha): close without flushing.
-}
-
-// isSyncClosing returns true if Close() was called i.e. the "closed" channel is closed.
-func (s *syncd) isSyncClosing() bool {
- select {
- case <-s.closed:
- return true
- default:
- return false
- }
-}
-
-// GetDeltas responds to the incoming request from a client by sending missing generations to the client.
-func (s *syncd) GetDeltas(call SyncGetDeltasServerCall, in map[ObjId]GenVector,
- syncGroups map[ObjId]GroupIdSet, clientID DeviceId) (map[ObjId]GenVector, error) {
-
- vlog.VI(1).Infof("GetDeltas:: Received vector %v from client %s", in, clientID)
-
- // Handle misconfiguration: the client cannot have the same ID as this node.
- if clientID == s.id {
- vlog.VI(1).Infof("GetDeltas:: impostor alert: client ID %s is the same as mine %s", clientID, s.id)
- return nil, fmt.Errorf("impostor: cannot be %s, for this node is %s", clientID, s.id)
- }
-
- out := make(map[ObjId]GenVector)
- for sr, gvIn := range in {
- gvOut, gens, gensInfo, err := s.prepareGensToReply(call, sr, syncGroups[sr], gvIn)
- if err != nil {
- vlog.Errorf("GetDeltas:: prepareGensToReply for sr %s failed with err %v",
- sr.String(), err)
- continue
- }
-
- for pos, v := range gens {
- gen := gensInfo[pos]
- var count uint64
- for i := SeqNum(0); i <= gen.MaxSeqNum; i++ {
- count++
- rec, err := s.getLogRec(v.devID, v.srID, v.genID, i)
- if err != nil {
- vlog.Fatalf("GetDeltas:: Couldn't get log record %s %s %d %d, err %v",
- v.devID, v.srID.String(), v.genID, i, err)
- }
- vlog.VI(1).Infof("Sending log record %v", rec)
- if err := call.SendStream().Send(*rec); err != nil {
- vlog.Errorf("GetDeltas:: Couldn't send stream err: %v", err)
- return nil, err
- }
- }
- if count != gen.Count {
- vlog.Fatalf("GetDeltas:: GenMetadata has incorrect log records for generation %s %s %d %v",
- v.devID, v.srID.String(), v.genID, gen)
- }
- }
- out[sr] = gvOut
- }
-
- // Notify the SyncGroup Refresher about the SyncGroup IDs associated
- // with the SyncRoots in the incoming request from the client.
- // Note: the initial slice capacity is only a lower-bound.
- // Note: by using the SyncRoots from the incoming "in" map and all
- // all SyncGroup IDs for each SyncRoot, the refresh is eager because
- // it does not filter out SyncRoots and SyncGroup IDs by the Permissions.
- // sgIDs := make([]syncgroup.Id, 0, len(in))
- // for sr := range in {
- // for _, id := range syncGroups[sr] {
- // sgIDs = append(sgIDs, id)
- // }
- // }
- // s.notifySyncGroupRefresher(devIDToName(clientID), sgIDs)
-
- return out, nil
-}
-
-// // updateDeviceInfo updates the remote device's information based on
-// // the incoming GetDeltas request.
-// func (s *syncd) updateDeviceInfo(ClientID DeviceId, In GenVector) error {
-// s.lock.Lock()
-// defer s.lock.Unlock()
-
-// // Note that the incoming client generation vector cannot be
-// // used for garbage collection. We can only garbage collect
-// // based on the generations we receive from other
-// // devices. Receiving a set of generations assures that all
-// // updates branching from those generations are also received
-// // and hence generations present on all devices can be
-// // GC'ed. This function sends generations to other devices and
-// // hence does not use the generation vector for GC.
-// //
-// // TODO(hpucha): Cache the client's incoming generation vector
-// // to assist in tracking missing generations and hence next
-// // peer to contact.
-// if !s.devtab.hasDevInfo(ClientID) {
-// if err := s.devtab.addDevice(ClientID); err != nil {
-// return err
-// }
-// }
-// return nil
-// }
-
-// prepareGensToReply verifies if the requestor has access to the
-// requested syncroot. If allowed, it processes the incoming
-// generation vector and returns the metadata of all the missing
-// generations between the incoming and the local generation vector
-// for that syncroot.
-func (s *syncd) prepareGensToReply(call rpc.ServerCall, srid ObjId, sgVec GroupIdSet, In GenVector) (GenVector, []*genOrder, []*genMetadata, error) {
- s.lock.RLock()
- defer s.lock.RUnlock()
-
- // Respond to the caller if any of the syncgroups
- // corresponding to the syncroot allow access.
- var allowedErr error
- // for _, sg := range sgVec {
- // // Check permissions for the syncgroup.
- // sgData, err := s.sgtab.getSyncGroupByID(sg)
- // if err != nil {
- // return GenVector{}, nil, nil, err
- // }
- // if vlog.V(2) {
- // b, _ := ctx.RemoteBlessings().ForContext(ctx)
- // vlog.Infof("prepareGensToReply:: matching acl %v, remote names %v", sgData.SrvInfo.Config.Permissions, b)
- // }
- // // TODO(ashankar): Consider changing TaggedPermissionsAuthorizer so that it doesn't return an error -
- // // instead of an error, it just locks down access completely. The error condition is really
- // // really rare and doing so will make for shorter code?
- // auth, err := access.TaggedPermissionsAuthorizer(sgData.SrvInfo.Config.Permissions, access.TypicalTagType())
- // if err != nil {
- // vlog.Errorf("unable to create authorizer: %v", err)
- // allowedErr = fmt.Errorf("server error: authorization policy misconfigured")
- // } else if allowedErr = auth.Authorize(ctx); allowedErr == nil {
- // break
- // }
- // }
- if allowedErr != nil {
- return GenVector{}, nil, nil, allowedErr
- }
-
- // // TODO(hpucha): Need only for GC.
- // if err := s.updateDeviceInfo(ClientID, gvIn); err != nil {
- // vlog.Fatalf("GetDeltas:: updateDeviceInfo failed with err %v", err)
- // }
-
- // Get local generation vector.
- out, err := s.devtab.getGenVec(s.id, srid)
- if err != nil {
- return GenVector{}, nil, nil, err
- }
-
- // Diff the two generation vectors.
- gens, err := s.devtab.diffGenVectors(out, In, srid)
- if err != nil {
- return GenVector{}, nil, nil, err
- }
-
- // Get the metadata for all the generations in the reply.
- gensInfo := make([]*genMetadata, len(gens))
- for pos, v := range gens {
- gen, err := s.log.getGenMetadata(v.devID, v.srID, v.genID)
- if err != nil || gen.Count <= 0 {
- return GenVector{}, nil, nil, err
- }
- gensInfo[pos] = gen
- }
-
- return out, gens, gensInfo, nil
-}
-
-// getLogRec gets the log record for a given generation and lsn.
-func (s *syncd) getLogRec(dev DeviceId, srid ObjId, gen GenId, lsn SeqNum) (*LogRec, error) {
- s.lock.RLock()
- defer s.lock.RUnlock()
- return s.log.getLogRec(dev, srid, gen, lsn)
-}
-
-// // GetSyncGroupStats gives the client high-level information on all SyncGroups.
-// // The streamed information is returned sorted by SyncGroup name.
-// func (s *syncd) GetSyncGroupStats(ctx SyncGetSyncGroupStatsContext) error {
-// // Get a snapshot of the SyncGroup names.
-// // The names are a copy, OK to unlock here.
-// s.lock.RLock()
-// names, err := s.sgtab.getAllSyncGroupNames()
-// s.lock.RUnlock()
-
-// if err != nil {
-// vlog.Errorf("GetSyncGroupStats:: cannot get SyncGroup names: %v", err)
-// return err
-// }
-// if len(names) == 0 {
-// return nil
-// }
-
-// // Send the SyncGroup stats in the stream in sorted order.
-// // Skip SyncGroups that got deleted in the meanwhile.
-// sort.Strings(names)
-// for _, name := range names {
-// // The SyncGroup object is a copy, OK to unlock here.
-// s.lock.RLock()
-// sg, err := s.sgtab.getSyncGroupByName(name)
-// s.lock.RUnlock()
-
-// if err != nil {
-// continue // skip deleted SyncGroup
-// }
-
-// stats := SyncGroupStats{
-// Name: name,
-// SGObjId: sg.SrvInfo.SGObjId,
-// Path: sg.LocalPath,
-// RootObjId: ObjId(sg.SrvInfo.RootObjId),
-// NumJoiners: uint32(len(sg.SrvInfo.Joiners)),
-// }
-
-// if err = ctx.SendStream().Send(stats); err != nil {
-// vlog.Errorf("GetSyncGroupStats:: cannot send stream data: %v", err)
-// return err
-// }
-// }
-
-// return nil
-// }
-
-// // GetSyncGroupMembers gives the client information on SyncGroup members.
-// // If SyncGroup names are specified, only their member information is sent.
-// // Otherwise information on members from all SyncGroups is sent.
-// func (s *syncd) GetSyncGroupMembers(ctx SyncGetSyncGroupMembersContext, sgNames []string) error {
-
-// // If SyncGroup names are given, fetch their IDs.
-// // Also get a snapshot of the SyncGroup members.
-// wantedGroupIds := make(map[syncgroup.Id]bool)
-
-// s.lock.RLock()
-// for _, name := range sgNames {
-// sgid, err := s.sgtab.getSyncGroupID(name)
-// if err != nil {
-// s.lock.RUnlock()
-// vlog.Errorf("GetSyncGroupMembers:: invalid SyncGroup %s: %v", name, err)
-// return err
-// }
-// wantedGroupIds[sgid] = true
-// }
-
-// memberMap, err := s.sgtab.getMembers() // The map is a copy, OK to unlock here.
-// s.lock.RUnlock()
-
-// if err != nil {
-// vlog.Errorf("GetSyncGroupMembers:: cannot get members: %v", err)
-// return err
-// }
-// if len(memberMap) == 0 {
-// return nil
-// }
-
-// // Send the member information in the stream in sorted order.
-// // Skip members that left in the meanwhile.
-// // If SyncGroup names are given, only return members in these groups.
-// members := make([]string, 0, len(memberMap))
-// for member := range memberMap {
-// members = append(members, member)
-// }
-// sort.Strings(members)
-
-// for _, member := range members {
-// // The member info points to in-memory data; cannot unlock
-// // while accessing that info.
-// s.lock.RLock()
-// info, err := s.sgtab.getMemberInfo(member)
-// if err != nil {
-// s.lock.RUnlock()
-// continue // skip member no longer there
-// }
-
-// replyData := make([]SyncGroupMember, 0, len(info.gids))
-// for sgid, meta := range info.gids {
-// if len(wantedGroupIds) > 0 && !wantedGroupIds[sgid] {
-// continue // skip unwanted SyncGroups
-// }
-
-// data := SyncGroupMember{
-// Name: member,
-// SGObjId: sgid,
-// Metadata: meta.metaData,
-// }
-// replyData = append(replyData, data)
-// }
-
-// s.lock.RUnlock()
-
-// for _, data := range replyData {
-// if err = ctx.SendStream().Send(data); err != nil {
-// vlog.Errorf("GetSyncGroupMembers:: cannot send stream data: %v", err)
-// return err
-// }
-// }
-// }
-
-// return nil
-// }
-
-// Dump writes to the Sync log internal information used for debugging.
-func (s *syncd) Dump(call rpc.ServerCall) error {
- s.lock.RLock()
- defer s.lock.RUnlock()
-
- vlog.VI(1).Infof("Dump: ==== BEGIN ====")
- s.devtab.dump()
- // s.sgtab.dump()
- s.log.dump()
- s.dag.dump()
- vlog.VI(1).Infof("Dump: ==== END ====")
- return nil
-}
-
-// GetDeviceStats gives the client information on devices that have synchronized
-// with the server.
-func (s *syncd) GetDeviceStats(call SyncGetDeviceStatsServerCall) error {
- // Get a snapshot of the device IDs.
- // The device IDs are copies, OK to unlock here.
- s.lock.RLock()
- myID := s.id
- devIDs, err := s.devtab.getDeviceIds()
- s.lock.RUnlock()
-
- if err != nil {
- vlog.Errorf("GetDeviceStats:: cannot get device IDs: %v", err)
- return err
- }
-
- // Send the device stats in the stream.
- for _, id := range devIDs {
- // The device info is a copy, OK to unlock here.
- s.lock.RLock()
- info, err := s.devtab.getDevInfo(id)
- s.lock.RUnlock()
-
- if err != nil {
- continue // skip unknown devices
- }
-
- stats := DeviceStats{
- DevId: id,
- LastSync: info.Ts.Unix(),
- GenVectors: info.Vectors,
- IsSelf: id == myID,
- }
-
- if err = call.SendStream().Send(stats); err != nil {
- vlog.Errorf("GetDeviceStats:: cannot send stream data: %v", err)
- return err
- }
- }
-
- return nil
-}
-
-// GetObjectHistory gives the client the mutation history of a store object.
-// It also returns the object version that is at the "head" of the graph (DAG).
-func (s *syncd) GetObjectHistory(call SyncGetObjectHistoryServerCall, oid ObjId) (Version, error) {
- // Get the object version numbers in reverse chronological order,
- // starting from the "head" version back through its ancestors.
- // Also get the DAG parents of each object version.
- var versions []Version
- parents := make(map[Version][]Version)
-
- s.lock.RLock()
- head, err := s.dag.getHead(oid)
- if err != nil {
- // For now ignore objects that do not yet have a "head" version.
- // They are either not part of any SyncGroup, or this Sync server
- // has not finished catching up (Sync or Watch operations).
- vlog.Errorf("GetObjectHistory:: unknown object %v: %v", oid, err)
- s.lock.RUnlock()
- return NoVersion, err
- }
-
- s.dag.ancestorIter(oid, []Version{head},
- func(oid ObjId, v Version, node *dagNode) error {
- versions = append(versions, v)
- parents[v] = node.Parents
- return nil
- })
- s.lock.RUnlock()
-
- // Stream the object log records in the given order.
- for _, v := range versions {
- // The log record is a copy, OK to unlock here.
- s.lock.RLock()
- rec, err := s.hdlInitiator.getLogRec(oid, v)
- s.lock.RUnlock()
-
- if err != nil {
- vlog.Errorf("GetObjectHistory:: cannot get log record for %v:%v: %v", oid, v, err)
- return NoVersion, err
- }
-
- // To give the client a full DAG, send the parents as seen in
- // the DAG node instead of the ones in the log record.
- // Note: DAG nodes contain the full sets of parents whereas
- // log records may have subsets due to the different types of
- // records: NodeRec-Parents + LinkRec-Parent == DAG-Parents.
- rec.Parents = parents[v]
-
- if err = call.SendStream().Send(*rec); err != nil {
- vlog.Errorf("GetObjectHistory:: cannot send stream data: %v", err)
- return NoVersion, err
- }
- }
-
- return head, nil
-}
-
-// allowEveryone implements the authorization policy of ... allowing every principal.
-type allowEveryone struct{}
-
-func (allowEveryone) Authorize(security.Call) error { return nil }
diff --git a/services/syncbase/sync/watcher.go b/services/syncbase/sync/watcher.go
deleted file mode 100644
index f9799ae..0000000
--- a/services/syncbase/sync/watcher.go
+++ /dev/null
@@ -1,257 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package vsync
-
-// Veyron Sync hook to the Store Watch API. When applications update objects
-// in the local Veyron Store, Sync learns about these via the Watch API stream
-// of object mutations. In turn, this Sync watcher thread updates the DAG and
-// ILog records to track the object change histories.
-
-import (
- "time"
-
- "v.io/v23/context"
- "v.io/x/lib/vlog"
-)
-
-var (
- // watchRetryDelay is how long the watcher waits before calling the Watch() RPC again
- // after the previous call fails.
- watchRetryDelay = 2 * time.Second
- // streamRetryDelay is how long the watcher waits before creating a new Watch stream
- // after the previous stream ends.
- streamRetryDelay = 1 * time.Second
-)
-
-// syncWatcher contains the metadata for the Sync Watcher thread.
-type syncWatcher struct {
- // syncd is a pointer to the Syncd instance owning this Watcher.
- syncd *syncd
-}
-
-// newWatcher creates a new Sync Watcher instance attached to the given Syncd instance.
-func newWatcher(syncd *syncd) *syncWatcher {
- return &syncWatcher{syncd: syncd}
-}
-
-func (w *syncWatcher) watchStreamCanceler(cancel context.CancelFunc, done <-chan struct{}) {
- select {
- case <-w.syncd.closed:
- vlog.VI(1).Info("watchStreamCanceler: Syncd channel closed, cancel stream and exit")
- cancel()
- case <-done:
- vlog.VI(1).Info("watchStreamCanceler: done, exit")
- }
-}
-
-// watchStore consumes change records obtained by watching the store
-// for updates and applies them to the sync DBs.
-func (w *syncWatcher) watchStore() {
- defer w.syncd.pending.Done()
-
- // // If no Veyron store is configured, there is nothing to watch.
- // if w.syncd.store == nil {
- // vlog.VI(1).Info("watchStore: Veyron store not configured; skipping the watcher")
- // return
- // }
-
- // // Get a Watch stream, process it, repeat till end-of-life.
- // for {
- // ctx, _ := vtrace.SetNewTrace(w.syncd.ctx)
- // ctx, cancel := context.WithCancel(ctx)
- // stream := w.getWatchStream(ctx)
- // if stream == nil {
- // return // Syncd is exiting.
- // }
-
- // // Spawn a goroutine to detect the Syncd "closed" channel and cancel the RPC stream
- // // to unblock the watcher. The "done" channel lets the watcher terminate the goroutine.
- // go w.watchStreamCanceler(cancel, ctx.Done())
-
- // // Process the stream of Watch updates until it closes (similar to "tail -f").
- // w.processWatchStream(stream)
-
- // if w.syncd.isSyncClosing() {
- // return // Syncd is exiting.
- // }
-
- // stream.Finish()
- // cancel()
-
- // // TODO(rdaoud): we need a rate-limiter here in case the stream closes too quickly.
- // // If the stream stays up long enough, no need to sleep before getting a new one.
- // time.Sleep(streamRetryDelay)
- // }
-}
-
-// // getWatchStream() returns a Watch API stream and handles retries if the Watch() call fails.
-// // If the stream is nil, it means Syncd is exiting cleanly and the caller should terminate.
-// func (w *syncWatcher) getWatchStream(ctx *context.T) watch.GlobWatcherWatchGlobCall {
-// for {
-// w.syncd.lock.RLock()
-// resmark := w.syncd.devtab.head.Resmark
-// w.syncd.lock.RUnlock()
-
-// req := raw.Request{}
-// if resmark != nil {
-// req.ResumeMarker = resmark
-// }
-
-// stream, err := w.syncd.store.Watch(ctx, req)
-// if err == nil {
-// return stream
-// }
-
-// if w.syncd.isSyncClosing() {
-// vlog.VI(1).Info("getWatchStream: exiting, Syncd closed its channel: ", err)
-// return nil
-// }
-
-// vlog.VI(1).Info("getWatchStream: call to Watch() failed, retrying a bit later: ", err)
-// time.Sleep(watchRetryDelay)
-// }
-// }
-
-// // processWatchStream reads the stream of Watch updates and applies the object mutations.
-// // Ideally this call does not return as the stream should be un-ending (like "tail -f").
-// // If the stream is closed, distinguish between the cases of end-of-stream vs Syncd canceling
-// // the stream to trigger a clean exit.
-// func (w *syncWatcher) processWatchStream(call watch.GlobWatcherWatchGlobCall) {
-// var txChanges []*types.Change = nil
-// var syncTime int64
-
-// stream := call.RecvStream()
-// for stream.Advance() {
-// change := stream.Value()
-
-// // Timestamp of the start of a batch of changes arriving at the Sync server.
-// if txChanges == nil {
-// syncTime = time.Now().UnixNano()
-// }
-// txChanges = append(txChanges, &change)
-
-// // Process the transaction when its last change record is received.
-// if !change.Continued {
-// if err := w.processTransaction(txChanges, syncTime); err != nil {
-// // TODO(rdaoud): don't crash, instead add retry policies to attempt some degree of
-// // self-healing from a data corruption where feasible, otherwise quarantine this device
-// // from the cluster and stop Syncd to avoid propagating data corruptions.
-// vlog.Fatal("processWatchStream:", err)
-// }
-// txChanges = nil
-// }
-// }
-
-// err := stream.Err()
-// if err == nil {
-// err = io.EOF
-// }
-
-// if w.syncd.isSyncClosing() {
-// vlog.VI(1).Info("processWatchStream: exiting, Syncd closed its channel: ", err)
-// } else {
-// vlog.VI(1).Info("processWatchStream: RPC stream error, re-issue Watch(): ", err)
-// }
-// }
-
-// // matchSyncRoot returns the SyncRoot (SyncGroup root ObjId) that matches the given object path.
-// // It traverses the object path looking if any of the IDs along the way are SyncRoots.
-// // If no match is found, it returns the invalid ID (zero).
-// // Note: the path IDs given by Watch are ordered: object, parent, grand-parent, etc.., root.
-// // As a result, this function returns the narrowest matching SyncGroup root ObjId.
-// func matchSyncRoot(objPathIDs []ObjId, sgObjIds map[ObjId]struct{}) ObjId {
-// for _, oid := range objPathIDs {
-// if _, ok := sgObjIds[oid]; ok {
-// return oid
-// }
-// }
-// return ObjId{}
-// }
-
-// // processTransaction applies a batch of changes (object mutations) that form a single transaction
-// // received from the Watch API. The function grabs the write-lock to access the Log and DAG DBs.
-// func (w *syncWatcher) processTransaction(txBatch []*types.Change, syncTime int64) error {
-// w.syncd.lock.Lock()
-// defer w.syncd.lock.Unlock()
-
-// count := uint32(len(txBatch))
-// if count == 0 {
-// return nil
-// }
-
-// // Get the current set of SyncGroup Root ObjIds and use it to determine for each object
-// // the SyncGroup it belongs to, if any.
-// sgRootObjIds, err := w.syncd.sgtab.getAllSyncRoots()
-// if err != nil {
-// return err
-// }
-
-// // If the batch has more than one mutation, start a DAG transaction for it.
-// txID := NoTxId
-// if count > 1 {
-// txID = w.syncd.dag.addNodeTxStart(txID)
-// if txID == NoTxId {
-// return fmt.Errorf("failed to generate transaction ID")
-// }
-// }
-
-// vlog.VI(1).Infof("processTransaction: ready to process a batch of %d changes for transaction %v", count, txID)
-
-// for _, ch := range txBatch {
-// rc, ok := ch.Value.(*raw.RawChange)
-// if !ok {
-// return fmt.Errorf("invalid change value, not a raw change: %#v", ch)
-// }
-
-// mu := &rc.Mutation
-// isDeleted := ch.State == types.DoesNotExist
-
-// // Determine the SyncGroup root ObjId that matches this object, if any.
-// rootObjId := matchSyncRoot(rc.PathIDs, sgRootObjIds)
-
-// if rootObjId.IsValid() {
-// // The object matches a SyncGroup: process its watch record to track it in the Sync logs.
-// // All LogValues belonging to the same transaction get the same timestamp.
-// val := &LogValue{
-// Mutation: *mu,
-// SyncTime: syncTime,
-// Delete: isDeleted,
-// TxId: txID,
-// TxCount: count,
-// }
-// vlog.VI(2).Infof("processTransaction: processing record %v, Tx %v", val, txID)
-
-// err = w.syncd.log.processWatchRecord(mu.Id, mu.Version, mu.PriorVersion, val, rootObjId)
-// } else {
-// // The object does not match any SyncGroup: stash it in the "private" table to save its info
-// // in case it becomes shared in the future, at which time it would be added to the Sync logs.
-// if isDeleted {
-// err = w.syncd.dag.delPrivNode(mu.Id)
-// } else {
-// priv := &privNode{Mutation: mu, PathIDs: rc.PathIDs, SyncTime: syncTime, TxId: txID, TxCount: count}
-// err = w.syncd.dag.setPrivNode(mu.Id, priv)
-// }
-// }
-
-// if err != nil {
-// return fmt.Errorf("cannot process mutation: %#v, mu %#v: %s", ch, mu, err)
-// }
-// }
-
-// // End the DAG transaction if any.
-// if txID != NoTxId {
-// // Note: if there are no shared objects in the transaction, the addNodeTxEnd() call
-// // does not persist the Tx state, it only cleans up the in-memory scaffolding.
-// // This is the desired behavior, Sync only needs to track the state of transactions
-// // that have at least one shared object.
-// if err := w.syncd.dag.addNodeTxEnd(txID, count); err != nil {
-// return err
-// }
-// }
-
-// // Update the device table with the new resume marker of the last record.
-// w.syncd.devtab.head.Resmark = txBatch[count-1].ResumeMarker
-// return nil
-// }
