syncbase: replace veyron/ with x/ref/
Change-Id: I246fa092651c95b64d5ed43a0f48d40050b46190
diff --git a/services/syncbase/server/database.go b/services/syncbase/server/database.go
new file mode 100644
index 0000000..5a0c4ae
--- /dev/null
+++ b/services/syncbase/server/database.go
@@ -0,0 +1,202 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package server
+
+import (
+ "v.io/syncbase/v23/services/syncbase"
+ "v.io/syncbase/x/ref/services/syncbase/store"
+ "v.io/v23/rpc"
+ "v.io/v23/security/access"
+ "v.io/v23/verror"
+)
+
+// TODO(sadovsky): Decide whether to use the same version-aware data layout that
+// we use for Items. Relatedly, decide whether the Database Permissions should get
+// synced. (If so, that suggests we should indeed use the same version-aware
+// data layout here, and perhaps everywhere.)
+
+type database struct {
+ name string
+ u *universe
+}
+
+var _ syncbase.DatabaseServerMethods = (*database)(nil)
+
+func (d *database) Create(call rpc.ServerCall, acl access.Permissions) error {
+ return store.RunInTransaction(d.u.s.st, func(st store.Store) error {
+ // Update universeData.
+ var uData *universeData
+ if err := d.u.update(call, st, true, func(data *universeData) error {
+ if _, ok := data.Databases[d.name]; ok {
+ return verror.New(verror.ErrExist, call.Context(), d.name)
+ }
+ // https://github.com/veyron/release-issues/issues/1145
+ if data.Databases == nil {
+ data.Databases = map[string]struct{}{}
+ }
+ data.Databases[d.name] = struct{}{}
+ uData = data
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ // Blind-write databaseData.
+ if acl == nil {
+ acl = uData.Permissions
+ }
+ data := &databaseData{
+ Name: d.name,
+ Permissions: acl,
+ }
+ return d.put(call, st, data)
+ })
+}
+
+func (d *database) Delete(call rpc.ServerCall) error {
+ return store.RunInTransaction(d.u.s.st, func(st store.Store) error {
+ // Read-check-delete databaseData.
+ if _, err := d.get(call, st, true); err != nil {
+ return err
+ }
+ // TODO(sadovsky): Delete all Tables (and Items) in this Database.
+ if err := d.del(call, st); err != nil {
+ return err
+ }
+
+ // Update universeData.
+ return d.u.update(call, st, false, func(data *universeData) error {
+ delete(data.Databases, d.name)
+ return nil
+ })
+ })
+}
+
+func (d *database) UpdateSchema(call rpc.ServerCall, schema syncbase.Schema, version string) error {
+ if schema == nil {
+ return verror.New(verror.ErrInternal, nil, "schema must be specified")
+ }
+
+ return store.RunInTransaction(d.u.s.st, func(st store.Store) error {
+ return d.update(call, st, true, func(data *databaseData) error {
+ if err := checkVersion(call, version, data.Version); err != nil {
+ return err
+ }
+ // TODO(sadovsky): Delete all Tables (and Items) that are no longer part
+ // of the schema.
+ data.Schema = schema
+ data.Version++
+ return nil
+ })
+ })
+}
+
+func (d *database) GetSchema(call rpc.ServerCall) (schema syncbase.Schema, version string, err error) {
+ data, err := d.get(call, d.u.s.st, true)
+ if err != nil {
+ return nil, "", err
+ }
+ return data.Schema, formatVersion(data.Version), nil
+}
+
+func (d *database) SetPermissions(call rpc.ServerCall, acl access.Permissions, version string) error {
+ return store.RunInTransaction(d.u.s.st, func(st store.Store) error {
+ return d.update(call, st, true, func(data *databaseData) error {
+ if err := checkVersion(call, version, data.Version); err != nil {
+ return err
+ }
+ data.Permissions = acl
+ data.Version++
+ return nil
+ })
+ })
+}
+
+func (d *database) GetPermissions(call rpc.ServerCall) (acl access.Permissions, version string, err error) {
+ data, err := d.get(call, d.u.s.st, true)
+ if err != nil {
+ return nil, "", err
+ }
+ return data.Permissions, formatVersion(data.Version), nil
+}
+
+// TODO(sadovsky): Implement Glob.
+
+////////////////////////////////////////
+// Internal helpers
+
+func (d *database) keyPart() string {
+ return joinKeyParts(d.u.keyPart(), d.name)
+}
+
+func (d *database) key() string {
+ return joinKeyParts("$database", d.keyPart())
+}
+
+// Note, the methods below use "x" as the receiver name to make find-replace
+// easier across the different levels of syncbase hierarchy.
+//
+// TODO(sadovsky): Is there any better way to share this code despite the lack
+// of generics?
+
+// Reads data from the storage engine.
+// Returns a VDL-compatible error.
+// checkAuth specifies whether to perform an authorization check.
+func (x *database) get(call rpc.ServerCall, st store.Store, checkAuth bool) (*databaseData, error) {
+ // TODO(kash): Get this to compile.
+ return nil, nil
+ // data := &databaseData{}
+ // if err := getObject(st, x.key(), data); err != nil {
+ // if _, ok := err.(*store.ErrUnknownKey); ok {
+ // // TODO(sadovsky): Return ErrNoExist if appropriate.
+ // return nil, verror.NewErrNoExistOrNoAccess(call.Context())
+ // }
+ // return nil, verror.New(verror.ErrInternal, call.Context(), err)
+ // }
+ // if checkAuth {
+ // auth, _ := access.PermissionsAuthorizer(data.Permissions, access.TypicalTagType())
+ // if err := auth.Authorize(call); err != nil {
+ // // TODO(sadovsky): Return ErrNoAccess if appropriate.
+ // return nil, verror.NewErrNoExistOrNoAccess(call.Context())
+ // }
+ // }
+ // return data, nil
+}
+
+// Writes data to the storage engine.
+// Returns a VDL-compatible error.
+// If you need to perform an authorization check, use update().
+func (x *database) put(call rpc.ServerCall, st store.Store, data *databaseData) error {
+ if err := putObject(st, x.key(), data); err != nil {
+ return verror.New(verror.ErrInternal, call.Context(), err)
+ }
+ return nil
+}
+
+// Deletes data from the storage engine.
+// Returns a VDL-compatible error.
+// If you need to perform an authorization check, call get() first.
+func (x *database) del(call rpc.ServerCall, st store.Store) error {
+ if err := st.Delete(x.key()); err != nil {
+ return verror.New(verror.ErrInternal, call.Context(), err)
+ }
+ return nil
+}
+
+// Updates data in the storage engine.
+// Returns a VDL-compatible error.
+// checkAuth specifies whether to perform an authorization check.
+// fn should perform the "modify" part of "read, modify, write", and should
+// return a VDL-compatible error.
+func (x *database) update(call rpc.ServerCall, st store.Store, checkAuth bool, fn func(data *databaseData) error) error {
+ data, err := x.get(call, st, checkAuth)
+ if err != nil {
+ return err
+ }
+ if err := fn(data); err != nil {
+ return err
+ }
+ return x.put(call, st, data)
+}
diff --git a/services/syncbase/server/dispatcher.go b/services/syncbase/server/dispatcher.go
new file mode 100644
index 0000000..5ea36d6
--- /dev/null
+++ b/services/syncbase/server/dispatcher.go
@@ -0,0 +1,78 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package server
+
+import (
+ "strings"
+
+ "v.io/syncbase/v23/services/syncbase"
+ "v.io/v23/rpc"
+ "v.io/v23/security"
+ "v.io/v23/verror"
+)
+
+type dispatcher struct {
+ s *service
+}
+
+var _ rpc.Dispatcher = (*dispatcher)(nil)
+
+func NewDispatcher(s *service) *dispatcher {
+ return &dispatcher{s: s}
+}
+
+// TODO(sadovsky): Return a real authorizer in various places below.
+func (d *dispatcher) Lookup(suffix string) (interface{}, security.Authorizer, error) {
+ suffix = strings.TrimPrefix(suffix, "/")
+ parts := strings.Split(suffix, "/")
+
+ // Validate all key atoms up front, so that we can avoid doing so in all our
+ // method implementations.
+ for _, s := range parts {
+ if !validKeyAtom(s) {
+ // TODO(sadovsky): Is it okay to pass a nil context to verror?
+ return nil, nil, syncbase.NewErrInvalidName(nil, suffix)
+ }
+ }
+
+ if len(parts) == 0 {
+ return syncbase.ServiceServer(d.s), nil, nil
+ }
+
+ universe := &universe{
+ name: parts[0],
+ s: d.s,
+ }
+ if len(parts) == 1 {
+ return syncbase.UniverseServer(universe), nil, nil
+ }
+
+ database := &database{
+ name: parts[1],
+ u: universe,
+ }
+ if len(parts) == 2 {
+ return syncbase.DatabaseServer(database), nil, nil
+ }
+
+ table := &table{
+ name: parts[2],
+ d: database,
+ }
+ if len(parts) == 3 {
+ return syncbase.TableServer(table), nil, nil
+ }
+
+ item := &item{
+ encodedKey: parts[3],
+ t: table,
+ }
+ if len(parts) == 4 {
+ return syncbase.ItemServer(item), nil, nil
+ }
+
+ // TODO(sadovsky): Is it okay to pass a nil context to verror?
+ return nil, nil, verror.NewErrNoExist(nil)
+}
diff --git a/services/syncbase/server/item.go b/services/syncbase/server/item.go
new file mode 100644
index 0000000..94f5df4
--- /dev/null
+++ b/services/syncbase/server/item.go
@@ -0,0 +1,115 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package server
+
+import (
+ "v.io/syncbase/v23/services/syncbase"
+ "v.io/syncbase/x/ref/services/syncbase/store"
+ "v.io/v23/rpc"
+ "v.io/v23/vdl"
+ "v.io/v23/verror"
+)
+
+// TODO(sadovsky): Extend data layout to support version tracking for sync.
+// See go/vanadium-local-structured-store.
+
+type item struct {
+ encodedKey string
+ t *table
+}
+
+var _ syncbase.ItemServerMethods = (*item)(nil)
+
+func (i *item) Get(call rpc.ServerCall) (*vdl.Value, error) {
+ var value *vdl.Value
+ if err := store.RunInTransaction(i.t.d.u.s.st, func(st store.Store) error {
+ var err error
+ value, err = i.get(call, st)
+ return err
+ }); err != nil {
+ return nil, err
+ }
+ return value, nil
+}
+
+func (i *item) Put(call rpc.ServerCall, value *vdl.Value) error {
+ return store.RunInTransaction(i.t.d.u.s.st, func(st store.Store) error {
+ return i.put(call, st, value)
+ })
+}
+
+func (i *item) Delete(call rpc.ServerCall) error {
+ return store.RunInTransaction(i.t.d.u.s.st, func(st store.Store) error {
+ return i.del(call, st)
+ })
+}
+
+////////////////////////////////////////
+// Internal helpers
+
+func (i *item) keyPart() string {
+ return joinKeyParts(i.t.keyPart(), i.encodedKey)
+}
+
+func (i *item) key() string {
+ return joinKeyParts("$item", i.keyPart())
+}
+
+// Performs authorization check (against the database Permissions) and checks that this
+// item's table exists in the database schema. Returns the schema and a
+// VDL-compatible error.
+func (i *item) checkAccess(call rpc.ServerCall, st store.Store) (syncbase.Schema, error) {
+ data, err := i.t.d.get(call, st, true)
+ if err != nil {
+ return nil, err
+ }
+ if _, ok := data.Schema[i.t.name]; !ok {
+ return nil, verror.NewErrNoExist(call.Context())
+ }
+ return data.Schema, nil
+}
+
+// Reads data from the storage engine.
+// Performs authorization check. Returns a VDL-compatible error.
+func (i *item) get(call rpc.ServerCall, st store.Store) (*vdl.Value, error) {
+ if _, err := i.checkAccess(call, st); err != nil {
+ return nil, err
+ }
+ value := &vdl.Value{}
+ if err := getObject(st, i.key(), value); err != nil {
+ if _, ok := err.(*store.ErrUnknownKey); ok {
+ // We've already done an auth check, so here we can safely return NoExist
+ // rather than NoExistOrNoAccess.
+ return nil, verror.NewErrNoExist(call.Context())
+ }
+ return nil, verror.New(verror.ErrInternal, call.Context(), err)
+ }
+ return value, nil
+}
+
+// Writes data to the storage engine.
+// Performs authorization check. Returns a VDL-compatible error.
+func (i *item) put(call rpc.ServerCall, st store.Store, value *vdl.Value) error {
+ // TODO(sadovsky): Check that value's primary key field matches i.encodedKey.
+ if _, err := i.checkAccess(call, st); err != nil {
+ return err
+ }
+ if err := putObject(st, i.key(), &value); err != nil {
+ return verror.New(verror.ErrInternal, call.Context(), err)
+ }
+ return nil
+}
+
+// Deletes data from the storage engine.
+// Performs authorization check. Returns a VDL-compatible error.
+func (i *item) del(call rpc.ServerCall, st store.Store) error {
+ if _, err := i.checkAccess(call, st); err != nil {
+ return err
+ }
+ if err := st.Delete(i.key()); err != nil {
+ return verror.New(verror.ErrInternal, call.Context(), err)
+ }
+ return nil
+}
diff --git a/services/syncbase/server/key_util.go b/services/syncbase/server/key_util.go
new file mode 100644
index 0000000..7506b38
--- /dev/null
+++ b/services/syncbase/server/key_util.go
@@ -0,0 +1,22 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package server
+
+import (
+ "regexp"
+ "strings"
+)
+
+// TODO(sadovsky): Consider loosening. Perhaps model after MySQL:
+// http://dev.mysql.com/doc/refman/5.7/en/identifiers.html
+var keyAtomRegexp *regexp.Regexp = regexp.MustCompile("^[a-zA-Z0-9_.-]+$")
+
+func validKeyAtom(s string) bool {
+ return keyAtomRegexp.MatchString(s)
+}
+
+func joinKeyParts(parts ...string) string {
+ return strings.Join(parts, ":")
+}
diff --git a/services/syncbase/server/server_test.go b/services/syncbase/server/server_test.go
new file mode 100644
index 0000000..e12f396
--- /dev/null
+++ b/services/syncbase/server/server_test.go
@@ -0,0 +1,106 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package server_test
+
+// Note: core/veyron/services/security/groups/server/server_test.go has some
+// helpful code snippets to model after.
+
+import (
+ "testing"
+
+ "v.io/v23"
+ "v.io/v23/context"
+ "v.io/v23/naming"
+ "v.io/v23/security"
+ "v.io/v23/security/access"
+ "v.io/x/lib/vlog"
+
+ "v.io/syncbase/x/ref/services/syncbase/server"
+ "v.io/syncbase/x/ref/services/syncbase/store/memstore"
+ _ "v.io/x/ref/profiles"
+ tsecurity "v.io/x/ref/test/testutil"
+)
+
+func defaultPermissions() access.Permissions {
+ acl := access.Permissions{}
+ for _, tag := range access.AllTypicalTags() {
+ acl.Add(security.BlessingPattern("server/client"), string(tag))
+ }
+ return acl
+}
+
+func newServer(ctx *context.T, acl access.Permissions) (string, func()) {
+ s, err := v23.NewServer(ctx)
+ if err != nil {
+ vlog.Fatal("v23.NewServer() failed: ", err)
+ }
+ eps, err := s.Listen(v23.GetListenSpec(ctx))
+ if err != nil {
+ vlog.Fatal("s.Listen() failed: ", err)
+ }
+
+ service := server.NewService(memstore.New())
+ if acl == nil {
+ acl = defaultPermissions()
+ }
+ if err := service.Create(acl); err != nil {
+ vlog.Fatal("service.Create() failed: ", err)
+ }
+ d := server.NewDispatcher(service)
+
+ if err := s.ServeDispatcher("", d); err != nil {
+ vlog.Fatal("s.ServeDispatcher() failed: ", err)
+ }
+
+ name := naming.JoinAddressName(eps[0].String(), "")
+ return name, func() {
+ s.Stop()
+ }
+}
+
+func setupOrDie(acl access.Permissions) (clientCtx *context.T, serverName string, cleanup func()) {
+ ctx, shutdown := v23.Init()
+ cp, sp := tsecurity.NewPrincipal("client"), tsecurity.NewPrincipal("server")
+
+ // Have the server principal bless the client principal as "client".
+ blessings, err := sp.Bless(cp.PublicKey(), sp.BlessingStore().Default(), "client", security.UnconstrainedUse())
+ if err != nil {
+ vlog.Fatal("sp.Bless() failed: ", err)
+ }
+ // Have the client present its "client" blessing when talking to the server.
+ if _, err := cp.BlessingStore().Set(blessings, "server"); err != nil {
+ vlog.Fatal("cp.BlessingStore().Set() failed: ", err)
+ }
+ // Have the client treat the server's public key as an authority on all
+ // blessings that match the pattern "server".
+ if err := cp.AddToRoots(blessings); err != nil {
+ vlog.Fatal("cp.AddToRoots() failed: ", err)
+ }
+
+ clientCtx, err = v23.SetPrincipal(ctx, cp)
+ if err != nil {
+ vlog.Fatal("v23.SetPrincipal() failed: ", err)
+ }
+ serverCtx, err := v23.SetPrincipal(ctx, sp)
+ if err != nil {
+ vlog.Fatal("v23.SetPrincipal() failed: ", err)
+ }
+
+ serverName, stopServer := newServer(serverCtx, acl)
+ cleanup = func() {
+ stopServer()
+ shutdown()
+ }
+ return
+}
+
+////////////////////////////////////////
+// Test cases
+
+// TODO(sadovsky): Write some tests.
+func TestSomething(t *testing.T) {
+ _, _, cleanup := setupOrDie(nil)
+ defer cleanup()
+}
diff --git a/services/syncbase/server/service.go b/services/syncbase/server/service.go
new file mode 100644
index 0000000..ef30019
--- /dev/null
+++ b/services/syncbase/server/service.go
@@ -0,0 +1,144 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package server
+
+import (
+ "v.io/syncbase/v23/services/syncbase"
+ "v.io/syncbase/x/ref/services/syncbase/store"
+ "v.io/v23/rpc"
+ "v.io/v23/security/access"
+ "v.io/v23/verror"
+)
+
+type service struct {
+ st store.TransactableStore
+}
+
+var _ syncbase.ServiceServerMethods = (*service)(nil)
+
+func NewService(st store.TransactableStore) *service {
+ return &service{st: st}
+}
+
+// Returns a VDL-compatible error.
+// Note, this is not an RPC method.
+func (s *service) Create(acl access.Permissions) error {
+ if acl == nil {
+ return verror.New(verror.ErrInternal, nil, "acl must be specified")
+ }
+
+ return store.RunInTransaction(s.st, func(st store.Store) error {
+ // TODO(sadovsky): Maybe add "has" method to storage engine.
+ data := &serviceData{}
+ if err := getObject(st, s.key(), data); err == nil {
+ return verror.NewErrExist(nil)
+ }
+
+ data = &serviceData{
+ Permissions: acl,
+ }
+ if err := putObject(st, s.key(), data); err != nil {
+ return verror.New(verror.ErrInternal, nil, "put failed")
+ }
+
+ return nil
+ })
+}
+
+func (s *service) SetPermissions(call rpc.ServerCall, acl access.Permissions, version string) error {
+ return store.RunInTransaction(s.st, func(st store.Store) error {
+ return s.update(call, st, true, func(data *serviceData) error {
+ if err := checkVersion(call, version, data.Version); err != nil {
+ return err
+ }
+ data.Permissions = acl
+ data.Version++
+ return nil
+ })
+ })
+}
+
+func (s *service) GetPermissions(call rpc.ServerCall) (acl access.Permissions, version string, err error) {
+ data, err := s.get(call, s.st, true)
+ if err != nil {
+ return nil, "", err
+ }
+ return data.Permissions, formatVersion(data.Version), nil
+}
+
+// TODO(sadovsky): Implement Glob.
+
+////////////////////////////////////////
+// Internal helpers
+
+func (s *service) key() string {
+ return "$service"
+}
+
+// Note, the methods below use "x" as the receiver name to make find-replace
+// easier across the different levels of syncbase hierarchy.
+//
+// TODO(sadovsky): Is there any better way to share this code despite the lack
+// of generics?
+
+// Reads data from the storage engine.
+// Returns a VDL-compatible error.
+// checkAuth specifies whether to perform an authorization check.
+func (x *service) get(call rpc.ServerCall, st store.Store, checkAuth bool) (*serviceData, error) {
+ // TODO(kash): Get this to compile.
+ return nil, nil
+ // data := &serviceData{}
+ // if err := getObject(st, x.key(), data); err != nil {
+ // if _, ok := err.(*store.ErrUnknownKey); ok {
+ // // TODO(sadovsky): Return ErrNoExist if appropriate.
+ // return nil, verror.New(verror.ErrNoExistOrNoAccess, call.Context())
+ // }
+ // return nil, verror.New(verror.ErrInternal, call.Context(), err)
+ // }
+ // if checkAuth {
+ // auth, _ := access.PermissionsAuthorizer(data.Permissions, access.TypicalTagType())
+ // if err := auth.Authorize(call); err != nil {
+ // // TODO(sadovsky): Return ErrNoAccess if appropriate.
+ // return nil, verror.New(verror.ErrNoExistOrNoAccess, call.Context(), err)
+ // }
+ // }
+ // return data, nil
+}
+
+// Writes data to the storage engine.
+// Returns a VDL-compatible error.
+// If you need to perform an authorization check, use update().
+func (x *service) put(call rpc.ServerCall, st store.Store, data *serviceData) error {
+ if err := putObject(st, x.key(), data); err != nil {
+ return verror.New(verror.ErrInternal, call.Context(), err)
+ }
+ return nil
+}
+
+// Deletes data from the storage engine.
+// Returns a VDL-compatible error.
+// If you need to perform an authorization check, call get() first.
+func (x *service) del(call rpc.ServerCall, st store.Store) error {
+ if err := st.Delete(x.key()); err != nil {
+ return verror.New(verror.ErrInternal, call.Context(), err)
+ }
+ return nil
+}
+
+// Updates data in the storage engine.
+// Returns a VDL-compatible error.
+// checkAuth specifies whether to perform an authorization check.
+// fn should perform the "modify" part of "read, modify, write", and should
+// return a VDL-compatible error.
+func (x *service) update(call rpc.ServerCall, st store.Store, checkAuth bool, fn func(data *serviceData) error) error {
+ data, err := x.get(call, st, checkAuth)
+ if err != nil {
+ return err
+ }
+ if err := fn(data); err != nil {
+ return err
+ }
+ return x.put(call, st, data)
+}
diff --git a/services/syncbase/server/store_util.go b/services/syncbase/server/store_util.go
new file mode 100644
index 0000000..fb09ad0
--- /dev/null
+++ b/services/syncbase/server/store_util.go
@@ -0,0 +1,42 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package server
+
+import (
+ "strconv"
+
+ "v.io/v23/rpc"
+ "v.io/v23/verror"
+ "v.io/v23/vom"
+
+ "v.io/syncbase/x/ref/services/syncbase/store"
+)
+
+func getObject(st store.Store, k string, v interface{}) error {
+ bytes, err := st.Get(k)
+ if err != nil {
+ return err
+ }
+ return vom.Decode(bytes, v)
+}
+
+func putObject(st store.Store, k string, v interface{}) error {
+ bytes, err := vom.Encode(v)
+ if err != nil {
+ return err
+ }
+ return st.Put(k, bytes)
+}
+
+func formatVersion(version uint64) string {
+ return strconv.FormatUint(version, 10)
+}
+
+func checkVersion(call rpc.ServerCall, presented string, actual uint64) error {
+ if presented != "" && presented != formatVersion(actual) {
+ return verror.NewErrBadVersion(call.Context())
+ }
+ return nil
+}
diff --git a/services/syncbase/server/table.go b/services/syncbase/server/table.go
new file mode 100644
index 0000000..2c1af6a
--- /dev/null
+++ b/services/syncbase/server/table.go
@@ -0,0 +1,23 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package server
+
+import "v.io/syncbase/v23/services/syncbase"
+
+type table struct {
+ name string
+ d *database
+}
+
+var _ syncbase.TableServerMethods = (*table)(nil)
+
+// TODO(sadovsky): Implement Glob.
+
+////////////////////////////////////////
+// Internal helpers
+
+func (t *table) keyPart() string {
+ return joinKeyParts(t.d.keyPart(), t.name)
+}
diff --git a/services/syncbase/server/types.vdl b/services/syncbase/server/types.vdl
new file mode 100644
index 0000000..8e7a751
--- /dev/null
+++ b/services/syncbase/server/types.vdl
@@ -0,0 +1,33 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package server
+
+import (
+ "v.io/syncbase/v23/services/syncbase"
+ "v.io/v23/security/access"
+)
+
+// serviceData represents the persistent state of a Service.
+type serviceData struct {
+ Universes set[string]
+ Version uint64 // covers the fields below
+ Permissions access.Permissions
+}
+
+// universeData represents the persistent state of a Universe.
+type universeData struct {
+ Name string
+ Databases set[string]
+ Version uint64 // covers the fields below
+ Permissions access.Permissions
+}
+
+// databaseData represents the persistent state of a Database.
+type databaseData struct {
+ Name string
+ Version uint64 // covers the fields below
+ Permissions access.Permissions
+ Schema syncbase.Schema
+}
diff --git a/services/syncbase/server/types.vdl.go b/services/syncbase/server/types.vdl.go
new file mode 100644
index 0000000..d029dd1
--- /dev/null
+++ b/services/syncbase/server/types.vdl.go
@@ -0,0 +1,61 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file was auto-generated by the vanadium vdl tool.
+// Source: types.vdl
+
+package server
+
+import (
+ // VDL system imports
+ "v.io/v23/vdl"
+
+ // VDL user imports
+ "v.io/syncbase/v23/services/syncbase"
+ "v.io/v23/security/access"
+)
+
+// serviceData represents the persistent state of a Service.
+type serviceData struct {
+ Universes map[string]struct{}
+ Version uint64 // covers the fields below
+ Permissions access.Permissions
+}
+
+func (serviceData) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/server.serviceData"
+}) {
+}
+
+// universeData represents the persistent state of a Universe.
+type universeData struct {
+ Name string
+ Databases map[string]struct{}
+ Version uint64 // covers the fields below
+ Permissions access.Permissions
+}
+
+func (universeData) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/server.universeData"
+}) {
+}
+
+// databaseData represents the persistent state of a Database.
+type databaseData struct {
+ Name string
+ Version uint64 // covers the fields below
+ Permissions access.Permissions
+ Schema syncbase.Schema
+}
+
+func (databaseData) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/server.databaseData"
+}) {
+}
+
+func init() {
+ vdl.Register((*serviceData)(nil))
+ vdl.Register((*universeData)(nil))
+ vdl.Register((*databaseData)(nil))
+}
diff --git a/services/syncbase/server/universe.go b/services/syncbase/server/universe.go
new file mode 100644
index 0000000..8f0f9b7
--- /dev/null
+++ b/services/syncbase/server/universe.go
@@ -0,0 +1,170 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package server
+
+import (
+ "v.io/syncbase/v23/services/syncbase"
+ "v.io/syncbase/x/ref/services/syncbase/store"
+ "v.io/v23/rpc"
+ "v.io/v23/security/access"
+ "v.io/v23/verror"
+)
+
+type universe struct {
+ name string
+ s *service
+}
+
+var _ syncbase.UniverseServerMethods = (*universe)(nil)
+
+func (u *universe) Create(call rpc.ServerCall, acl access.Permissions) error {
+ return store.RunInTransaction(u.s.st, func(st store.Store) error {
+ // Update serviceData.
+ var sData *serviceData
+ if err := u.s.update(call, st, true, func(data *serviceData) error {
+ if _, ok := data.Universes[u.name]; ok {
+ return verror.New(verror.ErrExist, call.Context(), u.name)
+ }
+ // https://github.com/veyron/release-issues/issues/1145
+ if data.Universes == nil {
+ data.Universes = map[string]struct{}{}
+ }
+ data.Universes[u.name] = struct{}{}
+ sData = data
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ // Blind-write universeData.
+ if acl == nil {
+ acl = sData.Permissions
+ }
+ data := &universeData{
+ Name: u.name,
+ Permissions: acl,
+ }
+ return u.put(call, st, data)
+ })
+}
+
+func (u *universe) Delete(call rpc.ServerCall) error {
+ return store.RunInTransaction(u.s.st, func(st store.Store) error {
+ // Read-check-delete universeData.
+ if _, err := u.get(call, st, true); err != nil {
+ return err
+ }
+ // TODO(sadovsky): Delete all Databases in this Universe.
+ if err := u.del(call, st); err != nil {
+ return err
+ }
+
+ // Update serviceData.
+ return u.s.update(call, st, false, func(data *serviceData) error {
+ delete(data.Universes, u.name)
+ return nil
+ })
+ })
+}
+
+func (u *universe) SetPermissions(call rpc.ServerCall, acl access.Permissions, version string) error {
+ return store.RunInTransaction(u.s.st, func(st store.Store) error {
+ return u.update(call, st, true, func(data *universeData) error {
+ if err := checkVersion(call, version, data.Version); err != nil {
+ return err
+ }
+ data.Permissions = acl
+ data.Version++
+ return nil
+ })
+ })
+}
+
+func (u *universe) GetPermissions(call rpc.ServerCall) (acl access.Permissions, version string, err error) {
+ data, err := u.get(call, u.s.st, true)
+ if err != nil {
+ return nil, "", err
+ }
+ return data.Permissions, formatVersion(data.Version), nil
+}
+
+// TODO(sadovsky): Implement Glob.
+
+////////////////////////////////////////
+// Internal helpers
+
+func (u *universe) keyPart() string {
+ return u.name
+}
+
+func (u *universe) key() string {
+ return joinKeyParts("$universe", u.keyPart())
+}
+
+// Note, the methods below use "x" as the receiver name to make find-replace
+// easier across the different levels of syncbase hierarchy.
+//
+// TODO(sadovsky): Is there any better way to share this code despite the lack
+// of generics?
+
+// Reads data from the storage engine.
+// Returns a VDL-compatible error.
+// checkAuth specifies whether to perform an authorization check.
+func (x *universe) get(call rpc.ServerCall, st store.Store, checkAuth bool) (*universeData, error) {
+ // TODO(kash): Get this to compile.
+ return nil, nil
+ // data := &universeData{}
+ // if err := getObject(st, x.key(), data); err != nil {
+ // if _, ok := err.(*store.ErrUnknownKey); ok {
+ // // TODO(sadovsky): Return ErrNoExist if appropriate.
+ // return nil, verror.NewErrNoExistOrNoAccess(call.Context())
+ // }
+ // return nil, verror.New(verror.ErrInternal, call.Context(), err)
+ // }
+ // if checkAuth {
+ // auth, _ := access.PermissionsAuthorizer(data.Permissions, access.TypicalTagType())
+ // if err := auth.Authorize(call); err != nil {
+ // // TODO(sadovsky): Return ErrNoAccess if appropriate.
+ // return nil, verror.NewErrNoExistOrNoAccess(call.Context())
+ // }
+ // }
+ // return data, nil
+}
+
+// Writes data to the storage engine.
+// Returns a VDL-compatible error.
+// If you need to perform an authorization check, use update().
+func (x *universe) put(call rpc.ServerCall, st store.Store, data *universeData) error {
+ if err := putObject(st, x.key(), data); err != nil {
+ return verror.New(verror.ErrInternal, call.Context(), err)
+ }
+ return nil
+}
+
+// Deletes data from the storage engine.
+// Returns a VDL-compatible error.
+// If you need to perform an authorization check, call get() first.
+func (x *universe) del(call rpc.ServerCall, st store.Store) error {
+ if err := st.Delete(x.key()); err != nil {
+ return verror.New(verror.ErrInternal, call.Context(), err)
+ }
+ return nil
+}
+
+// Updates data in the storage engine.
+// Returns a VDL-compatible error.
+// checkAuth specifies whether to perform an authorization check.
+// fn should perform the "modify" part of "read, modify, write", and should
+// return a VDL-compatible error.
+func (x *universe) update(call rpc.ServerCall, st store.Store, checkAuth bool, fn func(data *universeData) error) error {
+ data, err := x.get(call, st, checkAuth)
+ if err != nil {
+ return err
+ }
+ if err := fn(data); err != nil {
+ return err
+ }
+ return x.put(call, st, data)
+}
diff --git a/services/syncbase/store/memstore/memstore.go b/services/syncbase/store/memstore/memstore.go
new file mode 100644
index 0000000..f9de50a
--- /dev/null
+++ b/services/syncbase/store/memstore/memstore.go
@@ -0,0 +1,179 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package memstore provides a simple, in-memory implementation of
+// store.TransactableStore. Since it's a prototype implementation, it makes no
+// attempt to be performant.
+package memstore
+
+import (
+ "errors"
+ "sync"
+ "time"
+
+ "v.io/syncbase/x/ref/services/syncbase/store"
+)
+
+var (
+ txnTimeout = time.Duration(5) * time.Second
+ errExpiredTxn = errors.New("expired transaction")
+ errCommittedTxn = errors.New("committed transaction")
+ errAbortedTxn = errors.New("aborted transaction")
+ errConcurrentTxn = errors.New("concurrent transaction")
+)
+
+type transaction struct {
+ st *memstore
+ // The following fields are used to determine whether method calls should
+ // error out.
+ err error
+ seq uint64
+ createdTime time.Time
+ // The following fields track writes performed against this transaction.
+ puts map[string][]byte
+ deletes map[string]struct{}
+}
+
+var _ store.Transaction = (*transaction)(nil)
+
+type memstore struct {
+ mu sync.Mutex
+ data map[string][]byte
+ // Most recent sequence number handed out.
+ lastSeq uint64
+ // Value of lastSeq at the time of the most recent commit.
+ lastCommitSeq uint64
+}
+
+var _ store.TransactableStore = (*memstore)(nil)
+
+func New() store.TransactableStore {
+ return &memstore{data: map[string][]byte{}}
+}
+
+////////////////////////////////////////
+// transaction methods
+
+func newTxn(st *memstore, seq uint64) *transaction {
+ return &transaction{
+ st: st,
+ seq: seq,
+ createdTime: time.Now(),
+ puts: map[string][]byte{},
+ deletes: map[string]struct{}{},
+ }
+}
+
+func (tx *transaction) expired() bool {
+ return time.Now().After(tx.createdTime.Add(txnTimeout))
+}
+
+func (tx *transaction) checkError() error {
+ if tx.err != nil {
+ return tx.err
+ }
+ if tx.expired() {
+ return errExpiredTxn
+ }
+ if tx.seq <= tx.st.lastCommitSeq {
+ return errConcurrentTxn
+ }
+ return nil
+}
+
+func (tx *transaction) Get(k string) ([]byte, error) {
+ tx.st.mu.Lock()
+ defer tx.st.mu.Unlock()
+ if err := tx.checkError(); err != nil {
+ return nil, err
+ }
+ v, ok := tx.st.data[k]
+ if !ok {
+ return nil, &store.ErrUnknownKey{Key: k}
+ }
+ return v, nil
+}
+
+func (tx *transaction) Put(k string, v []byte) error {
+ tx.st.mu.Lock()
+ defer tx.st.mu.Unlock()
+ if err := tx.checkError(); err != nil {
+ return err
+ }
+ delete(tx.deletes, k)
+ tx.puts[k] = v
+ return nil
+}
+
+func (tx *transaction) Delete(k string) error {
+ tx.st.mu.Lock()
+ defer tx.st.mu.Unlock()
+ if err := tx.checkError(); err != nil {
+ return err
+ }
+ delete(tx.puts, k)
+ tx.deletes[k] = struct{}{}
+ return nil
+}
+
+func (tx *transaction) Commit() error {
+ tx.st.mu.Lock()
+ defer tx.st.mu.Unlock()
+ if err := tx.checkError(); err != nil {
+ return err
+ }
+ tx.err = errCommittedTxn
+ for k, v := range tx.puts {
+ tx.st.data[k] = v
+ }
+ for k := range tx.deletes {
+ delete(tx.st.data, k)
+ }
+ tx.st.lastCommitSeq = tx.st.lastSeq
+ return nil
+}
+
+func (tx *transaction) Abort() error {
+ tx.st.mu.Lock()
+ defer tx.st.mu.Unlock()
+ if err := tx.checkError(); err != nil {
+ return err
+ }
+ tx.err = errAbortedTxn
+ return nil
+}
+
+////////////////////////////////////////
+// memstore methods
+
+func (st *memstore) Get(k string) ([]byte, error) {
+ var v []byte
+ if err := store.RunInTransaction(st, func(st store.Store) error {
+ var err error
+ v, err = st.Get(k)
+ return err
+ }); err != nil {
+ return nil, err
+ }
+ return v, nil
+}
+
+func (st *memstore) Put(k string, v []byte) error {
+ return store.RunInTransaction(st, func(st store.Store) error {
+ return st.Put(k, v)
+ })
+}
+
+func (st *memstore) Delete(k string) error {
+ return store.RunInTransaction(st, func(st store.Store) error {
+ return st.Delete(k)
+ })
+}
+
+func (st *memstore) CreateTransaction() store.Transaction {
+ st.mu.Lock()
+ defer st.mu.Unlock()
+ st.lastSeq++
+ return newTxn(st, st.lastSeq)
+}
diff --git a/services/syncbase/store/model.go b/services/syncbase/store/model.go
new file mode 100644
index 0000000..ce18640
--- /dev/null
+++ b/services/syncbase/store/model.go
@@ -0,0 +1,81 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package store defines the API for the syncbase storage engine.
+// Currently, this API and its implementations are meant to be internal.
+package store
+
+// Store is a CRUD-capable storage engine.
+// TODO(sadovsky): For convenience, we use string (rather than []byte) as the
+// key type. Maybe revisit this.
+// TODO(sadovsky): Maybe separate Insert/Update/Upsert for implementation
+// convenience. Or, provide these as helpers on top of the API below.
+type Store interface {
+ // Get returns the value for the given key.
+ // Fails if the given key is unknown (ErrUnknownKey).
+ Get(k string) ([]byte, error)
+
+ // Put writes the given value for the given key.
+ Put(k string, v []byte) error
+
+ // Delete deletes the entry for the given key.
+ // Succeeds (no-op) if the given key is unknown.
+ Delete(k string) error
+}
+
+// TransactableStore is a Store that supports transactions.
+// It should be possible to implement using LevelDB, SQLite, or similar.
+type TransactableStore interface {
+ Store
+
+ // CreateTransaction creates a transaction.
+ CreateTransaction() Transaction
+
+ // TODO(sadovsky): Figure out how sync's "PutMutations" fits in here. In
+ // theory it avoids a read (since it knows the version, i.e. the etag, up
+ // front), but OTOH that version still must be read at commit time (for
+ // verification), so reading it in a transaction (and thus adding it to the
+ // transaction's read-set) incurs no extra cost if we use transaction
+ // implementation strategy #1 from the doc. That said, strategy #1 is not
+ // tenable in a cloud environment. Perhaps Transaction should provide some
+ // mechanism for directly adding to its read-set (without actually reading)?
+}
+
+// Transaction provides a mechanism for atomic reads and writes.
+// Reads don't reflect writes performed inside this transaction. (This
+// limitation is imposed for API parity with Spanner.)
+// Once a transaction has been committed or aborted, subsequent method calls
+// will fail with no effect.
+type Transaction interface {
+ Store
+
+ // Commit commits the transaction.
+ Commit() error
+
+ // Abort aborts the transaction.
+ Abort() error
+}
+
+// TODO(sadovsky): Maybe put this elsewhere.
+// TODO(sadovsky): Add retry loop.
+func RunInTransaction(st TransactableStore, fn func(st Store) error) error {
+ tx := st.CreateTransaction()
+ if err := fn(tx); err != nil {
+ tx.Abort()
+ return err
+ }
+ if err := tx.Commit(); err != nil {
+ tx.Abort()
+ return err
+ }
+ return nil
+}
+
+type ErrUnknownKey struct {
+ Key string
+}
+
+func (err *ErrUnknownKey) Error() string {
+ return "unknown key: " + err.Key
+}
diff --git a/services/syncbase/sync/dag.go b/services/syncbase/sync/dag.go
new file mode 100644
index 0000000..69f467b
--- /dev/null
+++ b/services/syncbase/sync/dag.go
@@ -0,0 +1,1183 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Veyron Sync DAG (directed acyclic graph) utility functions.
+// The DAG is used to track the version history of objects in order to
+// detect and resolve conflicts (concurrent changes on different devices).
+//
+// Terminology:
+// * An object is a unique value in the Veyron Store represented by its UID.
+// * As an object mutates, its version number is updated by the Store.
+// * Each (object, version) tuple is represented by a node in the Sync DAG.
+// * The previous version of an object is its parent in the DAG, i.e. the
+// new version is derived from that parent.
+// * When there are no conflicts, the node has a single reference back to
+// a parent node.
+// * When a conflict between two concurrent object versions is resolved,
+// the new version has references back to each of the two parents to
+// indicate that it is derived from both nodes.
+// * During a sync operation from a source device to a target device, the
+// target receives a DAG fragment from the source. That fragment has to
+// be incorporated (grafted) into the target device's DAG. It may be a
+// continuation of the DAG of an object, with the attachment (graft) point
+// being the current head of DAG, in which case there are no conflicts.
+// Or the graft point(s) may be older nodes, which means the new fragment
+// is a divergence in the graph causing a conflict that must be resolved
+// in order to re-converge the two DAG fragments.
+//
+// In the diagrams below:
+// (h) represents the head node in the local device.
+// (nh) represents the new head node received from the remote device.
+// (g) represents a graft node, where new nodes attach to the existing DAG.
+// <- represents a derived-from mutation, i.e. a child-to-parent pointer
+//
+// a- No-conflict example: the new nodes (v3, v4) attach to the head node (v2).
+// In this case the new head becomes the head node, the new DAG fragment
+// being a continuation of the existing DAG.
+//
+// Before:
+// v0 <- v1 <- v2(h)
+//
+// Sync updates applied, no conflict detected:
+// v0 <- v1 <- v2(h,g) <- v3 <- v4 (nh)
+//
+// After:
+// v0 <- v1 <- v2 <- v3 <- v4 (h)
+//
+// b- Conflict example: the new nodes (v3, v4) attach to an old node (v1).
+// The current head node (v2) and the new head node (v4) are divergent
+// (concurrent) mutations that need to be resolved. The conflict
+// resolution function is passed the old head (v2), new head (v4), and
+// the common ancestor (v1) and resolves the conflict with (v5) which
+// is represented in the DAG as derived from both v2 and v4 (2 parents).
+//
+// Before:
+// v0 <- v1 <- v2(h)
+//
+// Sync updates applied, conflict detected (v2 not a graft node):
+// v0 <- v1(g) <- v2(h)
+// <- v3 <- v4 (nh)
+//
+// After, conflict resolver creates v5 having 2 parents (v2, v4):
+// v0 <- v1(g) <- v2 <------- v5(h)
+// <- v3 <- v4 <-
+//
+// Note: the DAG does not grow indefinitely. During a sync operation each
+// device learns what the other device already knows -- where it's at in
+// the version history for the objects. When a device determines that all
+// devices that sync an object (as per the definitions of replication groups
+// in the Veyron Store) have moved past some version for that object, the
+// DAG for that object can be pruned, deleting all prior (ancestor) nodes.
+//
+// The DAG DB contains four tables persisted to disk (nodes, heads, trans,
+// priv) and three in-memory (ephemeral) maps (graft, txSet, txGC):
+// * nodes: one entry per (object, version) with references to the
+// parent node(s) it is derived from, a reference to the
+// log record identifying that change, a reference to its
+// transaction set (or NoTxId if none), and a boolean to
+// indicate whether this change was a deletion of the object.
+// * heads: one entry per object pointing to its most recent version
+// in the nodes table
+// * trans: one entry per transaction ID containing the set of objects
+// that forms the transaction and their versions.
+// * priv: one entry per object ID for objects that are private to the
+// store, not shared through any SyncGroup.
+// * graft: during a sync operation, it tracks the nodes where the new
+// DAG fragments are attached to the existing graph for each
+// mutated object. This map is used to determine whether a
+// conflict happened for an object and, if yes, select the most
+// recent common ancestor from these graft points to use when
+// resolving the conflict. At the end of a sync operation the
+// graft map is destroyed.
+// * txSet: used to incrementally construct the transaction sets that
+// are stored in the "trans" table once all the nodes of a
+// transaction have been added. Multiple transaction sets
+// can be constructed to support the concurrency between the
+// Sync Initiator and Watcher threads.
+// * txGC: used to track the transactions impacted by objects being
+// pruned. At the end of the pruning operation the records
+// of the "trans" table are updated from the txGC information.
+//
+// Note: for regular (no-conflict) changes, a node has a reference to
+// one parent from which it was derived. When a conflict is resolved,
+// the new node has references to the two concurrent parents that triggered
+// the conflict. The states of the parents[] array are:
+// * [] The earliest/first version of an object
+// * [XYZ] Regular non-conflict version derived from XYZ
+// * [XYZ, ABC] Resolution version caused by XYZ-vs-ABC conflict
+
+import (
+ "container/list"
+ "errors"
+ "fmt"
+ "math/rand"
+ "strconv"
+ "time"
+
+ "v.io/x/lib/vlog"
+ "v.io/x/ref/lib/stats"
+)
+
+const (
+ NoTxId = TxId(0)
+)
+
+var (
+ errBadDAG = errors.New("invalid DAG")
+)
+
+type dagTxMap map[ObjId]Version
+
+// dagTxState tracks the state of a transaction.
+type dagTxState struct {
+ TxMap dagTxMap
+ TxCount uint32
+}
+
+type dag struct {
+ fname string // file pathname
+ store *kvdb // underlying K/V store
+ heads *kvtable // pointer to "heads" table in the store
+ nodes *kvtable // pointer to "nodes" table in the store
+ trans *kvtable // pointer to "trans" table in the store
+ priv *kvtable // pointer to "priv" table in the store
+ graft map[ObjId]*graftInfo // in-memory state of DAG object grafting
+ txSet map[TxId]*dagTxState // in-memory construction of transaction sets
+ txGC map[TxId]dagTxMap // in-memory tracking of transaction sets to cleanup
+ txGen *rand.Rand // transaction ID random number generator
+
+ // DAG stats
+ numObj *stats.Integer // number of objects
+ numNode *stats.Integer // number of versions across all objects
+ numTx *stats.Integer // number of transactions tracked
+ numPriv *stats.Integer // number of private objects
+}
+
+type dagNode struct {
+ Level uint64 // node distance from root
+ Parents []Version // references to parent versions
+ Logrec string // reference to log record change
+ TxId TxId // ID of a transaction set
+ Deleted bool // true if the change was a delete
+}
+
+type graftInfo struct {
+ newNodes map[Version]struct{} // set of newly added nodes during a sync
+ graftNodes map[Version]uint64 // set of graft nodes and their level
+ newHeads map[Version]struct{} // set of candidate new head nodes
+}
+
+type privNode struct {
+ //Mutation *raw.Mutation // most recent store mutation for a private (unshared) object
+ PathIDs []ObjId // store IDs in the path from the object to the root of the store
+ SyncTime int64 // SyncTime is the timestamp of the mutation when it arrives at the Sync server.
+ TxId TxId // ID of the transaction in which this mutation was done
+ TxCount uint32 // total number of object mutations in that transaction
+}
+
+// openDAG opens or creates a DAG for the given filename.
+func openDAG(filename string) (*dag, error) {
+ // Open the file and create it if it does not exist.
+ // Also initialize the store and its tables.
+ db, tbls, err := kvdbOpen(filename, []string{"heads", "nodes", "trans", "priv"})
+ if err != nil {
+ return nil, err
+ }
+
+ d := &dag{
+ fname: filename,
+ store: db,
+ heads: tbls[0],
+ nodes: tbls[1],
+ trans: tbls[2],
+ priv: tbls[3],
+ txGen: rand.New(rand.NewSource(time.Now().UTC().UnixNano())),
+ txSet: make(map[TxId]*dagTxState),
+ numObj: stats.NewInteger(statsNumDagObj),
+ numNode: stats.NewInteger(statsNumDagNode),
+ numTx: stats.NewInteger(statsNumDagTx),
+ numPriv: stats.NewInteger(statsNumDagPrivNode),
+ }
+
+ // Initialize the stats counters from the tables.
+ d.numObj.Set(int64(d.heads.getNumKeys()))
+ d.numNode.Set(int64(d.nodes.getNumKeys()))
+ d.numTx.Set(int64(d.trans.getNumKeys()))
+ d.numPriv.Set(int64(d.priv.getNumKeys()))
+
+ d.clearGraft()
+ d.clearTxGC()
+
+ return d, nil
+}
+
+// close closes the DAG and invalidates its structure.
+func (d *dag) close() {
+ if d.store != nil {
+ d.store.close() // this also closes the tables
+ stats.Delete(statsNumDagObj)
+ stats.Delete(statsNumDagNode)
+ stats.Delete(statsNumDagTx)
+ stats.Delete(statsNumDagPrivNode)
+ }
+ *d = dag{} // zero out the DAG struct
+}
+
+// flush flushes the DAG store to disk.
+func (d *dag) flush() {
+ if d.store != nil {
+ d.store.flush()
+ }
+}
+
+// clearGraft clears the temporary in-memory grafting maps.
+func (d *dag) clearGraft() {
+ if d.store != nil {
+ d.graft = make(map[ObjId]*graftInfo)
+ }
+}
+
+// clearTxGC clears the temporary in-memory transaction garbage collection maps.
+func (d *dag) clearTxGC() {
+ if d.store != nil {
+ d.txGC = make(map[TxId]dagTxMap)
+ }
+}
+
+// getObjectGraft returns the graft structure for an object ID.
+// The graftInfo struct for an object is ephemeral (in-memory) and it
+// tracks the following information:
+// - newNodes: the set of newly added nodes used to detect the type of
+// edges between nodes (new-node to old-node or vice versa).
+// - newHeads: the set of new candidate head nodes used to detect conflicts.
+// - graftNodes: the set of nodes used to find common ancestors between
+// conflicting nodes.
+//
+// After the received Sync logs are applied, if there are two new heads in
+// the newHeads set, there is a conflict to be resolved for this object.
+// Otherwise if there is only one head, no conflict was triggered and the
+// new head becomes the current version for the object.
+//
+// In case of conflict, the graftNodes set is used to select the common
+// ancestor to pass to the conflict resolver.
+//
+// Note: if an object's graft structure does not exist only create it
+// if the "create" parameter is set to true.
+func (d *dag) getObjectGraft(oid ObjId, create bool) *graftInfo {
+ graft := d.graft[oid]
+ if graft == nil && create {
+ graft = &graftInfo{
+ newNodes: make(map[Version]struct{}),
+ graftNodes: make(map[Version]uint64),
+ newHeads: make(map[Version]struct{}),
+ }
+
+ // If a current head node exists for this object, initialize
+ // the set of candidate new heads to include it.
+ head, err := d.getHead(oid)
+ if err == nil {
+ graft.newHeads[head] = struct{}{}
+ }
+
+ d.graft[oid] = graft
+ }
+ return graft
+}
+
+// addNodeTxStart generates a transaction ID and returns it to the
+// caller if a TxId is not specified. This transaction ID is stored
+// as part of each log record. If a TxId is specified by the caller,
+// state corresponding to that TxId is instantiated. TxId is used to
+// track DAG nodes that are part of the same transaction.
+func (d *dag) addNodeTxStart(tid TxId) TxId {
+ if d.store == nil {
+ return NoTxId
+ }
+
+ // Check if "tid" already exists.
+ if tid != NoTxId {
+ if _, ok := d.txSet[tid]; ok {
+ return tid
+ }
+ txSt, err := d.getTransaction(tid)
+ if err == nil {
+ d.txSet[tid] = txSt
+ return tid
+ }
+ } else {
+ // Generate a random 64-bit transaction ID different than NoTxId.
+ // Also make sure the ID is not already being used.
+ for (tid == NoTxId) || (d.txSet[tid] != nil) {
+ // Generate an unsigned 64-bit random value by combining a
+ // random 63-bit value and a random 1-bit value.
+ tid = (TxId(d.txGen.Int63()) << 1) | TxId(d.txGen.Int63n(2))
+ }
+ }
+
+ // Initialize the in-memory object/version map for that transaction ID.
+ d.txSet[tid] = &dagTxState{TxMap: make(dagTxMap), TxCount: 0}
+ return tid
+}
+
+// addNodeTxEnd marks the end of a given transaction.
+// The DAG completes its internal tracking of the transaction information.
+func (d *dag) addNodeTxEnd(tid TxId, count uint32) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+ if tid == NoTxId || count == 0 {
+ return fmt.Errorf("invalid TxState: %v %v", tid, count)
+ }
+
+ txSt, ok := d.txSet[tid]
+ if !ok {
+ return fmt.Errorf("unknown transaction ID: %v", tid)
+ }
+
+ // The first time a transaction (TxId) is ended, TxCount is
+ // zero while "count" is not. Subsequently if this TxId is
+ // started and ended, TxCount should be the same as the
+ // incoming "count".
+ if txSt.TxCount != 0 && txSt.TxCount != count {
+ return fmt.Errorf("incorrect counts for transaction: %v (%v %v)", tid, txSt.TxCount, count)
+ }
+
+ // Only save non-empty transactions, i.e. those that have at least
+ // one mutation on a shared (non-private) object.
+ if len(txSt.TxMap) > 0 {
+ txSt.TxCount = count
+ if err := d.setTransaction(tid, txSt); err != nil {
+ return err
+ }
+ }
+
+ delete(d.txSet, tid)
+ return nil
+}
+
+// addNode adds a new node for an object in the DAG, linking it to its parent nodes.
+// It verifies that this node does not exist and that its parent nodes are valid.
+// It also determines the DAG level of the node from its parent nodes (max() + 1).
+//
+// If the node is due to a local change (from the Watcher API), no need to
+// update the grafting structure. Otherwise the node is due to a remote change
+// (from the Sync protocol) being grafted on the DAG:
+// - If a parent node is not new, mark it as a DAG graft point.
+// - Mark this version as a new node.
+// - Update the new head node pointer of the grafted DAG.
+//
+// If the transaction ID is set to NoTxId, this node is not part of a transaction.
+// Otherwise, track its membership in the given transaction ID.
+func (d *dag) addNode(oid ObjId, version Version, remote, deleted bool,
+ parents []Version, logrec string, tid TxId) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+
+ if parents != nil {
+ if len(parents) > 2 {
+ return fmt.Errorf("cannot have more than 2 parents, not %d", len(parents))
+ }
+ if len(parents) == 0 {
+ // Replace an empty array with a nil.
+ parents = nil
+ }
+ }
+
+ // The new node must not exist.
+ if d.hasNode(oid, version) {
+ return fmt.Errorf("node %d:%d already exists in the DAG", oid, version)
+ }
+
+ // A new root node (no parents) is allowed only for new objects.
+ if parents == nil {
+ _, err := d.getHead(oid)
+ if err == nil {
+ return fmt.Errorf("cannot add another root node %d:%d for this object in the DAG", oid, version)
+ }
+ }
+
+ // For a remote change, make sure the object has a graft info entry.
+ // During a sync operation, each mutated object gets new nodes added
+ // in its DAG. These new nodes are either derived from nodes that
+ // were previously known on this device (i.e. their parent nodes are
+ // pre-existing), or they are derived from other new DAG nodes being
+ // discovered during this sync (i.e. their parent nodes were also
+ // just added to the DAG).
+ //
+ // To detect a conflict and find the most recent common ancestor to
+ // pass to the conflict resolver callback, the DAG keeps track of the
+ // new nodes that have old parent nodes. These old-to-new edges are
+ // the points where new DAG fragments are attached (grafted) onto the
+ // existing DAG. The old nodes are the "graft nodes" and they form
+ // the set of possible common ancestors to use in case of conflict:
+ // 1- A conflict happens when the current "head node" for an object
+ // is not in the set of graft nodes. It means the object mutations
+ // were not derived from what the device knows, but where divergent
+ // changes from a prior point (from one of the graft nodes).
+ // 2- The most recent common ancestor to use in resolving the conflict
+ // is the object graft node with the deepest level (furthest from
+ // the origin root node), representing the most up-to-date common
+ // knowledge between this device and the divergent changes.
+ //
+ // Note: at the end of a sync operation between 2 devices, the whole
+ // graft info is cleared (Syncd calls clearGraft()) to prepare it for
+ // the new pairwise sync operation.
+ graft := d.getObjectGraft(oid, remote)
+
+ // Verify the parents and determine the node level.
+ // Update the graft info in the DAG for this object.
+ var level uint64
+ for _, parent := range parents {
+ node, err := d.getNode(oid, parent)
+ if err != nil {
+ return err
+ }
+ if level <= node.Level {
+ level = node.Level + 1
+ }
+ if remote {
+ // If this parent is an old node, it's a graft point in the DAG
+ // and may be a common ancestor used during conflict resolution.
+ if _, ok := graft.newNodes[parent]; !ok {
+ graft.graftNodes[parent] = node.Level
+ }
+
+ // The parent nodes can no longer be candidates for new head versions.
+ if _, ok := graft.newHeads[parent]; ok {
+ delete(graft.newHeads, parent)
+ }
+ }
+ }
+
+ if remote {
+ // This new node is a candidate for new head version.
+ graft.newNodes[version] = struct{}{}
+ graft.newHeads[version] = struct{}{}
+ }
+
+ // If this node is part of a transaction, add it to that set.
+ if tid != NoTxId {
+ txSt, ok := d.txSet[tid]
+ if !ok {
+ return fmt.Errorf("unknown transaction ID: %v", tid)
+ }
+
+ txSt.TxMap[oid] = version
+ }
+
+ // Insert the new node in the kvdb.
+ node := &dagNode{Level: level, Parents: parents, Logrec: logrec, TxId: tid, Deleted: deleted}
+ if err := d.setNode(oid, version, node); err != nil {
+ return err
+ }
+
+ d.numNode.Incr(1)
+ if parents == nil {
+ d.numObj.Incr(1)
+ }
+ return nil
+}
+
+// hasNode returns true if the node (oid, version) exists in the DAG DB.
+func (d *dag) hasNode(oid ObjId, version Version) bool {
+ if d.store == nil {
+ return false
+ }
+ key := objNodeKey(oid, version)
+ return d.nodes.hasKey(key)
+}
+
+// addParent adds to the DAG node (oid, version) linkage to this parent node.
+// If the parent linkage is due to a local change (from conflict resolution
+// by blessing an existing version), no need to update the grafting structure.
+// Otherwise a remote change (from the Sync protocol) updates the graft.
+//
+// TODO(rdaoud): recompute the levels of reachable child-nodes if the new
+// parent's level is greater or equal to the node's current level.
+func (d *dag) addParent(oid ObjId, version, parent Version, remote bool) error {
+ if version == parent {
+ return fmt.Errorf("addParent: object %v: node %d cannot be its own parent", oid, version)
+ }
+
+ node, err := d.getNode(oid, version)
+ if err != nil {
+ return err
+ }
+
+ pnode, err := d.getNode(oid, parent)
+ if err != nil {
+ vlog.VI(1).Infof("addParent: object %v, node %d, parent %d: parent node not found", oid, version, parent)
+ return err
+ }
+
+ // Check if the parent is already linked to this node.
+ found := false
+ for i := range node.Parents {
+ if node.Parents[i] == parent {
+ found = true
+ break
+ }
+ }
+
+ // If the parent is not yet linked (local or remote) add it.
+ if !found {
+ // Make sure that adding the link does not create a cycle in the DAG.
+ // This is done by verifying that the node is not an ancestor of the
+ // parent that it is being linked to.
+ err = d.ancestorIter(oid, pnode.Parents, func(oid ObjId, v Version, nd *dagNode) error {
+ if v == version {
+ return fmt.Errorf("addParent: cycle on object %v: node %d is an ancestor of parent node %d",
+ oid, version, parent)
+ }
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ node.Parents = append(node.Parents, parent)
+ err = d.setNode(oid, version, node)
+ if err != nil {
+ return err
+ }
+ }
+
+ // For local changes we are done, the grafting structure is not updated.
+ if !remote {
+ return nil
+ }
+
+ // If the node and its parent are new/old or old/new then add
+ // the parent as a graft point (a potential common ancestor).
+ graft := d.getObjectGraft(oid, true)
+
+ _, nodeNew := graft.newNodes[version]
+ _, parentNew := graft.newNodes[parent]
+ if (nodeNew && !parentNew) || (!nodeNew && parentNew) {
+ graft.graftNodes[parent] = pnode.Level
+ }
+
+ // The parent node can no longer be a candidate for a new head version.
+ // The addParent() function only removes candidates from newHeads that
+ // have become parents. It does not add the child nodes to newHeads
+ // because they are not necessarily new-head candidates. If they are
+ // new nodes, the addNode() function handles adding them to newHeads.
+ // For old nodes, only the current head could be a candidate and it is
+ // added to newHeads when the graft struct is initialized.
+ if _, ok := graft.newHeads[parent]; ok {
+ delete(graft.newHeads, parent)
+ }
+
+ return nil
+}
+
+// moveHead moves the object head node in the DAG.
+func (d *dag) moveHead(oid ObjId, head Version) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+
+ // Verify that the node exists.
+ if !d.hasNode(oid, head) {
+ return fmt.Errorf("node %d:%d does not exist in the DAG", oid, head)
+ }
+
+ return d.setHead(oid, head)
+}
+
+// hasConflict determines if there is a conflict for this object between its
+// new and old head nodes.
+// - Yes: return (true, newHead, oldHead, ancestor)
+// - No: return (false, newHead, oldHead, NoVersion)
+// A conflict exists when there are two new-head nodes. It means the newly
+// added object versions are not derived in part from this device's current
+// knowledge. If there is a single new-head, the object changes were applied
+// without triggering a conflict.
+func (d *dag) hasConflict(oid ObjId) (isConflict bool, newHead, oldHead, ancestor Version, err error) {
+ oldHead = NoVersion
+ newHead = NoVersion
+ ancestor = NoVersion
+ if d.store == nil {
+ err = errBadDAG
+ return
+ }
+
+ graft := d.graft[oid]
+ if graft == nil {
+ err = fmt.Errorf("node %d has no DAG graft information", oid)
+ return
+ }
+
+ numHeads := len(graft.newHeads)
+ if numHeads < 1 || numHeads > 2 {
+ err = fmt.Errorf("node %d has invalid number of new head candidates %d: %v", oid, numHeads, graft.newHeads)
+ return
+ }
+
+ // Fetch the current head for this object if it exists. The error from getHead()
+ // is ignored because a newly received object is not yet known on this device and
+ // will not trigger a conflict.
+ oldHead, _ = d.getHead(oid)
+
+ // If there is only one new head node there is no conflict.
+ // The new head is that single one, even if it might also be the same old node.
+ if numHeads == 1 {
+ for k := range graft.newHeads {
+ newHead = k
+ }
+ return
+ }
+
+ // With two candidate head nodes, the new one is the node that is
+ // not the current (old) head node.
+ for k := range graft.newHeads {
+ if k != oldHead {
+ newHead = k
+ break
+ }
+ }
+
+ // There is a conflict: the best choice ancestor is the graft point
+ // node with the largest level (farthest from the root). It is
+ // possible in some corner cases to have multiple graft nodes at
+ // the same level. This would still be a single conflict, but the
+ // multiple same-level graft points representing equivalent conflict
+ // resolutions on different devices that are now merging their
+ // resolutions. In such a case it does not matter which node is
+ // chosen as the ancestor because the conflict resolver function
+ // is assumed to be convergent. However it's nicer to make that
+ // selection deterministic so all devices see the same choice.
+ // For this the version number is used as a tie-breaker.
+ isConflict = true
+ var maxLevel uint64
+ for node, level := range graft.graftNodes {
+ if maxLevel < level ||
+ (maxLevel == level && ancestor < node) {
+ maxLevel = level
+ ancestor = node
+ }
+ }
+ return
+}
+
+// ancestorIter iterates over the DAG ancestor nodes for an object in a
+// breadth-first traversal starting from given version node(s). In its
+// traversal it invokes the callback function once for each node, passing
+// the object ID, version number and a pointer to the dagNode.
+func (d *dag) ancestorIter(oid ObjId, startVersions []Version,
+ cb func(ObjId, Version, *dagNode) error) error {
+ visited := make(map[Version]bool)
+ queue := list.New()
+ for _, version := range startVersions {
+ queue.PushBack(version)
+ visited[version] = true
+ }
+
+ for queue.Len() > 0 {
+ version := queue.Remove(queue.Front()).(Version)
+ node, err := d.getNode(oid, version)
+ if err != nil {
+ // Ignore it, the parent was previously pruned.
+ continue
+ }
+ for _, parent := range node.Parents {
+ if !visited[parent] {
+ queue.PushBack(parent)
+ visited[parent] = true
+ }
+ }
+ if err = cb(oid, version, node); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// hasDeletedDescendant returns true if the node (oid, version) exists in the
+// DAG DB and one of its descendants is a deleted node (i.e. has its "Deleted"
+// flag set true). This means that at some object mutation after this version,
+// the object was deleted.
+func (d *dag) hasDeletedDescendant(oid ObjId, version Version) bool {
+ if d.store == nil {
+ return false
+ }
+ if !d.hasNode(oid, version) {
+ return false
+ }
+
+ // Do a breadth-first traversal from the object's head node back to
+ // the given version. Along the way, track whether a deleted node is
+ // traversed. Return true only if a traversal reaches the given version
+ // and had seen a deleted node along the way.
+
+ // nodeStep tracks a step along a traversal. It stores the node to visit
+ // when taking that step and a boolean tracking whether a deleted node
+ // was seen so far along that trajectory.
+ head, err := d.getHead(oid)
+ if err != nil {
+ return false
+ }
+
+ type nodeStep struct {
+ node Version
+ deleted bool
+ }
+
+ visited := make(map[nodeStep]struct{})
+ queue := list.New()
+
+ step := nodeStep{node: head, deleted: false}
+ queue.PushBack(&step)
+ visited[step] = struct{}{}
+
+ for queue.Len() > 0 {
+ step := queue.Remove(queue.Front()).(*nodeStep)
+ if step.node == version {
+ if step.deleted {
+ return true
+ }
+ continue
+ }
+ node, err := d.getNode(oid, step.node)
+ if err != nil {
+ // Ignore it, the parent was previously pruned.
+ continue
+ }
+ nextDel := step.deleted || node.Deleted
+
+ for _, parent := range node.Parents {
+ nextStep := nodeStep{node: parent, deleted: nextDel}
+ if _, ok := visited[nextStep]; !ok {
+ queue.PushBack(&nextStep)
+ visited[nextStep] = struct{}{}
+ }
+ }
+ }
+
+ return false
+}
+
+// prune trims the DAG of an object at a given version (node) by deleting
+// all its ancestor nodes, making it the new root node. For each deleted
+// node it calls the given callback function to delete its log record.
+// This function should only be called when Sync determines that all devices
+// that know about the object have gotten past this version.
+// Also track any transaction sets affected by deleting DAG objects that
+// have transaction IDs. This is later used to do garbage collection
+// on transaction sets when pruneDone() is called.
+func (d *dag) prune(oid ObjId, version Version, delLogRec func(logrec string) error) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+
+ // Get the node at the pruning point and set its parents to nil.
+ // It will become the oldest DAG node (root) for the object.
+ node, err := d.getNode(oid, version)
+ if err != nil {
+ return err
+ }
+ if node.Parents == nil {
+ // Nothing to do, this node is already the root.
+ return nil
+ }
+
+ iterVersions := node.Parents
+
+ node.Parents = nil
+ if err = d.setNode(oid, version, node); err != nil {
+ return err
+ }
+
+ // Delete all ancestor nodes and their log records.
+ // Delete as many as possible and track the error counts.
+ // Keep track of objects deleted from transaction in order
+ // to cleanup transaction sets when pruneDone() is called.
+ numNodeErrs, numLogErrs := 0, 0
+ err = d.ancestorIter(oid, iterVersions, func(oid ObjId, v Version, node *dagNode) error {
+ nodeErrs, logErrs, err := d.removeNode(oid, v, node, delLogRec)
+ numNodeErrs += nodeErrs
+ numLogErrs += logErrs
+ return err
+ })
+ if err != nil {
+ return err
+ }
+ if numNodeErrs != 0 || numLogErrs != 0 {
+ return fmt.Errorf("prune failed to delete %d nodes and %d log records", numNodeErrs, numLogErrs)
+ }
+ return nil
+}
+
+// removeNode removes the state associated with a DAG node.
+func (d *dag) removeNode(oid ObjId, v Version, node *dagNode, delLogRec func(logrec string) error) (int, int, error) {
+ numNodeErrs, numLogErrs := 0, 0
+ if tid := node.TxId; tid != NoTxId {
+ if d.txGC[tid] == nil {
+ d.txGC[tid] = make(dagTxMap)
+ }
+ d.txGC[tid][oid] = v
+ }
+
+ if err := delLogRec(node.Logrec); err != nil {
+ numLogErrs++
+ }
+ if err := d.delNode(oid, v); err != nil {
+ numNodeErrs++
+ }
+ d.numNode.Incr(-1)
+ return numNodeErrs, numLogErrs, nil
+}
+
+// pruneAll prunes the entire DAG state corresponding to an object,
+// including the head.
+func (d *dag) pruneAll(oid ObjId, delLogRec func(logrec string) error) error {
+ vers, err := d.getHead(oid)
+ if err != nil {
+ return err
+ }
+ node, err := d.getNode(oid, vers)
+ if err != nil {
+ return err
+ }
+
+ if err := d.prune(oid, vers, delLogRec); err != nil {
+ return err
+ }
+
+ // Clean up the head.
+ numNodeErrs, numLogErrs, err := d.removeNode(oid, vers, node, delLogRec)
+ if err != nil {
+ return err
+ }
+ if numNodeErrs != 0 || numLogErrs != 0 {
+ return fmt.Errorf("pruneAll failed to delete %d nodes and %d log records", numNodeErrs, numLogErrs)
+ }
+
+ return d.delHead(oid)
+}
+
+// pruneDone is called when object pruning is finished within a single pass
+// of the Sync garbage collector. It updates the transaction sets affected
+// by the objects deleted by the prune() calls.
+func (d *dag) pruneDone() error {
+ if d.store == nil {
+ return errBadDAG
+ }
+
+ // Update transaction sets by removing from them the objects that
+ // were pruned. If the resulting set is empty, delete it.
+ for tid, txMapGC := range d.txGC {
+ txSt, err := d.getTransaction(tid)
+ if err != nil {
+ return err
+ }
+
+ for oid := range txMapGC {
+ delete(txSt.TxMap, oid)
+ }
+
+ if len(txSt.TxMap) > 0 {
+ err = d.setTransaction(tid, txSt)
+ } else {
+ err = d.delTransaction(tid)
+ }
+ if err != nil {
+ return err
+ }
+ }
+
+ d.clearTxGC()
+ return nil
+}
+
+// getLogrec returns the log record information for a given object version.
+func (d *dag) getLogrec(oid ObjId, version Version) (string, error) {
+ node, err := d.getNode(oid, version)
+ if err != nil {
+ return "", err
+ }
+ return node.Logrec, nil
+}
+
+// objNodeKey returns the key used to access the object node (oid, version)
+// in the DAG DB.
+func objNodeKey(oid ObjId, version Version) string {
+ return fmt.Sprintf("%s:%d", oid, version)
+}
+
+// setNode stores the dagNode structure for the object node (oid, version)
+// in the DAG DB.
+func (d *dag) setNode(oid ObjId, version Version, node *dagNode) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+ key := objNodeKey(oid, version)
+ return d.nodes.set(key, node)
+}
+
+// getNode retrieves the dagNode structure for the object node (oid, version)
+// from the DAG DB.
+func (d *dag) getNode(oid ObjId, version Version) (*dagNode, error) {
+ if d.store == nil {
+ return nil, errBadDAG
+ }
+ var node dagNode
+ key := objNodeKey(oid, version)
+ if err := d.nodes.get(key, &node); err != nil {
+ return nil, err
+ }
+ return &node, nil
+}
+
+// delNode deletes the object node (oid, version) from the DAG DB.
+func (d *dag) delNode(oid ObjId, version Version) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+ key := objNodeKey(oid, version)
+ return d.nodes.del(key)
+}
+
+// objHeadKey returns the key used to access the object head in the DAG DB.
+func objHeadKey(oid ObjId) string {
+ return oid.String()
+}
+
+// setHead stores version as the object head in the DAG DB.
+func (d *dag) setHead(oid ObjId, version Version) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+ key := objHeadKey(oid)
+ return d.heads.set(key, version)
+}
+
+// getHead retrieves the object head from the DAG DB.
+func (d *dag) getHead(oid ObjId) (Version, error) {
+ var version Version
+ if d.store == nil {
+ return version, errBadDAG
+ }
+ key := objHeadKey(oid)
+ err := d.heads.get(key, &version)
+ if err != nil {
+ version = NoVersion
+ }
+ return version, err
+}
+
+// delHead deletes the object head from the DAG DB.
+func (d *dag) delHead(oid ObjId) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+ key := objHeadKey(oid)
+ if err := d.heads.del(key); err != nil {
+ return err
+ }
+ d.numObj.Incr(-1)
+ return nil
+}
+
+// dagTransactionKey returns the key used to access the transaction in the DAG DB.
+func dagTransactionKey(tid TxId) string {
+ return fmt.Sprintf("%v", tid)
+}
+
+// setTransaction stores the transaction object/version map in the DAG DB.
+func (d *dag) setTransaction(tid TxId, txSt *dagTxState) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+ if tid == NoTxId {
+ return fmt.Errorf("invalid TxId: %v", tid)
+ }
+ key := dagTransactionKey(tid)
+ exists := d.trans.hasKey(key)
+
+ if err := d.trans.set(key, txSt); err != nil {
+ return err
+ }
+
+ if !exists {
+ d.numTx.Incr(1)
+ }
+ return nil
+}
+
+// getTransaction retrieves the transaction object/version map from the DAG DB.
+func (d *dag) getTransaction(tid TxId) (*dagTxState, error) {
+ if d.store == nil {
+ return nil, errBadDAG
+ }
+ if tid == NoTxId {
+ return nil, fmt.Errorf("invalid TxId: %v", tid)
+ }
+ var txSt dagTxState
+ key := dagTransactionKey(tid)
+ if err := d.trans.get(key, &txSt); err != nil {
+ return nil, err
+ }
+ return &txSt, nil
+}
+
+// delTransaction deletes the transation object/version map from the DAG DB.
+func (d *dag) delTransaction(tid TxId) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+ if tid == NoTxId {
+ return fmt.Errorf("invalid TxId: %v", tid)
+ }
+ key := dagTransactionKey(tid)
+ if err := d.trans.del(key); err != nil {
+ return err
+ }
+ d.numTx.Incr(-1)
+ return nil
+}
+
+// objPrivNodeKey returns the key used to access a private (unshared) node in the DAG DB.
+func objPrivNodeKey(oid ObjId) string {
+ return oid.String()
+}
+
+// setPrivNode stores the privNode structure for a private (unshared) object in the DAG DB.
+func (d *dag) setPrivNode(oid ObjId, priv *privNode) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+ key := objPrivNodeKey(oid)
+ exists := d.priv.hasKey(key)
+
+ if err := d.priv.set(key, priv); err != nil {
+ return err
+ }
+
+ if !exists {
+ d.numPriv.Incr(1)
+ }
+ return nil
+}
+
+// getPrivNode retrieves the privNode structure for a private (unshared) object from the DAG DB.
+func (d *dag) getPrivNode(oid ObjId) (*privNode, error) {
+ if d.store == nil {
+ return nil, errBadDAG
+ }
+ var priv privNode
+ key := objPrivNodeKey(oid)
+ if err := d.priv.get(key, &priv); err != nil {
+ return nil, err
+ }
+ return &priv, nil
+}
+
+// delPrivNode deletes a private (unshared) object from the DAG DB.
+func (d *dag) delPrivNode(oid ObjId) error {
+ if d.store == nil {
+ return errBadDAG
+ }
+ key := objPrivNodeKey(oid)
+ if err := d.priv.del(key); err != nil {
+ return err
+ }
+ d.numPriv.Incr(-1)
+ return nil
+}
+
+// getParentMap is a testing and debug helper function that returns for
+// an object a map of all the object version in the DAG and their parents.
+// The map represents the graph of the object version history.
+func (d *dag) getParentMap(oid ObjId) map[Version][]Version {
+ parentMap := make(map[Version][]Version)
+ var iterVersions []Version
+
+ if head, err := d.getHead(oid); err == nil {
+ iterVersions = append(iterVersions, head)
+ }
+ if graft := d.graft[oid]; graft != nil {
+ for k := range graft.newHeads {
+ iterVersions = append(iterVersions, k)
+ }
+ }
+
+ // Breadth-first traversal starting from the object head.
+ d.ancestorIter(oid, iterVersions, func(oid ObjId, v Version, node *dagNode) error {
+ parentMap[v] = node.Parents
+ return nil
+ })
+
+ return parentMap
+}
+
+// getGraftNodes is a testing and debug helper function that returns for
+// an object the graft information built and used during a sync operation.
+// The newHeads map identifies the candidate head nodes based on the data
+// reported by the other device during a sync operation. The graftNodes map
+// identifies the set of old nodes where the new DAG fragments were attached
+// and their depth level in the DAG.
+func (d *dag) getGraftNodes(oid ObjId) (map[Version]struct{}, map[Version]uint64) {
+ if d.store != nil {
+ if ginfo := d.graft[oid]; ginfo != nil {
+ return ginfo.newHeads, ginfo.graftNodes
+ }
+ }
+ return nil, nil
+}
+
+// strToTxId converts from a string to a transaction ID.
+func strToTxId(txStr string) (TxId, error) {
+ tx, err := strconv.ParseUint(txStr, 10, 64)
+ if err != nil {
+ return NoTxId, err
+ }
+ return TxId(tx), nil
+}
+
+// dump writes to the log file information on all DAG entries.
+func (d *dag) dump() {
+ if d.store == nil {
+ return
+ }
+
+ // Dump the head and ancestor information for DAG objects.
+ d.heads.keyIter(func(oidStr string) {
+ oid, err := strToObjId(oidStr)
+ if err != nil {
+ return
+ }
+
+ head, err := d.getHead(oid)
+ if err != nil {
+ return
+ }
+
+ vlog.VI(1).Infof("DUMP: DAG oid %v: head %v", oid, head)
+ start := []Version{head}
+ d.ancestorIter(oid, start, func(oid ObjId, v Version, node *dagNode) error {
+ vlog.VI(1).Infof("DUMP: DAG node %v:%v: tx %v, del %t, logrec %s --> %v",
+ oid, v, node.TxId, node.Deleted, node.Logrec, node.Parents)
+ return nil
+ })
+ })
+
+ // Dump the transactions.
+ d.trans.keyIter(func(tidStr string) {
+ tid, err := strToTxId(tidStr)
+ if err != nil {
+ return
+ }
+
+ txSt, err := d.getTransaction(tid)
+ if err != nil {
+ return
+ }
+
+ vlog.VI(1).Infof("DUMP: DAG tx %v: count %d, elem %v", tid, txSt.TxCount, txSt.TxMap)
+ })
+}
diff --git a/services/syncbase/sync/dag_test.go b/services/syncbase/sync/dag_test.go
new file mode 100644
index 0000000..a8ee7ee
--- /dev/null
+++ b/services/syncbase/sync/dag_test.go
@@ -0,0 +1,2128 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Tests for the Veyron Sync DAG component.
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+ "time"
+
+ "v.io/x/ref/lib/stats"
+)
+
+// dagFilename generates a filename for a temporary (per unit test) DAG file.
+// Do not replace this function with TempFile because TempFile creates the new
+// file and the tests must verify that the DAG can create a non-existing file.
+func dagFilename() string {
+ return fmt.Sprintf("%s/sync_dag_test_%d_%d", os.TempDir(), os.Getpid(), time.Now().UnixNano())
+}
+
+// TestDAGOpen tests the creation of a DAG, closing and re-opening it. It also
+// verifies that its backing file is created and that a 2nd close is safe.
+func TestDAGOpen(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ fsize := getFileSize(dagfile)
+ if fsize < 0 {
+ //t.Fatalf("DAG file %s not created", dagfile)
+ }
+
+ dag.flush()
+ oldfsize := fsize
+ fsize = getFileSize(dagfile)
+ if fsize <= oldfsize {
+ //t.Fatalf("DAG file %s not flushed", dagfile)
+ }
+
+ dag.close()
+
+ dag, err = openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot re-open existing DAG file %s", dagfile)
+ }
+
+ oldfsize = fsize
+ fsize = getFileSize(dagfile)
+ if fsize != oldfsize {
+ t.Fatalf("DAG file %s size changed across re-open", dagfile)
+ }
+
+ dag.close()
+ dag.close() // multiple closes should be a safe NOP
+
+ fsize = getFileSize(dagfile)
+ if fsize != oldfsize {
+ t.Fatalf("DAG file %s size changed across close", dagfile)
+ }
+
+ // Fail opening a DAG in a non-existent directory.
+ _, err = openDAG("/not/really/there/junk.dag")
+ if err == nil {
+ //t.Fatalf("openDAG() did not fail when using a bad pathname")
+ }
+}
+
+// TestInvalidDAG tests using DAG methods on an invalid (closed) DAG.
+func TestInvalidDAG(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ dag.close()
+
+ oid, err := strToObjId("6789")
+ if err != nil {
+ t.Error(err)
+ }
+
+ validateError := func(err error, funcName string) {
+ if err == nil || err.Error() != "invalid DAG" {
+ t.Errorf("%s() did not fail on a closed DAG: %v", funcName, err)
+ }
+ }
+
+ err = dag.addNode(oid, 4, false, false, []Version{2, 3}, "foobar", NoTxId)
+ validateError(err, "addNode")
+
+ err = dag.moveHead(oid, 4)
+ validateError(err, "moveHead")
+
+ _, _, _, _, err = dag.hasConflict(oid)
+ validateError(err, "hasConflict")
+
+ _, err = dag.getLogrec(oid, 4)
+ validateError(err, "getLogrec")
+
+ err = dag.prune(oid, 4, func(lr string) error {
+ return nil
+ })
+ validateError(err, "prune")
+
+ err = dag.pruneAll(oid, func(lr string) error {
+ return nil
+ })
+ validateError(err, "pruneAll")
+
+ err = dag.pruneDone()
+ validateError(err, "pruneDone")
+
+ node := &dagNode{Level: 15, Parents: []Version{444, 555}, Logrec: "logrec-23"}
+ err = dag.setNode(oid, 4, node)
+ validateError(err, "setNode")
+
+ _, err = dag.getNode(oid, 4)
+ validateError(err, "getNode")
+
+ err = dag.delNode(oid, 4)
+ validateError(err, "delNode")
+
+ err = dag.addParent(oid, 4, 2, true)
+ validateError(err, "addParent")
+
+ err = dag.setHead(oid, 4)
+ validateError(err, "setHead")
+
+ _, err = dag.getHead(oid)
+ validateError(err, "getHead")
+
+ err = dag.delHead(oid)
+ validateError(err, "delHead")
+
+ if tid := dag.addNodeTxStart(NoTxId); tid != NoTxId {
+ t.Errorf("addNodeTxStart() did not fail on a closed DAG: TxId %v", tid)
+ }
+
+ err = dag.addNodeTxEnd(1, 1)
+ validateError(err, "addNodeTxEnd")
+
+ err = dag.setTransaction(1, nil)
+ validateError(err, "setTransaction")
+
+ _, err = dag.getTransaction(1)
+ validateError(err, "getTransaction")
+
+ err = dag.delTransaction(1)
+ validateError(err, "delTransaction")
+
+ err = dag.setPrivNode(oid, nil)
+ validateError(err, "setPrivNode")
+
+ _, err = dag.getPrivNode(oid)
+ validateError(err, "getPrivNode")
+
+ err = dag.delPrivNode(oid)
+ validateError(err, "delPrivNode")
+
+ // These calls should be harmless NOPs.
+ dag.clearGraft()
+ dag.clearTxGC()
+ dag.dump()
+ dag.flush()
+ dag.close()
+ if dag.hasNode(oid, 4) {
+ t.Errorf("hasNode() found an object on a closed DAG")
+ }
+ if dag.hasDeletedDescendant(oid, 3) {
+ t.Errorf("hasDeletedDescendant() returned true on a closed DAG")
+ }
+ if pmap := dag.getParentMap(oid); len(pmap) != 0 {
+ t.Errorf("getParentMap() found data on a closed DAG: %v", pmap)
+ }
+ if hmap, gmap := dag.getGraftNodes(oid); hmap != nil || gmap != nil {
+ t.Errorf("getGraftNodes() found data on a closed DAG: head map: %v, graft map: %v", hmap, gmap)
+ }
+}
+
+// TestSetNode tests setting and getting a DAG node across DAG open/close/reopen.
+func TestSetNode(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ version := Version(0)
+ oid, err := strToObjId("1111")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ node, err := dag.getNode(oid, version)
+ if err == nil || node != nil {
+ t.Errorf("Found non-existent object %d:%d in DAG file %s: %v", oid, version, dagfile, node)
+ }
+
+ if dag.hasNode(oid, version) {
+ t.Errorf("hasNode() found non-existent object %d:%d in DAG file %s", oid, version, dagfile)
+ }
+
+ if logrec, err := dag.getLogrec(oid, version); err == nil || logrec != "" {
+ t.Errorf("Non-existent object %d:%d has a logrec in DAG file %s: %v", oid, version, dagfile, logrec)
+ }
+
+ node = &dagNode{Level: 15, Parents: []Version{444, 555}, Logrec: "logrec-23"}
+ if err = dag.setNode(oid, version, node); err != nil {
+ t.Fatalf("Cannot set object %d:%d (%v) in DAG file %s", oid, version, node, dagfile)
+ }
+
+ for i := 0; i < 2; i++ {
+ node2, err := dag.getNode(oid, version)
+ if err != nil || node2 == nil {
+ t.Errorf("Cannot find stored object %d:%d (i=%d) in DAG file %s", oid, version, i, dagfile)
+ }
+
+ if !dag.hasNode(oid, version) {
+ t.Errorf("hasNode() did not find object %d:%d (i=%d) in DAG file %s", oid, version, i, dagfile)
+ }
+
+ if !reflect.DeepEqual(node, node2) {
+ t.Errorf("Object %d:%d has wrong data (i=%d) in DAG file %s: %v instead of %v",
+ oid, version, i, dagfile, node2, node)
+ }
+
+ if logrec, err := dag.getLogrec(oid, version); err != nil || logrec != "logrec-23" {
+ t.Errorf("Object %d:%d has wrong logrec (i=%d) in DAG file %s: %v",
+ oid, version, i, dagfile, logrec)
+ }
+
+ if i == 0 {
+ dag.flush()
+ dag.close()
+ dag, err = openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot re-open DAG file %s", dagfile)
+ }
+ }
+ }
+
+ dag.close()
+}
+
+// TestDelNode tests deleting a DAG node across DAG open/close/reopen.
+func TestDelNode(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ version := Version(1)
+ oid, err := strToObjId("2222")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ node := &dagNode{Level: 123, Parents: []Version{333}, Logrec: "logrec-789"}
+ if err = dag.setNode(oid, version, node); err != nil {
+ t.Fatalf("Cannot set object %d:%d (%v) in DAG file %s", oid, version, node, dagfile)
+ }
+
+ dag.flush()
+
+ err = dag.delNode(oid, version)
+ if err != nil {
+ t.Fatalf("Cannot delete object %d:%d in DAG file %s", oid, version, dagfile)
+ }
+
+ dag.flush()
+
+ for i := 0; i < 2; i++ {
+ node2, err := dag.getNode(oid, version)
+ if err == nil || node2 != nil {
+ t.Errorf("Found deleted object %d:%d (%v) (i=%d) in DAG file %s", oid, version, node2, i, dagfile)
+ }
+
+ if dag.hasNode(oid, version) {
+ t.Errorf("hasNode() found deleted object %d:%d (i=%d) in DAG file %s", oid, version, i, dagfile)
+ }
+
+ if logrec, err := dag.getLogrec(oid, version); err == nil || logrec != "" {
+ t.Errorf("Deleted object %d:%d (i=%d) has logrec in DAG file %s: %v", oid, version, i, dagfile, logrec)
+ }
+
+ if i == 0 {
+ dag.close()
+ dag, err = openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot re-open DAG file %s", dagfile)
+ }
+ }
+ }
+
+ dag.close()
+}
+
+// TestAddParent tests adding parents to a DAG node.
+func TestAddParent(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ version := Version(7)
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err = dag.addParent(oid, version, 1, true); err == nil {
+ t.Errorf("addParent() did not fail for an unknown object %d:%d in DAG file %s", oid, version, dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ node := &dagNode{Level: 15, Logrec: "logrec-22"}
+ if err = dag.setNode(oid, version, node); err != nil {
+ t.Fatalf("Cannot set object %d:%d (%v) in DAG file %s", oid, version, node, dagfile)
+ }
+
+ if err = dag.addParent(oid, version, version, true); err == nil {
+ t.Errorf("addParent() did not fail on a self-parent for object %d:%d in DAG file %s", oid, version, dagfile)
+ }
+
+ for _, parent := range []Version{4, 5, 6} {
+ if err = dag.addParent(oid, version, parent, true); err == nil {
+ t.Errorf("addParent() did not reject invalid parent %d for object %d:%d in DAG file %s",
+ parent, oid, version, dagfile)
+ }
+
+ pnode := &dagNode{Level: 11, Logrec: fmt.Sprintf("logrec-%d", parent), Parents: []Version{3}}
+ if err = dag.setNode(oid, parent, pnode); err != nil {
+ t.Fatalf("Cannot set parent object %d:%d (%v) in DAG file %s", oid, parent, pnode, dagfile)
+ }
+
+ remote := parent%2 == 0
+ for i := 0; i < 2; i++ {
+ if err = dag.addParent(oid, version, parent, remote); err != nil {
+ t.Errorf("addParent() failed on parent %d, remote %t (i=%d) for object %d:%d in DAG file %s: %v",
+ parent, remote, i, oid, version, dagfile, err)
+ }
+ }
+ }
+
+ node2, err := dag.getNode(oid, version)
+ if err != nil || node2 == nil {
+ t.Errorf("Cannot find stored object %d:%d in DAG file %s", oid, version, dagfile)
+ }
+
+ expParents := []Version{4, 5, 6}
+ if !reflect.DeepEqual(node2.Parents, expParents) {
+ t.Errorf("invalid parents for object %d:%d in DAG file %s: %v instead of %v",
+ oid, version, dagfile, node2.Parents, expParents)
+ }
+
+ // Creating cycles should fail.
+ for v := Version(1); v < version; v++ {
+ if err = dag.addParent(oid, v, version, false); err == nil {
+ t.Errorf("addParent() failed to reject a cycle for object %d: from ancestor %d to node %d in DAG file %s",
+ oid, v, version, dagfile)
+ }
+ }
+
+ dag.close()
+}
+
+// TestSetHead tests setting and getting a DAG head node across DAG open/close/reopen.
+func TestSetHead(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ oid, err := strToObjId("3333")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ version, err := dag.getHead(oid)
+ if err == nil {
+ t.Errorf("Found non-existent object head %d in DAG file %s: %d", oid, dagfile, version)
+ }
+
+ version = 555
+ if err = dag.setHead(oid, version); err != nil {
+ t.Fatalf("Cannot set object head %d (%d) in DAG file %s", oid, version, dagfile)
+ }
+
+ dag.flush()
+
+ for i := 0; i < 3; i++ {
+ version2, err := dag.getHead(oid)
+ if err != nil {
+ t.Errorf("Cannot find stored object head %d (i=%d) in DAG file %s", oid, i, dagfile)
+ }
+ if version != version2 {
+ t.Errorf("Object %d has wrong head data (i=%d) in DAG file %s: %d instead of %d",
+ oid, i, dagfile, version2, version)
+ }
+
+ if i == 0 {
+ dag.close()
+ dag, err = openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot re-open DAG file %s", dagfile)
+ }
+ } else if i == 1 {
+ version = 888
+ if err = dag.setHead(oid, version); err != nil {
+ t.Fatalf("Cannot set new object head %d (%d) in DAG file %s", oid, version, dagfile)
+ }
+ dag.flush()
+ }
+ }
+
+ dag.close()
+}
+
+// checkEndOfSync simulates and check the end-of-sync operations: clear the
+// node grafting metadata and verify that it is empty and that HasConflict()
+// detects this case and fails, then close the DAG.
+func checkEndOfSync(d *dag, oid ObjId) error {
+ // Clear grafting info; this happens at the end of a sync log replay.
+ d.clearGraft()
+
+ // There should be no grafting or transaction info, and hasConflict() should fail.
+ newHeads, grafts := d.getGraftNodes(oid)
+ if newHeads != nil || grafts != nil {
+ return fmt.Errorf("Object %d: graft info not cleared: newHeads (%v), grafts (%v)", oid, newHeads, grafts)
+ }
+
+ if n := len(d.txSet); n != 0 {
+ return fmt.Errorf("transaction set not empty: %d entries found", n)
+ }
+
+ isConflict, newHead, oldHead, ancestor, errConflict := d.hasConflict(oid)
+ if errConflict == nil {
+ return fmt.Errorf("Object %d: conflict did not fail: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ d.dump()
+ d.close()
+ return nil
+}
+
+// checkDAGStats verifies the DAG stats counters.
+func checkDAGStats(t *testing.T, which string, numObj, numNode, numTx, numPriv int64) {
+ if num, err := stats.Value(statsNumDagObj); err != nil || num != numObj {
+ t.Errorf("num-dag-objects (%s): got %v (err: %v) instead of %v", which, num, err, numObj)
+ }
+ if num, err := stats.Value(statsNumDagNode); err != nil || num != numNode {
+ t.Errorf("num-dag-nodes (%s): got %v (err: %v) instead of %v", which, num, err, numNode)
+ }
+ if num, err := stats.Value(statsNumDagTx); err != nil || num != numTx {
+ t.Errorf("num-dag-tx (%s): got %v (err: %v) instead of %v", which, num, err, numTx)
+ }
+ if num, err := stats.Value(statsNumDagPrivNode); err != nil || num != numPriv {
+ t.Errorf("num-dag-privnodes (%s): got %v (err: %v) instead of %v", which, num, err, numPriv)
+ }
+}
+
+// TestLocalUpdates tests the sync handling of initial local updates: an object
+// is created (v0) and updated twice (v1, v2) on this device. The DAG should
+// show: v0 -> v1 -> v2 and the head should point to v2.
+func TestLocalUpdates(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ // The head must have moved to "v2" and the parent map shows the updated DAG.
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if head, e := dag.getHead(oid); e != nil || head != 2 {
+ t.Errorf("Invalid object %d head in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ pmap := dag.getParentMap(oid)
+
+ exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}}
+
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
+ }
+
+ // Make sure an existing node cannot be added again.
+ if err = dag.addNode(oid, 1, false, false, []Version{0, 2}, "foobar", NoTxId); err == nil {
+ t.Errorf("addNode() did not fail when given an existing node")
+ }
+
+ // Make sure a new node cannot have more than 2 parents.
+ if err = dag.addNode(oid, 3, false, false, []Version{0, 1, 2}, "foobar", NoTxId); err == nil {
+ t.Errorf("addNode() did not fail when given 3 parents")
+ }
+
+ // Make sure a new node cannot have an invalid parent.
+ if err = dag.addNode(oid, 3, false, false, []Version{0, 555}, "foobar", NoTxId); err == nil {
+ t.Errorf("addNode() did not fail when using an invalid parent")
+ }
+
+ // Make sure a new root node (no parents) cannot be added once a root exists.
+ // For the parents array, check both the "nil" and the empty array as input.
+ if err = dag.addNode(oid, 6789, false, false, nil, "foobar", NoTxId); err == nil {
+ t.Errorf("Adding a 2nd root node (nil parents) for object %d in DAG file %s did not fail", oid, dagfile)
+ }
+ if err = dag.addNode(oid, 6789, false, false, []Version{}, "foobar", NoTxId); err == nil {
+ t.Errorf("Adding a 2nd root node (empty parents) for object %d in DAG file %s did not fail", oid, dagfile)
+ }
+
+ checkDAGStats(t, "local-update", 1, 3, 0, 0)
+
+ if err := checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestRemoteUpdates tests the sync handling of initial remote updates:
+// an object is created (v0) and updated twice (v1, v2) on another device and
+// we learn about it during sync. The updated DAG should show: v0 -> v1 -> v2
+// and report no conflicts with the new head pointing at v2.
+func TestRemoteUpdates(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "remote-init-00.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ // The head must not have moved (i.e. still undefined) and the parent
+ // map shows the newly grafted DAG fragment.
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if head, e := dag.getHead(oid); e == nil {
+ t.Errorf("Object %d head found in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ pmap := dag.getParentMap(oid)
+
+ exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}}
+
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
+ }
+
+ // Verify the grafting of remote nodes.
+ newHeads, grafts := dag.getGraftNodes(oid)
+
+ expNewHeads := map[Version]struct{}{2: struct{}{}}
+ if !reflect.DeepEqual(newHeads, expNewHeads) {
+ t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
+ }
+
+ expgrafts := map[Version]uint64{}
+ if !reflect.DeepEqual(grafts, expgrafts) {
+ t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
+ }
+
+ // There should be no conflict.
+ isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
+ if !(!isConflict && newHead == 2 && oldHead == 0 && ancestor == 0 && errConflict == nil) {
+ t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-02" {
+ t.Errorf("Invalid logrec for newhead object %d:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
+ }
+
+ // Make sure an unknown node cannot become the new head.
+ if err = dag.moveHead(oid, 55); err == nil {
+ t.Errorf("moveHead() did not fail on an invalid node")
+ }
+
+ // Then we can move the head and clear the grafting data.
+ if err = dag.moveHead(oid, newHead); err != nil {
+ t.Errorf("Object %d cannot move head to %d in DAG file %s: %v", oid, newHead, dagfile, err)
+ }
+
+ checkDAGStats(t, "remote-update", 1, 3, 0, 0)
+
+ if err := checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestRemoteNoConflict tests sync of remote updates on top of a local initial
+// state without conflict. An object is created locally and updated twice
+// (v0 -> v1 -> v2). Another device, having gotten this info, makes 3 updates
+// on top of that (v2 -> v3 -> v4 -> v5) and sends this info in a later sync.
+// The updated DAG should show (v0 -> v1 -> v2 -> v3 -> v4 -> v5) and report
+// no conflicts with the new head pointing at v5. It should also report v2 as
+// the graft point on which the new fragment (v3 -> v4 -> v5) gets attached.
+func TestRemoteNoConflict(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
+ t.Fatal(err)
+ }
+ if err = dagReplayCommands(dag, "remote-noconf-00.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ // The head must not have moved (i.e. still at v2) and the parent map
+ // shows the newly grafted DAG fragment on top of the prior DAG.
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if head, e := dag.getHead(oid); e != nil || head != 2 {
+ t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ pmap := dag.getParentMap(oid)
+
+ exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}, 3: {2}, 4: {3}, 5: {4}}
+
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
+ }
+
+ // Verify the grafting of remote nodes.
+ newHeads, grafts := dag.getGraftNodes(oid)
+
+ expNewHeads := map[Version]struct{}{5: struct{}{}}
+ if !reflect.DeepEqual(newHeads, expNewHeads) {
+ t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
+ }
+
+ expgrafts := map[Version]uint64{2: 2}
+ if !reflect.DeepEqual(grafts, expgrafts) {
+ t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
+ }
+
+ // There should be no conflict.
+ isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
+ if !(!isConflict && newHead == 5 && oldHead == 2 && ancestor == 0 && errConflict == nil) {
+ t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ if logrec, e := dag.getLogrec(oid, oldHead); e != nil || logrec != "logrec-02" {
+ t.Errorf("Invalid logrec for oldhead object %d:%d in DAG file %s: %v", oid, oldHead, dagfile, logrec)
+ }
+ if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-05" {
+ t.Errorf("Invalid logrec for newhead object %d:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
+ }
+
+ // Then we can move the head and clear the grafting data.
+ if err = dag.moveHead(oid, newHead); err != nil {
+ t.Errorf("Object %d cannot move head to %d in DAG file %s: %v", oid, newHead, dagfile, err)
+ }
+
+ // Clear the grafting data and verify that hasConflict() fails without it.
+ dag.clearGraft()
+ isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
+ if errConflict == nil {
+ t.Errorf("hasConflict() on %d did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ checkDAGStats(t, "remote-noconf", 1, 6, 0, 0)
+
+ if err := checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestRemoteConflict tests sync handling remote updates that build on the
+// local initial state and trigger a conflict. An object is created locally
+// and updated twice (v0 -> v1 -> v2). Another device, having only gotten
+// the v0 -> v1 history, makes 3 updates on top of v1 (v1 -> v3 -> v4 -> v5)
+// and sends this info during a later sync. Separately, the local device
+// makes a conflicting (concurrent) update v1 -> v2. The updated DAG should
+// show the branches: (v0 -> v1 -> v2) and (v0 -> v1 -> v3 -> v4 -> v5) and
+// report the conflict between v2 and v5 (current and new heads). It should
+// also report v1 as the graft point and the common ancestor in the conflict.
+// The conflict is resolved locally by creating v6 that is derived from both
+// v2 and v5 and it becomes the new head.
+func TestRemoteConflict(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
+ t.Fatal(err)
+ }
+ if err = dagReplayCommands(dag, "remote-conf-00.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ // The head must not have moved (i.e. still at v2) and the parent map
+ // shows the newly grafted DAG fragment on top of the prior DAG.
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if head, e := dag.getHead(oid); e != nil || head != 2 {
+ t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ pmap := dag.getParentMap(oid)
+
+ exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}, 3: {1}, 4: {3}, 5: {4}}
+
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
+ }
+
+ // Verify the grafting of remote nodes.
+ newHeads, grafts := dag.getGraftNodes(oid)
+
+ expNewHeads := map[Version]struct{}{2: struct{}{}, 5: struct{}{}}
+ if !reflect.DeepEqual(newHeads, expNewHeads) {
+ t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
+ }
+
+ expgrafts := map[Version]uint64{1: 1}
+ if !reflect.DeepEqual(grafts, expgrafts) {
+ t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
+ }
+
+ // There should be a conflict between v2 and v5 with v1 as ancestor.
+ isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
+ if !(isConflict && newHead == 5 && oldHead == 2 && ancestor == 1 && errConflict == nil) {
+ t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ if logrec, e := dag.getLogrec(oid, oldHead); e != nil || logrec != "logrec-02" {
+ t.Errorf("Invalid logrec for oldhead object %d:%d in DAG file %s: %v", oid, oldHead, dagfile, logrec)
+ }
+ if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-05" {
+ t.Errorf("Invalid logrec for newhead object %d:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
+ }
+ if logrec, e := dag.getLogrec(oid, ancestor); e != nil || logrec != "logrec-01" {
+ t.Errorf("Invalid logrec for ancestor object %d:%d in DAG file %s: %v", oid, ancestor, dagfile, logrec)
+ }
+
+ checkDAGStats(t, "remote-conf-pre", 1, 6, 0, 0)
+
+ // Resolve the conflict by adding a new local v6 derived from v2 and v5 (this replay moves the head).
+ if err = dagReplayCommands(dag, "local-resolve-00.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ // Verify that the head moved to v6 and the parent map shows the resolution.
+ if head, e := dag.getHead(oid); e != nil || head != 6 {
+ t.Errorf("Object %d has wrong head after conflict resolution in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ exp[6] = []Version{2, 5}
+ pmap = dag.getParentMap(oid)
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map after conflict resolution in DAG file %s: (%v) instead of (%v)",
+ oid, dagfile, pmap, exp)
+ }
+
+ checkDAGStats(t, "remote-conf-post", 1, 7, 0, 0)
+
+ if err := checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestRemoteConflictTwoGrafts tests sync handling remote updates that build
+// on the local initial state and trigger a conflict with 2 graft points.
+// An object is created locally and updated twice (v0 -> v1 -> v2). Another
+// device, first learns about v0 and makes it own conflicting update v0 -> v3.
+// That remote device later learns about v1 and resolves the v1/v3 confict by
+// creating v4. Then it makes a last v4 -> v5 update -- which will conflict
+// with v2 but it doesn't know that.
+// Now the sync order is reversed and the local device learns all of what
+// happened on the remote device. The local DAG should get be augmented by
+// a subtree with 2 graft points: v0 and v1. It receives this new branch:
+// v0 -> v3 -> v4 -> v5. Note that v4 is also derived from v1 as a remote
+// conflict resolution. This should report a conflict between v2 and v5
+// (current and new heads), with v0 and v1 as graft points, and v1 as the
+// most-recent common ancestor for that conflict. The conflict is resolved
+// locally by creating v6, derived from both v2 and v5, becoming the new head.
+func TestRemoteConflictTwoGrafts(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-00.sync"); err != nil {
+ t.Fatal(err)
+ }
+ if err = dagReplayCommands(dag, "remote-conf-01.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ // The head must not have moved (i.e. still at v2) and the parent map
+ // shows the newly grafted DAG fragment on top of the prior DAG.
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if head, e := dag.getHead(oid); e != nil || head != 2 {
+ t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ pmap := dag.getParentMap(oid)
+
+ exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}, 3: {0}, 4: {1, 3}, 5: {4}}
+
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
+ }
+
+ // Verify the grafting of remote nodes.
+ newHeads, grafts := dag.getGraftNodes(oid)
+
+ expNewHeads := map[Version]struct{}{2: struct{}{}, 5: struct{}{}}
+ if !reflect.DeepEqual(newHeads, expNewHeads) {
+ t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
+ }
+
+ expgrafts := map[Version]uint64{0: 0, 1: 1}
+ if !reflect.DeepEqual(grafts, expgrafts) {
+ t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
+ }
+
+ // There should be a conflict between v2 and v5 with v1 as ancestor.
+ isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
+ if !(isConflict && newHead == 5 && oldHead == 2 && ancestor == 1 && errConflict == nil) {
+ t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ if logrec, e := dag.getLogrec(oid, oldHead); e != nil || logrec != "logrec-02" {
+ t.Errorf("Invalid logrec for oldhead object %d:%d in DAG file %s: %v", oid, oldHead, dagfile, logrec)
+ }
+ if logrec, e := dag.getLogrec(oid, newHead); e != nil || logrec != "logrec-05" {
+ t.Errorf("Invalid logrec for newhead object %d:%d in DAG file %s: %v", oid, newHead, dagfile, logrec)
+ }
+ if logrec, e := dag.getLogrec(oid, ancestor); e != nil || logrec != "logrec-01" {
+ t.Errorf("Invalid logrec for ancestor object %d:%d in DAG file %s: %v", oid, ancestor, dagfile, logrec)
+ }
+
+ checkDAGStats(t, "remote-conf2-pre", 1, 6, 0, 0)
+
+ // Resolve the conflict by adding a new local v6 derived from v2 and v5 (this replay moves the head).
+ if err = dagReplayCommands(dag, "local-resolve-00.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ // Verify that the head moved to v6 and the parent map shows the resolution.
+ if head, e := dag.getHead(oid); e != nil || head != 6 {
+ t.Errorf("Object %d has wrong head after conflict resolution in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ exp[6] = []Version{2, 5}
+ pmap = dag.getParentMap(oid)
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map after conflict resolution in DAG file %s: (%v) instead of (%v)",
+ oid, dagfile, pmap, exp)
+ }
+
+ checkDAGStats(t, "remote-conf2-post", 1, 7, 0, 0)
+
+ if err := checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestAncestorIterator checks that the iterator goes over the correct set
+// of ancestor nodes for an object given a starting node. It should traverse
+// reconvergent DAG branches only visiting each ancestor once:
+// v0 -> v1 -> v2 -> v4 -> v5 -> v7 -> v8
+// |--> v3 ---| |
+// +--> v6 ---------------+
+// - Starting at v0 it should only cover v0.
+// - Starting at v2 it should only cover v0-v2.
+// - Starting at v5 it should only cover v0-v5.
+// - Starting at v8 it should cover all nodes (v0-v8).
+func TestAncestorIterator(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-01.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Loop checking the iteration behavior for different starting nodes.
+ for _, start := range []Version{0, 2, 5, 8} {
+ visitCount := make(map[Version]int)
+ err = dag.ancestorIter(oid, []Version{start},
+ func(oid ObjId, v Version, node *dagNode) error {
+ visitCount[v]++
+ return nil
+ })
+
+ // Check that all prior nodes are visited only once.
+ for i := Version(0); i < (start + 1); i++ {
+ if visitCount[i] != 1 {
+ t.Errorf("wrong visit count for iter on object %d node %d starting from node %d: %d instead of 1",
+ oid, i, start, visitCount[i])
+ }
+ }
+ }
+
+ // Make sure an error in the callback is returned through the iterator.
+ cbErr := errors.New("callback error")
+ err = dag.ancestorIter(oid, []Version{8}, func(oid ObjId, v Version, node *dagNode) error {
+ if v == 0 {
+ return cbErr
+ }
+ return nil
+ })
+ if err != cbErr {
+ t.Errorf("wrong error returned from callback: %v instead of %v", err, cbErr)
+ }
+
+ checkDAGStats(t, "ancestor-iter", 1, 9, 0, 0)
+
+ if err = checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestPruning tests sync pruning of the DAG for an object with 3 concurrent
+// updates (i.e. 2 conflict resolution convergent points). The pruning must
+// get rid of the DAG branches across the reconvergence points:
+// v0 -> v1 -> v2 -> v4 -> v5 -> v7 -> v8
+// |--> v3 ---| |
+// +--> v6 ---------------+
+// By pruning at v0, nothing is deleted.
+// Then by pruning at v1, only v0 is deleted.
+// Then by pruning at v5, v1-v4 are deleted leaving v5 and "v6 -> v7 -> v8".
+// Then by pruning at v7, v5-v6 are deleted leaving "v7 -> v8".
+// Then by pruning at v8, v7 is deleted leaving v8 as the head.
+// Then by pruning again at v8 nothing changes.
+func TestPruning(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-01.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ checkDAGStats(t, "prune-init", 1, 9, 0, 0)
+
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ exp := map[Version][]Version{0: nil, 1: {0}, 2: {1}, 3: {1}, 4: {2, 3}, 5: {4}, 6: {1}, 7: {5, 6}, 8: {7}}
+
+ // Loop pruning at an invalid version (333) then at v0, v5, v8 and again at v8.
+ testVersions := []Version{333, 0, 1, 5, 7, 8, 8}
+ delCounts := []int{0, 0, 1, 4, 2, 1, 0}
+ which := "prune-snip-"
+ remain := 9
+
+ for i, version := range testVersions {
+ del := 0
+ err = dag.prune(oid, version, func(lr string) error {
+ del++
+ return nil
+ })
+
+ if i == 0 && err == nil {
+ t.Errorf("pruning non-existent object %d:%d did not fail in DAG file %s", oid, version, dagfile)
+ } else if i > 0 && err != nil {
+ t.Errorf("pruning object %d:%d failed in DAG file %s: %v", oid, version, dagfile, err)
+ }
+
+ if del != delCounts[i] {
+ t.Errorf("pruning object %d:%d deleted %d log records instead of %d", oid, version, del, delCounts[i])
+ }
+
+ which += "*"
+ remain -= del
+ checkDAGStats(t, which, 1, int64(remain), 0, 0)
+
+ if head, err := dag.getHead(oid); err != nil || head != 8 {
+ t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ err = dag.pruneDone()
+ if err != nil {
+ t.Errorf("pruneDone() failed in DAG file %s: %v", dagfile, err)
+ }
+
+ // Remove pruned nodes from the expected parent map used to validate
+ // and set the parents of the pruned node to nil.
+ if version < 10 {
+ for j := Version(0); j < version; j++ {
+ delete(exp, j)
+ }
+ exp[version] = nil
+ }
+
+ pmap := dag.getParentMap(oid)
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
+ }
+ }
+
+ checkDAGStats(t, "prune-end", 1, 1, 0, 0)
+
+ err = dag.pruneAll(oid, func(lr string) error {
+ return nil
+ })
+ if err != nil {
+ t.Errorf("pruneAll() for object %d failed in DAG file %s: %v", oid, dagfile, err)
+ }
+
+ if err = checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestPruningCallbackError tests sync pruning of the DAG when the callback
+// function returns an error. The pruning must try to delete as many nodes
+// and log records as possible and properly adjust the parent pointers of
+// the pruning node. The object DAG is:
+// v0 -> v1 -> v2 -> v4 -> v5 -> v7 -> v8
+// |--> v3 ---| |
+// +--> v6 ---------------+
+// By pruning at v8 and having the callback function fail for v3, all other
+// nodes must be deleted and only v8 remains as the head.
+func TestPruningCallbackError(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-01.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ checkDAGStats(t, "prune-cb-init", 1, 9, 0, 0)
+
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ exp := map[Version][]Version{8: nil}
+
+ // Prune at v8 with a callback function that fails for v3.
+ del, expDel := 0, 8
+ version := Version(8)
+ err = dag.prune(oid, version, func(lr string) error {
+ del++
+ if lr == "logrec-03" {
+ return fmt.Errorf("refuse to delete %s", lr)
+ }
+ return nil
+ })
+
+ if err == nil {
+ t.Errorf("pruning object %d:%d did not fail in DAG file %s", oid, version, dagfile)
+ }
+ if del != expDel {
+ t.Errorf("pruning object %d:%d deleted %d log records instead of %d", oid, version, del, expDel)
+ }
+
+ err = dag.pruneDone()
+ if err != nil {
+ t.Errorf("pruneDone() failed in DAG file %s: %v", dagfile, err)
+ }
+
+ if head, err := dag.getHead(oid); err != nil || head != 8 {
+ t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ pmap := dag.getParentMap(oid)
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
+ }
+
+ checkDAGStats(t, "prune-cb-end", 1, 1, 0, 0)
+
+ if err = checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestRemoteLinkedNoConflictSameHead tests sync of remote updates that contain
+// linked nodes (conflict resolution by selecting an existing version) on top of
+// a local initial state without conflict. An object is created locally and
+// updated twice (v1 -> v2 -> v3). Another device has learned about v1, created
+// (v1 -> v4), then learned about (v1 -> v2) and resolved that conflict by selecting
+// v2 over v4. Now it sends that new info (v4 and the v2/v4 link) back to the
+// original (local) device. Instead of a v3/v4 conflict, the device sees that
+// v2 was chosen over v4 and resolves it as a no-conflict case.
+func TestRemoteLinkedNoConflictSameHead(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
+ t.Fatal(err)
+ }
+ if err = dagReplayCommands(dag, "remote-noconf-link-00.log.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ checkDAGStats(t, "linked-noconf", 1, 4, 0, 0)
+
+ // The head must not have moved (i.e. still at v3) and the parent map
+ // shows the newly grafted DAG fragment on top of the prior DAG.
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if head, e := dag.getHead(oid); e != nil || head != 3 {
+ t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ pmap := dag.getParentMap(oid)
+
+ exp := map[Version][]Version{1: nil, 2: {1, 4}, 3: {2}, 4: {1}}
+
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
+ }
+
+ // Verify the grafting of remote nodes.
+ newHeads, grafts := dag.getGraftNodes(oid)
+
+ expNewHeads := map[Version]struct{}{3: struct{}{}}
+ if !reflect.DeepEqual(newHeads, expNewHeads) {
+ t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
+ }
+
+ expgrafts := map[Version]uint64{1: 0, 4: 1}
+ if !reflect.DeepEqual(grafts, expgrafts) {
+ t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
+ }
+
+ // There should be no conflict.
+ isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
+ if !(!isConflict && newHead == 3 && oldHead == 3 && ancestor == NoVersion && errConflict == nil) {
+ t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ // Clear the grafting data and verify that hasConflict() fails without it.
+ dag.clearGraft()
+ isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
+ if errConflict == nil {
+ t.Errorf("hasConflict() on %d did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ if err := checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestRemoteLinkedConflict tests sync of remote updates that contain linked
+// nodes (conflict resolution by selecting an existing version) on top of a local
+// initial state triggering a local conflict. An object is created locally and
+// updated twice (v1 -> v2 -> v3). Another device has along the way learned about v1,
+// created (v1 -> v4), then learned about (v1 -> v2) and resolved that conflict by
+// selecting v4 over v2. Now it sends that new info (v4 and the v4/v2 link) back
+// to the original (local) device. The device sees a v3/v4 conflict.
+func TestRemoteLinkedConflict(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
+ t.Fatal(err)
+ }
+ if err = dagReplayCommands(dag, "remote-conf-link.log.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ checkDAGStats(t, "linked-conf", 1, 4, 0, 0)
+
+ // The head must not have moved (i.e. still at v2) and the parent map
+ // shows the newly grafted DAG fragment on top of the prior DAG.
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if head, e := dag.getHead(oid); e != nil || head != 3 {
+ t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ pmap := dag.getParentMap(oid)
+
+ exp := map[Version][]Version{1: nil, 2: {1}, 3: {2}, 4: {1, 2}}
+
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
+ }
+
+ // Verify the grafting of remote nodes.
+ newHeads, grafts := dag.getGraftNodes(oid)
+
+ expNewHeads := map[Version]struct{}{3: struct{}{}, 4: struct{}{}}
+ if !reflect.DeepEqual(newHeads, expNewHeads) {
+ t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
+ }
+
+ expgrafts := map[Version]uint64{1: 0, 2: 1}
+ if !reflect.DeepEqual(grafts, expgrafts) {
+ t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
+ }
+
+ // There should be a conflict.
+ isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
+ if !(isConflict && newHead == 4 && oldHead == 3 && ancestor == 2 && errConflict == nil) {
+ t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ // Clear the grafting data and verify that hasConflict() fails without it.
+ dag.clearGraft()
+ isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
+ if errConflict == nil {
+ t.Errorf("hasConflict() on %d did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ if err := checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestRemoteLinkedNoConflictNewHead tests sync of remote updates that contain
+// linked nodes (conflict resolution by selecting an existing version) on top of
+// a local initial state without conflict, but moves the head node to a new one.
+// An object is created locally and updated twice (v1 -> v2 -> v3). Another device
+// has along the way learned about v1, created (v1 -> v4), then learned about
+// (v1 -> v2 -> v3) and resolved that conflict by selecting v4 over v3. Now it
+// sends that new info (v4 and the v4/v3 link) back to the original (local) device.
+// The device sees that the new head v4 is "derived" from v3 thus no conflict.
+func TestRemoteLinkedConflictNewHead(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
+ t.Fatal(err)
+ }
+ if err = dagReplayCommands(dag, "remote-noconf-link-01.log.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ checkDAGStats(t, "linked-conf2", 1, 4, 0, 0)
+
+ // The head must not have moved (i.e. still at v2) and the parent map
+ // shows the newly grafted DAG fragment on top of the prior DAG.
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if head, e := dag.getHead(oid); e != nil || head != 3 {
+ t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ pmap := dag.getParentMap(oid)
+
+ exp := map[Version][]Version{1: nil, 2: {1}, 3: {2}, 4: {1, 3}}
+
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
+ }
+
+ // Verify the grafting of remote nodes.
+ newHeads, grafts := dag.getGraftNodes(oid)
+
+ expNewHeads := map[Version]struct{}{4: struct{}{}}
+ if !reflect.DeepEqual(newHeads, expNewHeads) {
+ t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
+ }
+
+ expgrafts := map[Version]uint64{1: 0, 3: 2}
+ if !reflect.DeepEqual(grafts, expgrafts) {
+ t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
+ }
+
+ // There should be no conflict.
+ isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
+ if !(!isConflict && newHead == 4 && oldHead == 3 && ancestor == NoVersion && errConflict == nil) {
+ t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ // Clear the grafting data and verify that hasConflict() fails without it.
+ dag.clearGraft()
+ isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
+ if errConflict == nil {
+ t.Errorf("hasConflict() on %d did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ if err := checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestRemoteLinkedNoConflictNewHeadOvertake tests sync of remote updates that
+// contain linked nodes (conflict resolution by selecting an existing version)
+// on top of a local initial state without conflict, but moves the head node
+// to a new one that overtook the linked node.
+// An object is created locally and updated twice (v1 -> v2 -> v3). Another
+// device has along the way learned about v1, created (v1 -> v4), then learned
+// about (v1 -> v2 -> v3) and resolved that conflict by selecting v3 over v4.
+// Then it creates a new update v5 from v3 (v3 -> v5). Now it sends that new
+// info (v4, the v3/v4 link, and v5) back to the original (local) device.
+// The device sees that the new head v5 is "derived" from v3 thus no conflict.
+func TestRemoteLinkedConflictNewHeadOvertake(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-00.log.sync"); err != nil {
+ t.Fatal(err)
+ }
+ if err = dagReplayCommands(dag, "remote-noconf-link-02.log.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ checkDAGStats(t, "linked-conf3-pre", 1, 5, 0, 0)
+
+ // The head must not have moved (i.e. still at v2) and the parent map
+ // shows the newly grafted DAG fragment on top of the prior DAG.
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if head, e := dag.getHead(oid); e != nil || head != 3 {
+ t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ pmap := dag.getParentMap(oid)
+
+ exp := map[Version][]Version{1: nil, 2: {1}, 3: {2, 4}, 4: {1}, 5: {3}}
+
+ if !reflect.DeepEqual(pmap, exp) {
+ t.Errorf("Invalid object %d parent map in DAG file %s: (%v) instead of (%v)", oid, dagfile, pmap, exp)
+ }
+
+ // Verify the grafting of remote nodes.
+ newHeads, grafts := dag.getGraftNodes(oid)
+
+ expNewHeads := map[Version]struct{}{5: struct{}{}}
+ if !reflect.DeepEqual(newHeads, expNewHeads) {
+ t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
+ }
+
+ expgrafts := map[Version]uint64{1: 0, 3: 2, 4: 1}
+ if !reflect.DeepEqual(grafts, expgrafts) {
+ t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
+ }
+
+ // There should be no conflict.
+ isConflict, newHead, oldHead, ancestor, errConflict := dag.hasConflict(oid)
+ if !(!isConflict && newHead == 5 && oldHead == 3 && ancestor == NoVersion && errConflict == nil) {
+ t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ // Then we can move the head and clear the grafting data.
+ if err = dag.moveHead(oid, newHead); err != nil {
+ t.Errorf("Object %d cannot move head to %d in DAG file %s: %v", oid, newHead, dagfile, err)
+ }
+
+ // Clear the grafting data and verify that hasConflict() fails without it.
+ dag.clearGraft()
+ isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
+ if errConflict == nil {
+ t.Errorf("hasConflict() on %d did not fail w/o graft info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ // Now new info comes from another device repeating the v2/v3 link.
+ // Verify that it is a NOP (no changes).
+ if err = dagReplayCommands(dag, "remote-noconf-link-repeat.log.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ if head, e := dag.getHead(oid); e != nil || head != 5 {
+ t.Errorf("Object %d has wrong head in DAG file %s: %d", oid, dagfile, head)
+ }
+
+ newHeads, grafts = dag.getGraftNodes(oid)
+ if !reflect.DeepEqual(newHeads, expNewHeads) {
+ t.Errorf("Object %d has invalid newHeads in DAG file %s: (%v) instead of (%v)", oid, dagfile, newHeads, expNewHeads)
+ }
+
+ expgrafts = map[Version]uint64{}
+ if !reflect.DeepEqual(grafts, expgrafts) {
+ t.Errorf("Invalid object %d graft in DAG file %s: (%v) instead of (%v)", oid, dagfile, grafts, expgrafts)
+ }
+
+ isConflict, newHead, oldHead, ancestor, errConflict = dag.hasConflict(oid)
+ if !(!isConflict && newHead == 5 && oldHead == 5 && ancestor == NoVersion && errConflict == nil) {
+ t.Errorf("Object %d wrong conflict info: flag %t, newHead %d, oldHead %d, ancestor %d, err %v",
+ oid, isConflict, newHead, oldHead, ancestor, errConflict)
+ }
+
+ checkDAGStats(t, "linked-conf3-post", 1, 5, 0, 0)
+
+ if err := checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestAddNodeTransactional tests adding multiple DAG nodes grouped within a transaction.
+func TestAddNodeTransactional(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-02.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ checkDAGStats(t, "add-tx-init", 3, 5, 0, 0)
+
+ oid_a, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+ oid_b, err := strToObjId("6789")
+ if err != nil {
+ t.Fatal(err)
+ }
+ oid_c, err := strToObjId("2222")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Verify NoTxId is reported as an error.
+ if err := dag.addNodeTxEnd(NoTxId, 0); err == nil {
+ t.Errorf("addNodeTxEnd() did not fail for invalid 'NoTxId' value")
+ }
+ if _, err := dag.getTransaction(NoTxId); err == nil {
+ t.Errorf("getTransaction() did not fail for invalid 'NoTxId' value")
+ }
+ if err := dag.setTransaction(NoTxId, nil); err == nil {
+ t.Errorf("setTransaction() did not fail for invalid 'NoTxId' value")
+ }
+ if err := dag.delTransaction(NoTxId); err == nil {
+ t.Errorf("delTransaction() did not fail for invalid 'NoTxId' value")
+ }
+
+ // Mutate 2 objects within a transaction.
+ tid_1 := dag.addNodeTxStart(NoTxId)
+ if tid_1 == NoTxId {
+ t.Fatal("Cannot start 1st DAG addNode() transaction")
+ }
+ if err := dag.addNodeTxEnd(tid_1, 0); err == nil {
+ t.Errorf("addNodeTxEnd() did not fail for a zero-count transaction")
+ }
+
+ txSt, ok := dag.txSet[tid_1]
+ if !ok {
+ t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_1, dagfile)
+ }
+ if n := len(txSt.TxMap); n != 0 {
+ t.Errorf("Transactions map for Tx ID %v has length %d instead of 0 in DAG file %s", tid_1, n, dagfile)
+ }
+
+ if err := dag.addNode(oid_a, 3, false, false, []Version{2}, "logrec-a-03", tid_1); err != nil {
+ t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_a, tid_1, dagfile, err)
+ }
+
+ if tTmp := dag.addNodeTxStart(tid_1); tTmp != tid_1 {
+ t.Fatal("restarting transaction failed")
+ }
+
+ if err := dag.addNode(oid_b, 3, false, false, []Version{2}, "logrec-b-03", tid_1); err != nil {
+ t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_b, tid_1, dagfile, err)
+ }
+
+ // At the same time mutate the 3rd object in another transaction.
+ tid_2 := dag.addNodeTxStart(NoTxId)
+ if tid_2 == NoTxId {
+ t.Fatal("Cannot start 2nd DAG addNode() transaction")
+ }
+
+ txSt, ok = dag.txSet[tid_2]
+ if !ok {
+ t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_2, dagfile)
+ }
+ if n := len(txSt.TxMap); n != 0 {
+ t.Errorf("Transactions map for Tx ID %v has length %d instead of 0 in DAG file %s", tid_2, n, dagfile)
+ }
+
+ if err := dag.addNode(oid_c, 2, false, false, []Version{1}, "logrec-c-02", tid_2); err != nil {
+ t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_c, tid_2, dagfile, err)
+ }
+
+ // Verify the in-memory transaction sets constructed.
+ txSt, ok = dag.txSet[tid_1]
+ if !ok {
+ t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_1, dagfile)
+ }
+ expTxSt := &dagTxState{dagTxMap{oid_a: 3, oid_b: 3}, 0}
+ if !reflect.DeepEqual(txSt, expTxSt) {
+ t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_1, dagfile, txSt, expTxSt)
+ }
+
+ txSt, ok = dag.txSet[tid_2]
+ if !ok {
+ t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_2, dagfile)
+ }
+ expTxSt = &dagTxState{dagTxMap{oid_c: 2}, 0}
+ if !reflect.DeepEqual(txSt, expTxSt) {
+ t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txSt, expTxSt)
+ }
+
+ // Verify failing to use a Tx ID not returned by addNodeTxStart().
+ bad_tid := tid_1 + 1
+ for bad_tid == NoTxId || bad_tid == tid_2 {
+ bad_tid++
+ }
+
+ if err := dag.addNode(oid_c, 3, false, false, []Version{2}, "logrec-c-03", bad_tid); err == nil {
+ t.Errorf("addNode() did not fail on object %d for a bad Tx ID %v in DAG file %s", oid_c, bad_tid, dagfile)
+ }
+ if err := dag.addNodeTxEnd(bad_tid, 1); err == nil {
+ t.Errorf("addNodeTxEnd() did not fail for a bad Tx ID %v in DAG file %s", bad_tid, dagfile)
+ }
+
+ // End the 1st transaction and verify the in-memory and in-DAG data.
+ if err := dag.addNodeTxEnd(tid_1, 2); err != nil {
+ t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
+ }
+
+ checkDAGStats(t, "add-tx-1", 3, 8, 1, 0)
+
+ if _, ok = dag.txSet[tid_1]; ok {
+ t.Errorf("Transactions state for Tx ID %v still exists in DAG file %s", tid_1, dagfile)
+ }
+
+ txSt, err = dag.getTransaction(tid_1)
+ if err != nil {
+ t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
+ }
+
+ expTxSt = &dagTxState{dagTxMap{oid_a: 3, oid_b: 3}, 2}
+ if !reflect.DeepEqual(txSt, expTxSt) {
+ t.Errorf("Invalid transaction state from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
+ tid_1, dagfile, txSt, expTxSt)
+ }
+
+ txSt, ok = dag.txSet[tid_2]
+ if !ok {
+ t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_2, dagfile)
+ }
+
+ expTxSt = &dagTxState{dagTxMap{oid_c: 2}, 0}
+ if !reflect.DeepEqual(txSt, expTxSt) {
+ t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txSt, expTxSt)
+ }
+
+ // End the 2nd transaction and re-verify the in-memory and in-DAG data.
+ if err := dag.addNodeTxEnd(tid_2, 1); err != nil {
+ t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
+ }
+
+ checkDAGStats(t, "add-tx-2", 3, 8, 2, 0)
+
+ if _, ok = dag.txSet[tid_2]; ok {
+ t.Errorf("Transactions state for Tx ID %v still exists in DAG file %s", tid_2, dagfile)
+ }
+
+ txSt, err = dag.getTransaction(tid_2)
+ if err != nil {
+ t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
+ }
+
+ expTxSt = &dagTxState{dagTxMap{oid_c: 2}, 1}
+ if !reflect.DeepEqual(txSt, expTxSt) {
+ t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txSt, expTxSt)
+ }
+
+ if n := len(dag.txSet); n != 0 {
+ t.Errorf("Transaction sets in-memory: %d entries found, should be empty in DAG file %s", n, dagfile)
+ }
+
+ // Test incrementally filling up a transaction.
+ tid_3 := TxId(100)
+ if _, ok = dag.txSet[tid_3]; ok {
+ t.Errorf("Transactions state for Tx ID %v found in DAG file %s", tid_3, dagfile)
+ }
+
+ if tTmp := dag.addNodeTxStart(tid_3); tTmp != tid_3 {
+ t.Fatalf("Cannot start transaction %v", tid_3)
+ }
+
+ txSt, ok = dag.txSet[tid_3]
+ if !ok {
+ t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_3, dagfile)
+ }
+ if n := len(txSt.TxMap); n != 0 {
+ t.Errorf("Transactions map for Tx ID %v has length %d instead of 0 in DAG file %s", tid_3, n, dagfile)
+ }
+
+ if err := dag.addNode(oid_a, 4, false, false, []Version{3}, "logrec-a-04", tid_3); err != nil {
+ t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_a, tid_3, dagfile, err)
+ }
+
+ if err := dag.addNodeTxEnd(tid_3, 2); err != nil {
+ t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_3, dagfile, err)
+ }
+
+ checkDAGStats(t, "add-tx-3", 3, 9, 3, 0)
+
+ if _, ok = dag.txSet[tid_3]; ok {
+ t.Errorf("Transactions state for Tx ID %v still exists in DAG file %s", tid_3, dagfile)
+ }
+
+ txSt, err = dag.getTransaction(tid_3)
+ if err != nil {
+ t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_3, dagfile, err)
+ }
+
+ expTxSt = &dagTxState{dagTxMap{oid_a: 4}, 2}
+ if !reflect.DeepEqual(txSt, expTxSt) {
+ t.Errorf("Invalid transaction state from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
+ tid_3, dagfile, txSt, expTxSt)
+ }
+
+ if tTmp := dag.addNodeTxStart(tid_3); tTmp != tid_3 {
+ t.Fatalf("Cannot start transaction %v", tid_3)
+ }
+
+ txSt, ok = dag.txSet[tid_3]
+ if !ok {
+ t.Errorf("Transactions state for Tx ID %v not found in DAG file %s", tid_3, dagfile)
+ }
+
+ if !reflect.DeepEqual(txSt, expTxSt) {
+ t.Errorf("Invalid transaction state from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
+ tid_3, dagfile, txSt, expTxSt)
+ }
+
+ if err := dag.addNode(oid_b, 4, false, false, []Version{3}, "logrec-b-04", tid_3); err != nil {
+ t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_b, tid_3, dagfile, err)
+ }
+
+ if err := dag.addNodeTxEnd(tid_3, 3); err == nil {
+ t.Errorf("addNodeTxEnd() didn't fail for Tx ID %v in DAG file %s: %v", tid_3, dagfile, err)
+ }
+
+ if err := dag.addNodeTxEnd(tid_3, 2); err != nil {
+ t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_3, dagfile, err)
+ }
+
+ checkDAGStats(t, "add-tx-4", 3, 10, 3, 0)
+
+ txSt, err = dag.getTransaction(tid_3)
+ if err != nil {
+ t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_3, dagfile, err)
+ }
+
+ expTxSt = &dagTxState{dagTxMap{oid_a: 4, oid_b: 4}, 2}
+ if !reflect.DeepEqual(txSt, expTxSt) {
+ t.Errorf("Invalid transaction state from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
+ tid_3, dagfile, txSt, expTxSt)
+ }
+
+ // Get the 3 new nodes from the DAG and verify their Tx IDs.
+ node, err := dag.getNode(oid_a, 3)
+ if err != nil {
+ t.Errorf("Cannot find object %d:3 in DAG file %s: %v", oid_a, dagfile, err)
+ }
+ if node.TxId != tid_1 {
+ t.Errorf("Invalid TxId for object %d:3 in DAG file %s: %v instead of %v", oid_a, dagfile, node.TxId, tid_1)
+ }
+ node, err = dag.getNode(oid_a, 4)
+ if err != nil {
+ t.Errorf("Cannot find object %d:4 in DAG file %s: %v", oid_a, dagfile, err)
+ }
+ if node.TxId != tid_3 {
+ t.Errorf("Invalid TxId for object %d:4 in DAG file %s: %v instead of %v", oid_a, dagfile, node.TxId, tid_3)
+ }
+ node, err = dag.getNode(oid_b, 3)
+ if err != nil {
+ t.Errorf("Cannot find object %d:3 in DAG file %s: %v", oid_b, dagfile, err)
+ }
+ if node.TxId != tid_1 {
+ t.Errorf("Invalid TxId for object %d:3 in DAG file %s: %v instead of %v", oid_b, dagfile, node.TxId, tid_1)
+ }
+ node, err = dag.getNode(oid_b, 4)
+ if err != nil {
+ t.Errorf("Cannot find object %d:4 in DAG file %s: %v", oid_b, dagfile, err)
+ }
+ if node.TxId != tid_3 {
+ t.Errorf("Invalid TxId for object %d:4 in DAG file %s: %v instead of %v", oid_b, dagfile, node.TxId, tid_3)
+ }
+ node, err = dag.getNode(oid_c, 2)
+ if err != nil {
+ t.Errorf("Cannot find object %d:2 in DAG file %s: %v", oid_c, dagfile, err)
+ }
+ if node.TxId != tid_2 {
+ t.Errorf("Invalid TxId for object %d:2 in DAG file %s: %v instead of %v", oid_c, dagfile, node.TxId, tid_2)
+ }
+
+ for _, oid := range []ObjId{oid_a, oid_b, oid_c} {
+ if err := checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+// TestPruningTransactions tests pruning DAG nodes grouped within transactions.
+func TestPruningTransactions(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-02.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ checkDAGStats(t, "prune-tx-init", 3, 5, 0, 0)
+
+ oid_a, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+ oid_b, err := strToObjId("6789")
+ if err != nil {
+ t.Fatal(err)
+ }
+ oid_c, err := strToObjId("2222")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Mutate objects in 2 transactions then add non-transactional mutations
+ // to act as the pruning points. Before pruning the DAG is:
+ // a1 -- a2 -- (a3) --- a4
+ // b1 -- b2 -- (b3) -- (b4) -- b5
+ // c1 ---------------- (c2)
+ // Now by pruning at (a4, b5, c2), the new DAG should be:
+ // a4
+ // b5
+ // (c2)
+ // Transaction 1 (a3, b3) gets deleted, but transaction 2 (b4, c2) still
+ // has (c2) dangling waiting for a future pruning.
+ tid_1 := dag.addNodeTxStart(NoTxId)
+ if tid_1 == NoTxId {
+ t.Fatal("Cannot start 1st DAG addNode() transaction")
+ }
+ if err := dag.addNode(oid_a, 3, false, false, []Version{2}, "logrec-a-03", tid_1); err != nil {
+ t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_a, tid_1, dagfile, err)
+ }
+ if err := dag.addNode(oid_b, 3, false, false, []Version{2}, "logrec-b-03", tid_1); err != nil {
+ t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_b, tid_1, dagfile, err)
+ }
+ if err := dag.addNodeTxEnd(tid_1, 2); err != nil {
+ t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
+ }
+
+ checkDAGStats(t, "prune-tx-1", 3, 7, 1, 0)
+
+ tid_2 := dag.addNodeTxStart(NoTxId)
+ if tid_2 == NoTxId {
+ t.Fatal("Cannot start 2nd DAG addNode() transaction")
+ }
+ if err := dag.addNode(oid_b, 4, false, false, []Version{3}, "logrec-b-04", tid_2); err != nil {
+ t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_b, tid_2, dagfile, err)
+ }
+ if err := dag.addNode(oid_c, 2, false, false, []Version{1}, "logrec-c-02", tid_2); err != nil {
+ t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_c, tid_2, dagfile, err)
+ }
+ if err := dag.addNodeTxEnd(tid_2, 2); err != nil {
+ t.Errorf("Cannot addNodeTxEnd() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
+ }
+
+ checkDAGStats(t, "prune-tx-2", 3, 9, 2, 0)
+
+ if err := dag.addNode(oid_a, 4, false, false, []Version{3}, "logrec-a-04", NoTxId); err != nil {
+ t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_a, tid_1, dagfile, err)
+ }
+ if err := dag.addNode(oid_b, 5, false, false, []Version{4}, "logrec-b-05", NoTxId); err != nil {
+ t.Errorf("Cannot addNode() on object %d and Tx ID %v in DAG file %s: %v", oid_b, tid_2, dagfile, err)
+ }
+
+ if err = dag.moveHead(oid_a, 4); err != nil {
+ t.Errorf("Object %d cannot move head in DAG file %s: %v", oid_a, dagfile, err)
+ }
+ if err = dag.moveHead(oid_b, 5); err != nil {
+ t.Errorf("Object %d cannot move head in DAG file %s: %v", oid_b, dagfile, err)
+ }
+ if err = dag.moveHead(oid_c, 2); err != nil {
+ t.Errorf("Object %d cannot move head in DAG file %s: %v", oid_c, dagfile, err)
+ }
+
+ checkDAGStats(t, "prune-tx-3", 3, 11, 2, 0)
+
+ // Verify the transaction sets.
+ txSt, err := dag.getTransaction(tid_1)
+ if err != nil {
+ t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_1, dagfile, err)
+ }
+
+ expTxSt := &dagTxState{dagTxMap{oid_a: 3, oid_b: 3}, 2}
+ if !reflect.DeepEqual(txSt, expTxSt) {
+ t.Errorf("Invalid transaction state from DAG storage for Tx ID %v in DAG file %s: %v instead of %v",
+ tid_1, dagfile, txSt, expTxSt)
+ }
+
+ txSt, err = dag.getTransaction(tid_2)
+ if err != nil {
+ t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
+ }
+
+ expTxSt = &dagTxState{dagTxMap{oid_b: 4, oid_c: 2}, 2}
+ if !reflect.DeepEqual(txSt, expTxSt) {
+ t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txSt, expTxSt)
+ }
+
+ // Prune the 3 objects at their head nodes.
+ for _, oid := range []ObjId{oid_a, oid_b, oid_c} {
+ head, err := dag.getHead(oid)
+ if err != nil {
+ t.Errorf("Cannot getHead() on object %d in DAG file %s: %v", oid, dagfile, err)
+ }
+ err = dag.prune(oid, head, func(lr string) error {
+ return nil
+ })
+ if err != nil {
+ t.Errorf("Cannot prune() on object %d in DAG file %s: %v", oid, dagfile, err)
+ }
+ }
+
+ if err = dag.pruneDone(); err != nil {
+ t.Errorf("pruneDone() failed in DAG file %s: %v", dagfile, err)
+ }
+
+ if n := len(dag.txGC); n != 0 {
+ t.Errorf("Transaction GC map not empty after pruneDone() in DAG file %s: %d", dagfile, n)
+ }
+
+ // Verify that Tx-1 was deleted and Tx-2 still has c2 in it.
+ checkDAGStats(t, "prune-tx-4", 3, 3, 1, 0)
+
+ txSt, err = dag.getTransaction(tid_1)
+ if err == nil {
+ t.Errorf("getTransaction() did not fail for Tx ID %v in DAG file %s: %v", tid_1, dagfile, txSt)
+ }
+
+ txSt, err = dag.getTransaction(tid_2)
+ if err != nil {
+ t.Errorf("Cannot getTransaction() for Tx ID %v in DAG file %s: %v", tid_2, dagfile, err)
+ }
+
+ expTxSt = &dagTxState{dagTxMap{oid_c: 2}, 2}
+ if !reflect.DeepEqual(txSt, expTxSt) {
+ t.Errorf("Invalid transaction state for Tx ID %v in DAG file %s: %v instead of %v", tid_2, dagfile, txSt, expTxSt)
+ }
+
+ // Add c3 as a new head and prune at that point. This should GC Tx-2.
+ if err := dag.addNode(oid_c, 3, false, false, []Version{2}, "logrec-c-03", NoTxId); err != nil {
+ t.Errorf("Cannot addNode() on object %d in DAG file %s: %v", oid_c, dagfile, err)
+ }
+ if err = dag.moveHead(oid_c, 3); err != nil {
+ t.Errorf("Object %d cannot move head in DAG file %s: %v", oid_c, dagfile, err)
+ }
+
+ checkDAGStats(t, "prune-tx-5", 3, 4, 1, 0)
+
+ err = dag.prune(oid_c, 3, func(lr string) error {
+ return nil
+ })
+ if err != nil {
+ t.Errorf("Cannot prune() on object %d in DAG file %s: %v", oid_c, dagfile, err)
+ }
+ if err = dag.pruneDone(); err != nil {
+ t.Errorf("pruneDone() #2 failed in DAG file %s: %v", dagfile, err)
+ }
+ if n := len(dag.txGC); n != 0 {
+ t.Errorf("Transaction GC map not empty after pruneDone() in DAG file %s: %d", dagfile, n)
+ }
+
+ checkDAGStats(t, "prune-tx-6", 3, 3, 0, 0)
+
+ txSt, err = dag.getTransaction(tid_2)
+ if err == nil {
+ t.Errorf("getTransaction() did not fail for Tx ID %v in DAG file %s: %v", tid_2, dagfile, txSt)
+ }
+
+ for _, oid := range []ObjId{oid_a, oid_b, oid_c} {
+ if err := checkEndOfSync(dag, oid); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+// TestHasDeletedDescendant tests lookup of DAG deleted nodes descending from a given node.
+func TestHasDeletedDescendant(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ if err = dagReplayCommands(dag, "local-init-03.sync"); err != nil {
+ t.Fatal(err)
+ }
+
+ oid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Delete node v3 to create a dangling parent link from v7 (increase code coverage).
+ if err = dag.delNode(oid, 3); err != nil {
+ t.Errorf("cannot delete node %d:3 in DAG file %s: %v", oid, dagfile, err)
+ }
+
+ type hasDelDescTest struct {
+ node Version
+ result bool
+ }
+ tests := []hasDelDescTest{
+ {NoVersion, false},
+ {999, false},
+ {1, true},
+ {2, true},
+ {3, false},
+ {4, false},
+ {5, false},
+ {6, false},
+ {7, false},
+ {8, false},
+ }
+
+ for _, test := range tests {
+ result := dag.hasDeletedDescendant(oid, test.node)
+ if result != test.result {
+ t.Errorf("hasDeletedDescendant() for node %d in DAG file %s: %v instead of %v",
+ test.node, dagfile, result, test.result)
+ }
+ }
+
+ dag.close()
+}
+
+// TestPrivNode tests access to the private nodes table in a DAG.
+func TestPrivNode(t *testing.T) {
+ dagfile := dagFilename()
+ defer os.Remove(dagfile)
+
+ dag, err := openDAG(dagfile)
+ if err != nil {
+ t.Fatalf("Cannot open new DAG file %s", dagfile)
+ }
+
+ oid, err := strToObjId("2222")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ priv, err := dag.getPrivNode(oid)
+ if err == nil || priv != nil {
+ t.Errorf("Found non-existing private object %d in DAG file %s: %v, err %v", oid, dagfile, priv, err)
+ }
+
+ priv = &privNode{
+ //Mutation: &raw.Mutation{ID: oid, PriorVersion: 0x0, Version: 0x55104dc76695721d, Value: "value-foobar"},
+ PathIDs: []ObjId{oid, ObjId("haha"), ObjId("foobar")},
+ TxId: 56789,
+ }
+
+ if err = dag.setPrivNode(oid, priv); err != nil {
+ t.Fatalf("Cannot set private object %d (%v) in DAG file %s: %v", oid, priv, dagfile, err)
+ }
+
+ checkDAGStats(t, "priv-1", 0, 0, 0, 1)
+
+ priv2, err := dag.getPrivNode(oid)
+ if err != nil {
+ t.Fatalf("Cannot get private object %d from DAG file %s: %v", oid, dagfile, err)
+ }
+ if !reflect.DeepEqual(priv2, priv) {
+ t.Errorf("Private object %d has wrong data in DAG file %s: %v instead of %v", oid, dagfile, priv2, priv)
+ }
+
+ //priv.Mutation.PriorVersion = priv.Mutation.Version
+ //priv.Mutation.Version = 0x55555ddddd345abc
+ //priv.Mutation.Value = "value-new"
+ priv.TxId = 98765
+
+ if err = dag.setPrivNode(oid, priv); err != nil {
+ t.Fatalf("Cannot overwrite private object %d (%v) in DAG file %s: %v", oid, priv, dagfile, err)
+ }
+
+ checkDAGStats(t, "priv-1", 0, 0, 0, 1)
+
+ priv2, err = dag.getPrivNode(oid)
+ if err != nil {
+ t.Fatalf("Cannot get updated private object %d from DAG file %s: %v", oid, dagfile, err)
+ }
+ if !reflect.DeepEqual(priv2, priv) {
+ t.Errorf("Private object %d has wrong data post-update in DAG file %s: %v instead of %v", oid, dagfile, priv2, priv)
+ }
+
+ err = dag.delPrivNode(oid)
+ if err != nil {
+ t.Fatalf("Cannot delete private object %d in DAG file %s: %v", oid, dagfile, err)
+ }
+
+ checkDAGStats(t, "priv-1", 0, 0, 0, 0)
+
+ priv3, err := dag.getPrivNode(oid)
+ if err == nil || priv3 != nil {
+ t.Errorf("Found deleted private object %d in DAG file %s: %v, err %v", oid, dagfile, priv3, err)
+ }
+
+ dag.close()
+}
diff --git a/services/syncbase/sync/devtable.go b/services/syncbase/sync/devtable.go
new file mode 100644
index 0000000..9a463a1
--- /dev/null
+++ b/services/syncbase/sync/devtable.go
@@ -0,0 +1,546 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Package vsync provides veyron sync DevTable utility functions.
+// DevTable is indexed by the device id and stores device level
+// information needed by sync. Main component of a device's info are
+// a set of generation vectors: one per SyncRoot. Generation vector
+// is the version vector for a device's view of a SyncRoot,
+// representing all the different generations (from different devices)
+// seen by that device for that SyncRoot. A generation represents a
+// collection of updates that originated on a device during an
+// interval of time. It serves as a checkpoint when communicating with
+// other devices. Generations do not overlap and all updates belong to
+// a generation.
+//
+// Synchronization for a given SyncRoot between two devices A and B
+// uses generation vectors as follows:
+// A B
+// <== B's generation vector
+// diff(A's generation vector, B's generation vector)
+// log records of missing generations ==>
+// cache B's generation vector (for space reclamation)
+//
+// Implementation notes: DevTable is stored in a persistent K/V
+// database in the current implementation. Generation vector is
+// implemented as a map of (Device ID -> Generation ID), one entry for
+// every known device. If the generation vector contains an entry
+// (Device ID -> Generation ID), it implies that the device has
+// learned of all the generations until and including Generation
+// ID. Generation IDs start from 1. A generation ID of 0 is a
+// reserved bootstrap value, and indicates the device has no updates.
+import (
+ "errors"
+ "fmt"
+ "sort"
+ "time"
+
+ "v.io/x/lib/vlog"
+)
+
+var (
+ errInvalidDTab = errors.New("invalid devtable db")
+)
+
+// devInfo is the information stored per device.
+type devInfo struct {
+ Vectors map[ObjId]GenVector // device generation vectors.
+ Ts time.Time // last communication time stamp.
+}
+
+// devTableHeader contains the header metadata.
+type devTableHeader struct {
+ Resmark []byte // resume marker for watch.
+ // Generation vector for space reclamation. All generations
+ // less than this generation vector are deleted from storage.
+ ReclaimVec GenVector
+}
+
+// devTable contains the metadata for the device table db.
+type devTable struct {
+ fname string // file pathname.
+ db *kvdb // underlying K/V DB.
+ devices *kvtable // pointer to the "devices" table in the kvdb. Contains device info.
+
+ // Key:"Head" Value:devTableHeader
+ header *kvtable // pointer to the "header" table in the kvdb. Contains device table header.
+ head *devTableHeader // devTable head cached in memory.
+
+ s *syncd // pointer to the sync daemon object.
+}
+
+// genOrder represents a generation along with its position in the log.
+type genOrder struct {
+ devID DeviceId
+ srID ObjId
+ genID GenId
+ order uint32
+}
+
+// byOrder is used to sort the genOrder array.
+type byOrder []*genOrder
+
+func (a byOrder) Len() int {
+ return len(a)
+}
+
+func (a byOrder) Swap(i, j int) {
+ a[i], a[j] = a[j], a[i]
+}
+
+func (a byOrder) Less(i, j int) bool {
+ return a[i].order < a[j].order
+}
+
+// openDevTable opens or creates a devTable for the given filename.
+func openDevTable(filename string, sin *syncd) (*devTable, error) {
+ dtab := &devTable{
+ fname: filename,
+ s: sin,
+ }
+ // Open the file and create it if it does not exist.
+ // Also initialize the kvdb and its collection.
+ db, tbls, err := kvdbOpen(filename, []string{"devices", "header"})
+ if err != nil {
+ return nil, err
+ }
+
+ dtab.db = db
+ dtab.devices = tbls[0]
+ dtab.header = tbls[1]
+
+ // Initialize the devTable header.
+ dtab.head = &devTableHeader{
+ ReclaimVec: GenVector{
+ dtab.s.id: 0,
+ },
+ }
+ // If header already exists in db, read it back from db.
+ if dtab.hasHead() {
+ if err := dtab.getHead(); err != nil {
+ dtab.db.close() // this also closes the tables.
+ return nil, err
+ }
+ }
+
+ return dtab, nil
+}
+
+// close closes the devTable and invalidates its struct.
+func (dt *devTable) close() error {
+ if dt.db == nil {
+ return errInvalidDTab
+ }
+ // Flush the dirty data.
+ if err := dt.flush(); err != nil {
+ return err
+ }
+ dt.db.close() // this also closes the tables.
+
+ *dt = devTable{} // zero out the devTable struct.
+ return nil
+}
+
+// flush flushes the devTable db to storage.
+func (dt *devTable) flush() error {
+ if dt.db == nil {
+ return errInvalidDTab
+ }
+ // Set the head from memory before flushing.
+ if err := dt.putHead(); err != nil {
+ return err
+ }
+ dt.db.flush()
+ return nil
+}
+
+// initSyncRoot initializes the local generation vector for this SyncRoot.
+func (dt *devTable) initSyncRoot(srid ObjId) error {
+ if dt.db == nil {
+ return errInvalidDTab
+ }
+ if _, err := dt.getGenVec(dt.s.id, srid); err == nil {
+ return fmt.Errorf("syncroot already exists %v", srid)
+ }
+ return dt.putGenVec(dt.s.id, srid, GenVector{dt.s.id: 0})
+}
+
+// delSyncRoot deletes the generation vector for this SyncRoot.
+func (dt *devTable) delSyncRoot(srid ObjId) error {
+ if dt.db == nil {
+ return errInvalidDTab
+ }
+ info, err := dt.getDevInfo(dt.s.id)
+ if err != nil {
+ return fmt.Errorf("dev doesn't exists %v", dt.s.id)
+ }
+ if _, ok := info.Vectors[srid]; !ok {
+ return fmt.Errorf("syncroot doesn't exist %v", srid)
+ }
+ delete(info.Vectors, srid)
+ if len(info.Vectors) == 0 {
+ return dt.delDevInfo(dt.s.id)
+ }
+ return dt.putDevInfo(dt.s.id, info)
+}
+
+// putHead puts the devTable head into the devTable db.
+func (dt *devTable) putHead() error {
+ return dt.header.set("Head", dt.head)
+}
+
+// getHead gets the devTable head from the devTable db.
+func (dt *devTable) getHead() error {
+ if dt.head == nil {
+ return errors.New("nil devTable header")
+ }
+ return dt.header.get("Head", dt.head)
+}
+
+// hasHead returns true if the devTable db has a devTable head.
+func (dt *devTable) hasHead() bool {
+ return dt.header.hasKey("Head")
+}
+
+// putDevInfo puts a devInfo struct in the devTable db.
+func (dt *devTable) putDevInfo(devid DeviceId, info *devInfo) error {
+ if dt.db == nil {
+ return errInvalidDTab
+ }
+ return dt.devices.set(string(devid), info)
+}
+
+// getDevInfo gets a devInfo struct from the devTable db.
+func (dt *devTable) getDevInfo(devid DeviceId) (*devInfo, error) {
+ if dt.db == nil {
+ return nil, errInvalidDTab
+ }
+ var info devInfo
+ if err := dt.devices.get(string(devid), &info); err != nil {
+ return nil, err
+ }
+ if info.Vectors == nil {
+ return nil, errors.New("nil genvectors")
+ }
+ return &info, nil
+}
+
+// hasDevInfo returns true if the device (devid) has any devInfo in the devTable db.
+func (dt *devTable) hasDevInfo(devid DeviceId) bool {
+ if dt.db == nil {
+ return false
+ }
+ return dt.devices.hasKey(string(devid))
+}
+
+// delDevInfo deletes devInfo struct in the devTable db.
+func (dt *devTable) delDevInfo(devid DeviceId) error {
+ if dt.db == nil {
+ return errInvalidDTab
+ }
+ return dt.devices.del(string(devid))
+}
+
+// putGenVec puts a generation vector in the devTable db.
+func (dt *devTable) putGenVec(devid DeviceId, srid ObjId, v GenVector) error {
+ if dt.db == nil {
+ return errInvalidDTab
+ }
+ var info *devInfo
+ if dt.hasDevInfo(devid) {
+ var err error
+ if info, err = dt.getDevInfo(devid); err != nil {
+ return err
+ }
+ } else {
+ info = &devInfo{
+ Vectors: make(map[ObjId]GenVector),
+ }
+ }
+ info.Vectors[srid] = v
+ info.Ts = time.Now().UTC()
+ return dt.putDevInfo(devid, info)
+}
+
+// getGenVec gets a generation vector from the devTable db.
+func (dt *devTable) getGenVec(devid DeviceId, srid ObjId) (GenVector, error) {
+ if dt.db == nil {
+ return nil, errInvalidDTab
+ }
+ info, err := dt.getDevInfo(devid)
+ if err != nil {
+ return nil, err
+ }
+ v, ok := info.Vectors[srid]
+ if !ok {
+ return nil, fmt.Errorf("srid %s doesn't exist", srid.String())
+ }
+ return v, nil
+}
+
+// populateGenOrderEntry populates a genOrder entry.
+func (dt *devTable) populateGenOrderEntry(e *genOrder, id DeviceId, srid ObjId, gnum GenId) error {
+ e.devID = id
+ e.srID = srid
+ e.genID = gnum
+
+ o, err := dt.s.log.getGenMetadata(id, srid, gnum)
+ if err != nil {
+ return err
+ }
+ e.order = o.Pos
+ return nil
+}
+
+// updateGeneration updates a single generation (upID, upGen) in a device's generation vector for SyncRoot srID.
+func (dt *devTable) updateGeneration(key DeviceId, srID ObjId, upID DeviceId, upGen GenId) error {
+ if dt.db == nil {
+ return errInvalidDTab
+ }
+ info, err := dt.getDevInfo(key)
+ if err != nil {
+ return err
+ }
+
+ v, ok := info.Vectors[srID]
+ if !ok {
+ return fmt.Errorf("srid %s doesn't exist", srID.String())
+ }
+ v[upID] = upGen
+ return dt.putDevInfo(key, info)
+}
+
+// updateLocalGenVector updates local generation vector based on the remote generation vector.
+func (dt *devTable) updateLocalGenVector(local, remote GenVector) error {
+ if dt.db == nil {
+ return errInvalidDTab
+ }
+ if local == nil || remote == nil {
+ return errors.New("invalid input args to function")
+ }
+ for rid, rgen := range remote {
+ lgen, ok := local[rid]
+ if !ok || lgen < rgen {
+ local[rid] = rgen
+ }
+ }
+ return nil
+}
+
+// diffGenVectors diffs generation vectors for a given SyncRoot belonging
+// to src and dest and returns the generations known to src and not known to
+// dest. In addition, sync needs to maintain the order in which device
+// generations are created/received. Hence, when two generation
+// vectors are diffed, the differing generations are returned in a
+// sorted order based on their position in the src's log. genOrder
+// array consists of every generation that is missing between src and
+// dest sorted using its position in the src's log.
+// Example: Generation vector for device A (src) AVec = {A:10, B:5, C:1}
+// Generation vector for device B (dest) BVec = {A:5, B:10, D:2}
+// Missing generations in unsorted order: {A:6, A:7, A:8, A:9, A:10, C:1}
+//
+// TODO(hpucha): Revisit for the case of a lot of generations to
+// send back (say during bootstrap).
+func (dt *devTable) diffGenVectors(srcVec, destVec GenVector, srid ObjId) ([]*genOrder, error) {
+ if dt.db == nil {
+ return nil, errInvalidDTab
+ }
+
+ // Create an array for the generations that need to be returned.
+ var gens []*genOrder
+
+ // Compute missing generations for devices that are in destination and source vector.
+ for devid, genid := range destVec {
+ srcGenId, ok := srcVec[devid]
+ // Skip since src doesn't know of this device.
+ if !ok {
+ continue
+ }
+ // Need to include all generations in the interval [genid+1, srcGenId],
+ // genid+1 and srcGenId inclusive.
+ // Check against reclaimVec to see if required generations are already GCed.
+ // Starting gen is then max(oldGen, genid+1)
+ startGen := genid + 1
+ oldGen := dt.getOldestGen(devid) + 1
+ if startGen < oldGen {
+ vlog.VI(1).Infof("diffGenVectors:: Adjusting starting generations from %d to %d",
+ startGen, oldGen)
+ startGen = oldGen
+ }
+ for i := startGen; i <= srcGenId; i++ {
+ // Populate the genorder entry.
+ var entry genOrder
+ if err := dt.populateGenOrderEntry(&entry, devid, srid, i); err != nil {
+ return nil, err
+ }
+ gens = append(gens, &entry)
+ }
+ }
+ // Compute missing generations for devices not in destination vector but in source vector.
+ for devid, genid := range srcVec {
+ // Add devices destination does not know about.
+ if _, ok := destVec[devid]; !ok {
+ // Bootstrap generation to oldest available.
+ destGenId := dt.getOldestGen(devid) + 1
+ // Need to include all generations in the interval [destGenId, genid],
+ // destGenId and genid inclusive.
+ for i := destGenId; i <= genid; i++ {
+ // Populate the genorder entry.
+ var entry genOrder
+ if err := dt.populateGenOrderEntry(&entry, devid, srid, i); err != nil {
+ return nil, err
+ }
+ gens = append(gens, &entry)
+ }
+ }
+ }
+
+ // Sort generations in log order.
+ sort.Sort(byOrder(gens))
+ return gens, nil
+}
+
+// getOldestGen returns the most recent gc'ed generation for the device "dev".
+func (dt *devTable) getOldestGen(dev DeviceId) GenId {
+ return dt.head.ReclaimVec[dev]
+}
+
+// // computeReclaimVector computes a generation vector such that the
+// // generations less than or equal to those in the vector can be
+// // garbage collected. Caller holds a lock on s.lock.
+// //
+// // Approach: For each device in the system, we compute its maximum
+// // generation known to all the other devices in the system. This is a
+// // O(N^2) algorithm where N is the number of devices in the system. N
+// // is assumed to be small, of the order of hundreds of devices.
+// func (dt *devTable) computeReclaimVector() (GenVector, error) {
+// // Get local generation vector to create the set of devices in
+// // the system. Local generation vector is a good bootstrap
+// // device set since it contains all the devices whose log
+// // records were ever stored locally.
+// devSet, err := dt.getGenVec(dt.s.id)
+// if err != nil {
+// return nil, err
+// }
+//
+// newReclaimVec := GenVector{}
+// for devid := range devSet {
+// if !dt.hasDevInfo(devid) {
+// // This node knows of devid, but hasn't yet
+// // contacted the device. Do not garbage
+// // collect any further. For instance, when
+// // node A learns of node C's generations from
+// // node B, node A may not have an entry for
+// // node C yet, but node C will be part of its
+// // devSet.
+// for dev := range devSet {
+// newReclaimVec[dev] = dt.getOldestGen(dev)
+// }
+// return newReclaimVec, nil
+// }
+//
+// vec, err := dt.getGenVec(devid)
+// if err != nil {
+// return nil, err
+// }
+// for dev := range devSet {
+// gen1, ok := vec[dev]
+// // Device "devid" does not know about device "dev".
+// if !ok {
+// newReclaimVec[dev] = dt.getOldestGen(dev)
+// continue
+// }
+// gen2, ok := newReclaimVec[dev]
+// if !ok || (gen1 < gen2) {
+// newReclaimVec[dev] = gen1
+// }
+// }
+// }
+// return newReclaimVec, nil
+// }
+//
+// // addDevice adds a newly learned device to the devTable state.
+// func (dt *devTable) addDevice(newDev DeviceId) error {
+// // Create an entry in the device table for the new device.
+// vector := GenVector{
+// newDev: 0,
+// }
+// if err := dt.putGenVec(newDev, vector); err != nil {
+// return err
+// }
+//
+// // Update local generation vector with the new device.
+// local, err := dt.getDevInfo(dt.s.id)
+// if err != nil {
+// return err
+// }
+// if err := dt.updateLocalGenVector(local.Vector, vector); err != nil {
+// return err
+// }
+// if err := dt.putDevInfo(dt.s.id, local); err != nil {
+// return err
+// }
+// return nil
+// }
+//
+// // updateReclaimVec updates the reclaim vector to track gc'ed generations.
+// func (dt *devTable) updateReclaimVec(minGens GenVector) error {
+// for dev, min := range minGens {
+// gen, ok := dt.head.ReclaimVec[dev]
+// if !ok {
+// if min < 1 {
+// vlog.Errorf("updateReclaimVec:: Received bad generation %s %d",
+// dev, min)
+// dt.head.ReclaimVec[dev] = 0
+// } else {
+// dt.head.ReclaimVec[dev] = min - 1
+// }
+// continue
+// }
+//
+// // We obtained a generation that is already reclaimed.
+// if min <= gen {
+// return errors.New("requested gen smaller than GC'ed gen")
+// }
+// }
+// return nil
+// }
+
+// getDeviceIds returns the IDs of all devices that are involved in synchronization.
+func (dt *devTable) getDeviceIds() ([]DeviceId, error) {
+ if dt.db == nil {
+ return nil, errInvalidDTab
+ }
+
+ devIDs := make([]DeviceId, 0)
+ dt.devices.keyIter(func(devStr string) {
+ devIDs = append(devIDs, DeviceId(devStr))
+ })
+
+ return devIDs, nil
+}
+
+// dump writes to the log file information on all device table entries.
+func (dt *devTable) dump() {
+ if dt.db == nil {
+ return
+ }
+
+ vlog.VI(1).Infof("DUMP: Dev: self %v: resmark %v, reclaim %v",
+ dt.s.id, dt.head.Resmark, dt.head.ReclaimVec)
+
+ dt.devices.keyIter(func(devStr string) {
+ info, err := dt.getDevInfo(DeviceId(devStr))
+ if err != nil {
+ return
+ }
+
+ vlog.VI(1).Infof("DUMP: Dev: %s: #SR %d, time %v", devStr, len(info.Vectors), info.Ts)
+ for sr, vec := range info.Vectors {
+ vlog.VI(1).Infof("DUMP: Dev: %s: SR %v, vec %v", devStr, sr, vec)
+ }
+ })
+}
diff --git a/services/syncbase/sync/devtable_test.go b/services/syncbase/sync/devtable_test.go
new file mode 100644
index 0000000..8cf5307
--- /dev/null
+++ b/services/syncbase/sync/devtable_test.go
@@ -0,0 +1,1227 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Tests for the Veyron Sync devTable component.
+import (
+ "os"
+ "reflect"
+ "runtime"
+ "testing"
+ "time"
+)
+
+// TestDevTabStore tests creating a backing file for devTable.
+func TestDevTabStore(t *testing.T) {
+ devfile := getFileName()
+ defer os.Remove(devfile)
+
+ s := &syncd{id: "VeyronPhone"}
+ dtab, err := openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+
+ fsize := getFileSize(devfile)
+ if fsize < 0 {
+ //t.Errorf("DevTable file %s not created", devfile)
+ }
+
+ if err := dtab.flush(); err != nil {
+ t.Errorf("Cannot flush devTable file %s, err %v", devfile, err)
+ }
+
+ oldfsize := fsize
+ fsize = getFileSize(devfile)
+ if fsize <= oldfsize {
+ //t.Errorf("DevTable file %s not flushed", devfile)
+ }
+
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+
+ oldfsize = getFileSize(devfile)
+
+ dtab, err = openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot re-open existing devTable file %s, err %v", devfile, err)
+ }
+
+ fsize = getFileSize(devfile)
+ if fsize != oldfsize {
+ t.Errorf("DevTable file %s size changed across re-open (%d %d)", devfile, fsize, oldfsize)
+ }
+
+ if err := dtab.flush(); err != nil {
+ t.Errorf("Cannot flush devTable file %s, err %v", devfile, err)
+ }
+
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+}
+
+// TestInvalidDTab tests devTable methods on an invalid (closed) devTable ptr.
+func TestInvalidDTab(t *testing.T) {
+ devfile := getFileName()
+ defer os.Remove(devfile)
+
+ s := &syncd{id: "VeyronPhone"}
+ dtab, err := openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+
+ validateError := func(err error, funcName string) {
+ _, file, line, _ := runtime.Caller(1)
+ if err == nil || err != errInvalidDTab {
+ t.Errorf("%s:%d %s() did not fail on a closed devTable: %v", file, line, funcName, err)
+ }
+ }
+
+ err = dtab.close()
+ validateError(err, "close")
+
+ err = dtab.flush()
+ validateError(err, "flush")
+
+ srid := ObjId("foo")
+
+ err = dtab.initSyncRoot(srid)
+ validateError(err, "initSyncRoot")
+
+ err = dtab.putDevInfo(s.id, &devInfo{})
+ validateError(err, "putDevInfo")
+
+ _, err = dtab.getDevInfo(s.id)
+ validateError(err, "getDevInfo")
+
+ err = dtab.delDevInfo(s.id)
+ validateError(err, "delDevInfo")
+
+ if dtab.hasDevInfo(s.id) {
+ t.Errorf("hasDevInfo() did not fail on a closed devTable: %v", err)
+ }
+
+ err = dtab.putGenVec(s.id, srid, GenVector{})
+ validateError(err, "putGenVec")
+
+ _, err = dtab.getGenVec(s.id, srid)
+ validateError(err, "getGenVec")
+
+ err = dtab.updateGeneration(s.id, srid, s.id, 0)
+ validateError(err, "updateGeneration")
+
+ err = dtab.updateLocalGenVector(GenVector{}, GenVector{})
+ validateError(err, "updateLocalGenVector")
+
+ _, err = dtab.diffGenVectors(GenVector{}, GenVector{}, srid)
+ validateError(err, "diffGenVectors")
+
+ _, err = dtab.getDeviceIds()
+ validateError(err, "getDeviceIds")
+
+ // Harmless NOP.
+ dtab.dump()
+}
+
+// TestPutGetDevTableHeader tests setting and getting devTable header across devTable open/close/reopen.
+func TestPutGetDevTableHeader(t *testing.T) {
+ devfile := getFileName()
+ defer os.Remove(devfile)
+
+ s := &syncd{id: "VeyronPhone"}
+ dtab, err := openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+
+ // In memory head should be initialized.
+ if dtab.head.Resmark != nil {
+ t.Errorf("First time log create should reset header: %v", dtab.head.Resmark)
+ }
+ expVec := GenVector{dtab.s.id: 0}
+ if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
+ t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
+ devfile, dtab.head.ReclaimVec, expVec)
+ }
+
+ // No head should be there in db.
+ if err = dtab.getHead(); err == nil {
+ t.Errorf("getHead() found non-existent head in devTable file %s, err %v", devfile, err)
+ }
+
+ if dtab.hasHead() {
+ t.Errorf("hasHead() found non-existent head in devTable file %s", devfile)
+ }
+
+ expMark := []byte{1, 2, 3}
+ expVec = GenVector{
+ "VeyronTab": 30,
+ "VeyronPhone": 10,
+ }
+ dtab.head = &devTableHeader{
+ Resmark: expMark,
+ ReclaimVec: expVec,
+ }
+
+ if err := dtab.putHead(); err != nil {
+ t.Errorf("Cannot put head %v in devTable file %s, err %v", dtab.head, devfile, err)
+ }
+
+ // Reset values.
+ dtab.head.Resmark = nil
+ dtab.head.ReclaimVec = GenVector{}
+
+ for i := 0; i < 2; i++ {
+ if err := dtab.getHead(); err != nil {
+ t.Fatalf("getHead() can not find head (i=%d) in devTable file %s, err %v", i, devfile, err)
+ }
+
+ if !dtab.hasHead() {
+ t.Errorf("hasHead() can not find head (i=%d) in devTable file %s", i, devfile)
+ }
+
+ if !reflect.DeepEqual(dtab.head.Resmark, expMark) {
+ t.Errorf("Data mismatch for resmark (i=%d) in devTable file %s: %v instead of %v",
+ i, devfile, dtab.head.Resmark, expMark)
+ }
+ if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
+ t.Errorf("Data mismatch for reclaimVec (i=%d) in devTable file %s: %v instead of %v",
+ i, devfile, dtab.head.ReclaimVec, expVec)
+ }
+
+ if i == 0 {
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+ dtab, err = openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot re-open devTable file %s, err %v", devfile, err)
+ }
+ }
+ }
+
+ dtab.dump()
+
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+}
+
+// TestPersistDevTableHeader tests that devTable header is
+// automatically persisted across devTable open/close/reopen.
+func TestPersistDevTableHeader(t *testing.T) {
+ devfile := getFileName()
+ defer os.Remove(devfile)
+
+ s := &syncd{id: "VeyronPhone"}
+ dtab, err := openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+
+ // In memory head should be initialized.
+ if dtab.head.Resmark != nil {
+ t.Errorf("First time log create should reset header: %v", dtab.head.Resmark)
+ }
+ expVec := GenVector{dtab.s.id: 0}
+ if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
+ t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
+ devfile, dtab.head.ReclaimVec, expVec)
+ }
+
+ expMark := []byte{0, 2, 255}
+ expVec = GenVector{
+ "VeyronTab": 100,
+ "VeyronPhone": 10000,
+ }
+ dtab.head = &devTableHeader{
+ Resmark: expMark,
+ ReclaimVec: expVec,
+ }
+
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+
+ dtab, err = openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+
+ // In memory head should be initialized from db.
+ if !reflect.DeepEqual(dtab.head.Resmark, expMark) {
+ t.Errorf("Data mismatch for resmark in devTable file %s: %v instead of %v",
+ devfile, dtab.head.Resmark, expMark)
+ }
+ if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
+ t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
+ devfile, dtab.head.ReclaimVec, expVec)
+ }
+
+ expMark = []byte{60, 180, 7}
+ expVec = GenVector{
+ "VeyronTab": 1,
+ "VeyronPhone": 1987,
+ }
+ dtab.head = &devTableHeader{
+ Resmark: expMark,
+ ReclaimVec: expVec,
+ }
+
+ if err := dtab.flush(); err != nil {
+ t.Errorf("Cannot flush devTable file %s, err %v", devfile, err)
+ }
+
+ // Reset values.
+ dtab.head.Resmark = nil
+ dtab.head.ReclaimVec = GenVector{}
+
+ if err := dtab.getHead(); err != nil {
+ t.Fatalf("getHead() can not find head in devTable file %s, err %v", devfile, err)
+ }
+
+ // In memory head should be initialized from db.
+ if !reflect.DeepEqual(dtab.head.Resmark, expMark) {
+ t.Errorf("Data mismatch for resmark in devTable file %s: %v instead of %v",
+ devfile, dtab.head.Resmark, expMark)
+ }
+ if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
+ t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
+ devfile, dtab.head.ReclaimVec, expVec)
+ }
+
+ dtab.dump()
+
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+}
+
+// TestDTabInitDelSyncRoot tests initing and deleting a new SyncRoot.
+func TestDTabInitDelSyncRoot(t *testing.T) {
+ devfile := getFileName()
+ defer os.Remove(devfile)
+
+ s := &syncd{id: "VeyronPhone"}
+ dtab, err := openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+ srid1 := ObjId("foo")
+ if err := dtab.initSyncRoot(srid1); err != nil {
+ t.Fatalf("Cannot create new SyncRoot %s, err %v", srid1.String(), err)
+ }
+ srid2 := ObjId("bar")
+ if err := dtab.initSyncRoot(srid2); err != nil {
+ t.Fatalf("Cannot create new SyncRoot %s, err %v", srid2.String(), err)
+ }
+ if err := dtab.initSyncRoot(srid2); err == nil {
+ t.Fatalf("Creating existing SyncRoot didn't fail %s", srid2.String())
+ }
+
+ info, err := dtab.getDevInfo(s.id)
+ if err != nil {
+ t.Errorf("GetDevInfo() can not find device %s in devTable file %s err %v",
+ s.id, devfile, err)
+ }
+ expVec := map[ObjId]GenVector{
+ srid1: {s.id: 0},
+ srid2: {s.id: 0},
+ }
+ if !reflect.DeepEqual(info.Vectors, expVec) {
+ t.Errorf("Data mismatch for device %s %v instead of %v",
+ s.id, info.Vectors, expVec)
+ }
+ if err := dtab.delSyncRoot(srid1); err != nil {
+ t.Fatalf("Cannot delete SyncRoot %s, err %v", srid1.String(), err)
+ }
+ if err := dtab.delSyncRoot(srid1); err == nil {
+ t.Fatalf("Deleting non-existent SyncRoot didn't fail %s", srid1.String())
+ }
+ if err := dtab.delSyncRoot(srid2); err != nil {
+ t.Fatalf("Cannot delete SyncRoot %s, err %v", srid2.String(), err)
+ }
+
+ if _, err = dtab.getDevInfo(s.id); err == nil {
+ t.Errorf("GetDevInfo() found device %s in devTable file %s err %v",
+ s.id, devfile, err)
+ }
+}
+
+// TestPutGetDevInfo tests setting and getting devInfo across devTable open/close/reopen.
+func TestPutGetDevInfo(t *testing.T) {
+ devfile := getFileName()
+ defer os.Remove(devfile)
+
+ s := &syncd{id: "VeyronPhone"}
+ dtab, err := openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+
+ var devid DeviceId = "VeyronTab"
+
+ info, err := dtab.getDevInfo(devid)
+ if err == nil || info != nil {
+ t.Errorf("GetDevInfo() found non-existent device %s in devTable file %s: %v, err %v",
+ devid, devfile, info, err)
+ }
+
+ if dtab.hasDevInfo(devid) {
+ t.Errorf("HasDevInfo() found non-existent device %s in devTable file %s",
+ devid, devfile)
+ }
+ info = &devInfo{
+ Vectors: map[ObjId]GenVector{ObjId("haha"): GenVector{"VeyronTab": 0, "VeyronPhone": 10},
+ ObjId("hello"): GenVector{"VeyronLaptop": 20, "VeyronPhone": 30, "VeyronTab": 80}},
+ Ts: time.Now().UTC(),
+ }
+
+ if err := dtab.putDevInfo(devid, info); err != nil {
+ t.Errorf("Cannot put device %s (%v) in devTable file %s, err %v", devid, info, devfile, err)
+ }
+
+ for i := 0; i < 2; i++ {
+ curInfo, err := dtab.getDevInfo(devid)
+ if err != nil || curInfo == nil {
+ t.Fatalf("GetDevInfo() can not find device %s (i=%d) in devTable file %s: %v, err: %v",
+ devid, i, devfile, curInfo, err)
+ }
+
+ if !dtab.hasDevInfo(devid) {
+ t.Errorf("HasDevInfo() can not find device %s (i=%d) in devTable file %s",
+ devid, i, devfile)
+ }
+
+ if !reflect.DeepEqual(curInfo, info) {
+ t.Errorf("Data mismatch for device %s (i=%d) in devTable file %s: %v instead of %v",
+ devid, i, devfile, curInfo, info)
+ }
+
+ if i == 0 {
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+ dtab, err = openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot re-open devTable file %s, err %v", devfile, err)
+ }
+ }
+ }
+
+ dtab.dump()
+
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+}
+
+// TestPutGetGenVec tests setting and getting generation vector across dtab open/close/reopen.
+func TestPutGetGenVec(t *testing.T) {
+ devfile := getFileName()
+ defer os.Remove(devfile)
+
+ s := &syncd{id: "VeyronPhone"}
+ dtab, err := openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+
+ var devid DeviceId = "VeyronTab"
+ srids := []ObjId{"foo", "bar"}
+ vecs := []GenVector{GenVector{"VeyronTab": 0, "VeyronPhone": 10, "VeyronDesktop": 20, "VeyronLaptop": 2},
+ GenVector{"VeyronTab": 400, "VeyronPhone": 100, "VeyronDesktop": 200, "VeyronLaptop": 20}}
+
+ for i, sr := range srids {
+ vec, err := dtab.getGenVec(devid, sr)
+ if err == nil || vec != nil {
+ t.Errorf("GetGenVec() found non-existent device %s in devTable file %s: %v, err %v",
+ devid, devfile, vec, err)
+ }
+
+ if err := dtab.putGenVec(devid, sr, vecs[i]); err != nil {
+ t.Errorf("Cannot put device %s (%s %v) in devTable file %s, err %v", devid, sr.String(),
+ vecs[i], devfile, err)
+ }
+ }
+
+ for k, sr := range srids {
+ for i := 0; i < 2; i++ {
+ // Check for devid.
+ curVec, err := dtab.getGenVec(devid, sr)
+ if err != nil || curVec == nil {
+ t.Fatalf("GetGenVec() can not find device %s (i=%d) in devTable file %s, err %v",
+ devid, i, devfile, err)
+ }
+
+ if !reflect.DeepEqual(curVec, vecs[k]) {
+ t.Errorf("Data mismatch for device %s srid %s (i=%d) in devTable file %s: %v instead of %v",
+ devid, sr.String(), i, devfile, curVec, vecs[k])
+ }
+
+ if i == 0 {
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+ dtab, err = openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot re-open devTable file %s, err %v", devfile, err)
+ }
+ }
+ }
+ }
+
+ dtab.dump()
+
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+}
+
+// TestUpdateGeneration tests updating a generation.
+func TestUpdateGeneration(t *testing.T) {
+ devfile := getFileName()
+ defer os.Remove(devfile)
+
+ s := &syncd{id: "VeyronPhone"}
+ dtab, err := openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+
+ var devid DeviceId = "VeyronTab"
+ srid := ObjId("foo")
+
+ err = dtab.updateGeneration(devid, srid, devid, 10)
+ if err == nil {
+ t.Errorf("UpdateGeneration() found non-existent device %s in devTable file %s, err %v",
+ devid, devfile, err)
+ }
+ v := GenVector{
+ "VeyronTab": 0,
+ "VeyronPhone": 10,
+ "VeyronDesktop": 20,
+ "VeyronLaptop": 2,
+ }
+
+ if err := dtab.putGenVec(devid, srid, v); err != nil {
+ t.Errorf("Cannot put device %s (%s %v) in devTable file %s, err %v", devid,
+ srid.String(), v, devfile, err)
+ }
+
+ // Try a non-existent SyncRoot ID.
+ if err = dtab.updateGeneration(devid, ObjId("haha"), devid, 10); err == nil {
+ t.Errorf("UpdateGeneration() did not fail on a wrong SyncRoot for %s in devTable file %s", devid, devfile)
+ }
+
+ err = dtab.updateGeneration(devid, srid, devid, 10)
+ if err != nil {
+ t.Errorf("UpdateGeneration() failed for %s in devTable file %s with error %v",
+ devid, devfile, err)
+ }
+ err = dtab.updateGeneration(devid, srid, "VeyronLaptop", 18)
+ if err != nil {
+ t.Errorf("UpdateGeneration() failed for %s in devTable file %s with error %v",
+ devid, devfile, err)
+ }
+ curVec, err := dtab.getGenVec(devid, srid)
+ if err != nil || curVec == nil {
+ t.Fatalf("GetGenVec() can not find device %s in devTable file %s, err %v",
+ devid, devfile, err)
+ }
+ vExp := GenVector{
+ "VeyronTab": 10,
+ "VeyronPhone": 10,
+ "VeyronDesktop": 20,
+ "VeyronLaptop": 18,
+ }
+
+ if !reflect.DeepEqual(curVec, vExp) {
+ t.Errorf("Data mismatch for device %s srid %s in devTable file %s: %v instead of %v",
+ devid, srid.String(), devfile, v, vExp)
+ }
+
+ dtab.dump()
+
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+}
+
+// TestUpdateLocalGenVector tests updating a gen vector.
+func TestUpdateLocalGenVector(t *testing.T) {
+ devfile := getFileName()
+ defer os.Remove(devfile)
+
+ s := &syncd{id: "VeyronPhone"}
+ dtab, err := openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+
+ // Test nil args.
+ if err := dtab.updateLocalGenVector(nil, nil); err == nil {
+ t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
+ devfile, err)
+ }
+
+ // Nothing to update.
+ local := GenVector{
+ "VeyronTab": 0,
+ "VeyronPhone": 1,
+ }
+ remote := GenVector{
+ "VeyronTab": 0,
+ "VeyronPhone": 1,
+ }
+ if err := dtab.updateLocalGenVector(local, remote); err != nil {
+ t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
+ devfile, err)
+ }
+
+ if !reflect.DeepEqual(local, remote) {
+ t.Errorf("Data mismatch for object %v instead of %v",
+ local, remote)
+ }
+
+ // local is missing a generation.
+ local = GenVector{
+ "VeyronPhone": 1,
+ }
+ if err := dtab.updateLocalGenVector(local, remote); err != nil {
+ t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
+ devfile, err)
+ }
+ if !reflect.DeepEqual(local, remote) {
+ t.Errorf("Data mismatch for object %v instead of %v",
+ local, remote)
+ }
+
+ // local is stale compared to remote.
+ local = GenVector{
+ "VeyronTab": 0,
+ "VeyronPhone": 0,
+ }
+ remote = GenVector{
+ "VeyronTab": 1,
+ "VeyronPhone": 0,
+ "VeyronLaptop": 2,
+ }
+ if err := dtab.updateLocalGenVector(local, remote); err != nil {
+ t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
+ devfile, err)
+ }
+ if !reflect.DeepEqual(local, remote) {
+ t.Errorf("Data mismatch for object %v instead of %v",
+ local, remote)
+ }
+
+ // local is partially stale.
+ local = GenVector{
+ "VeyronTab": 0,
+ "VeyronPhone": 0,
+ "VeyronDesktop": 20,
+ }
+ remote = GenVector{
+ "VeyronTab": 1,
+ "VeyronPhone": 10,
+ "VeyronLaptop": 2,
+ }
+ localExp := GenVector{
+ "VeyronTab": 1,
+ "VeyronPhone": 10,
+ "VeyronDesktop": 20,
+ "VeyronLaptop": 2,
+ }
+ if err := dtab.updateLocalGenVector(local, remote); err != nil {
+ t.Errorf("UpdateLocalGenVector() failed in devTable file %s with error %v",
+ devfile, err)
+ }
+ if !reflect.DeepEqual(local, localExp) {
+ t.Errorf("Data mismatch for object %v instead of %v",
+ local, localExp)
+ }
+
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+}
+
+// TestDiffGenVectors tests diffing gen vectors.
+func TestDiffGenVectors(t *testing.T) {
+ logOrder := []DeviceId{"VeyronTab", "VeyronPhone", "VeyronDesktop", "VeyronLaptop"}
+ var expGens []*genOrder
+ srid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // set reclaimVec such that it doesn't affect diffs.
+ reclaimVec := GenVector{
+ "VeyronTab": 0,
+ "VeyronPhone": 0,
+ "VeyronDesktop": 0,
+ "VeyronLaptop": 0,
+ }
+
+ // src and dest are identical vectors.
+ vec := GenVector{
+ "VeyronTab": 1,
+ "VeyronPhone": 10,
+ "VeyronDesktop": 20,
+ "VeyronLaptop": 2,
+ }
+ setupAndTestDiff(t, vec, vec, reclaimVec, logOrder, expGens)
+
+ // src has no updates.
+ srcVec := GenVector{
+ "VeyronTab": 0,
+ }
+ remoteVec := GenVector{
+ "VeyronTab": 5,
+ "VeyronPhone": 10,
+ "VeyronDesktop": 20,
+ "VeyronLaptop": 8,
+ }
+ setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, []DeviceId{}, expGens)
+
+ // src and remote have no updates.
+ srcVec = GenVector{
+ "VeyronTab": 0,
+ }
+ remoteVec = GenVector{
+ "VeyronTab": 0,
+ }
+ setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, []DeviceId{}, expGens)
+
+ // set reclaimVec such that it doesn't affect diffs.
+ reclaimVec = GenVector{
+ "VeyronTab": 0,
+ }
+
+ // src is staler than remote.
+ srcVec = GenVector{
+ "VeyronTab": 1,
+ "VeyronPhone": 10,
+ "VeyronDesktop": 20,
+ "VeyronLaptop": 2,
+ }
+ remoteVec = GenVector{
+ "VeyronTab": 5,
+ "VeyronPhone": 10,
+ "VeyronDesktop": 20,
+ "VeyronLaptop": 8,
+ }
+ setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
+
+ // src is fresher than remote.
+ srcVec = GenVector{
+ "VeyronTab": 5,
+ "VeyronPhone": 10,
+ "VeyronDesktop": 20,
+ "VeyronLaptop": 2,
+ }
+ remoteVec = GenVector{
+ "VeyronTab": 1,
+ "VeyronPhone": 10,
+ "VeyronDesktop": 20,
+ "VeyronLaptop": 2,
+ }
+ expGens = make([]*genOrder, 4)
+ for i := 0; i < 4; i++ {
+ expGens[i] = &genOrder{
+ devID: "VeyronTab",
+ srID: srid,
+ genID: GenId(i + 2),
+ order: uint32(i + 1),
+ }
+ }
+ setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
+
+ // src is fresher than remote in all but one device.
+ srcVec = GenVector{
+ "VeyronTab": 5,
+ "VeyronPhone": 10,
+ "VeyronDesktop": 22,
+ "VeyronLaptop": 2,
+ }
+ remoteVec = GenVector{
+ "VeyronTab": 1,
+ "VeyronPhone": 10,
+ "VeyronDesktop": 20,
+ "VeyronLaptop": 2,
+ "VeyronCloud": 40,
+ }
+ expGens = make([]*genOrder, 6)
+ for i := 0; i < 6; i++ {
+ switch {
+ case i < 4:
+ expGens[i] = &genOrder{
+ devID: "VeyronTab",
+ srID: srid,
+ genID: GenId(i + 2),
+ order: uint32(i + 1),
+ }
+ default:
+ expGens[i] = &genOrder{
+ devID: "VeyronDesktop",
+ srID: srid,
+ genID: GenId(i - 4 + 21),
+ order: uint32(i - 4 + 35),
+ }
+ }
+ }
+ setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
+
+ // src is fresher than dest, scramble log order.
+ o := []DeviceId{"VeyronTab", "VeyronLaptop", "VeyronPhone", "VeyronDesktop"}
+ srcVec = GenVector{
+ "VeyronTab": 1,
+ "VeyronPhone": 2,
+ "VeyronDesktop": 3,
+ "VeyronLaptop": 4,
+ }
+ remoteVec = GenVector{
+ "VeyronTab": 0,
+ "VeyronPhone": 2,
+ "VeyronDesktop": 0,
+ }
+ expGens = make([]*genOrder, 8)
+ for i := 0; i < 8; i++ {
+ switch {
+ case i < 1:
+ expGens[i] = &genOrder{
+ devID: "VeyronTab",
+ srID: srid,
+ genID: GenId(i + 1),
+ order: uint32(i),
+ }
+ case i >= 1 && i < 5:
+ expGens[i] = &genOrder{
+ devID: "VeyronLaptop",
+ srID: srid,
+ genID: GenId(i),
+ order: uint32(i),
+ }
+ default:
+ expGens[i] = &genOrder{
+ devID: "VeyronDesktop",
+ srID: srid,
+ genID: GenId(i - 4),
+ order: uint32(i - 5 + 7),
+ }
+ }
+ }
+ setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, o, expGens)
+
+ // remote has no updates.
+ srcVec = GenVector{
+ "VeyronTab": 1,
+ "VeyronPhone": 2,
+ "VeyronDesktop": 3,
+ "VeyronLaptop": 4,
+ }
+ remoteVec = GenVector{
+ "VeyronPhone": 0,
+ }
+ expGens = make([]*genOrder, 10)
+ for i := 0; i < 10; i++ {
+ switch {
+ case i < 1:
+ expGens[i] = &genOrder{
+ devID: "VeyronTab",
+ srID: srid,
+ genID: GenId(i + 1),
+ order: uint32(i),
+ }
+ case i >= 1 && i < 3:
+ expGens[i] = &genOrder{
+ devID: "VeyronPhone",
+ srID: srid,
+ genID: GenId(i),
+ order: uint32(i),
+ }
+ case i >= 3 && i < 6:
+ expGens[i] = &genOrder{
+ devID: "VeyronDesktop",
+ srID: srid,
+ genID: GenId(i - 2),
+ order: uint32(i),
+ }
+ default:
+ expGens[i] = &genOrder{
+ devID: "VeyronLaptop",
+ srID: srid,
+ genID: GenId(i - 5),
+ order: uint32(i),
+ }
+ }
+ }
+ setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
+
+ // Test with reclaimVec fast-fwded.
+ reclaimVec = GenVector{
+ "VeyronPhone": 1,
+ "VeyronLaptop": 2,
+ }
+ srcVec = GenVector{
+ "VeyronTab": 1,
+ "VeyronPhone": 2,
+ "VeyronDesktop": 3,
+ "VeyronLaptop": 4,
+ }
+ remoteVec = GenVector{
+ "VeyronPhone": 0,
+ }
+ expGens = make([]*genOrder, 7)
+ for i := 0; i < 7; i++ {
+ switch {
+ case i < 1:
+ expGens[i] = &genOrder{
+ devID: "VeyronTab",
+ srID: srid,
+ genID: GenId(i + 1),
+ order: uint32(i),
+ }
+ case i == 1:
+ expGens[i] = &genOrder{
+ devID: "VeyronPhone",
+ srID: srid,
+ genID: GenId(i + 1),
+ order: uint32(i + 1),
+ }
+ case i >= 2 && i < 5:
+ expGens[i] = &genOrder{
+ devID: "VeyronDesktop",
+ srID: srid,
+ genID: GenId(i - 1),
+ order: uint32(i + 1),
+ }
+ default:
+ expGens[i] = &genOrder{
+ devID: "VeyronLaptop",
+ srID: srid,
+ genID: GenId(i - 2),
+ order: uint32(i + 3),
+ }
+ }
+ }
+ setupAndTestDiff(t, srcVec, remoteVec, reclaimVec, logOrder, expGens)
+}
+
+// setupAndTestDiff is an utility function to test diffing generation vectors.
+func setupAndTestDiff(t *testing.T, srcVec, remoteVec, reclaimVec GenVector, logOrder []DeviceId, expGens []*genOrder) {
+ devfile := getFileName()
+ defer os.Remove(devfile)
+
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ var srcid DeviceId = "VeyronTab"
+ var destid DeviceId = "VeyronPhone"
+
+ var err error
+ s := &syncd{id: srcid}
+ s.log, err = openILog(logfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+ dtab, err := openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+ dtab.head.ReclaimVec = reclaimVec
+
+ srid, err := strToObjId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Populate generations in log order.
+ var order uint32
+ for _, k := range logOrder {
+ v, ok := (srcVec)[k]
+ if !ok {
+ t.Errorf("Cannot find key %s in srcVec %v", k, srcVec)
+ }
+ for i := GenId(1); i <= v; i++ {
+ val := &genMetadata{Pos: order}
+ if err := s.log.putGenMetadata(k, srid, i, val); err != nil {
+ t.Errorf("Cannot put object %s:%d in log file %s, err %v", k, v, logfile, err)
+ }
+ order++
+ }
+ }
+ gens, err := dtab.diffGenVectors(srcVec, remoteVec, srid)
+ if err != nil {
+ t.Fatalf("DiffGenVectors() failed src: %s %v dest: %s %v in devTable file %s, err %v",
+ srcid, srcVec, destid, remoteVec, devfile, err)
+ }
+
+ if !reflect.DeepEqual(gens, expGens) {
+ t.Fatalf("Data mismatch for genorder %v instead of %v, src %v dest %v reclaim %v",
+ gens, expGens, srcVec, remoteVec, reclaimVec)
+ }
+
+ if err := dtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+}
+
+// TestGetOldestGen tests obtaining generations from reclaimVec.
+func TestGetOldestGen(t *testing.T) {
+ devfile := getFileName()
+ defer os.Remove(devfile)
+
+ var srcid DeviceId = "VeyronTab"
+ s := &syncd{id: srcid}
+ var err error
+ s.devtab, err = openDevTable(devfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+ }
+
+ if s.devtab.getOldestGen(srcid) != 0 {
+ t.Errorf("Cannot get generation for device %s in devTable file %s",
+ srcid, devfile)
+ }
+
+ var destid DeviceId = "VeyronPhone"
+ if s.devtab.getOldestGen(destid) != 0 {
+ t.Errorf("Cannot get generation for device %s in devTable file %s",
+ destid, devfile)
+ }
+
+ s.devtab.head.ReclaimVec[srcid] = 10
+ if s.devtab.getOldestGen(srcid) != 10 {
+ t.Errorf("Cannot get generation for device %s in devTable file %s",
+ srcid, devfile)
+ }
+ if s.devtab.getOldestGen(destid) != 0 {
+ t.Errorf("Cannot get generation for device %s in devTable file %s",
+ destid, devfile)
+ }
+
+ if err := s.devtab.close(); err != nil {
+ t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+ }
+}
+
+// // TestComputeReclaimVector tests reclaim vector computation.
+// func TestComputeReclaimVector(t *testing.T) {
+// devArr := []DeviceId{"VeyronTab", "VeyronPhone", "VeyronDesktop", "VeyronLaptop"}
+// genVecArr := make([]GenVector, 4)
+//
+// // All devices are up-to-date.
+// genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// setupAndTestReclaimVector(t, devArr, genVecArr, nil, genVecArr[0])
+//
+// // Every device is missing at least one other device. Not possible to gc.
+// genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronLaptop": 4}
+// genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3}
+// genVecArr[3] = GenVector{"VeyronDesktop": 3, "VeyronLaptop": 4}
+// expReclaimVec := GenVector{"VeyronTab": 0, "VeyronPhone": 0, "VeyronDesktop": 0, "VeyronLaptop": 0}
+// setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
+//
+// // All devices know at least one generation from other devices.
+// genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 2, "VeyronLaptop": 2}
+// genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 3, "VeyronLaptop": 1}
+// genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 1, "VeyronLaptop": 4}
+// expReclaimVec = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 1, "VeyronLaptop": 1}
+// setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
+//
+// // One device is missing from one other device.
+// genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 2}
+// genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 3, "VeyronLaptop": 1}
+// genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 1, "VeyronLaptop": 4}
+// expReclaimVec = GenVector{"VeyronTab": 1, "VeyronPhone": 1, "VeyronDesktop": 1, "VeyronLaptop": 0}
+// setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
+//
+// // All devices know at least "n" generations from other devices.
+// var n GenId = 10
+// genVecArr[0] = GenVector{"VeyronTab": n + 10, "VeyronPhone": n,
+// "VeyronDesktop": n + 8, "VeyronLaptop": n + 4}
+// genVecArr[1] = GenVector{"VeyronTab": n + 6, "VeyronPhone": n + 10,
+// "VeyronDesktop": n, "VeyronLaptop": n + 3}
+// genVecArr[2] = GenVector{"VeyronTab": n, "VeyronPhone": n + 2,
+// "VeyronDesktop": n + 10, "VeyronLaptop": n}
+// genVecArr[3] = GenVector{"VeyronTab": n + 7, "VeyronPhone": n + 1,
+// "VeyronDesktop": n + 5, "VeyronLaptop": n + 10}
+// expReclaimVec = GenVector{"VeyronTab": n, "VeyronPhone": n, "VeyronDesktop": n, "VeyronLaptop": n}
+// setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
+//
+// // Never contacted a device.
+// devArr = []DeviceId{"VeyronTab", "VeyronDesktop", "VeyronLaptop"}
+// genVecArr[0] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// genVecArr[1] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// genVecArr[2] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// expReclaimVec = GenVector{"VeyronTab": 0, "VeyronPhone": 0, "VeyronDesktop": 0, "VeyronLaptop": 0}
+// setupAndTestReclaimVector(t, devArr, genVecArr, nil, expReclaimVec)
+//
+// // Start from existing reclaim vector.
+// devArr = []DeviceId{"VeyronTab", "VeyronPhone", "VeyronDesktop", "VeyronLaptop"}
+// reclaimVec := GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// genVecArr[0] = GenVector{"VeyronTab": 6, "VeyronPhone": 6, "VeyronDesktop": 6, "VeyronLaptop": 6}
+// genVecArr[1] = GenVector{"VeyronTab": 6, "VeyronPhone": 6, "VeyronDesktop": 3, "VeyronLaptop": 6}
+// genVecArr[2] = GenVector{"VeyronTab": 6, "VeyronPhone": 6, "VeyronDesktop": 6, "VeyronLaptop": 4}
+// genVecArr[3] = GenVector{"VeyronTab": 1, "VeyronPhone": 2, "VeyronDesktop": 6, "VeyronLaptop": 6}
+//
+// setupAndTestReclaimVector(t, devArr, genVecArr, reclaimVec, reclaimVec)
+// }
+//
+// // setupAndTestReclaimVector is an utility function to test reclaim vector computation.
+// func setupAndTestReclaimVector(t *testing.T, devArr []DeviceId, genVecArr []GenVector, reclaimStart, expReclaimVec GenVector) {
+// devfile := getFileName()
+// defer os.Remove(devfile)
+//
+// s := &syncd{id: "VeyronTab"}
+// dtab, err := openDevTable(devfile, s)
+// if err != nil {
+// t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+// }
+// if reclaimStart != nil {
+// dtab.head.ReclaimVec = reclaimStart
+// }
+//
+// for i := range devArr {
+// if err := dtab.putGenVec(devArr[i], genVecArr[i]); err != nil {
+// t.Errorf("Cannot put object %s (%v) in devTable file %s, err %v",
+// devArr[i], genVecArr[i], devfile, err)
+// }
+// }
+//
+// reclaimVec, err := dtab.computeReclaimVector()
+// if err != nil {
+// t.Fatalf("computeReclaimVector() failed devices: %v, vectors: %v in devTable file %s, err %v",
+// devArr, genVecArr, devfile, err)
+// }
+//
+// if !reflect.DeepEqual(reclaimVec, expReclaimVec) {
+// t.Fatalf("Data mismatch for reclaimVec %v instead of %v",
+// reclaimVec, expReclaimVec)
+// }
+//
+// if err := dtab.close(); err != nil {
+// t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+// }
+// }
+//
+// // TestAddDevice tests adding a device to the devTable.
+// func TestAddDevice(t *testing.T) {
+// devfile := getFileName()
+// defer os.Remove(devfile)
+//
+// s := &syncd{id: "VeyronPhone"}
+// dtab, err := openDevTable(devfile, s)
+// if err != nil {
+// t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+// }
+//
+// var dev DeviceId = "VeyronLaptop"
+// if err := dtab.addDevice(dev); err != nil {
+// t.Fatalf("Cannot add new device in devTable file %s, err %v", devfile, err)
+// }
+//
+// vec, err := dtab.getGenVec(dev)
+// if err != nil || vec == nil {
+// t.Fatalf("GetGenVec() can not find object %s in devTable file %s, err %v",
+// dev, devfile, err)
+// }
+// expVec := GenVector{dev: 0}
+// if !reflect.DeepEqual(vec, expVec) {
+// t.Errorf("Data mismatch for object %s in devTable file %s: %v instead of %v",
+// dev, devfile, vec, expVec)
+// }
+//
+// vec, err = dtab.getGenVec(dtab.s.id)
+// if err != nil || vec == nil {
+// t.Fatalf("GetGenVec() can not find object %s in devTable file %s, err %v",
+// dtab.s.id, devfile, err)
+// }
+// expVec = GenVector{dtab.s.id: 0, dev: 0}
+// if !reflect.DeepEqual(vec, expVec) {
+// t.Errorf("Data mismatch for object %s in devTable file %s: %v instead of %v",
+// dtab.s.id, devfile, vec, expVec)
+// }
+//
+// expVec = GenVector{dtab.s.id: 10, "VeyronDesktop": 40, dev: 80}
+// if err := dtab.putGenVec(dtab.s.id, expVec); err != nil {
+// t.Fatalf("PutGenVec() can not put object %s in devTable file %s, err %v",
+// dtab.s.id, devfile, err)
+// }
+// dev = "VeyronTab"
+// if err := dtab.addDevice(dev); err != nil {
+// t.Fatalf("Cannot add new device in devTable file %s, err %v", devfile, err)
+// }
+// expVec[dev] = 0
+//
+// vec, err = dtab.getGenVec(dtab.s.id)
+// if err != nil || vec == nil {
+// t.Fatalf("GetGenVec() can not find object %s in devTable file %s, err %v",
+// dtab.s.id, devfile, err)
+// }
+// if !reflect.DeepEqual(vec, expVec) {
+// t.Errorf("Data mismatch for object %s in devTable file %s: %v instead of %v",
+// dtab.s.id, devfile, vec, expVec)
+// }
+//
+// if err := dtab.close(); err != nil {
+// t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+// }
+// }
+//
+// // TestUpdateReclaimVec tests updating the reclaim vector.
+// func TestUpdateReclaimVec(t *testing.T) {
+// devfile := getFileName()
+// defer os.Remove(devfile)
+//
+// s := &syncd{id: "VeyronPhone"}
+// dtab, err := openDevTable(devfile, s)
+// if err != nil {
+// t.Fatalf("Cannot open new devTable file %s, err %v", devfile, err)
+// }
+//
+// minGens := GenVector{"VeyronTab": 1, "VeyronDesktop": 3, "VeyronLaptop": 4}
+// if err := dtab.updateReclaimVec(minGens); err != nil {
+// t.Fatalf("Cannot update reclaimvec in devTable file %s, err %v", devfile, err)
+// }
+// expVec := GenVector{dtab.s.id: 0, "VeyronTab": 0, "VeyronDesktop": 2, "VeyronLaptop": 3}
+// if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
+// t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
+// devfile, dtab.head.ReclaimVec, expVec)
+// }
+//
+// dtab.head.ReclaimVec[DeviceId("VeyronTab")] = 4
+// minGens = GenVector{"VeyronTab": 1}
+// if err := dtab.updateReclaimVec(minGens); err == nil {
+// t.Fatalf("Update reclaimvec didn't fail in devTable file %s", devfile)
+// }
+//
+// minGens = GenVector{"VeyronTab": 5}
+// if err := dtab.updateReclaimVec(minGens); err != nil {
+// t.Fatalf("Cannot update reclaimvec in devTable file %s, err %v", devfile, err)
+// }
+// expVec = GenVector{dtab.s.id: 0, "VeyronTab": 4, "VeyronDesktop": 2, "VeyronLaptop": 3}
+// if !reflect.DeepEqual(dtab.head.ReclaimVec, expVec) {
+// t.Errorf("Data mismatch for reclaimVec in devTable file %s: %v instead of %v",
+// devfile, dtab.head.ReclaimVec, expVec)
+// }
+//
+// if err := dtab.close(); err != nil {
+// t.Errorf("Cannot close devTable file %s, err %v", devfile, err)
+// }
+// }
diff --git a/services/syncbase/sync/ilog.go b/services/syncbase/sync/ilog.go
new file mode 100644
index 0000000..9e35c24
--- /dev/null
+++ b/services/syncbase/sync/ilog.go
@@ -0,0 +1,627 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Package vsync provides veyron sync ILog utility functions. ILog
+// (Indexed Log) provides log functionality with indexing support.
+// ILog stores log records that are locally generated or obtained over
+// the network. Indexing is needed since sync needs to selectively
+// retrieve log records that belong to a particular device, syncroot
+// and generation during synchronization.
+//
+// When a device receives a request to send log records, it first
+// computes the missing generations between itself and the incoming
+// request on a per-syncroot basis. It then sends all the log records
+// belonging to each missing generation. A device that receives log
+// records over the network replays all the records received from
+// another device in a single batch. Each replayed log record adds a
+// new version to the dag of the object contained in the log
+// record. At the end of replaying all the log records, conflict
+// detection and resolution is carried out for all the objects learned
+// during this iteration. Conflict detection and resolution is carried
+// out after a batch of log records are replayed, instead of
+// incrementally after each record is replayed, to avoid repeating
+// conflict resolution already performed by other devices.
+//
+// New log records are created when objects in the local store are
+// created/updated. Local log records are also replayed to keep the
+// per-object dags consistent with the local store state.
+//
+// Note that syncgroups are mainly tracked between syncd/store and the
+// client. Sync-related metadata (log records, generations and
+// generation vectors) is unaware of syncgroups, and uses syncroots
+// instead. Each syncgroup contains a root object and a unique root
+// object ID. In case of peer syncgroups, all the peer syncgroups
+// contain the same root object ID. Sync translates any given
+// syncgroup to a syncroot rooted at this root object ID and syncs all
+// the syncroots the device is part of. Thus, although a device may
+// be part of more than 1 peer syncgroup, at the sync level, a single
+// syncroot is synced on behalf of all the peer syncgroups.
+//
+// Implementation notes: ILog records are stored in a persistent K/V
+// database in the current implementation. ILog db consists of 3
+// tables:
+// ** records: table consists of all the log records indexed
+// by deviceid:srid:genid:lsn referring to
+// the device that creates the log record
+// the root oid of the syncroot to which the log record belongs
+// the generation on the syncroot the log record is part of
+// the sequence number in that generation
+// Note that lsn in each generation starts from 0 and genid starts from 1.
+// ** gens: table consists of the generation metadata for each
+// generation, and is indexed by deviceid:srid:genid.
+// ** head: table consists of the log header.
+import (
+ "errors"
+ "fmt"
+ "strconv"
+ "strings"
+
+ "v.io/x/lib/vlog"
+)
+
+var (
+ errNoUpdates = errors.New("no new local updates")
+ errInvalidLog = errors.New("invalid log db")
+)
+
+// curLocalGen contains metadata re. the current local generation for a SyncRoot.
+type curLocalGen struct {
+ CurGenNum GenId // generation id for a SyncRoot's current generation.
+ CurSeqNum SeqNum // log sequence number for a SyncRoot's current generation.
+}
+
+// iLogHeader contains the log header metadata.
+type iLogHeader struct {
+ CurSRGens map[ObjId]*curLocalGen // local generation info per SyncRoot.
+ Curorder uint32 // position in log for the next generation.
+}
+
+// genMetadata contains the metadata for a generation.
+type genMetadata struct {
+ // All generations stored in the log are ordered wrt each
+ // other and this order needs to be preserved.
+ // Position of this generation in the log.
+ Pos uint32
+
+ // Number of log records in this generation still stored in the log.
+ // This count is used during garbage collection.
+ Count uint64
+
+ // Maximum SeqNum that was part of this generation.
+ // This is useful during garbage collection to track any unclaimed log records.
+ MaxSeqNum SeqNum
+}
+
+// iLog contains the metadata for the ILog db.
+type iLog struct {
+ fname string // file pathname.
+ db *kvdb // underlying k/v db.
+
+ // Key:deviceid:srid:genid:lsn Value:LogRecord. Pointer to
+ // the "records" table in the kvdb. Contains log records.
+ records *kvtable
+
+ // Key:deviceid:srid:genid Value:genMetadata. Pointer to the
+ // "gens" table in the kvdb. Contains generation metadata for
+ // each generation.
+ gens *kvtable
+
+ // Key:"Head" Value:iLogHeader. Pointer to the "header" table
+ // in the kvdb. Contains logheader.
+ header *kvtable
+
+ head *iLogHeader // log head cached in memory.
+
+ s *syncd // pointer to the sync daemon object.
+}
+
+// openILog opens or creates a ILog for the given filename.
+func openILog(filename string, sin *syncd) (*iLog, error) {
+ ilog := &iLog{
+ fname: filename,
+ head: &iLogHeader{},
+ s: sin,
+ }
+ // Open the file and create it if it does not exist.
+ // Also initialize the kvdb and its three collections.
+ db, tbls, err := kvdbOpen(filename, []string{"records", "gens", "header"})
+ if err != nil {
+ return nil, err
+ }
+
+ ilog.db = db
+ ilog.records = tbls[0]
+ ilog.gens = tbls[1]
+ ilog.header = tbls[2]
+
+ // If header already exists in db, read it back from db.
+ if ilog.hasHead() {
+ if err := ilog.getHead(); err != nil {
+ ilog.db.close() // this also closes the tables.
+ return nil, err
+ }
+ } else {
+ ilog.head.CurSRGens = make(map[ObjId]*curLocalGen)
+ }
+
+ return ilog, nil
+}
+
+// close closes the ILog and invalidate its struct.
+func (l *iLog) close() error {
+ if l.db == nil {
+ return errInvalidLog
+ }
+ // Flush the dirty data.
+ if err := l.flush(); err != nil {
+ return err
+ }
+
+ l.db.close() // this also closes the tables.
+
+ *l = iLog{} // zero out the ILog struct.
+ return nil
+}
+
+// flush flushes the ILog db to disk.
+func (l *iLog) flush() error {
+ if l.db == nil {
+ return errInvalidLog
+ }
+ // Set the head from memory before flushing.
+ if err := l.putHead(); err != nil {
+ return err
+ }
+
+ l.db.flush()
+
+ return nil
+}
+
+// initSyncRoot initializes the log header for this SyncRoot.
+func (l *iLog) initSyncRoot(srid ObjId) error {
+ if l.db == nil {
+ return errInvalidLog
+ }
+ if _, ok := l.head.CurSRGens[srid]; ok {
+ return fmt.Errorf("syncroot already exists %v", srid)
+ }
+ // First generation to be created is generation 1. A
+ // generation of 0 represents no updates on the device.
+ l.head.CurSRGens[srid] = &curLocalGen{CurGenNum: 1}
+ return nil
+}
+
+// delSyncRoot deletes the log header for this SyncRoot.
+func (l *iLog) delSyncRoot(srid ObjId) error {
+ if l.db == nil {
+ return errInvalidLog
+ }
+ if _, ok := l.head.CurSRGens[srid]; !ok {
+ return fmt.Errorf("syncroot doesn't exist %v", srid)
+ }
+
+ delete(l.head.CurSRGens, srid)
+ return nil
+}
+
+// putHead puts the log head into the ILog db.
+func (l *iLog) putHead() error {
+ return l.header.set("Head", l.head)
+}
+
+// getHead gets the log head from the ILog db.
+func (l *iLog) getHead() error {
+ if l.head == nil {
+ return errors.New("nil log header")
+ }
+ if err := l.header.get("Head", l.head); err != nil {
+ return err
+ }
+ // When we put an empty map in kvdb, we get back a "nil" map
+ // (due to VOM encoding/decoding). To keep putHead/getHead
+ // symmetrical, we add this additional initialization.
+ if l.head.CurSRGens == nil {
+ l.head.CurSRGens = make(map[ObjId]*curLocalGen)
+ }
+ return nil
+}
+
+// hasHead returns true if the ILog db has a log head.
+func (l *iLog) hasHead() bool {
+ return l.header.hasKey("Head")
+}
+
+// logRecKey creates a key for a log record.
+func logRecKey(devid DeviceId, srid ObjId, gnum GenId, lsn SeqNum) string {
+ return fmt.Sprintf("%s:%s:%d:%d", devid, srid.String(), uint64(gnum), uint64(lsn))
+}
+
+// strToGenId converts a string to GenId.
+func strToGenId(genIDStr string) (GenId, error) {
+ id, err := strconv.ParseUint(genIDStr, 10, 64)
+ if err != nil {
+ return 0, err
+ }
+ return GenId(id), nil
+}
+
+// strToObjId converts a string to ObjId.
+func strToObjId(objIDStr string) (ObjId, error) {
+ return ObjId(objIDStr), nil
+}
+
+// splitLogRecKey splits a : separated logrec key into its components.
+func splitLogRecKey(key string) (DeviceId, ObjId, GenId, SeqNum, error) {
+ args := strings.Split(key, ":")
+ if len(args) != 4 {
+ return "", NoObjId, 0, 0, fmt.Errorf("bad logrec key %s", key)
+ }
+ srid, err := strToObjId(args[1])
+ if err != nil {
+ return "", NoObjId, 0, 0, err
+ }
+ gnum, err := strToGenId(args[2])
+ if err != nil {
+ return "", NoObjId, 0, 0, err
+ }
+ lsn, err := strconv.ParseUint(args[3], 10, 64)
+ if err != nil {
+ return "", NoObjId, 0, 0, err
+ }
+ return DeviceId(args[0]), srid, gnum, SeqNum(lsn), nil
+}
+
+// putLogRec puts the log record into the ILog db.
+func (l *iLog) putLogRec(rec *LogRec) (string, error) {
+ if l.db == nil {
+ return "", errInvalidLog
+ }
+ key := logRecKey(rec.DevId, rec.SyncRootId, rec.GenNum, rec.SeqNum)
+ return key, l.records.set(key, rec)
+}
+
+// getLogRec gets the log record from the ILog db.
+func (l *iLog) getLogRec(devid DeviceId, srid ObjId, gnum GenId, lsn SeqNum) (*LogRec, error) {
+ if l.db == nil {
+ return nil, errInvalidLog
+ }
+ key := logRecKey(devid, srid, gnum, lsn)
+ var rec LogRec
+ if err := l.records.get(key, &rec); err != nil {
+ return nil, err
+ }
+ return &rec, nil
+}
+
+// hasLogRec returns true if the ILog db has a log record matching (devid, srid, gnum, lsn).
+func (l *iLog) hasLogRec(devid DeviceId, srid ObjId, gnum GenId, lsn SeqNum) bool {
+ if l.db == nil {
+ return false
+ }
+ key := logRecKey(devid, srid, gnum, lsn)
+ return l.records.hasKey(key)
+}
+
+// delLogRec deletes the log record matching (devid, srid, gnum, lsn) from the ILog db.
+func (l *iLog) delLogRec(devid DeviceId, srid ObjId, gnum GenId, lsn SeqNum) error {
+ if l.db == nil {
+ return errInvalidLog
+ }
+ key := logRecKey(devid, srid, gnum, lsn)
+ return l.records.del(key)
+}
+
+// generationKey creates a key for a generation.
+func generationKey(devid DeviceId, srid ObjId, gnum GenId) string {
+ return fmt.Sprintf("%s:%s:%d", devid, srid.String(), gnum)
+}
+
+// splitGenerationKey splits a : separated generation key into its components.
+func splitGenerationKey(key string) (DeviceId, ObjId, GenId, error) {
+ args := strings.Split(key, ":")
+ if len(args) != 3 {
+ return "", NoObjId, 0, fmt.Errorf("bad generation key %s", key)
+ }
+ srid, err := strToObjId(args[1])
+ if err != nil {
+ return "", NoObjId, 0, err
+ }
+ gnum, err := strToGenId(args[2])
+ if err != nil {
+ return "", NoObjId, 0, err
+ }
+ return DeviceId(args[0]), srid, gnum, nil
+}
+
+// putGenMetadata puts the metadata of the generation (devid, srid, gnum) into the ILog db.
+func (l *iLog) putGenMetadata(devid DeviceId, srid ObjId, gnum GenId, val *genMetadata) error {
+ key := generationKey(devid, srid, gnum)
+ return l.gens.set(key, val)
+}
+
+// getGenMetadata gets the metadata of the generation (devid, srid, gnum) from the ILog db.
+func (l *iLog) getGenMetadata(devid DeviceId, srid ObjId, gnum GenId) (*genMetadata, error) {
+ if l.db == nil {
+ return nil, errInvalidLog
+ }
+ key := generationKey(devid, srid, gnum)
+ var val genMetadata
+ if err := l.gens.get(key, &val); err != nil {
+ return nil, err
+ }
+ return &val, nil
+}
+
+// hasGenMetadata returns true if the ILog db has the generation (devid, srid, gnum).
+func (l *iLog) hasGenMetadata(devid DeviceId, srid ObjId, gnum GenId) bool {
+ key := generationKey(devid, srid, gnum)
+ return l.gens.hasKey(key)
+}
+
+// delGenMetadata deletes the generation (devid, srid, gnum) metadata from the ILog db.
+func (l *iLog) delGenMetadata(devid DeviceId, srid ObjId, gnum GenId) error {
+ if l.db == nil {
+ return errInvalidLog
+ }
+ key := generationKey(devid, srid, gnum)
+ return l.gens.del(key)
+}
+
+// getLocalGenInfo gets the local generation info for the given syncroot.
+func (l *iLog) getLocalGenInfo(srid ObjId) (*curLocalGen, error) {
+ gen, ok := l.head.CurSRGens[srid]
+ if !ok {
+ return nil, fmt.Errorf("no gen info found for srid %s", srid.String())
+ }
+ return gen, nil
+}
+
+// createLocalLogRec creates a new local log record of type NodeRec.
+func (l *iLog) createLocalLogRec(obj ObjId, vers Version,
+ par []Version, val *LogValue, srid ObjId) (*LogRec, error) {
+ gen, err := l.getLocalGenInfo(srid)
+ if err != nil {
+ return nil, err
+ }
+ rec := &LogRec{
+ DevId: l.s.id,
+ SyncRootId: srid,
+ GenNum: gen.CurGenNum,
+ SeqNum: gen.CurSeqNum,
+ RecType: NodeRec,
+
+ ObjId: obj,
+ CurVers: vers,
+ Parents: par,
+ Value: *val,
+ }
+
+ // Increment the SeqNum for the local log.
+ gen.CurSeqNum++
+
+ return rec, nil
+}
+
+// createLocalLinkLogRec creates a new local log record of type LinkRec.
+func (l *iLog) createLocalLinkLogRec(obj ObjId, vers, par Version, srid ObjId) (*LogRec, error) {
+ gen, err := l.getLocalGenInfo(srid)
+ if err != nil {
+ return nil, err
+ }
+ rec := &LogRec{
+ DevId: l.s.id,
+ SyncRootId: srid,
+ GenNum: gen.CurGenNum,
+ SeqNum: gen.CurSeqNum,
+ RecType: LinkRec,
+
+ ObjId: obj,
+ CurVers: vers,
+ Parents: []Version{par},
+ Value: LogValue{},
+ }
+
+ // Increment the SeqNum for the local log.
+ gen.CurSeqNum++
+
+ return rec, nil
+}
+
+// createRemoteGeneration adds a new remote generation.
+func (l *iLog) createRemoteGeneration(dev DeviceId, srid ObjId, gnum GenId, gen *genMetadata) error {
+ if l.db == nil {
+ return errInvalidLog
+ }
+
+ if gen.Count != uint64(gen.MaxSeqNum+1) {
+ return errors.New("mismatch in count and lsn")
+ }
+
+ vlog.VI(2).Infof("createRemoteGeneration:: dev %s srid %s gen %d %v", dev, srid.String(), gnum, gen)
+
+ gen.Pos = l.head.Curorder
+ l.head.Curorder++
+
+ return l.putGenMetadata(dev, srid, gnum, gen)
+}
+
+// createLocalGeneration creates a new local generation.
+func (l *iLog) createLocalGeneration(srid ObjId) (GenId, error) {
+ if l.db == nil {
+ return 0, errInvalidLog
+ }
+
+ g, err := l.getLocalGenInfo(srid)
+ if err != nil {
+ return 0, err
+ }
+
+ gnum := g.CurGenNum
+
+ // If there are no updates, there will be no new generation.
+ if g.CurSeqNum == 0 {
+ return gnum - 1, errNoUpdates
+ }
+
+ // Add the current generation to the db.
+ val := &genMetadata{
+ Pos: l.head.Curorder,
+ Count: uint64(g.CurSeqNum),
+ MaxSeqNum: g.CurSeqNum - 1,
+ }
+
+ err = l.putGenMetadata(l.s.id, srid, gnum, val)
+
+ vlog.VI(2).Infof("createLocalGeneration:: created srid %s gen %d %v", srid.String(), gnum, val)
+ // Move to the next generation irrespective of err.
+ l.head.Curorder++
+ g.CurGenNum++
+ g.CurSeqNum = 0
+
+ return gnum, err
+}
+
+// processWatchRecord processes new object versions obtained from the local store.
+func (l *iLog) processWatchRecord(objID ObjId, vers, parent Version, val *LogValue, srid ObjId) error {
+ if l.db == nil {
+ return errInvalidLog
+ }
+
+ vlog.VI(2).Infof("processWatchRecord:: adding for object %v %v (srid %v)", objID, vers, srid)
+
+ if !srid.IsValid() {
+ return errors.New("invalid syncroot id")
+ }
+
+ // Filter out the echo from the watcher. When syncd puts mutations into store,
+ // it hears back these mutations once again on the watch stream. We need to
+ // filter out these echoes and only process brand new watch changes.
+ if vers != NoVersion {
+ // Check if the object's vers already exists in the DAG.
+ if l.s.dag.hasNode(objID, vers) {
+ return nil
+ }
+
+ // When we successfully join a SyncGroup, Store
+ // creates an empty directory with object ID as the
+ // rootObjId of the SyncGroup joined. We do not care
+ // about this version of the directory. We will start
+ // accepting local mutations on this object only after
+ // we hear about it remotely first.
+ //
+ // NOTE: Since this new directory is protected from
+ // any local updates via an Permissions until after the first
+ // sync, waiting to hear remotely first works without
+ // dropping any updates. However, we cache in the
+ // "priv" table its version in the Store so that we
+ // can correctly specify prior version when we put
+ // mutations into the store.
+ if _, err := l.s.dag.getHead(objID); err != nil && objID == srid {
+ priv := &privNode{ /*Mutation: &val.Mutation, */ SyncTime: val.SyncTime, TxId: val.TxId, TxCount: val.TxCount}
+ return l.s.dag.setPrivNode(objID, priv)
+ }
+ } else {
+ // Check if the parent version has a deleted
+ // descendant already in the DAG.
+ if l.s.dag.hasDeletedDescendant(objID, parent) {
+ return nil
+ }
+ }
+
+ var pars []Version
+ if parent != NoVersion {
+ pars = []Version{parent}
+ }
+
+ // If the current version is a deletion, generate a new version number.
+ if val.Delete {
+ if vers != NoVersion {
+ return fmt.Errorf("deleted vers is %v", vers)
+ }
+ vers = NewVersion()
+ //val.Mutation.Version = vers
+ }
+
+ // Create a log record from Watch's Change Record.
+ rec, err := l.createLocalLogRec(objID, vers, pars, val, srid)
+ if err != nil {
+ return err
+ }
+
+ // Insert the new log record into the log.
+ logKey, err := l.putLogRec(rec)
+ if err != nil {
+ return err
+ }
+
+ // Insert the new log record into dag.
+ if err = l.s.dag.addNode(rec.ObjId, rec.CurVers, false, val.Delete, rec.Parents, logKey, val.TxId); err != nil {
+ return err
+ }
+
+ // Move the head.
+ return l.s.dag.moveHead(rec.ObjId, rec.CurVers)
+}
+
+// dump writes to the log file information on ILog internals.
+func (l *iLog) dump() {
+ if l.db == nil {
+ return
+ }
+
+ vlog.VI(1).Infof("DUMP: ILog: #SR %d, cur %d", len(l.head.CurSRGens), l.head.Curorder)
+ for sr, gen := range l.head.CurSRGens {
+ vlog.VI(1).Infof("DUMP: ILog: SR %v: gen %v, lsn %v", sr, gen.CurGenNum, gen.CurSeqNum)
+ }
+
+ // Find for each (devid, srid) pair its lowest and highest generation numbers.
+ type genInfo struct{ min, max GenId }
+ devs := make(map[DeviceId]map[ObjId]*genInfo)
+
+ l.gens.keyIter(func(genKey string) {
+ devid, srid, gnum, err := splitGenerationKey(genKey)
+ if err != nil {
+ return
+ }
+
+ srids, ok := devs[devid]
+ if !ok {
+ srids = make(map[ObjId]*genInfo)
+ devs[devid] = srids
+ }
+
+ info, ok := srids[srid]
+ if ok {
+ if gnum > info.max {
+ info.max = gnum
+ }
+ if gnum < info.min {
+ info.min = gnum
+ }
+ } else {
+ srids[srid] = &genInfo{min: gnum, max: gnum}
+ }
+ })
+
+ // For each (devid, srid) pair dump its generation info in order from
+ // min to max generation numbers, inclusive.
+ for devid, srids := range devs {
+ for srid, info := range srids {
+ for gnum := info.min; gnum <= info.max; gnum++ {
+ meta, err := l.getGenMetadata(devid, srid, gnum)
+ if err == nil {
+ vlog.VI(1).Infof(
+ "DUMP: ILog: gen (%v, %v, %v): pos %v, count %v, maxlsn %v",
+ devid, srid, gnum, meta.Pos, meta.Count, meta.MaxSeqNum)
+ } else {
+ vlog.VI(1).Infof("DUMP: ILog: gen (%v, %v, %v): missing generation",
+ devid, srid, gnum)
+ }
+ }
+ }
+ }
+}
diff --git a/services/syncbase/sync/ilog_test.go b/services/syncbase/sync/ilog_test.go
new file mode 100644
index 0000000..ba12bf0
--- /dev/null
+++ b/services/syncbase/sync/ilog_test.go
@@ -0,0 +1,1024 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Tests for the Veyron Sync ILog component.
+import (
+ "fmt"
+ "math/rand"
+ "os"
+ "reflect"
+ "runtime"
+ "testing"
+)
+
+// TestILogStore tests creating a backing file for ILog.
+func TestILogStore(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ log, err := openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot open new ILog file %s, err %v", logfile, err)
+ }
+
+ fsize := getFileSize(logfile)
+ if fsize < 0 {
+ //t.Errorf("Log file %s not created", logfile)
+ }
+
+ if err := log.flush(); err != nil {
+ t.Errorf("Cannot flush ILog file %s, err %v", logfile, err)
+ }
+
+ oldfsize := fsize
+ fsize = getFileSize(logfile)
+ if fsize <= oldfsize {
+ //t.Errorf("Log file %s not flushed", logfile)
+ }
+
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close ILog file %s, err %v", logfile, err)
+ }
+
+ oldfsize = getFileSize(logfile)
+
+ log, err = openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot re-open existing log file %s, err %v", logfile, err)
+ }
+
+ fsize = getFileSize(logfile)
+ if fsize != oldfsize {
+ t.Errorf("Log file %s size changed across re-open (%d %d)", logfile, fsize, oldfsize)
+ }
+
+ if err := log.flush(); err != nil {
+ t.Errorf("Cannot flush ILog file %s, err %v", logfile, err)
+ }
+
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close ILog file %s, err %v", logfile, err)
+ }
+}
+
+// TestInvalidLog tests log methods on an invalid (closed) log ptr.
+func TestInvalidLog(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ log, err := openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot open new ILog file %s, err %v", logfile, err)
+ }
+
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close ILog file %s, err %v", logfile, err)
+ }
+
+ validateError := func(err error, funcName string) {
+ _, file, line, _ := runtime.Caller(1)
+ if err == nil || err != errInvalidLog {
+ t.Errorf("%s:%d %s() did not fail on a closed log: %v", file, line, funcName, err)
+ }
+ }
+
+ err = log.close()
+ validateError(err, "close")
+
+ err = log.flush()
+ validateError(err, "flush")
+
+ srid := ObjId("haha")
+
+ err = log.initSyncRoot(srid)
+ validateError(err, "initSyncRoot")
+
+ _, err = log.putLogRec(&LogRec{})
+ validateError(err, "putLogRec")
+
+ var devid DeviceId = "VeyronPhone"
+ var gnum GenId = 1
+ var lsn SeqNum
+
+ _, err = log.getLogRec(devid, srid, gnum, lsn)
+ validateError(err, "getLogRec")
+
+ if log.hasLogRec(devid, srid, gnum, lsn) {
+ t.Errorf("hasLogRec() did not fail on a closed log: %v", err)
+ }
+
+ err = log.delLogRec(devid, srid, gnum, lsn)
+ validateError(err, "delLogRec")
+
+ _, err = log.getGenMetadata(devid, srid, gnum)
+ validateError(err, "getGenMetadata")
+
+ err = log.delGenMetadata(devid, srid, gnum)
+ validateError(err, "delGenMetadata")
+
+ err = log.createRemoteGeneration(devid, srid, gnum, &genMetadata{})
+ validateError(err, "createRemoteGeneration")
+
+ _, err = log.createLocalGeneration(srid)
+ validateError(err, "createLocalGeneration")
+
+ err = log.processWatchRecord(ObjId("foobar"), 2, Version(999), &LogValue{}, srid)
+ validateError(err, "processWatchRecord")
+
+ // Harmless NOP.
+ log.dump()
+}
+
+// TestPutGetLogHeader tests setting and getting log header across log open/close/reopen.
+func TestPutGetLogHeader(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ log, err := openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ // In memory head should be initialized.
+ if len(log.head.CurSRGens) != 0 || log.head.Curorder != 0 {
+ t.Errorf("First time log create should reset header")
+ }
+
+ // No head should be there in db.
+ if err = log.getHead(); err == nil {
+ t.Errorf("getHead() found non-existent head in log file %s, err %v", logfile, err)
+ }
+
+ if log.hasHead() {
+ t.Errorf("hasHead() found non-existent head in log file %s", logfile)
+ }
+
+ expVal := map[ObjId]*curLocalGen{ObjId("bla"): &curLocalGen{CurGenNum: 10, CurSeqNum: 100},
+ ObjId("qwerty"): &curLocalGen{CurGenNum: 40, CurSeqNum: 80}}
+
+ log.head = &iLogHeader{
+ CurSRGens: expVal,
+ Curorder: 1000,
+ }
+
+ if err := log.putHead(); err != nil {
+ t.Errorf("Cannot put head %v in log file %s, err %v", log.head, logfile, err)
+ }
+
+ // Reset values.
+ log.head.CurSRGens = nil
+ log.head.Curorder = 0
+
+ for i := 0; i < 2; i++ {
+ if err := log.getHead(); err != nil {
+ t.Fatalf("getHead() can not find head (i=%d) in log file %s, err %v", i, logfile, err)
+ }
+
+ if !log.hasHead() {
+ t.Errorf("hasHead() can not find head (i=%d) in log file %s", i, logfile)
+ }
+
+ if !reflect.DeepEqual(log.head.CurSRGens, expVal) || log.head.Curorder != 1000 {
+ t.Errorf("Data mismatch for head (i=%d) in log file %s: %v",
+ i, logfile, log.head)
+ }
+
+ if i == 0 {
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+ log, err = openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
+ }
+ }
+ }
+
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+}
+
+// TestPersistLogHeader tests that log header is automatically persisted across log open/close/reopen.
+func TestPersistLogHeader(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ log, err := openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ // In memory head should be initialized.
+ if len(log.head.CurSRGens) != 0 || log.head.Curorder != 0 {
+ t.Errorf("First time log create should reset header")
+ }
+
+ expVal := map[ObjId]*curLocalGen{ObjId("blabla"): &curLocalGen{CurGenNum: 10, CurSeqNum: 100},
+ ObjId("xyz"): &curLocalGen{CurGenNum: 40, CurSeqNum: 80}}
+ log.head = &iLogHeader{
+ CurSRGens: expVal,
+ Curorder: 1000,
+ }
+
+ if err = log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+
+ log, err = openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ // In memory head should be initialized from db.
+ if !reflect.DeepEqual(log.head.CurSRGens, expVal) || log.head.Curorder != 1000 {
+ t.Errorf("Data mismatch for head in log file %s: %v", logfile, log.head)
+ }
+
+ for sr := range expVal {
+ expVal[sr] = &curLocalGen{CurGenNum: 1000, CurSeqNum: 10}
+ break
+ }
+ log.head = &iLogHeader{
+ CurSRGens: expVal,
+ Curorder: 100,
+ }
+
+ if err := log.flush(); err != nil {
+ t.Errorf("Cannot flush ILog file %s, err %v", logfile, err)
+ }
+
+ // Reset values.
+ log.head.CurSRGens = nil
+ log.head.Curorder = 0
+
+ if err := log.getHead(); err != nil {
+ t.Fatalf("getHead() can not find head in log file %s, err %v", logfile, err)
+ }
+
+ // In memory head should be initialized from db.
+ if !reflect.DeepEqual(log.head.CurSRGens, expVal) || log.head.Curorder != 100 {
+ t.Errorf("Data mismatch for head in log file %s: %v", logfile, log.head)
+ }
+
+ if err = log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+}
+
+// TestLogInitDelSyncRoot tests initing and deleting a new SyncRoot.
+func TestLogInitDelSyncRoot(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ log, err := openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ srid1 := ObjId("haha")
+ if err := log.initSyncRoot(srid1); err != nil {
+ t.Fatalf("Cannot create new SyncRoot %s, err %v", srid1.String(), err)
+ }
+ srid2 := ObjId("asdf")
+ if err := log.initSyncRoot(srid2); err != nil {
+ t.Fatalf("Cannot create new SyncRoot %s, err %v", srid2.String(), err)
+ }
+ if err := log.initSyncRoot(srid2); err == nil {
+ t.Fatalf("Creating existing SyncRoot didn't fail %s, err %v", srid2.String(), err)
+ }
+ expGen := &curLocalGen{CurGenNum: 1, CurSeqNum: 0}
+ expSRGens := map[ObjId]*curLocalGen{
+ srid1: expGen,
+ srid2: expGen,
+ }
+ if !reflect.DeepEqual(log.head.CurSRGens, expSRGens) {
+ t.Errorf("Data mismatch for head in log file %s: %v instead of %v",
+ logfile, log.head.CurSRGens, expSRGens)
+ }
+
+ if err := log.delSyncRoot(srid1); err != nil {
+ t.Fatalf("Cannot delete SyncRoot %s, err %v", srid1.String(), err)
+ }
+ if err := log.delSyncRoot(srid2); err != nil {
+ t.Fatalf("Cannot delete SyncRoot %s, err %v", srid1.String(), err)
+ }
+ if err := log.delSyncRoot(srid1); err == nil {
+ t.Fatalf("Deleting non-existent SyncRoot didn't fail %s", srid1.String())
+ }
+ expSRGens = map[ObjId]*curLocalGen{}
+ if !reflect.DeepEqual(log.head.CurSRGens, expSRGens) {
+ t.Errorf("Data mismatch for head in log file %s: %v instead of %v",
+ logfile, log.head.CurSRGens, expSRGens)
+ }
+
+ if err := log.putHead(); err != nil {
+ t.Errorf("Cannot put head %v in log file %s, err %v", log.head, logfile, err)
+ }
+
+ // Reset values.
+ log.head.CurSRGens = nil
+ log.head.Curorder = 0
+
+ if err := log.getHead(); err != nil {
+ t.Fatalf("getHead() can not find head in log file %s, err %v", logfile, err)
+ }
+
+ if !reflect.DeepEqual(log.head.CurSRGens, expSRGens) {
+ t.Errorf("Data mismatch for head in log file %s: %v instead of %v",
+ logfile, log.head.CurSRGens, expSRGens)
+ }
+}
+
+// TestStringAndParseKeys tests conversion to/from string for log record and generation keys.
+func TestStringAndParseKeys(t *testing.T) {
+ var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+ randString := func(n int) string {
+ b := make([]rune, n)
+ for i := range b {
+ b[i] = letters[rand.Intn(len(letters))]
+ }
+ return string(b)
+ }
+
+ for i := 0; i < 10; i++ {
+ devid := DeviceId(randString(rand.Intn(20) + 1))
+ srid := ObjId(fmt.Sprintf("haha-%d", i))
+ gnum := GenId(rand.Int63())
+ lsn := SeqNum(rand.Int63())
+
+ devid1, srid1, gnum1, lsn1, err := splitLogRecKey(logRecKey(devid, srid, gnum, lsn))
+ if err != nil || devid1 != devid || srid1 != srid || gnum1 != gnum || lsn1 != lsn {
+ t.Fatalf("LogRec conversion failed in iter %d: got %s %v %v %v want %s %v %v %v, err %v",
+ i, devid1, srid1, gnum1, lsn1, devid, srid, gnum, lsn, err)
+ }
+
+ devid2, srid2, gnum2, err := splitGenerationKey(generationKey(devid, srid, gnum))
+ if err != nil || devid2 != devid || srid2 != srid || gnum2 != gnum {
+ t.Fatalf("Gen conversion failed in iter %d: got %s %v %v want %s %v %v, err %v",
+ i, devid2, srid2, gnum2, devid, srid, gnum, err)
+ }
+ }
+
+ //badStrs := []string{"abc:3:4", "abc:123:4:5", "abc:1234:-1:10", "ijk:7890:1000:-1", "abc:3:4:5:6", "abc::3:4:5"}
+ badStrs := []string{"abc:3:4", "abc:1234:-1:10", "ijk:7890:1000:-1", "abc:3:4:5:6", "abc::3:4:5"}
+ for _, s := range badStrs {
+ if _, _, _, _, err := splitLogRecKey(s); err == nil {
+ t.Fatalf("LogRec conversion didn't fail for str %s", s)
+ }
+ }
+
+ //badStrs = []string{"abc:3", "abc:123:4", "abc:1234:-1", "abc:3:4:5", "abc:4::5"}
+ badStrs = []string{"abc:3", "abc:1234:-1", "abc:3:4:5", "abc:4::5"}
+ for _, s := range badStrs {
+ if _, _, _, err := splitGenerationKey(s); err == nil {
+ t.Fatalf("Gen conversion didn't fail for str %s", s)
+ }
+ }
+}
+
+// TestPutGetLogRec tests setting and getting a log record across log open/close/reopen.
+func TestPutGetLogRec(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ log, err := openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ var devid DeviceId = "VeyronTab"
+ srid := ObjId("haha")
+ var gnum GenId = 100
+ var lsn SeqNum = 1000
+
+ rec, err := log.getLogRec(devid, srid, gnum, lsn)
+ if err == nil || rec != nil {
+ t.Errorf("GetLogRec() found non-existent object %s:%s:%d:%d in log file %s: %v, err %v",
+ devid, srid.String(), gnum, lsn, logfile, rec, err)
+ }
+
+ if log.hasLogRec(devid, srid, gnum, lsn) {
+ t.Errorf("HasLogRec() found non-existent object %s:%s:%d:%d in log file %s",
+ devid, srid.String(), gnum, lsn, logfile)
+ }
+
+ rec = &LogRec{
+ DevId: devid,
+ SyncRootId: srid,
+ GenNum: gnum,
+ SeqNum: lsn,
+ ObjId: ObjId("bla"),
+ CurVers: 2,
+ Parents: []Version{0, 1},
+ Value: LogValue{},
+ }
+
+ if _, err := log.putLogRec(rec); err != nil {
+ t.Errorf("Cannot put object %s:%s:%d:%d (%v) in log file %s, err %v", devid, srid.String(), gnum, lsn, rec, logfile, err)
+ }
+
+ for i := 0; i < 2; i++ {
+ curRec, err := log.getLogRec(devid, srid, gnum, lsn)
+ if err != nil || curRec == nil {
+ t.Fatalf("GetLogRec() can not find object %s:%s:%d:%d (i=%d) in log file %s, err %v",
+ devid, srid.String(), gnum, lsn, i, logfile, err)
+ }
+
+ if !log.hasLogRec(devid, srid, gnum, lsn) {
+ t.Errorf("HasLogRec() can not find object %s:%s:%d:%d (i=%d) in log file %s",
+ devid, srid.String(), gnum, lsn, i, logfile)
+ }
+
+ if !reflect.DeepEqual(curRec, rec) {
+ t.Errorf("Data mismatch for object %s:%d:%d (i=%d) in log file %s: %v instead of %v",
+ devid, gnum, lsn, i, logfile, curRec, rec)
+ }
+
+ if i == 0 {
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+ log, err = openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
+ }
+ }
+ }
+
+ log.dump()
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+}
+
+// TestDelLogRec tests deleting a log record across log open/close/reopen.
+func TestDelLogRec(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ log, err := openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ var devid DeviceId = "VeyronTab"
+ srid := ObjId("haha")
+ var gnum GenId = 100
+ var lsn SeqNum = 1000
+
+ rec := &LogRec{
+ DevId: devid,
+ SyncRootId: srid,
+ GenNum: gnum,
+ SeqNum: lsn,
+ ObjId: ObjId("bla"),
+ CurVers: 2,
+ Parents: []Version{0, 1},
+ Value: LogValue{},
+ }
+
+ if _, err := log.putLogRec(rec); err != nil {
+ t.Errorf("Cannot put object %s:%s:%d:%d (%v) in log file %s, err %v", devid, srid.String(), gnum, lsn, rec, logfile, err)
+ }
+
+ curRec, err := log.getLogRec(devid, srid, gnum, lsn)
+ if err != nil || curRec == nil {
+ t.Fatalf("GetLogRec() can not find object %s:%s:%d:%d in log file %s, err %v",
+ devid, srid.String(), gnum, lsn, logfile, err)
+ }
+
+ if err := log.delLogRec(devid, srid, gnum, lsn); err != nil {
+ t.Fatalf("DelLogRec() can not delete object %s:%s:%d:%d in log file %s, err %v",
+ devid, srid.String(), gnum, lsn, logfile, err)
+ }
+
+ for i := 0; i < 2; i++ {
+ curRec, err = log.getLogRec(devid, srid, gnum, lsn)
+ if err == nil || curRec != nil {
+ t.Fatalf("GetLogRec() finds deleted object %s:%s:%d:%d (i=%d) in log file %s, err %v",
+ devid, srid.String(), gnum, lsn, i, logfile, err)
+ }
+
+ if log.hasLogRec(devid, srid, gnum, lsn) {
+ t.Errorf("HasLogRec() finds deleted object %s:%s:%d:%d (i=%d) in log file %s",
+ devid, srid.String(), gnum, lsn, i, logfile)
+ }
+
+ if i == 0 {
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+ log, err = openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
+ }
+ }
+ }
+
+ log.dump()
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+
+}
+
+// TestPutGetGenMetadata tests setting and getting generation metadata across log open/close/reopen.
+func TestPutGetGenMetadata(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ log, err := openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ var devid DeviceId = "VeyronTab"
+ srid := ObjId("haha")
+ var gnum GenId = 100
+
+ val, err := log.getGenMetadata(devid, srid, gnum)
+ if err == nil || val != nil {
+ t.Errorf("GetGenMetadata() found non-existent object %s:%s:%d in log file %s: %v, err %v",
+ devid, srid.String(), gnum, logfile, val, err)
+ }
+
+ if log.hasGenMetadata(devid, srid, gnum) {
+ t.Errorf("hasGenMetadata() found non-existent object %s:%s:%d in log file %s",
+ devid, srid.String(), gnum, logfile)
+ }
+
+ val = &genMetadata{Pos: 40, Count: 100, MaxSeqNum: 99}
+ if err := log.putGenMetadata(devid, srid, gnum, val); err != nil {
+ t.Errorf("Cannot put object %s:%s:%d in log file %s, err %v", devid, srid.String(), gnum, logfile, err)
+ }
+
+ for i := 0; i < 2; i++ {
+ curVal, err := log.getGenMetadata(devid, srid, gnum)
+ if err != nil || curVal == nil {
+ t.Fatalf("GetGenMetadata() can not find object %s:%s:%d (i=%d) in log file %s, err %v",
+ devid, srid.String(), gnum, i, logfile, err)
+ }
+
+ if !log.hasGenMetadata(devid, srid, gnum) {
+ t.Errorf("hasGenMetadata() can not find object %s:%s:%d (i=%d) in log file %s",
+ devid, srid.String(), gnum, i, logfile)
+ }
+
+ if !reflect.DeepEqual(curVal, val) {
+ t.Errorf("Data mismatch for object %s:%s:%d (i=%d) in log file %s: %v instead of %v",
+ devid, srid.String(), gnum, i, logfile, curVal, val)
+ }
+
+ if i == 0 {
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+ log, err = openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
+ }
+ }
+ }
+
+ log.dump()
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+}
+
+// TestDelGenMetadata tests deleting generation metadata across log open/close/reopen.
+func TestDelGenMetadata(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ log, err := openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ var devid DeviceId = "VeyronTab"
+ srid := ObjId("haha")
+ var gnum GenId = 100
+
+ val := &genMetadata{Pos: 40, Count: 100, MaxSeqNum: 99}
+ if err := log.putGenMetadata(devid, srid, gnum, val); err != nil {
+ t.Errorf("Cannot put object %s:%s:%d in log file %s, err %v", devid, srid.String(), gnum, logfile, err)
+ }
+
+ curVal, err := log.getGenMetadata(devid, srid, gnum)
+ if err != nil || curVal == nil {
+ t.Fatalf("GetGenMetadata() can not find object %s:%s:%d in log file %s, err %v",
+ devid, srid.String(), gnum, logfile, err)
+ }
+
+ if err := log.delGenMetadata(devid, srid, gnum); err != nil {
+ t.Fatalf("DelGenMetadata() can not delete object %s:%s:%d in log file %s, err %v",
+ devid, srid.String(), gnum, logfile, err)
+ }
+
+ for i := 0; i < 2; i++ {
+ curVal, err := log.getGenMetadata(devid, srid, gnum)
+ if err == nil || curVal != nil {
+ t.Fatalf("GetGenMetadata() finds deleted object %s:%s:%d (i=%d) in log file %s, err %v",
+ devid, srid.String(), gnum, i, logfile, err)
+ }
+
+ if log.hasGenMetadata(devid, srid, gnum) {
+ t.Errorf("hasGenMetadata() finds deleted object %s:%s:%d (i=%d) in log file %s",
+ devid, srid.String(), gnum, i, logfile)
+ }
+
+ if i == 0 {
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+ log, err = openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
+ }
+ }
+ }
+
+ log.dump()
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+}
+
+// TestPersistLogState tests that generation metadata and record state
+// is persisted across log open/close/reopen.
+func TestPersistLogState(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ log, err := openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ var devid DeviceId = "VeyronTab"
+ srid := ObjId("haha")
+
+ // Add several generations.
+ for i := uint32(0); i < 10; i++ {
+ val := &genMetadata{Pos: i}
+ if err := log.putGenMetadata(devid, srid, GenId(i+10), val); err != nil {
+ t.Errorf("Cannot put object %s:%s:%d in log file %s, err %v", devid, srid.String(),
+ i, logfile, err)
+ }
+ }
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+ log, err = openILog(logfile, nil)
+ if err != nil {
+ t.Fatalf("Cannot re-open log file %s, err %v", logfile, err)
+ }
+ for i := uint32(0); i < 10; i++ {
+ curVal, err := log.getGenMetadata(devid, srid, GenId(i+10))
+ if err != nil || curVal == nil {
+ t.Fatalf("GetGenMetadata() can not find object %s:%s:%d in log file %s, err %v",
+ devid, srid.String(), i, logfile, err)
+ }
+ if curVal.Pos != i {
+ t.Errorf("Data mismatch for object %s:%s:%d in log file %s: %v",
+ devid, srid.String(), i, logfile, curVal)
+ }
+ // Should safely overwrite the same keys.
+ curVal.Pos = i + 10
+ if err := log.putGenMetadata(devid, srid, GenId(i+10), curVal); err != nil {
+ t.Errorf("Cannot put object %s:%s:%d in log file %s, err %v", devid, srid.String(),
+ i, logfile, err)
+ }
+ }
+ for i := uint32(0); i < 10; i++ {
+ curVal, err := log.getGenMetadata(devid, srid, GenId(i+10))
+ if err != nil || curVal == nil {
+ t.Fatalf("GetGenMetadata() can not find object %s:%s:%d in log file %s, err %v",
+ devid, srid.String(), i, logfile, err)
+ }
+ if curVal.Pos != (i + 10) {
+ t.Errorf("Data mismatch for object %s:%s:%d in log file %s: %v, err %v",
+ devid, srid.String(), i, logfile, curVal, err)
+ }
+ }
+
+ log.dump()
+ if err := log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+}
+
+// fillFakeLogRecords fills fake log records for testing purposes.
+func (l *iLog) fillFakeLogRecords(srid ObjId) {
+ const num = 10
+ var parvers []Version
+ id := ObjId("haha")
+ for i := int(0); i < num; i++ {
+ // Create a local log record.
+ curvers := Version(i)
+ rec, err := l.createLocalLogRec(id, curvers, parvers, &LogValue{}, srid)
+ if err != nil {
+ return
+ }
+ // Insert the new log record into the log.
+ _, err = l.putLogRec(rec)
+ if err != nil {
+ return
+ }
+ parvers = []Version{curvers}
+ }
+}
+
+// TestCreateGeneration tests that local log records and local
+// generations are created uniquely and remote generations are
+// correctly inserted in log order.
+func TestCreateGeneration(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ s := &syncd{id: "VeyronTab"}
+ log, err := openILog(logfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ // Try using a wrong SyncRoot ID.
+ bad_srid := ObjId("xyz")
+ if _, err := log.createLocalLogRec(ObjId("asdf"), NoVersion, nil, &LogValue{}, bad_srid); err == nil {
+ t.Errorf("createLocalLogRec did not fail when using an invalid SyncRoot ID %v", bad_srid)
+ }
+ if _, err := log.createLocalLinkLogRec(ObjId("qwerty"), NoVersion, NoVersion, bad_srid); err == nil {
+ t.Errorf("createLocalLinkLogRec did not fail when using an invalid SyncRoot ID %v", bad_srid)
+ }
+ if _, err := log.createLocalGeneration(bad_srid); err == nil {
+ t.Errorf("createLocalGeneration did not fail when using an invalid SyncRoot ID %v", bad_srid)
+ }
+
+ srids := []ObjId{ObjId("foo"), ObjId("bar")}
+ num := []uint64{10, 20}
+ for pos, sr := range srids {
+ if err := log.initSyncRoot(sr); err != nil {
+ t.Fatalf("Cannot create new SyncRoot %s, err %v", sr.String(), err)
+ }
+ if g, err := log.createLocalGeneration(sr); err != errNoUpdates {
+ t.Errorf("Should not find local updates gen %d with error %v", g, err)
+ }
+
+ var parvers []Version
+ id := ObjId(fmt.Sprintf("haha-%d", pos))
+ for i := uint64(0); i < num[pos]; i++ {
+ // Create a local log record.
+ curvers := Version(i)
+ rec, err := log.createLocalLogRec(id, curvers, parvers, &LogValue{}, sr)
+ if err != nil {
+ t.Fatalf("Cannot create local log rec ObjId: %v Current: %v Parents: %v Error: %v",
+ id, curvers, parvers, err)
+ }
+
+ temprec := &LogRec{
+ DevId: log.s.id,
+ SyncRootId: sr,
+ GenNum: GenId(1),
+ SeqNum: SeqNum(i),
+ ObjId: id,
+ CurVers: curvers,
+ Parents: parvers,
+ Value: LogValue{},
+ }
+ // Verify that the log record has the right values.
+ if !reflect.DeepEqual(rec, temprec) {
+ t.Errorf("Data mismtach in log record %v instead of %v", rec, temprec)
+ }
+
+ // Insert the new log record into the log.
+ _, err = log.putLogRec(rec)
+ if err != nil {
+ t.Errorf("Cannot put log record:: failed with err %v", err)
+ }
+
+ parvers = []Version{curvers}
+ }
+ }
+ if err = log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+
+ log, err = openILog(logfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ for pos, sr := range srids {
+ if log.head.CurSRGens[sr].CurGenNum != 1 || log.head.CurSRGens[sr].CurSeqNum != SeqNum(num[pos]) {
+ t.Errorf("Data mismatch in log header %v (pos=%d)", log.head, pos)
+ }
+
+ g, err := log.createLocalGeneration(sr)
+ if g != 1 || err != nil {
+ t.Errorf("Could not create local generation srid %s gen %d (pos=%d) with error %v",
+ sr.String(), g, pos, err)
+ }
+ curVal, err := log.getGenMetadata(log.s.id, sr, g)
+ if err != nil || curVal == nil {
+ t.Fatalf("GetGenMetadata() can not find object %s:%s:%d (pos=%d) in log file %s, err %v",
+ log.s.id, sr.String(), g, pos, logfile, err)
+ }
+ expVal := &genMetadata{Pos: uint32(pos), Count: num[pos], MaxSeqNum: SeqNum(num[pos] - 1)}
+ if !reflect.DeepEqual(curVal, expVal) {
+ t.Errorf("Data mismatch for object %s:%s:%d (pos=%d) in log file %s: %v instead of %v",
+ log.s.id, sr.String(), g, pos, logfile, curVal, expVal)
+ }
+ if log.head.CurSRGens[sr].CurGenNum != 2 || log.head.CurSRGens[sr].CurSeqNum != 0 ||
+ log.head.Curorder != uint32(pos+1) {
+ t.Errorf("Data mismatch in log header (pos=%d) %v %v",
+ pos, log.head.CurSRGens[sr], log.head.Curorder)
+ }
+
+ if g, err := log.createLocalGeneration(sr); err != errNoUpdates {
+ t.Errorf("Should not find local updates sr %s gen %d (pos=%d) with error %v",
+ sr.String(), g, pos, err)
+ }
+ }
+
+ // Populate one more generation for only one syncroot.
+ log.fillFakeLogRecords(srids[0])
+
+ if g, err := log.createLocalGeneration(srids[0]); g != 2 || err != nil {
+ t.Errorf("Could not create local generation sgid %s gen %d with error %v",
+ srids[0].String(), g, err)
+ }
+
+ if g, err := log.createLocalGeneration(srids[1]); err != errNoUpdates {
+ t.Errorf("Should not find local updates sr %s gen %d with error %v", srids[1].String(), g, err)
+ }
+
+ // Create a remote generation.
+ expGen := &genMetadata{Count: 1, MaxSeqNum: 50}
+ if err = log.createRemoteGeneration("VeyronPhone", srids[0], 1, expGen); err == nil {
+ t.Errorf("Remote generation create should have failed")
+ }
+ expGen.MaxSeqNum = 0
+ if err = log.createRemoteGeneration("VeyronPhone", srids[0], 1, expGen); err != nil {
+ t.Errorf("createRemoteGeneration failed with err %v", err)
+ }
+ if expGen.Pos != 3 {
+ t.Errorf("createRemoteGeneration created incorrect log order %d", expGen.Pos)
+ }
+
+ if err = log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+
+ // Reopen the log and ensure that all log records for generations exist.
+ // Also ensure that generation metadata is accurate.
+ log, err = openILog(logfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ gens := []GenId{2, 1}
+ order := map[ObjId][]uint32{srids[0]: []uint32{0, 2},
+ srids[1]: []uint32{1}}
+ for pos, sr := range srids {
+ for g := GenId(1); g <= gens[pos]; g++ {
+ // Check all log records.
+ for i := SeqNum(0); i < SeqNum(num[pos]); i++ {
+ curRec, err := log.getLogRec(log.s.id, sr, g, i)
+ if err != nil || curRec == nil {
+ t.Fatalf("GetLogRec() can not find object %s:%s:%d:%d in log file %s, err %v",
+ log.s.id, sr.String(), g, i, logfile, err)
+ }
+ }
+ // Check generation metadata.
+ curVal, err := log.getGenMetadata(log.s.id, sr, g)
+ if err != nil || curVal == nil {
+ t.Fatalf("GetGenMetadata() can not find object %s:%s:%d in log file %s, err %v",
+ log.s.id, sr.String(), g, logfile, err)
+ }
+ expVal := &genMetadata{Count: num[pos], MaxSeqNum: SeqNum(num[pos] - 1), Pos: order[sr][g-1]}
+ if !reflect.DeepEqual(curVal, expVal) {
+ t.Errorf("Data mismatch for object %s:%s:%d in log file %s: %v instead of %v",
+ log.s.id, sr.String(), g, logfile, curVal, expVal)
+ }
+ }
+ }
+
+ // Check remote generation metadata.
+ curVal, err := log.getGenMetadata("VeyronPhone", srids[0], 1)
+ if err != nil || curVal == nil {
+ t.Fatalf("GetGenMetadata() can not find object in log file %s, err %v",
+ logfile, err)
+ }
+ if !reflect.DeepEqual(curVal, expGen) {
+ t.Errorf("Data mismatch for object in log file %s: %v instead of %v",
+ logfile, curVal, expGen)
+ }
+
+ expSRGens := map[ObjId]*curLocalGen{srids[0]: &curLocalGen{3, 0},
+ srids[1]: &curLocalGen{2, 0}}
+
+ if !reflect.DeepEqual(log.head.CurSRGens, expSRGens) || log.head.Curorder != 4 {
+ t.Errorf("Data mismatch in log header %v %v", log.head.CurSRGens, log.head.Curorder)
+ }
+
+ log.dump()
+ if err = log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+}
+
+// TestProcessWatchRecord tests that local updates are correctly handled.
+// Commands are in file testdata/<local-init-00.log.sync, local-init-02.log.sync>.
+/*
+func TestProcessWatchRecord(t *testing.T) {
+ logfile := getFileName()
+ defer os.Remove(logfile)
+
+ dagfile := getFileName()
+ defer os.Remove(dagfile)
+
+ var err error
+ s := &syncd{id: "VeyronTab"}
+
+ if s.dag, err = openDAG(dagfile); err != nil {
+ t.Fatalf("Cannot open new dag file %s, err %v", logfile, err)
+ }
+ log, err := openILog(logfile, s)
+ if err != nil {
+ t.Fatalf("Cannot open new log file %s, err %v", logfile, err)
+ }
+
+ srids := []ObjId{"foo", "bar"}
+ fnames := []string{"local-init-00.log.sync", "local-init-02.log.sync"}
+ num := []uint32{3, 5}
+
+ // Try using an invalid SyncRoot ID.
+ if err = log.processWatchRecord(ObjId("haha"), 2, Version(999), &LogValue{}, NoObjId); err == nil {
+ t.Errorf("processWatchRecord did not fail when using an invalid SyncRoot ID")
+ }
+
+ // Test echo suppression on JoinSyncGroup.
+ if err := log.processWatchRecord(srids[0], 5, NoVersion, &LogValue{}, srids[0]); err != nil {
+ t.Fatalf("Echo suppression on JoinSyncGroup failed with err %v", err)
+ }
+
+ for pos, sr := range srids {
+ if err := log.initSyncRoot(sr); err != nil {
+ t.Fatalf("Cannot create new SyncRoot %s, err %v", sr.String(), err)
+ }
+
+ if err := logReplayCommands(log, fnames[pos], sr); err != nil {
+ t.Error(err)
+ }
+ }
+
+ checkObject := func(obj string, expHead Version) {
+ _, file, line, _ := runtime.Caller(1)
+ objid, err := strToObjId(obj)
+ if err != nil {
+ t.Errorf("%s:%d Could not create objid %v", file, line, err)
+ }
+
+ // Verify DAG state.
+ if head, err := log.s.dag.getHead(objid); err != nil || head != expHead {
+ t.Errorf("%s:%d Invalid object %d head in DAG %v want %v, err %v", file, line,
+ objid, head, expHead, err)
+ }
+ }
+ for pos, sr := range srids {
+ if log.head.CurSRGens[sr].CurGenNum != 1 || log.head.CurSRGens[sr].CurSeqNum != SeqNum(num[pos]) ||
+ log.head.Curorder != 0 {
+ t.Errorf("Data mismatch in log header %v", log.head)
+ }
+
+ // Check all log records.
+ for i := SeqNum(0); i < SeqNum(num[pos]); i++ {
+ curRec, err := log.getLogRec(log.s.id, sr, GenId(1), i)
+ if err != nil || curRec == nil {
+ t.Fatalf("GetLogRec() can not find object %s:%s:1:%d in log file %s, err %v",
+ log.s.id, sr.String(), i, logfile, err)
+ }
+ }
+
+ if pos == 0 {
+ checkObject("1234", 3)
+ } else if pos == 1 {
+ checkObject("12", 3)
+ checkObject("45", 20)
+ }
+ }
+
+ s.dag.flush()
+ s.dag.close()
+
+ log.dump()
+ if err = log.close(); err != nil {
+ t.Errorf("Cannot close log file %s, err %v", logfile, err)
+ }
+}
+*/
diff --git a/services/syncbase/sync/initiator.go b/services/syncbase/sync/initiator.go
new file mode 100644
index 0000000..057a2e8
--- /dev/null
+++ b/services/syncbase/sync/initiator.go
@@ -0,0 +1,943 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+import (
+ "errors"
+ "fmt"
+ "math/rand"
+ "os"
+ "time"
+
+ "v.io/v23/context"
+ "v.io/v23/naming"
+ "v.io/v23/vtrace"
+ "v.io/x/lib/vlog"
+)
+
+// Policies to pick a peer to sync with.
+const (
+ // Picks a peer at random from the available set.
+ selectRandom = iota
+
+ // TODO(hpucha): implement other policies.
+ // Picks a peer with most differing generations.
+ selectMostDiff
+
+ // Picks a peer that was synced with the furthest in the past.
+ selectOldest
+)
+
+// Policies for conflict resolution.
+const (
+ // Resolves conflicts by picking the mutation with the most recent timestamp.
+ useTime = iota
+
+ // TODO(hpucha): implement other policies.
+ // Resolves conflicts by using the app conflict resolver callbacks via store.
+ useCallback
+)
+
+var (
+ // peerSyncInterval is the duration between two consecutive
+ // sync events. In every sync event, the initiator contacts
+ // one of its peers to obtain any pending updates.
+ peerSyncInterval = 50 * time.Millisecond
+
+ // peerSelectionPolicy is the policy used to select a peer when
+ // the initiator gets a chance to sync.
+ peerSelectionPolicy = selectRandom
+
+ // conflictResolutionPolicy is the policy used to resolve conflicts.
+ conflictResolutionPolicy = useTime
+
+ errNoUsefulPeer = errors.New("no useful peer to contact")
+)
+
+// syncInitiator contains the metadata and state for the initiator thread.
+type syncInitiator struct {
+ syncd *syncd
+
+ // State accumulated during an initiation round.
+ iState *initiationState
+}
+
+// initiationState accumulated during an initiation round.
+type initiationState struct {
+
+ // Veyron name of peer being synced with.
+ peer string
+
+ // Local generation vector.
+ local map[ObjId]GenVector
+
+ // SyncGroups being requested in the initiation round.
+ syncGroups map[ObjId]GroupIdSet
+
+ // Map to track new generations received in the RPC reply.
+ newGens map[string]*genMetadata
+
+ // Array to track order of arrival for the generations.
+ // We need to preserve this order for the replay.
+ orderGens []string
+
+ // Generation vector to track the oldest generation received
+ // in the RPC reply per device, for garbage collection.
+ minGens map[ObjId]GenVector
+
+ // Generation vector from the remote peer.
+ remote map[ObjId]GenVector
+
+ // Tmp kvdb state.
+ tmpFile string
+ tmpDB *kvdb
+ tmpTbl *kvtable
+
+ // State to track updated objects during a log replay.
+ updObjects map[ObjId]*objConflictState
+
+ // State to delete objects from the "priv" table.
+ delPrivObjs map[ObjId]struct{}
+}
+
+// objConflictState contains the conflict state for objects that are
+// updated during an initiator run.
+type objConflictState struct {
+ isConflict bool
+ newHead Version
+ oldHead Version
+ ancestor Version
+ resolvVal *LogValue
+ srID ObjId
+}
+
+// newInitiator creates a new initiator instance attached to the given syncd instance.
+func newInitiator(syncd *syncd, syncTick time.Duration) *syncInitiator {
+ i := &syncInitiator{syncd: syncd}
+
+ // Override the default peerSyncInterval value if syncTick is specified.
+ if syncTick > 0 {
+ peerSyncInterval = syncTick
+ }
+
+ vlog.VI(1).Infof("newInitiator: My device ID: %s", i.syncd.id)
+ vlog.VI(1).Infof("newInitiator: Sync interval: %v", peerSyncInterval)
+
+ return i
+}
+
+// contactPeers wakes up every peerSyncInterval to contact peers and get deltas from them.
+func (i *syncInitiator) contactPeers() {
+ ticker := time.NewTicker(peerSyncInterval)
+ for {
+ select {
+ case <-i.syncd.closed:
+ ticker.Stop()
+ i.syncd.pending.Done()
+ return
+ case <-ticker.C:
+ }
+
+ peerRelName, err := i.pickPeer()
+ if err != nil {
+ continue
+ }
+
+ i.getDeltasFromPeer(peerRelName)
+ }
+}
+
+// pickPeer picks a sync endpoint to sync with.
+func (i *syncInitiator) pickPeer() (string, error) {
+ myID := string(i.syncd.relName)
+
+ switch peerSelectionPolicy {
+ case selectRandom:
+ // TODO(hpucha): Eliminate reaching into syncd's lock.
+ i.syncd.lock.RLock()
+ // neighbors, err := i.syncd.sgtab.getMembers()
+ neighbors := make(map[string]uint32)
+ var err error
+ i.syncd.lock.RUnlock()
+
+ if err != nil {
+ return "", err
+ }
+
+ // Remove myself from the set so only neighbors are counted.
+ delete(neighbors, myID)
+
+ if len(neighbors) == 0 {
+ return "", errNoUsefulPeer
+ }
+
+ // Pick a neighbor at random.
+ ind := rand.Intn(len(neighbors))
+ for k := range neighbors {
+ if ind == 0 {
+ return k, nil
+ }
+ ind--
+ }
+ return "", fmt.Errorf("random selection didn't succeed")
+ default:
+ return "", fmt.Errorf("unknown peer selection policy")
+ }
+}
+
+// getDeltasFromPeer performs an initiation round to the specified
+// peer. An initiation round consists of:
+// * Creating local generation for syncroots that are going to be requested in this round.
+// * Contacting the peer to receive all the deltas based on the local gen vector.
+// * Processing those deltas to discover objects which have been updated.
+// * Processing updated objects to detect and resolve any conflicts if needed.
+// * Communicating relevant object updates to the store.
+func (i *syncInitiator) getDeltasFromPeer(peerRelName string) {
+
+ vlog.VI(2).Infof("getDeltasFromPeer:: %s", peerRelName)
+ // Initialize initiation state.
+ i.newInitiationState()
+ defer i.clearInitiationState()
+
+ // Freeze the most recent batch of local changes
+ // before fetching remote changes from a peer.
+ //
+ // We only allow an initiator to create new local
+ // generations (not responders/watcher) in order to
+ // maintain a static baseline for the duration of a
+ // sync. This addresses the following race condition:
+ // If we allow responders to create new local
+ // generations while the initiator is in progress,
+ // they may beat the initiator and send these new
+ // generations to remote devices. These remote
+ // devices in turn can send these generations back to
+ // the initiator in progress which was started with
+ // older generation information.
+ err := i.updateLocalGeneration(peerRelName)
+ if err != nil {
+ vlog.Fatalf("getDeltasFromPeer:: error updating local generation: err %v", err)
+ }
+
+ // Obtain deltas from the peer over the network. These deltas
+ // are stored in a tmp kvdb.
+ if err := i.getDeltas(); err != nil {
+ vlog.Errorf("getDeltasFromPeer:: error getting deltas: err %v", err)
+ return
+ }
+
+ i.syncd.sgOp.Lock()
+ defer i.syncd.sgOp.Unlock()
+
+ if err := i.processDeltas(); err != nil {
+ vlog.Fatalf("getDeltasFromPeer:: error processing logs: err %v", err)
+ }
+
+ if err := i.processUpdatedObjects(); err != nil {
+ vlog.Fatalf("getDeltasFromPeer:: error processing objects: err %v", err)
+ }
+}
+
+// newInitiationState creates new initiation state.
+func (i *syncInitiator) newInitiationState() {
+ st := &initiationState{}
+ st.local = make(map[ObjId]GenVector)
+ st.syncGroups = make(map[ObjId]GroupIdSet)
+ st.newGens = make(map[string]*genMetadata)
+ st.minGens = make(map[ObjId]GenVector)
+ st.remote = make(map[ObjId]GenVector)
+ st.updObjects = make(map[ObjId]*objConflictState)
+ st.delPrivObjs = make(map[ObjId]struct{})
+
+ i.iState = st
+}
+
+// clearInitiationState cleans up the state from the current initiation round.
+func (i *syncInitiator) clearInitiationState() {
+ if i.iState.tmpDB != nil {
+ i.iState.tmpDB.close()
+ }
+ if i.iState.tmpFile != "" {
+ os.Remove(i.iState.tmpFile)
+ }
+
+ for o := range i.iState.delPrivObjs {
+ i.syncd.dag.delPrivNode(o)
+ }
+
+ i.syncd.dag.clearGraft()
+
+ i.iState = nil
+}
+
+// updateLocalGeneration creates a new local generation if needed and
+// populates the newest local generation vector.
+func (i *syncInitiator) updateLocalGeneration(peerRelName string) error {
+ // TODO(hpucha): Eliminate reaching into syncd's lock.
+ i.syncd.lock.Lock()
+ defer i.syncd.lock.Unlock()
+
+ // peerInfo, err := i.syncd.sgtab.getMemberInfo(peerRelName)
+ // if err != nil {
+ // return err
+ // }
+
+ // Re-construct all mount table possibilities.
+ mtTables := make(map[string]struct{})
+
+ // for sg := range peerInfo.gids {
+ // sgData, err := i.syncd.sgtab.getSyncGroupByID(sg)
+ // if err != nil {
+ // return err
+ // }
+ // sr := ObjId(sgData.SrvInfo.RootObjId)
+ // for _, mt := range sgData.SrvInfo.Config.MountTables {
+ // mtTables[mt] = struct{}{}
+ // }
+ // i.iState.syncGroups[sr] = append(i.iState.syncGroups[sr], sg)
+ // }
+
+ // Create a new local generation if there are any local updates
+ // for every syncroot that is common with the "peer" to be
+ // contacted.
+ for sr := range i.iState.syncGroups {
+ gen, err := i.syncd.log.createLocalGeneration(sr)
+ if err != nil && err != errNoUpdates {
+ return err
+ }
+
+ if err == nil {
+ vlog.VI(2).Infof("updateLocalGeneration:: created gen for sr %s at %d", sr.String(), gen)
+
+ // Update local generation vector in devTable.
+ if err = i.syncd.devtab.updateGeneration(i.syncd.id, sr, i.syncd.id, gen); err != nil {
+ return err
+ }
+ }
+
+ v, err := i.syncd.devtab.getGenVec(i.syncd.id, sr)
+ if err != nil {
+ return err
+ }
+
+ i.iState.local[sr] = v
+ }
+
+ // Check if the name is absolute, and if so, use the name as-is.
+ if naming.Rooted(peerRelName) {
+ i.iState.peer = peerRelName
+ return nil
+ }
+
+ // Pick any global name to contact the peer.
+ for mt := range mtTables {
+ i.iState.peer = naming.Join(mt, peerRelName)
+ vlog.VI(2).Infof("updateLocalGeneration:: contacting peer %s", i.iState.peer)
+ return nil
+ }
+
+ return fmt.Errorf("no mounttables found")
+}
+
+// getDeltas obtains the deltas from the peer and stores them in a tmp kvdb.
+func (i *syncInitiator) getDeltas() error {
+ // As log records are streamed, they are saved
+ // in a tmp kvdb so that they can be processed in one batch.
+ if err := i.initTmpKVDB(); err != nil {
+ return err
+ }
+
+ ctx, _ := vtrace.SetNewTrace(i.syncd.ctx)
+ ctx, cancel := context.WithTimeout(ctx, time.Minute)
+ defer cancel()
+
+ vlog.VI(1).Infof("getDeltas:: From peer with DeviceId %s at %v", i.iState.peer, time.Now().UTC())
+
+ // Construct a new stub that binds to peer endpoint.
+ c := SyncClient(naming.JoinAddressName(i.iState.peer, SyncSuffix))
+
+ vlog.VI(1).Infof("GetDeltasFromPeer:: Sending local information: %v", i.iState.local)
+
+ // Issue a GetDeltas() rpc.
+ stream, err := c.GetDeltas(ctx, i.iState.local, i.iState.syncGroups, i.syncd.id)
+ if err != nil {
+ return err
+ }
+
+ return i.recvLogStream(stream)
+}
+
+// initTmpKVDB initializes the tmp kvdb to store the log record stream.
+func (i *syncInitiator) initTmpKVDB() error {
+ i.iState.tmpFile = fmt.Sprintf("%s/tmp_%d", i.syncd.kvdbPath, time.Now().UnixNano())
+ tmpDB, tbls, err := kvdbOpen(i.iState.tmpFile, []string{"tmprec"})
+ if err != nil {
+ return err
+ }
+ i.iState.tmpDB = tmpDB
+ i.iState.tmpTbl = tbls[0]
+ return nil
+}
+
+// recvLogStream receives the log records from the GetDeltas stream
+// and puts them in tmp kvdb for later processing.
+func (i *syncInitiator) recvLogStream(stream SyncGetDeltasClientCall) error {
+ rStream := stream.RecvStream()
+ for rStream.Advance() {
+ rec := rStream.Value()
+
+ // Insert log record in tmpTbl.
+ if err := i.iState.tmpTbl.set(logRecKey(rec.DevId, rec.SyncRootId, rec.GenNum, rec.SeqNum), &rec); err != nil {
+ // TODO(hpucha): do we need to cancel stream?
+ return err
+ }
+
+ // Populate the generation metadata.
+ genKey := generationKey(rec.DevId, rec.SyncRootId, rec.GenNum)
+ if gen, ok := i.iState.newGens[genKey]; !ok {
+ // New generation in the stream.
+ i.iState.orderGens = append(i.iState.orderGens, genKey)
+ i.iState.newGens[genKey] = &genMetadata{
+ Count: 1,
+ MaxSeqNum: rec.SeqNum,
+ }
+ if _, ok := i.iState.minGens[rec.SyncRootId]; !ok {
+ i.iState.minGens[rec.SyncRootId] = GenVector{}
+ }
+ g, ok := i.iState.minGens[rec.SyncRootId][rec.DevId]
+ if !ok || g > rec.GenNum {
+ i.iState.minGens[rec.SyncRootId][rec.DevId] = rec.GenNum
+ }
+ } else {
+ gen.Count++
+ if rec.SeqNum > gen.MaxSeqNum {
+ gen.MaxSeqNum = rec.SeqNum
+ }
+ }
+ }
+
+ if err := rStream.Err(); err != nil {
+ return err
+ }
+
+ var err error
+ if i.iState.remote, err = stream.Finish(); err != nil {
+ return err
+ }
+ vlog.VI(1).Infof("recvLogStream:: Local vector %v", i.iState.local)
+ vlog.VI(1).Infof("recvLogStream:: Remote vector %v", i.iState.remote)
+ vlog.VI(2).Infof("recvLogStream:: orderGens %v", i.iState.orderGens)
+ return nil
+}
+
+// processDeltas replays an entire log stream spanning multiple
+// generations from different devices received from a single GetDeltas
+// call. It does not perform any conflict resolution during replay.
+// This avoids resolving conflicts that have already been resolved by
+// other devices.
+func (i *syncInitiator) processDeltas() error {
+ // TODO(hpucha): Eliminate reaching into syncd's lock.
+ i.syncd.lock.Lock()
+ defer i.syncd.lock.Unlock()
+
+ // Track received transactions.
+ txMap := make(map[TxId]uint32)
+
+ // Loop through each received generation in order.
+ for _, key := range i.iState.orderGens {
+ gen := i.iState.newGens[key]
+ dev, sr, gnum, err := splitGenerationKey(key)
+ if err != nil {
+ return err
+ }
+
+ // If "sr" has been left since getDeltas, skip processing.
+ // if !i.syncd.sgtab.isSyncRoot(sr) {
+ // continue
+ // }
+
+ for l := SeqNum(0); l <= gen.MaxSeqNum; l++ {
+ var rec LogRec
+ if err := i.iState.tmpTbl.get(logRecKey(dev, sr, gnum, l), &rec); err != nil {
+ return err
+ }
+
+ // Begin a new transaction if needed.
+ curTx := rec.Value.TxId
+ if curTx != NoTxId {
+ if cnt, ok := txMap[curTx]; !ok {
+ if i.syncd.dag.addNodeTxStart(curTx) != curTx {
+ return fmt.Errorf("failed to create transaction")
+ }
+ txMap[curTx] = rec.Value.TxCount
+ vlog.VI(2).Infof("processDeltas:: Begin Tx %v, count %d", curTx, rec.Value.TxCount)
+ } else if cnt != rec.Value.TxCount {
+ return fmt.Errorf("inconsistent counts for tid %v %d, %d", curTx, cnt, rec.Value.TxCount)
+ }
+ }
+
+ if err := i.insertRecInLogAndDag(&rec, curTx); err != nil {
+ return err
+ }
+
+ // Mark object dirty.
+ i.iState.updObjects[rec.ObjId] = &objConflictState{
+ srID: rec.SyncRootId,
+ }
+ }
+ // Insert the generation metadata.
+ if err := i.syncd.log.createRemoteGeneration(dev, sr, gnum, gen); err != nil {
+ return err
+ }
+ }
+
+ // End the started transactions if any.
+ for t, cnt := range txMap {
+ if err := i.syncd.dag.addNodeTxEnd(t, cnt); err != nil {
+ return err
+ }
+ vlog.VI(2).Infof("processLogStream:: End Tx %v %v", t, cnt)
+ }
+
+ return nil
+}
+
+// insertRecInLogAndDag adds a new log record to log and dag data structures.
+func (i *syncInitiator) insertRecInLogAndDag(rec *LogRec, txID TxId) error {
+ logKey, err := i.syncd.log.putLogRec(rec)
+ if err != nil {
+ return err
+ }
+
+ vlog.VI(2).Infof("insertRecInLogAndDag:: Adding log record %v, Tx %v", rec, txID)
+ switch rec.RecType {
+ case NodeRec:
+ return i.syncd.dag.addNode(rec.ObjId, rec.CurVers, true, rec.Value.Delete, rec.Parents, logKey, txID)
+ case LinkRec:
+ return i.syncd.dag.addParent(rec.ObjId, rec.CurVers, rec.Parents[0], true)
+ default:
+ return fmt.Errorf("unknown log record type")
+ }
+}
+
+// processUpdatedObjects processes all the updates received by the
+// initiator, one object at a time. For each updated object, we first
+// check if the object has any conflicts, resulting in three
+// possibilities:
+//
+// * There is no conflict, and no updates are needed to the store
+// (isConflict=false, newHead == oldHead). All changes received convey
+// information that still keeps the local head as the most recent
+// version. This occurs when conflicts are resolved by blessings.
+//
+// * There is no conflict, but a remote version is discovered that
+// builds on the local head (isConflict=false, newHead != oldHead). In
+// this case, we generate a store mutation to simply update the store
+// to the latest value.
+//
+// * There is a conflict and we call into the app or the system to
+// resolve the conflict, resulting in three possibilties: (a) conflict
+// was resolved by blessing the local version. In this case, store
+// need not be updated, but a link is added to record the
+// blessing. (b) conflict was resolved by blessing the remote
+// version. In this case, store is updated with the remote version and
+// a link is added as well. (c) conflict was resolved by generating a
+// new store mutation. In this case, store is updated with the new
+// version.
+//
+// We then put all these mutations in the store. If the put succeeds,
+// we update the log and dag state suitably (move the head ptr of the
+// object in the dag to the latest version, and create a new log
+// record reflecting conflict resolution if any). Puts to store can
+// fail since preconditions on the objects may have been violated. In
+// this case, we wait to get the latest versions of objects from the
+// store, and recheck if the object has any conflicts and repeat the
+// above steps, until put to store succeeds.
+func (i *syncInitiator) processUpdatedObjects() error {
+ for {
+ if err := i.detectConflicts(); err != nil {
+ return err
+ }
+
+ if err := i.resolveConflicts(); err != nil {
+ return err
+ }
+
+ err := i.updateStoreAndSync()
+ if err == nil {
+ break
+ }
+
+ vlog.Errorf("PutMutations failed %v. Will retry", err)
+ // TODO(hpucha): Sleeping and retrying is a temporary
+ // solution. Next iteration will have coordination
+ // with watch thread to intelligently retry. Hence
+ // this value is not a config param.
+ time.Sleep(1 * time.Second)
+ }
+
+ return nil
+}
+
+// detectConflicts iterates through all the updated objects to detect
+// conflicts.
+func (i *syncInitiator) detectConflicts() error {
+ // TODO(hpucha): Eliminate reaching into syncd's lock.
+ i.syncd.lock.RLock()
+ defer i.syncd.lock.RUnlock()
+
+ for obj, st := range i.iState.updObjects {
+ // Check if object has a conflict.
+ var err error
+ st.isConflict, st.newHead, st.oldHead, st.ancestor, err = i.syncd.dag.hasConflict(obj)
+ vlog.VI(2).Infof("detectConflicts:: object %v state %v err %v",
+ obj, st, err)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// resolveConflicts resolves conflicts for updated objects. Conflicts
+// may be resolved by adding new versions or blessing either the local
+// or the remote version.
+func (i *syncInitiator) resolveConflicts() error {
+ for obj, st := range i.iState.updObjects {
+ if !st.isConflict {
+ continue
+ }
+
+ res, err := i.resolveObjConflict(obj, st.oldHead, st.newHead, st.ancestor)
+ if err != nil {
+ return err
+ }
+
+ st.resolvVal = res
+ }
+
+ return nil
+}
+
+// resolveObjConflict resolves a conflict for an object given its ID and
+// the 3 versions that express the conflict: the object's local version, its
+// remote version (from the device contacted), and the common ancestor from
+// which both versions branched away. The function returns the new object
+// value according to the conflict resolution policy.
+func (i *syncInitiator) resolveObjConflict(oid ObjId,
+ local, remote, ancestor Version) (*LogValue, error) {
+
+ // Fetch the log records of the 3 object versions.
+ versions := []Version{local, remote, ancestor}
+ lrecs, err := i.getLogRecsBatch(oid, versions)
+ if err != nil {
+ return nil, err
+ }
+
+ // Resolve the conflict according to the resolution policy.
+ var res *LogValue
+
+ switch conflictResolutionPolicy {
+ case useTime:
+ res, err = i.resolveObjConflictByTime(oid, lrecs[0], lrecs[1], lrecs[2])
+ default:
+ err = fmt.Errorf("unknown conflict resolution policy: %v", conflictResolutionPolicy)
+ }
+
+ if err != nil {
+ return nil, err
+ }
+
+ resCopy := *res
+ // resCopy.Mutation.Version = NewVersion()
+ // resCopy.Mutation.Dir = resDir
+ resCopy.SyncTime = time.Now().UnixNano()
+ resCopy.TxId = NoTxId
+ resCopy.TxCount = 0
+ return &resCopy, nil
+}
+
+// resolveObjConflictByTime resolves conflicts using the timestamps
+// of the conflicting mutations. It picks a mutation with the larger
+// timestamp, i.e. the most recent update. If the timestamps are equal,
+// it uses the mutation version numbers as a tie-breaker, picking the
+// mutation with the larger version.
+// Instead of creating a new version that resolves the conflict, we are
+// blessing an existing version as the conflict resolution.
+func (i *syncInitiator) resolveObjConflictByTime(oid ObjId,
+ local, remote, ancestor *LogRec) (*LogValue, error) {
+
+ var res *LogValue
+ switch {
+ case local.Value.SyncTime > remote.Value.SyncTime:
+ res = &local.Value
+ case local.Value.SyncTime < remote.Value.SyncTime:
+ res = &remote.Value
+ // case local.Value.Mutation.Version > remote.Value.Mutation.Version:
+ // res = &local.Value
+ // case local.Value.Mutation.Version < remote.Value.Mutation.Version:
+ // res = &remote.Value
+ }
+
+ return res, nil
+}
+
+// getLogRecsBatch gets the log records for an array of versions.
+func (i *syncInitiator) getLogRecsBatch(obj ObjId, versions []Version) ([]*LogRec, error) {
+ // TODO(hpucha): Eliminate reaching into syncd's lock.
+ i.syncd.lock.RLock()
+ defer i.syncd.lock.RUnlock()
+
+ lrecs := make([]*LogRec, len(versions))
+ var err error
+ for p, v := range versions {
+ lrecs[p], err = i.getLogRec(obj, v)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return lrecs, nil
+}
+
+// updateStoreAndSync updates the store, and if that is successful,
+// updates log and dag data structures.
+func (i *syncInitiator) updateStoreAndSync() error {
+
+ // TODO(hpucha): Eliminate reaching into syncd's lock.
+ i.syncd.lock.Lock()
+ defer i.syncd.lock.Unlock()
+
+ // var m []raw.Mutation
+ // for obj, st := range i.iState.updObjects {
+ // if !st.isConflict {
+ // rec, err := i.getLogRec(obj, st.newHead)
+ // if err != nil {
+ // return err
+ // }
+ // st.resolvVal = &rec.Value
+ // // Sanity check.
+ // if st.resolvVal.Mutation.Version != st.newHead {
+ // return fmt.Errorf("bad mutation %d %d",
+ // st.resolvVal.Mutation.Version, st.newHead)
+ // }
+ // }
+
+ // // If the local version is picked, no further updates
+ // // to the store are needed. If the remote version is
+ // // picked, we put it in the store.
+ // if st.resolvVal.Mutation.Version != st.oldHead {
+ // st.resolvVal.Mutation.PriorVersion = st.oldHead
+
+ // // Convert resolvVal.Mutation into a mutation for the Store.
+ // stMutation, err := i.storeMutation(obj, st.resolvVal)
+ // if err != nil {
+ // return err
+ // }
+
+ // vlog.VI(2).Infof("updateStoreAndSync:: Try to append mutation %v (%v) for obj %v (nh %v, oh %v)",
+ // st.resolvVal.Mutation, stMutation, obj, st.newHead, st.oldHead)
+
+ // // Append to mutations, skipping a delete following a delete mutation.
+ // if stMutation.Version != NoVersion ||
+ // stMutation.PriorVersion != NoVersion {
+ // vlog.VI(2).Infof("updateStoreAndSync:: appending mutation %v for obj %v",
+ // stMutation, obj)
+ // m = append(m, stMutation)
+ // }
+ // }
+ // }
+
+ // TODO(hpucha): We will hold the lock across PutMutations rpc
+ // to prevent a race with watcher. The next iteration will
+ // clean up this coordination.
+ // if store := i.syncd.store; store != nil && len(m) > 0 {
+ // ctx, _ := vtrace.SetNewTrace(i.syncd.ctx)
+ // ctx, cancel := context.WithTimeout(ctx, time.Minute)
+ // defer cancel()
+
+ // stream, err := store.PutMutations(ctx)
+ // if err != nil {
+ // vlog.Errorf("updateStoreAndSync:: putmutations err %v", err)
+ // return err
+ // }
+ // sender := stream.SendStream()
+ // for i := range m {
+ // if err := sender.Send(m[i]); err != nil {
+ // vlog.Errorf("updateStoreAndSync:: send err %v", err)
+ // return err
+ // }
+ // }
+ // if err := sender.Close(); err != nil {
+ // vlog.Errorf("updateStoreAndSync:: closesend err %v", err)
+ // return err
+ // }
+ // if err := stream.Finish(); err != nil {
+ // vlog.Errorf("updateStoreAndSync:: finish err %v", err)
+ // return err
+ // }
+ // }
+
+ vlog.VI(2).Infof("updateStoreAndSync:: putmutations succeeded")
+ if err := i.updateLogAndDag(); err != nil {
+ return err
+ }
+
+ if err := i.updateGenVecs(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// storeMutation converts a resolved mutation generated by syncd to
+// one that can be sent to the store. To send to the store, it
+// converts the version numbers corresponding to object deletions to
+// NoVersion when required. It also converts the version number
+// appropriately to handle SyncGroup join.
+// func (i *syncInitiator) storeMutation(obj ObjId, resolvVal *LogValue) (raw.Mutation, error) {
+// curDelete := resolvVal.Delete
+// priorDelete := false
+// if resolvVal.Mutation.PriorVersion != raw.NoVersion {
+// oldRec, err := i.getLogRec(obj, resolvVal.Mutation.PriorVersion)
+// if err != nil {
+// return raw.Mutation{}, err
+// }
+// priorDelete = oldRec.Value.Delete
+// }
+
+// // Handle the versioning of a SyncGroup's root ObjId during join.
+// if resolvVal.Mutation.PriorVersion == raw.NoVersion {
+// if i.syncd.sgtab.isSyncRoot(obj) {
+// node, err := i.syncd.dag.getPrivNode(obj)
+// if err != nil {
+// return raw.Mutation{}, err
+// }
+// resolvVal.Mutation.PriorVersion = node.Mutation.Version
+// i.iState.delPrivObjs[obj] = struct{}{}
+// }
+// }
+
+// // Current version and prior versions are not deletes.
+// if !curDelete && !priorDelete {
+// return resolvVal.Mutation, nil
+// }
+
+// // Creating a new copy of the mutation to adjust version
+// // numbers when handling deletions.
+// stMutation := resolvVal.Mutation
+// // Adjust the current version if this a deletion.
+// if curDelete {
+// stMutation.Version = NoVersion
+// }
+// // Adjust the prior version if it is a deletion.
+// if priorDelete {
+// stMutation.PriorVersion = NoVersion
+// }
+
+// return stMutation, nil
+// }
+
+// getLogRec returns the log record corresponding to a given object and its version.
+func (i *syncInitiator) getLogRec(obj ObjId, vers Version) (*LogRec, error) {
+ logKey, err := i.syncd.dag.getLogrec(obj, vers)
+ vlog.VI(2).Infof("getLogRec:: logkey from dag is %s", logKey)
+ if err != nil {
+ return nil, err
+ }
+ dev, sg, gen, lsn, err := splitLogRecKey(logKey)
+ if err != nil {
+ return nil, err
+ }
+ vlog.VI(2).Infof("getLogRec:: splitting logkey %s to %s %v %v %v", logKey, dev, sg, gen, lsn)
+ rec, err := i.syncd.log.getLogRec(dev, sg, gen, lsn)
+ if err != nil {
+ return nil, err
+ }
+ return rec, nil
+}
+
+// updateLogAndDag updates the log and dag data structures on a successful store put.
+func (i *syncInitiator) updateLogAndDag() error {
+ for obj, st := range i.iState.updObjects {
+ if st.isConflict {
+ // Object had a conflict, which was resolved successfully.
+ // Put is successful, create a log record.
+ var err error
+ var rec *LogRec
+
+ // switch {
+ // case st.resolvVal.Mutation.Version == st.oldHead:
+ // // Local version was blessed as the conflict resolution.
+ // rec, err = i.syncd.log.createLocalLinkLogRec(obj, st.oldHead, st.newHead, st.srID)
+ // case st.resolvVal.Mutation.Version == st.newHead:
+ // // Remote version was blessed as the conflict resolution.
+ // rec, err = i.syncd.log.createLocalLinkLogRec(obj, st.newHead, st.oldHead, st.srID)
+ // default:
+ // // New version was created to resolve the conflict.
+ // parents := []Version{st.newHead, st.oldHead}
+ // rec, err = i.syncd.log.createLocalLogRec(obj, st.resolvVal.Mutation.Version, parents, st.resolvVal, st.srID)
+ // }
+ if err != nil {
+ return err
+ }
+ logKey, err := i.syncd.log.putLogRec(rec)
+ if err != nil {
+ return err
+ }
+ // Add a new DAG node.
+ switch rec.RecType {
+ case NodeRec:
+ // TODO(hpucha): addNode operations arising out of conflict resolution
+ // may need to be part of a transaction when app-driven resolution
+ // is introduced.
+ err = i.syncd.dag.addNode(obj, rec.CurVers, false, rec.Value.Delete, rec.Parents, logKey, NoTxId)
+ case LinkRec:
+ err = i.syncd.dag.addParent(obj, rec.CurVers, rec.Parents[0], false)
+ default:
+ return fmt.Errorf("unknown log record type")
+ }
+ if err != nil {
+ return err
+ }
+ }
+
+ // Move the head. This should be idempotent. We may
+ // move head to the local head in some cases.
+ // if err := i.syncd.dag.moveHead(obj, st.resolvVal.Mutation.Version); err != nil {
+ // return err
+ // }
+ }
+ return nil
+}
+
+// updateGenVecs updates local, reclaim and remote vectors at the end of an initiator cycle.
+func (i *syncInitiator) updateGenVecs() error {
+ // Update the local gen vector and put it in kvdb only if we have new updates.
+ if len(i.iState.updObjects) > 0 {
+ // remote can be a subset of local.
+ for sr, rVec := range i.iState.remote {
+ lVec := i.iState.local[sr]
+
+ if err := i.syncd.devtab.updateLocalGenVector(lVec, rVec); err != nil {
+ return err
+ }
+
+ if err := i.syncd.devtab.putGenVec(i.syncd.id, sr, lVec); err != nil {
+ return err
+ }
+
+ // if err := i.syncd.devtab.updateReclaimVec(minGens); err != nil {
+ // return err
+ // }
+ }
+ }
+
+ for sr, rVec := range i.iState.remote {
+ // Cache the remote generation vector for space reclamation.
+ if err := i.syncd.devtab.putGenVec(i.syncd.nameToDevId(i.iState.peer), sr, rVec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/services/syncbase/sync/kvdb.go b/services/syncbase/sync/kvdb.go
new file mode 100644
index 0000000..a15542c
--- /dev/null
+++ b/services/syncbase/sync/kvdb.go
@@ -0,0 +1,137 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Helpful wrappers to a persistent key/value (K/V) DB used by Veyron Sync.
+// The current underlying DB is an in-memory map.
+
+import (
+ "bytes"
+ "fmt"
+ "sort"
+
+ "v.io/v23/vom"
+)
+
+var memStore map[string]*kvdb
+
+type kvdb struct {
+ tables map[string]*kvtable
+}
+
+type kvtable struct {
+ data map[string][]byte
+}
+
+// kvdbOpen opens or creates a K/V DB for the given filename and table names
+// within the DB. It returns the DB handler and handlers for each table.
+func kvdbOpen(filename string, tables []string) (*kvdb, []*kvtable, error) {
+ if memStore == nil {
+ memStore = make(map[string]*kvdb)
+ }
+
+ db := memStore[filename]
+ if db == nil {
+ db = &kvdb{tables: make(map[string]*kvtable)}
+ memStore[filename] = db
+ }
+
+ tbls := make([]*kvtable, len(tables))
+
+ for i, table := range tables {
+ t := db.tables[table]
+ if t == nil {
+ t = &kvtable{data: make(map[string][]byte)}
+ db.tables[table] = t
+ }
+ tbls[i] = t
+ }
+
+ return db, tbls, nil
+}
+
+// close closes the given K/V DB.
+func (db *kvdb) close() {
+}
+
+// flush flushes the given K/V DB to disk.
+func (db *kvdb) flush() {
+}
+
+// set stores (or overwrites) the given key/value pair in the DB table.
+func (t *kvtable) set(key string, value interface{}) error {
+ var val bytes.Buffer
+ enc, err := vom.NewEncoder(&val)
+ if err != nil {
+ return err
+ }
+ if enc.Encode(value); err != nil {
+ return err
+ }
+ t.data[key] = val.Bytes()
+ return nil
+}
+
+// create stores the given key/value pair in the DB table only if
+// the key does not already exist. Otherwise it returns an error.
+func (t *kvtable) create(key string, value interface{}) error {
+ if t.hasKey(key) {
+ return fmt.Errorf("key %s exists", key)
+ }
+ return t.set(key, value)
+}
+
+// update stores the given key/value pair in the DB table only if
+// the key already exists. Otherwise it returns an error.
+func (t *kvtable) update(key string, value interface{}) error {
+ if !t.hasKey(key) {
+ return fmt.Errorf("key %s does not exist", key)
+ }
+ return t.set(key, value)
+}
+
+// get retrieves the value of a key from the DB table.
+func (t *kvtable) get(key string, value interface{}) error {
+ val := t.data[key]
+ if val == nil {
+ return fmt.Errorf("entry %s not found in the K/V DB table", key)
+ }
+ dec, err := vom.NewDecoder(bytes.NewBuffer(val))
+ if err != nil {
+ return err
+ }
+ return dec.Decode(value)
+}
+
+// del deletes the entry in the DB table given its key.
+func (t *kvtable) del(key string) error {
+ delete(t.data, key)
+ return nil
+}
+
+// hasKey returns true if the given key exists in the DB table.
+func (t *kvtable) hasKey(key string) bool {
+ val, ok := t.data[key]
+ return ok && val != nil
+}
+
+// keyIter iterates over all keys in a DB table invoking the given callback
+// function for each one. The key iterator callback is passed the item key.
+func (t *kvtable) keyIter(keyIterCB func(key string)) error {
+ keys := make([]string, 0, len(t.data))
+ for k := range t.data {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ keyIterCB(k)
+ }
+ return nil
+}
+
+// getNumKeys returns the number of keys (entries) in the DB table.
+func (t *kvtable) getNumKeys() uint64 {
+ return uint64(len(t.data))
+}
diff --git a/services/syncbase/sync/kvdb_test.go b/services/syncbase/sync/kvdb_test.go
new file mode 100644
index 0000000..69116e3
--- /dev/null
+++ b/services/syncbase/sync/kvdb_test.go
@@ -0,0 +1,407 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Tests for the Veyron Sync K/V DB component.
+
+import (
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+ "time"
+)
+
+// A user structure stores info in the "users" table.
+type user struct {
+ Username string
+ Drinks []string
+}
+
+// A drink structure stores info in the "drinks" table.
+type drink struct {
+ Name string
+ Alcohol bool
+}
+
+var (
+ users = []user{
+ {Username: "lancelot", Drinks: []string{"beer", "coffee"}},
+ {Username: "arthur", Drinks: []string{"coke", "beer", "coffee"}},
+ {Username: "robin", Drinks: []string{"pepsi"}},
+ {Username: "galahad"},
+ }
+ drinks = []drink{
+ {Name: "coke", Alcohol: false},
+ {Name: "pepsi", Alcohol: false},
+ {Name: "beer", Alcohol: true},
+ {Name: "coffee", Alcohol: false},
+ }
+)
+
+// createTestDB creates a K/V DB with 2 tables.
+func createTestDB(t *testing.T) (fname string, db *kvdb, usersTbl, drinksTbl *kvtable) {
+ fname = fmt.Sprintf("%s/sync_kvdb_test_%d_%d", os.TempDir(), os.Getpid(), time.Now().UnixNano())
+ db, tbls, err := kvdbOpen(fname, []string{"users", "drinks"})
+ if err != nil {
+ os.Remove(fname)
+ t.Fatalf("cannot create new K/V DB file %s: %v", fname, err)
+ }
+
+ usersTbl, drinksTbl = tbls[0], tbls[1]
+ return
+}
+
+// initTestTables initializes the K/V tables used by the tests.
+func initTestTables(t *testing.T, usersTbl, drinksTbl *kvtable, useCreate bool) {
+ userPut, drinkPut, funcName := usersTbl.set, drinksTbl.set, "set()"
+ if useCreate {
+ userPut, drinkPut, funcName = usersTbl.create, drinksTbl.create, "create()"
+ }
+
+ for _, uu := range users {
+ if err := userPut(uu.Username, &uu); err != nil {
+ t.Fatalf("%s failed for user %s", funcName, uu.Username)
+ }
+ }
+
+ for _, dd := range drinks {
+ if err := drinkPut(dd.Name, &dd); err != nil {
+ t.Fatalf("%s failed for drink %s", funcName, dd.Name)
+ }
+ }
+
+ return
+}
+
+// checkTestTables verifies the contents of the K/V tables.
+func checkTestTables(t *testing.T, usersTbl, drinksTbl *kvtable) {
+ for _, uu := range users {
+ var u2 user
+ if err := usersTbl.get(uu.Username, &u2); err != nil {
+ t.Fatalf("get() failed for user %s", uu.Username)
+ }
+ if !reflect.DeepEqual(u2, uu) {
+ t.Fatalf("got wrong data for user %s: %#v instead of %#v", uu.Username, u2, uu)
+ }
+ if !usersTbl.hasKey(uu.Username) {
+ t.Fatalf("hasKey() did not find user %s", uu.Username)
+ }
+ }
+ for _, dd := range drinks {
+ var d2 drink
+ if err := drinksTbl.get(dd.Name, &d2); err != nil {
+ t.Fatalf("get() failed for drink %s", dd.Name)
+ }
+ if !reflect.DeepEqual(d2, dd) {
+ t.Fatalf("got wrong data for drink %s: %#v instead of %#v", dd.Name, d2, dd)
+ }
+ if !drinksTbl.hasKey(dd.Name) {
+ t.Fatalf("hasKey() did not find drink %s", dd.Name)
+ }
+ }
+
+ if num := usersTbl.getNumKeys(); num != uint64(len(users)) {
+ t.Fatalf("getNumKeys(): wrong user count: got %v instead of %v", num, len(users))
+ }
+ if num := drinksTbl.getNumKeys(); num != uint64(len(drinks)) {
+ t.Fatalf("getNumKeys(): wrong drink count: got %v instead of %v", num, len(drinks))
+ }
+}
+
+func TestKVDBSet(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+ defer db.close()
+
+ initTestTables(t, usersTbl, drinksTbl, false)
+
+ db.flush()
+}
+
+func TestKVDBCreate(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+ defer db.close()
+
+ initTestTables(t, usersTbl, drinksTbl, true)
+
+ db.flush()
+}
+
+func TestKVDBBadGet(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+ defer db.close()
+
+ // The DB is empty, all gets must fail.
+ for _, uu := range users {
+ var u2 user
+ if err := usersTbl.get(uu.Username, &u2); err == nil {
+ t.Fatalf("get() found non-existent user %s in file %s: %v", uu.Username, kvdbfile, u2)
+ }
+ }
+ for _, dd := range drinks {
+ var d2 drink
+ if err := drinksTbl.get(dd.Name, &d2); err == nil {
+ t.Fatalf("get() found non-existent drink %s in file %s: %v", dd.Name, kvdbfile, d2)
+ }
+ }
+}
+
+func TestKVDBBadUpdate(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+ defer db.close()
+
+ // The DB is empty, all updates must fail.
+ for _, uu := range users {
+ u2 := user{Username: uu.Username}
+ if err := usersTbl.update(uu.Username, &u2); err == nil {
+ t.Fatalf("update() worked for a non-existent user %s in file %s", uu.Username, kvdbfile)
+ }
+ }
+ for _, dd := range drinks {
+ d2 := drink{Name: dd.Name}
+ if err := drinksTbl.update(dd.Name, &d2); err == nil {
+ t.Fatalf("update() worked for a non-existent drink %s in file %s", dd.Name, kvdbfile)
+ }
+ }
+}
+
+func TestKVDBBadHasKey(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+ defer db.close()
+
+ // The DB is empty, all key-checks must fail.
+ for _, uu := range users {
+ if usersTbl.hasKey(uu.Username) {
+ t.Fatalf("hasKey() found non-existent user %s in file %s", uu.Username, kvdbfile)
+ }
+ }
+ for _, dd := range drinks {
+ if drinksTbl.hasKey(dd.Name) {
+ t.Fatalf("hasKey() found non-existent drink %s in file %s", dd.Name, kvdbfile)
+ }
+ }
+}
+
+func TestKVDBGet(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+ defer db.close()
+
+ initTestTables(t, usersTbl, drinksTbl, false)
+ checkTestTables(t, usersTbl, drinksTbl)
+
+ db.flush()
+ checkTestTables(t, usersTbl, drinksTbl)
+}
+
+func TestKVDBBadCreate(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+ defer db.close()
+
+ initTestTables(t, usersTbl, drinksTbl, false)
+
+ // Must not be able to re-create the same entries.
+ for _, uu := range users {
+ u2 := user{Username: uu.Username}
+ if err := usersTbl.create(uu.Username, &u2); err == nil {
+ t.Fatalf("create() worked for an existing user %s in file %s", uu.Username, kvdbfile)
+ }
+ }
+ for _, dd := range drinks {
+ d2 := drink{Name: dd.Name}
+ if err := drinksTbl.create(dd.Name, &d2); err == nil {
+ t.Fatalf("create() worked for an existing drink %s in file %s", dd.Name, kvdbfile)
+ }
+ }
+
+ db.flush()
+}
+
+func TestKVDBReopen(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+
+ initTestTables(t, usersTbl, drinksTbl, true)
+
+ // Close the re-open the file.
+ db.flush()
+ db.close()
+
+ db, tbls, err := kvdbOpen(kvdbfile, []string{"users", "drinks"})
+ if err != nil {
+ t.Fatalf("Cannot re-open existing K/V DB file %s", kvdbfile)
+ }
+ defer db.close()
+
+ usersTbl, drinksTbl = tbls[0], tbls[1]
+ checkTestTables(t, usersTbl, drinksTbl)
+}
+
+func TestKVDBKeyIter(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+ defer db.close()
+
+ initTestTables(t, usersTbl, drinksTbl, false)
+
+ // Get the list of all entry keys in each table.
+ keylist := ""
+ err := usersTbl.keyIter(func(key string) {
+ keylist += key + ","
+ })
+ if err != nil || keylist != "arthur,galahad,lancelot,robin," {
+ t.Fatalf("keyIter() failed in file %s: err %v, user names: %v", kvdbfile, err, keylist)
+ }
+ keylist = ""
+ err = drinksTbl.keyIter(func(key string) {
+ keylist += key + ","
+ })
+ if err != nil || keylist != "beer,coffee,coke,pepsi," {
+ t.Fatalf("keyIter() failed in file %s: err %v, drink names: %v", kvdbfile, err, keylist)
+ }
+
+ db.flush()
+}
+
+func TestKVDBUpdate(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+
+ initTestTables(t, usersTbl, drinksTbl, false)
+ db.flush()
+ db.close()
+
+ db, tbls, err := kvdbOpen(kvdbfile, []string{"users", "drinks"})
+ if err != nil {
+ t.Fatalf("Cannot re-open existing K/V DB file %s", kvdbfile)
+ }
+ defer db.close()
+
+ usersTbl, drinksTbl = tbls[0], tbls[1]
+
+ for _, uu := range users {
+ key := uu.Username
+ u2 := uu
+ u2.Username += "-New"
+
+ if err = usersTbl.update(key, &u2); err != nil {
+ t.Fatalf("update() failed for user %s in file %s", key, kvdbfile)
+ }
+
+ var u3 user
+ if err = usersTbl.get(key, &u3); err != nil {
+ t.Fatalf("get() failed for user %s in file %s", key, kvdbfile)
+ }
+ if !reflect.DeepEqual(u3, u2) {
+ t.Fatalf("got wrong new data for user %s in file %s: %#v instead of %#v", key, kvdbfile, u3, u2)
+ }
+ }
+
+ for _, dd := range drinks {
+ key := dd.Name
+ d2 := dd
+ d2.Alcohol = !d2.Alcohol
+
+ if err = drinksTbl.update(key, &d2); err != nil {
+ t.Fatalf("update() failed for drink %s in file %s", key, kvdbfile)
+ }
+
+ var d3 drink
+ if err = drinksTbl.get(key, &d3); err != nil {
+ t.Fatalf("get() failed for drink %s in file %s", key, kvdbfile)
+ }
+ if !reflect.DeepEqual(d3, d2) {
+ t.Fatalf("got wrong new data for drink %s in file %s: %#v instead of %#v", key, kvdbfile, d3, d2)
+ }
+ }
+
+ db.flush()
+}
+
+func TestKVDBSetAgain(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+ defer db.close()
+
+ initTestTables(t, usersTbl, drinksTbl, false)
+
+ for _, uu := range users {
+ key := uu.Username
+ u2 := uu
+ u2.Username += "-New"
+
+ if err := usersTbl.set(key, &u2); err != nil {
+ t.Fatalf("set() again failed for user %s in file %s", key, kvdbfile)
+ }
+
+ var u3 user
+ if err := usersTbl.get(key, &u3); err != nil {
+ t.Fatalf("get() failed for user %s in file %s", key, kvdbfile)
+ }
+ if !reflect.DeepEqual(u3, u2) {
+ t.Fatalf("got wrong new data for user %s in file %s: %#v instead of %#v", key, kvdbfile, u3, u2)
+ }
+ }
+
+ for _, dd := range drinks {
+ key := dd.Name
+ d2 := dd
+ d2.Alcohol = !d2.Alcohol
+
+ if err := drinksTbl.update(key, &d2); err != nil {
+ t.Fatalf("set() again failed for drink %s in file %s", key, kvdbfile)
+ }
+
+ var d3 drink
+ if err := drinksTbl.get(key, &d3); err != nil {
+ t.Fatalf("get() failed for drink %s in file %s", key, kvdbfile)
+ }
+ if !reflect.DeepEqual(d3, d2) {
+ t.Fatalf("got wrong new data for drink %s in file %s: %#v instead of %#v", key, kvdbfile, d3, d2)
+ }
+ }
+
+ db.flush()
+}
+
+func TestKVDBDelete(t *testing.T) {
+ kvdbfile, db, usersTbl, drinksTbl := createTestDB(t)
+ defer os.Remove(kvdbfile)
+ defer db.close()
+
+ initTestTables(t, usersTbl, drinksTbl, false)
+
+ db.flush()
+
+ // Delete entries and verify that they no longer exist.
+
+ for _, uu := range users {
+ key := uu.Username
+ if err := usersTbl.del(key); err != nil {
+ t.Errorf("del() failed for user %s in file %s", key, kvdbfile)
+ }
+ if usersTbl.hasKey(key) {
+ t.Errorf("hasKey() still finds deleted user %s in file %s", key, kvdbfile)
+ }
+ }
+
+ for _, dd := range drinks {
+ key := dd.Name
+ if err := drinksTbl.del(key); err != nil {
+ t.Errorf("del() failed for drink %s in file %s", key, kvdbfile)
+ }
+ if drinksTbl.hasKey(key) {
+ t.Errorf("hasKey() still finds deleted drink %s in file %s", key, kvdbfile)
+ }
+ }
+
+ db.flush()
+}
diff --git a/services/syncbase/sync/replay_test.go b/services/syncbase/sync/replay_test.go
new file mode 100644
index 0000000..3c86a69
--- /dev/null
+++ b/services/syncbase/sync/replay_test.go
@@ -0,0 +1,226 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Used to ease the setup of Veyron Sync test scenarios.
+// Parses a sync command file and returns a vector of commands to execute.
+//
+// Used by different test replay engines:
+// - dagReplayCommands() executes the parsed commands at the DAG API level.
+// - logReplayCommands() executes the parsed commands at the Log API level.
+
+import (
+ "bufio"
+ "fmt"
+ "os"
+ "strconv"
+ "strings"
+)
+
+const (
+ addLocal = iota
+ addRemote
+ setDevTable
+ linkLocal
+ linkRemote
+)
+
+type syncCommand struct {
+ cmd int
+ objID ObjId
+ version Version
+ parents []Version
+ logrec string
+ devID DeviceId
+ genVec GenVector
+ txID TxId
+ txCount uint32
+ deleted bool
+}
+
+func strToVersion(verStr string) (Version, error) {
+ ver, err := strconv.ParseUint(verStr, 10, 64)
+ if err != nil {
+ return 0, err
+ }
+ return Version(ver), nil
+}
+
+func parseSyncCommands(file string) ([]syncCommand, error) {
+ cmds := []syncCommand{}
+ sf, err := os.Open("testdata/" + file)
+ if err != nil {
+ return nil, err
+ }
+ defer sf.Close()
+
+ scanner := bufio.NewScanner(sf)
+ lineno := 0
+ for scanner.Scan() {
+ lineno++
+ line := strings.TrimSpace(scanner.Text())
+ if line == "" || line[0] == '#' {
+ continue
+ }
+
+ args := strings.Split(line, "|")
+ nargs := len(args)
+
+ switch args[0] {
+ case "addl", "addr":
+ expNargs := 9
+ if nargs != expNargs {
+ return nil, fmt.Errorf("%s:%d: need %d args instead of %d", file, lineno, expNargs, nargs)
+ }
+ version, err := strToVersion(args[2])
+ if err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid version: %s", file, lineno, args[2])
+ }
+ var parents []Version
+ for i := 3; i <= 4; i++ {
+ if args[i] != "" {
+ pver, err := strToVersion(args[i])
+ if err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid parent: %s", file, lineno, args[i])
+ }
+ parents = append(parents, pver)
+ }
+ }
+
+ txID, err := strToTxId(args[6])
+ if err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid TxId: %s", file, lineno, args[6])
+ }
+ txCount, err := strconv.ParseUint(args[7], 10, 32)
+ if err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid tx count: %s", file, lineno, args[7])
+ }
+ del, err := strconv.ParseBool(args[8])
+ if err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid deleted bit: %s", file, lineno, args[8])
+ }
+ cmd := syncCommand{
+ version: version,
+ parents: parents,
+ logrec: args[5],
+ txID: txID,
+ txCount: uint32(txCount),
+ deleted: del,
+ }
+ if args[0] == "addl" {
+ cmd.cmd = addLocal
+ } else {
+ cmd.cmd = addRemote
+ }
+ if cmd.objID, err = strToObjId(args[1]); err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid object ID: %s", file, lineno, args[1])
+ }
+ cmds = append(cmds, cmd)
+
+ case "setdev":
+ expNargs := 3
+ if nargs != expNargs {
+ return nil, fmt.Errorf("%s:%d: need %d args instead of %d", file, lineno, expNargs, nargs)
+ }
+
+ genVec := make(GenVector)
+ for _, elem := range strings.Split(args[2], ",") {
+ kv := strings.Split(elem, ":")
+ if len(kv) != 2 {
+ return nil, fmt.Errorf("%s:%d: invalid gen vector key/val: %s", file, lineno, elem)
+ }
+ genID, err := strToGenId(kv[1])
+ if err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid gen ID: %s", file, lineno, kv[1])
+ }
+ genVec[DeviceId(kv[0])] = genID
+ }
+
+ cmd := syncCommand{cmd: setDevTable, devID: DeviceId(args[1]), genVec: genVec}
+ cmds = append(cmds, cmd)
+
+ case "linkl", "linkr":
+ expNargs := 6
+ if nargs != expNargs {
+ return nil, fmt.Errorf("%s:%d: need %d args instead of %d", file, lineno, expNargs, nargs)
+ }
+
+ version, err := strToVersion(args[2])
+ if err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid version: %s", file, lineno, args[2])
+ }
+ if args[3] == "" {
+ return nil, fmt.Errorf("%s:%d: parent (to-node) version not specified", file, lineno)
+ }
+ if args[4] != "" {
+ return nil, fmt.Errorf("%s:%d: cannot specify a 2nd parent (to-node): %s", file, lineno, args[4])
+ }
+ parent, err := strToVersion(args[3])
+ if err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid parent (to-node) version: %s", file, lineno, args[3])
+ }
+
+ cmd := syncCommand{version: version, parents: []Version{parent}, logrec: args[5]}
+ if args[0] == "linkl" {
+ cmd.cmd = linkLocal
+ } else {
+ cmd.cmd = linkRemote
+ }
+ if cmd.objID, err = strToObjId(args[1]); err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid object ID: %s", file, lineno, args[1])
+ }
+ cmds = append(cmds, cmd)
+
+ default:
+ return nil, fmt.Errorf("%s:%d: invalid operation: %s", file, lineno, args[0])
+ }
+ }
+
+ err = scanner.Err()
+ return cmds, err
+}
+
+func dagReplayCommands(dag *dag, syncfile string) error {
+ cmds, err := parseSyncCommands(syncfile)
+ if err != nil {
+ return err
+ }
+
+ for _, cmd := range cmds {
+ switch cmd.cmd {
+ case addLocal:
+ err = dag.addNode(cmd.objID, cmd.version, false, cmd.deleted, cmd.parents, cmd.logrec, NoTxId)
+ if err != nil {
+ return fmt.Errorf("cannot add local node %d:%d to DAG: %v", cmd.objID, cmd.version, err)
+ }
+ if err := dag.moveHead(cmd.objID, cmd.version); err != nil {
+ return fmt.Errorf("cannot move head to %d:%d in DAG: %v", cmd.objID, cmd.version, err)
+ }
+ dag.flush()
+
+ case addRemote:
+ err = dag.addNode(cmd.objID, cmd.version, true, cmd.deleted, cmd.parents, cmd.logrec, NoTxId)
+ if err != nil {
+ return fmt.Errorf("cannot add remote node %d:%d to DAG: %v", cmd.objID, cmd.version, err)
+ }
+ dag.flush()
+
+ case linkLocal:
+ if err = dag.addParent(cmd.objID, cmd.version, cmd.parents[0], false); err != nil {
+ return fmt.Errorf("cannot add local parent %d to DAG node %d:%d: %v",
+ cmd.parents[0], cmd.objID, cmd.version, err)
+ }
+ dag.flush()
+
+ case linkRemote:
+ if err = dag.addParent(cmd.objID, cmd.version, cmd.parents[0], true); err != nil {
+ return fmt.Errorf("cannot add remote parent %d to DAG node %d:%d: %v",
+ cmd.parents[0], cmd.objID, cmd.version, err)
+ }
+ dag.flush()
+ }
+ }
+ return nil
+}
diff --git a/services/syncbase/sync/sgtable.go b/services/syncbase/sync/sgtable.go
new file mode 100644
index 0000000..41818c1
--- /dev/null
+++ b/services/syncbase/sync/sgtable.go
@@ -0,0 +1,500 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// The SyncGroup Table stores the group information in a K/V DB. It also
+// maintains an index to provide access by SyncGroup ID or name.
+//
+// The SyncGroup info is fetched from the SyncGroup server by the create or
+// join operations, and is regularly updated after that.
+//
+// The DB contains two tables persisted to disk (data, names) and one
+// in-memory (ephemeral) map (members):
+// * data: one entry per SyncGroup ID containing the SyncGroup data
+// * names: one entry per SyncGroup name pointing to its SyncGroup ID
+// * members: an inverted index of SyncGroup members to SyncGroup IDs
+// built from the list of SyncGroup joiners
+
+import (
+ "errors"
+ "fmt"
+ "path"
+ "strconv"
+
+ "v.io/x/lib/vlog"
+ "v.io/x/ref/lib/stats"
+)
+
+var (
+ errBadSGTable = errors.New("invalid SyncGroup Table")
+)
+
+type syncGroupTable struct {
+ fname string // file pathname
+ store *kvdb // underlying K/V store
+ sgData *kvtable // pointer to "data" table in the kvdb
+ sgNames *kvtable // pointer to "names" table in the kvdb
+ members map[string]*memberInfo // in-memory tracking of SyncGroup member info
+
+ // SyncGroup Table stats
+ numSGs *stats.Integer // number of SyncGroups
+ numMembers *stats.Integer // number of Sync members
+}
+
+type syncGroupData struct {
+ SrvInfo SyncGroupInfo // SyncGroup info from SyncGroupServer
+ LocalPath string // local path of the SyncGroup in the Store
+}
+
+type memberInfo struct {
+ gids map[GroupId]*memberMetaData // map of SyncGroup IDs joined and their metadata
+}
+
+type memberMetaData struct {
+ metaData JoinerMetaData // joiner metadata at the SyncGroup server
+}
+
+type sgSet map[GroupId]struct{} // a set of SyncGroups
+
+// strToGroupId converts a SyncGroup ID in string format to an GroupId.
+func strToGroupId(str string) (GroupId, error) {
+ id, err := strconv.ParseUint(str, 10, 64)
+ if err != nil {
+ return NoGroupId, err
+ }
+ return GroupId(id), nil
+}
+
+// openSyncGroupTable opens or creates a syncGroupTable for the given filename.
+func openSyncGroupTable(filename string) (*syncGroupTable, error) {
+ // Open the file and create it if it does not exist.
+ // Also initialize the store and its tables.
+ db, tbls, err := kvdbOpen(filename, []string{"data", "names"})
+ if err != nil {
+ return nil, err
+ }
+
+ s := &syncGroupTable{
+ fname: filename,
+ store: db,
+ sgData: tbls[0],
+ sgNames: tbls[1],
+ members: make(map[string]*memberInfo),
+ numSGs: stats.NewInteger(statsNumSyncGroup),
+ numMembers: stats.NewInteger(statsNumMember),
+ }
+
+ // Reconstruct the in-memory tracking maps by iterating over the SyncGroups.
+ // This is needed when an existing SyncGroup Table file is re-opened.
+ s.sgData.keyIter(func(gidStr string) {
+ // Get the SyncGroup data given the group ID in string format (as the data table key).
+ gid, err := strToGroupId(gidStr)
+ if err != nil {
+ return
+ }
+
+ data, err := s.getSyncGroupByID(gid)
+ if err != nil {
+ return
+ }
+
+ s.numSGs.Incr(1)
+
+ // Add all SyncGroup members to the members inverted index.
+ s.addAllMembers(data)
+ })
+
+ return s, nil
+}
+
+// close closes the syncGroupTable and invalidates its structure.
+func (s *syncGroupTable) close() {
+ if s.store != nil {
+ s.store.close() // this also closes the tables
+ stats.Delete(statsNumSyncGroup)
+ stats.Delete(statsNumMember)
+ }
+ *s = syncGroupTable{} // zero out the structure
+}
+
+// flush flushes the syncGroupTable store to disk.
+func (s *syncGroupTable) flush() {
+ if s.store != nil {
+ s.store.flush()
+ }
+}
+
+// addSyncGroup adds a new SyncGroup given its information.
+func (s *syncGroupTable) addSyncGroup(sgData *syncGroupData) error {
+ if s.store == nil {
+ return errBadSGTable
+ }
+ if sgData == nil {
+ return errors.New("group information not specified")
+ }
+ gid, name := sgData.SrvInfo.Id, path.Join(sgData.SrvInfo.ServerName, sgData.SrvInfo.GroupName)
+ if name == "" {
+ return errors.New("group name not specified")
+ }
+ if sgData.LocalPath == "" {
+ return errors.New("group local path not specified")
+ }
+ if len(sgData.SrvInfo.Joiners) == 0 {
+ return errors.New("group has no joiners")
+ }
+
+ if s.hasSGDataEntry(gid) {
+ return fmt.Errorf("group %d already exists", gid)
+ }
+ if s.hasSGNameEntry(name) {
+ return fmt.Errorf("group name %s already exists", name)
+ }
+
+ // Add the group name and data entries.
+ if err := s.setSGNameEntry(name, gid); err != nil {
+ return err
+ }
+
+ if err := s.setSGDataEntry(gid, sgData); err != nil {
+ s.delSGNameEntry(name)
+ return err
+ }
+
+ s.numSGs.Incr(1)
+ s.addAllMembers(sgData)
+ return nil
+}
+
+// getSyncGroupID retrieves the SyncGroup ID given its name.
+func (s *syncGroupTable) getSyncGroupID(name string) (GroupId, error) {
+ return s.getSGNameEntry(name)
+}
+
+// getSyncGroupName retrieves the SyncGroup name given its ID.
+func (s *syncGroupTable) getSyncGroupName(gid GroupId) (string, error) {
+ data, err := s.getSyncGroupByID(gid)
+ if err != nil {
+ return "", err
+ }
+
+ return path.Join(data.SrvInfo.ServerName, data.SrvInfo.GroupName), nil
+}
+
+// getSyncGroupByID retrieves the SyncGroup given its ID.
+func (s *syncGroupTable) getSyncGroupByID(gid GroupId) (*syncGroupData, error) {
+ return s.getSGDataEntry(gid)
+}
+
+// getSyncGroupByName retrieves the SyncGroup given its name.
+func (s *syncGroupTable) getSyncGroupByName(name string) (*syncGroupData, error) {
+ gid, err := s.getSyncGroupID(name)
+ if err != nil {
+ return nil, err
+ }
+ return s.getSyncGroupByID(gid)
+}
+
+// updateSyncGroup updates the SyncGroup data.
+func (s *syncGroupTable) updateSyncGroup(data *syncGroupData) error {
+ if s.store == nil {
+ return errBadSGTable
+ }
+ if data == nil {
+ return errors.New("SyncGroup data not specified")
+ }
+ if data.SrvInfo.GroupName == "" {
+ return errors.New("group name not specified")
+ }
+ if len(data.SrvInfo.Joiners) == 0 {
+ return errors.New("group has no joiners")
+ }
+
+ fullGroupName := path.Join(data.SrvInfo.ServerName, data.SrvInfo.GroupName)
+ oldData, err := s.getSyncGroupByName(fullGroupName)
+ if err != nil {
+ return err
+ }
+
+ if data.SrvInfo.Id != oldData.SrvInfo.Id {
+ return fmt.Errorf("cannot change ID of SyncGroup name %s", fullGroupName)
+ }
+ if data.LocalPath == "" {
+ data.LocalPath = oldData.LocalPath
+ } else if data.LocalPath != oldData.LocalPath {
+ return fmt.Errorf("cannot change local path of SyncGroup name %s", fullGroupName)
+ }
+
+ // Get the old set of SyncGroup joiners and diff it with the new set.
+ // Add all the current members because this inserts the new members and
+ // updates the metadata of the existing ones (addMember() is like a "put").
+ // Delete the members that are no longer part of the SyncGroup.
+ gid := oldData.SrvInfo.Id
+ newJoiners, oldJoiners := data.SrvInfo.Joiners, oldData.SrvInfo.Joiners
+
+ for member, memberData := range newJoiners {
+ s.addMember(member, gid, memberData)
+ }
+
+ for member := range oldJoiners {
+ if _, ok := newJoiners[member]; !ok {
+ s.delMember(member, gid)
+ }
+ }
+
+ return s.setSGDataEntry(gid, data)
+}
+
+// delSyncGroupByID deletes the SyncGroup given its ID.
+func (s *syncGroupTable) delSyncGroupByID(gid GroupId) error {
+ data, err := s.getSyncGroupByID(gid)
+ if err != nil {
+ return err
+ }
+ if err = s.delSGNameEntry(path.Join(data.SrvInfo.ServerName, data.SrvInfo.GroupName)); err != nil {
+ return err
+ }
+
+ s.numSGs.Incr(-1)
+ s.delAllMembers(data)
+ return s.delSGDataEntry(gid)
+}
+
+// delSyncGroupByName deletes the SyncGroup given its name.
+func (s *syncGroupTable) delSyncGroupByName(name string) error {
+ gid, err := s.getSyncGroupID(name)
+ if err != nil {
+ return err
+ }
+
+ return s.delSyncGroupByID(gid)
+}
+
+// getAllSyncGroupNames returns the names of all SyncGroups.
+func (s *syncGroupTable) getAllSyncGroupNames() ([]string, error) {
+ if s.store == nil {
+ return nil, errBadSGTable
+ }
+
+ names := make([]string, 0)
+
+ err := s.sgNames.keyIter(func(name string) {
+ names = append(names, name)
+ })
+
+ if err != nil {
+ return nil, err
+ }
+ return names, nil
+}
+
+// getMembers returns all SyncGroup members and the count of SyncGroups each one joined.
+func (s *syncGroupTable) getMembers() (map[string]uint32, error) {
+ if s.store == nil {
+ return nil, errBadSGTable
+ }
+
+ members := make(map[string]uint32)
+ for member, info := range s.members {
+ members[member] = uint32(len(info.gids))
+ }
+
+ return members, nil
+}
+
+// getMemberInfo returns SyncGroup information for a given member.
+func (s *syncGroupTable) getMemberInfo(member string) (*memberInfo, error) {
+ if s.store == nil {
+ return nil, errBadSGTable
+ }
+
+ info, ok := s.members[member]
+ if !ok {
+ return nil, fmt.Errorf("unknown member: %s", member)
+ }
+
+ return info, nil
+}
+
+// addMember inserts or updates a (member, group ID) entry in the in-memory
+// structure that indexes SyncGroup memberships based on member names and stores
+// in it the member's joiner metadata.
+func (s *syncGroupTable) addMember(member string, gid GroupId, metadata JoinerMetaData) {
+ if s.store == nil {
+ return
+ }
+
+ info, ok := s.members[member]
+ if !ok {
+ info = &memberInfo{gids: make(map[GroupId]*memberMetaData)}
+ s.members[member] = info
+ s.numMembers.Incr(1)
+ }
+
+ info.gids[gid] = &memberMetaData{metaData: metadata}
+}
+
+// delMember removes a (member, group ID) entry from the in-memory structure
+// that indexes SyncGroup memberships based on member names.
+func (s *syncGroupTable) delMember(member string, gid GroupId) {
+ if s.store == nil {
+ return
+ }
+
+ info, ok := s.members[member]
+ if !ok {
+ return
+ }
+
+ delete(info.gids, gid)
+ if len(info.gids) == 0 {
+ delete(s.members, member)
+ s.numMembers.Incr(-1)
+ }
+}
+
+// addAllMembers inserts all members of a SyncGroup in the in-memory structure
+// that indexes SyncGroup memberships based on member names.
+func (s *syncGroupTable) addAllMembers(data *syncGroupData) {
+ if s.store == nil || data == nil {
+ return
+ }
+
+ gid := data.SrvInfo.Id
+ for member, memberData := range data.SrvInfo.Joiners {
+ s.addMember(member, gid, memberData)
+ }
+}
+
+// delAllMembers removes all members of a SyncGroup from the in-memory structure
+// that indexes SyncGroup memberships based on member names.
+func (s *syncGroupTable) delAllMembers(data *syncGroupData) {
+ if s.store == nil || data == nil {
+ return
+ }
+
+ gid := data.SrvInfo.Id
+ for member := range data.SrvInfo.Joiners {
+ s.delMember(member, gid)
+ }
+}
+
+// Low-level functions to access the tables in the K/V DB.
+// They directly access the table entries without tracking their relationships.
+
+// sgDataKey returns the key used to access the SyncGroup data in the DB.
+func sgDataKey(gid GroupId) string {
+ return fmt.Sprintf("%d", gid)
+}
+
+// hasSGDataEntry returns true if the SyncGroup data entry exists in the DB.
+func (s *syncGroupTable) hasSGDataEntry(gid GroupId) bool {
+ if s.store == nil {
+ return false
+ }
+ key := sgDataKey(gid)
+ return s.sgData.hasKey(key)
+}
+
+// setSGDataEntry stores the SyncGroup data in the DB.
+func (s *syncGroupTable) setSGDataEntry(gid GroupId, data *syncGroupData) error {
+ if s.store == nil {
+ return errBadSGTable
+ }
+ key := sgDataKey(gid)
+ return s.sgData.set(key, data)
+}
+
+// getSGDataEntry retrieves from the DB the SyncGroup data for a given group ID.
+func (s *syncGroupTable) getSGDataEntry(gid GroupId) (*syncGroupData, error) {
+ if s.store == nil {
+ return nil, errBadSGTable
+ }
+ var data syncGroupData
+ key := sgDataKey(gid)
+ if err := s.sgData.get(key, &data); err != nil {
+ return nil, err
+ }
+ return &data, nil
+}
+
+// delSGDataEntry deletes the SyncGroup data from the DB.
+func (s *syncGroupTable) delSGDataEntry(gid GroupId) error {
+ if s.store == nil {
+ return errBadSGTable
+ }
+ key := sgDataKey(gid)
+ return s.sgData.del(key)
+}
+
+// sgNameKey returns the key used to access the SyncGroup name in the DB.
+func sgNameKey(name string) string {
+ return name
+}
+
+// hasSGNameEntry returns true if the SyncGroup name entry exists in the DB.
+func (s *syncGroupTable) hasSGNameEntry(name string) bool {
+ if s.store == nil {
+ return false
+ }
+ key := sgNameKey(name)
+ return s.sgNames.hasKey(key)
+}
+
+// setSGNameEntry stores the SyncGroup name to ID mapping in the DB.
+func (s *syncGroupTable) setSGNameEntry(name string, gid GroupId) error {
+ if s.store == nil {
+ return errBadSGTable
+ }
+ key := sgNameKey(name)
+ return s.sgNames.set(key, gid)
+}
+
+// getSGNameEntry retrieves the SyncGroup name to ID mapping from the DB.
+func (s *syncGroupTable) getSGNameEntry(name string) (GroupId, error) {
+ var gid GroupId
+ if s.store == nil {
+ return gid, errBadSGTable
+ }
+ key := sgNameKey(name)
+ err := s.sgNames.get(key, &gid)
+ return gid, err
+}
+
+// delSGNameEntry deletes the SyncGroup name to ID mapping from the DB.
+func (s *syncGroupTable) delSGNameEntry(name string) error {
+ if s.store == nil {
+ return errBadSGTable
+ }
+ key := sgNameKey(name)
+ return s.sgNames.del(key)
+}
+
+// dump writes to the log file information on all SyncGroups.
+func (s *syncGroupTable) dump() {
+ if s.store == nil {
+ return
+ }
+
+ s.sgData.keyIter(func(gidStr string) {
+ // Get the SyncGroup data given the group ID in string format (as the data table key).
+ gid, err := strToGroupId(gidStr)
+ if err != nil {
+ return
+ }
+
+ data, err := s.getSyncGroupByID(gid)
+ if err != nil {
+ return
+ }
+
+ members := make([]string, 0, len(data.SrvInfo.Joiners))
+ for joiner := range data.SrvInfo.Joiners {
+ members = append(members, joiner)
+ }
+ vlog.VI(1).Infof("DUMP: SyncGroup %s: id %v, path %s, members: %s",
+ path.Join(data.SrvInfo.ServerName, data.SrvInfo.GroupName),
+ gid, data.LocalPath, members)
+ })
+}
diff --git a/services/syncbase/sync/sgtable_test.go b/services/syncbase/sync/sgtable_test.go
new file mode 100644
index 0000000..c80a2a9
--- /dev/null
+++ b/services/syncbase/sync/sgtable_test.go
@@ -0,0 +1,769 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Tests for the Veyron SyncGroup Table.
+
+import (
+ "os"
+ "reflect"
+ "testing"
+
+ "v.io/x/ref/lib/stats"
+)
+
+// TestSyncGroupTableOpen tests the creation of a SyncGroup Table, closing and re-opening it.
+// It also verifies that its backing file is created and that a 2nd close is safe.
+func TestSyncGroupTableOpen(t *testing.T) {
+ sgfile := getFileName()
+ defer os.Remove(sgfile)
+
+ sg, err := openSyncGroupTable(sgfile)
+ if err != nil {
+ t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
+ }
+
+ fsize := getFileSize(sgfile)
+ if fsize < 0 {
+ //t.Fatalf("SyncGroup Table file %s not created", sgfile)
+ }
+
+ sg.flush()
+ oldfsize := fsize
+ fsize = getFileSize(sgfile)
+ if fsize <= oldfsize {
+ //t.Fatalf("SyncGroup Table file %s not flushed", sgfile)
+ }
+
+ sg.close()
+
+ sg, err = openSyncGroupTable(sgfile)
+ if err != nil {
+ t.Fatalf("cannot re-open existing SyncGroup Table file %s", sgfile)
+ }
+
+ oldfsize = fsize
+ fsize = getFileSize(sgfile)
+ if fsize != oldfsize {
+ t.Fatalf("SyncGroup Table file %s size changed across re-open", sgfile)
+ }
+
+ sg.close()
+ sg.close() // multiple closes should be a safe NOP
+
+ fsize = getFileSize(sgfile)
+ if fsize != oldfsize {
+ t.Fatalf("SyncGroup Table file %s size changed across close", sgfile)
+ }
+
+ // Fail opening a SyncGroup Table in a non-existent directory.
+ _, err = openSyncGroupTable("/not/really/there/junk.sg")
+ if err == nil {
+ //t.Fatalf("openSyncGroupTable() did not fail when using a bad pathname")
+ }
+}
+
+// TestInvalidSyncGroupTable tests using methods on an invalid (closed) SyncGroup Table.
+func TestInvalidSyncGroupTable(t *testing.T) {
+ sgfile := getFileName()
+ defer os.Remove(sgfile)
+
+ sg, err := openSyncGroupTable(sgfile)
+ if err != nil {
+ t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
+ }
+
+ sg.close()
+
+ sgid, err := strToGroupId("1234")
+ if err != nil {
+ t.Error(err)
+ }
+
+ validateError := func(t *testing.T, err error, funcName string) {
+ if err == nil || err.Error() != "invalid SyncGroup Table" {
+ t.Errorf("%s() did not fail on a closed SyncGroup Table: %v", funcName, err)
+ }
+ }
+
+ err = sg.addSyncGroup(&syncGroupData{})
+ validateError(t, err, "addSyncGroup")
+
+ _, err = sg.getSyncGroupID("foobar")
+ validateError(t, err, "getSyncGroupID")
+
+ _, err = sg.getSyncGroupName(sgid)
+ validateError(t, err, "getSyncGroupName")
+
+ _, err = sg.getSyncGroupByID(sgid)
+ validateError(t, err, "getSyncGroupByID")
+
+ _, err = sg.getSyncGroupByName("foobar")
+ validateError(t, err, "getSyncGroupByName")
+
+ err = sg.updateSyncGroup(&syncGroupData{})
+ validateError(t, err, "updateSyncGroup")
+
+ err = sg.delSyncGroupByID(sgid)
+ validateError(t, err, "delSyncGroupByID")
+
+ err = sg.delSyncGroupByName("foobar")
+ validateError(t, err, "delSyncGroupByName")
+
+ _, err = sg.getAllSyncGroupNames()
+ validateError(t, err, "getAllSyncGroupNames")
+
+ _, err = sg.getMembers()
+ validateError(t, err, "getMembers")
+
+ _, err = sg.getMemberInfo("foobar")
+ validateError(t, err, "getMemberInfo")
+
+ err = sg.setSGDataEntry(sgid, &syncGroupData{})
+ validateError(t, err, "setSGDataEntry")
+
+ _, err = sg.getSGDataEntry(sgid)
+ validateError(t, err, "getSGDataEntry")
+
+ err = sg.delSGDataEntry(sgid)
+ validateError(t, err, "delSGDataEntry")
+
+ err = sg.setSGNameEntry("foobar", sgid)
+ validateError(t, err, "setSGNameEntry")
+
+ _, err = sg.getSGNameEntry("foobar")
+ validateError(t, err, "getSGNameEntry")
+
+ err = sg.delSGNameEntry("foobar")
+ validateError(t, err, "delSGNameEntry")
+
+ // These calls should be harmless NOPs.
+ sg.dump()
+ sg.flush()
+ sg.close()
+ sg.addMember("foobar", sgid, JoinerMetaData{})
+ sg.delMember("foobar", sgid)
+ sg.addAllMembers(&syncGroupData{})
+ sg.delAllMembers(&syncGroupData{})
+
+ if sg.hasSGDataEntry(sgid) {
+ t.Errorf("hasSGDataEntry() found an entry on a closed SyncGroup Table")
+ }
+ if sg.hasSGNameEntry("foobar") {
+ t.Errorf("hasSGNameEntry() found an entry on a closed SyncGroup Table")
+ }
+}
+
+// checkSGStats verifies the SyncGroup Table stats counters.
+func checkSGStats(t *testing.T, which string, numSG, numMembers int64) {
+ if num, err := stats.Value(statsNumSyncGroup); err != nil || num != numSG {
+ t.Errorf("num-syncgroups (%s): got %v (err: %v) instead of %v", which, num, err, numSG)
+ }
+ if num, err := stats.Value(statsNumMember); err != nil || num != numMembers {
+ t.Errorf("num-members (%s): got %v (err: %v) instead of %v", which, num, err, numMembers)
+ }
+}
+
+// TestAddSyncGroup tests adding SyncGroups.
+func TestAddSyncGroup(t *testing.T) {
+ sgfile := getFileName()
+ defer os.Remove(sgfile)
+
+ sg, err := openSyncGroupTable(sgfile)
+ if err != nil {
+ t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
+ }
+
+ checkSGStats(t, "add-1", 0, 0)
+
+ sgname := "foobar"
+ sgid, err := strToGroupId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ sgData := &syncGroupData{
+ SrvInfo: SyncGroupInfo{
+ Id: sgid,
+ GroupName: sgname,
+ Joiners: map[string]JoinerMetaData{
+ "phone": JoinerMetaData{SyncPriority: 10},
+ "tablet": JoinerMetaData{SyncPriority: 25},
+ "cloud": JoinerMetaData{SyncPriority: 1},
+ },
+ },
+ LocalPath: "/foo/bar",
+ }
+
+ err = sg.addSyncGroup(sgData)
+ if err != nil {
+ t.Errorf("adding SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
+ }
+
+ // Verify SyncGroup ID, name, and data.
+ if id, err := sg.getSyncGroupID(sgname); err != nil || id != sgid {
+ t.Errorf("cannot get back ID of SyncGroup %s: got ID %d instead of %d; err: %v", sgname, id, sgid, err)
+ }
+ if name, err := sg.getSyncGroupName(sgid); err != nil || name != sgname {
+ t.Errorf("cannot get back name of SyncGroup ID %d: got %s instead of %s; err: %v", sgid, name, sgname, err)
+ }
+
+ data, err := sg.getSyncGroupByID(sgid)
+ if err != nil {
+ t.Errorf("cannot get SyncGroup by ID %d: %v", sgid, err)
+ }
+ if !reflect.DeepEqual(data, sgData) {
+ t.Errorf("invalid SyncGroup data for group ID %d: got %v instead of %v", sgid, data, sgData)
+ }
+
+ data, err = sg.getSyncGroupByName(sgname)
+ if err != nil {
+ t.Errorf("cannot get SyncGroup by Name %s: %v", sgname, err)
+ }
+ if !reflect.DeepEqual(data, sgData) {
+ t.Errorf("invalid SyncGroup data for group name %s: got %v instead of %v", sgname, data, sgData)
+ }
+
+ // Verify membership data.
+ members, err := sg.getMembers()
+ if err != nil {
+ t.Errorf("cannot get all SyncGroup members: %v", err)
+ }
+ expMembers := map[string]uint32{"phone": 1, "tablet": 1, "cloud": 1}
+ if !reflect.DeepEqual(members, expMembers) {
+ t.Errorf("invalid SyncGroup members: got %v instead of %v", members, expMembers)
+ }
+
+ expMetaData := map[string]*memberMetaData{
+ "phone": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 10}},
+ "tablet": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 25}},
+ "cloud": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 1}},
+ }
+ for mm := range members {
+ info, err := sg.getMemberInfo(mm)
+ if err != nil || info == nil {
+ t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
+ }
+ if len(info.gids) != 1 {
+ t.Errorf("invalid info for SyncGroup member %s: %v", mm, info)
+ }
+ expJoinerMetaData := expMetaData[mm]
+ joinerMetaData := info.gids[sgid]
+ if !reflect.DeepEqual(joinerMetaData, expJoinerMetaData) {
+ t.Errorf("invalid joiner Data for SyncGroup member %s under group ID %d: got %v instead of %v",
+ mm, sgid, joinerMetaData, expJoinerMetaData)
+ }
+ }
+
+ checkSGStats(t, "add-2", 1, 3)
+
+ // Use a non-existent member.
+ if info, err := sg.getMemberInfo("should-not-be-there"); err == nil {
+ t.Errorf("found info for invalid SyncGroup member: %v", info)
+ }
+
+ // Adding a SyncGroup for a pre-existing group ID or name should fail.
+ err = sg.addSyncGroup(sgData)
+ if err == nil {
+ t.Errorf("re-adding SyncGroup %d did not fail", sgid)
+ }
+
+ sgData.SrvInfo.Id, err = strToGroupId("5555")
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = sg.addSyncGroup(sgData)
+ if err == nil {
+ t.Errorf("adding SyncGroup %s with a different ID did not fail", sgname)
+ }
+
+ checkSGStats(t, "add-3", 1, 3)
+
+ sg.dump()
+ sg.close()
+}
+
+// TestInvalidAddSyncGroup tests adding SyncGroups.
+func TestInvalidAddSyncGroup(t *testing.T) {
+ sgfile := getFileName()
+ defer os.Remove(sgfile)
+
+ sg, err := openSyncGroupTable(sgfile)
+ if err != nil {
+ t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
+ }
+
+ sgname := "foobar"
+ sgid, err := strToGroupId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = sg.addSyncGroup(nil)
+ if err == nil {
+ t.Errorf("adding a nil SyncGroup did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ sgData := &syncGroupData{}
+ sgData.SrvInfo.Id = sgid
+
+ err = sg.addSyncGroup(sgData)
+ if err == nil {
+ t.Errorf("adding a SyncGroup with an empty name did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ sgData.SrvInfo.GroupName = sgname
+
+ err = sg.addSyncGroup(sgData)
+ if err == nil {
+ t.Errorf("adding a SyncGroup with no local path did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ sgData.LocalPath = "/foo/bar"
+
+ err = sg.addSyncGroup(sgData)
+ if err == nil {
+ t.Errorf("adding a SyncGroup with no joiners did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ sg.dump()
+ sg.close()
+}
+
+// TestUpdateSyncGroup tests updating a SyncGroup.
+func TestUpdateSyncGroup(t *testing.T) {
+ sgfile := getFileName()
+ defer os.Remove(sgfile)
+
+ sg, err := openSyncGroupTable(sgfile)
+ if err != nil {
+ t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
+ }
+
+ err = sg.updateSyncGroup(nil)
+ if err == nil {
+ t.Errorf("updating a nil SyncGroup did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ sgData := &syncGroupData{}
+ err = sg.updateSyncGroup(sgData)
+ if err == nil {
+ t.Errorf("updating a SyncGroup with an empty name did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ sgData.SrvInfo.GroupName = "blabla"
+ err = sg.updateSyncGroup(sgData)
+ if err == nil {
+ t.Errorf("updating a SyncGroup with no joiners did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ sgData.SrvInfo.Joiners = map[string]JoinerMetaData{
+ "phone": JoinerMetaData{SyncPriority: 10},
+ }
+ err = sg.updateSyncGroup(sgData)
+ if err == nil {
+ t.Errorf("updating a SyncGroup with a non-existing name did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ // Create the SyncGroup to update later.
+ sgname := "foobar"
+ sgid, err := strToGroupId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ sgData = &syncGroupData{
+ SrvInfo: SyncGroupInfo{
+ Id: sgid,
+ GroupName: sgname,
+ Joiners: map[string]JoinerMetaData{
+ "phone": JoinerMetaData{SyncPriority: 10},
+ "tablet": JoinerMetaData{SyncPriority: 25},
+ "cloud": JoinerMetaData{SyncPriority: 1},
+ },
+ },
+ LocalPath: "/foo/bar",
+ }
+
+ err = sg.addSyncGroup(sgData)
+ if err != nil {
+ t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
+ }
+
+ checkSGStats(t, "up-1", 1, 3)
+
+ // Update it using different group or root IDs, which is not allowed.
+ xid, err := strToGroupId("9999")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ sgData.SrvInfo.Id = xid
+
+ err = sg.updateSyncGroup(sgData)
+ if err == nil {
+ t.Errorf("updating a SyncGroup with an ID mismatch did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ sgData.SrvInfo.Id = sgid
+ sgData.LocalPath = "hahahaha"
+ err = sg.updateSyncGroup(sgData)
+ if err == nil {
+ t.Errorf("updating a SyncGroup with a local path mismatch did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ checkSGStats(t, "up-2", 1, 3)
+
+ // Update it using a modified set of joiners.
+ // An empty string indicates no change to the local path.
+ sgData.LocalPath = ""
+ sgData.SrvInfo.Joiners["universe"] = JoinerMetaData{SyncPriority: 0}
+ delete(sgData.SrvInfo.Joiners, "cloud")
+
+ err = sg.updateSyncGroup(sgData)
+ if err != nil {
+ t.Errorf("updating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
+ }
+
+ // Do some NOP member deletions (bad member, bad group ID).
+ // SyncGroup verification (below) should see the expected info asserting these were NOPs.
+ sg.delMember("blablablablabla", sgid)
+ sg.delMember("phone", xid)
+
+ checkSGStats(t, "up-3", 1, 3)
+
+ // Verify updated SyncGroup.
+ if id, err := sg.getSyncGroupID(sgname); err != nil || id != sgid {
+ t.Errorf("cannot get back ID of updated SyncGroup %s: got ID %d instead of %d; err: %v", sgname, id, sgid, err)
+ }
+ if name, err := sg.getSyncGroupName(sgid); err != nil || name != sgname {
+ t.Errorf("cannot get back name of updated SyncGroup ID %d: got %s instead of %s; err: %v", sgid, name, sgname, err)
+ }
+
+ expData := &syncGroupData{
+ SrvInfo: SyncGroupInfo{
+ Id: sgid,
+ GroupName: sgname,
+ Joiners: map[string]JoinerMetaData{
+ "phone": JoinerMetaData{SyncPriority: 10},
+ "tablet": JoinerMetaData{SyncPriority: 25},
+ "universe": JoinerMetaData{SyncPriority: 0},
+ },
+ },
+ LocalPath: "/foo/bar",
+ }
+
+ data, err := sg.getSyncGroupByID(sgid)
+ if err != nil {
+ t.Errorf("cannot get updated SyncGroup by ID %d: %v", sgid, err)
+ }
+ if !reflect.DeepEqual(data, expData) {
+ t.Errorf("invalid SyncGroup data for updated group ID %d: got %v instead of %v", sgid, data, expData)
+ }
+
+ data, err = sg.getSyncGroupByName(sgname)
+ if err != nil {
+ t.Errorf("cannot get updated SyncGroup by Name %s: %v", sgname, err)
+ }
+ if !reflect.DeepEqual(data, expData) {
+ t.Errorf("invalid SyncGroup data for updated group name %s: got %v instead of %v", sgname, data, expData)
+ }
+
+ // Verify membership data.
+ members, err := sg.getMembers()
+ if err != nil {
+ t.Errorf("cannot get all SyncGroup members after update: %v", err)
+ }
+ expMembers := map[string]uint32{"phone": 1, "tablet": 1, "universe": 1}
+ if !reflect.DeepEqual(members, expMembers) {
+ t.Errorf("invalid SyncGroup members after update: got %v instead of %v", members, expMembers)
+ }
+
+ expMetaData := map[string]*memberMetaData{
+ "phone": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 10}},
+ "tablet": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 25}},
+ "universe": &memberMetaData{metaData: JoinerMetaData{SyncPriority: 0}},
+ }
+ for mm := range members {
+ info, err := sg.getMemberInfo(mm)
+ if err != nil || info == nil {
+ t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
+ }
+ if len(info.gids) != 1 {
+ t.Errorf("invalid info for SyncGroup member %s: %v", mm, info)
+ }
+ expJoinerMetaData := expMetaData[mm]
+ joinerMetaData := info.gids[sgid]
+ if !reflect.DeepEqual(joinerMetaData, expJoinerMetaData) {
+ t.Errorf("invalid joiner Data for SyncGroup member %s under group ID %d: got %v instead of %v",
+ mm, sgid, joinerMetaData, expJoinerMetaData)
+ }
+ }
+
+ sg.dump()
+ sg.close()
+}
+
+// TestDeleteSyncGroup tests deleting a SyncGroup.
+func TestDeleteSyncGroup(t *testing.T) {
+ sgfile := getFileName()
+ defer os.Remove(sgfile)
+
+ sg, err := openSyncGroupTable(sgfile)
+ if err != nil {
+ t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
+ }
+
+ sgname := "foobar"
+ sgid, err := strToGroupId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Delete non-existing SyncGroups.
+ err = sg.delSyncGroupByID(sgid)
+ if err == nil {
+ t.Errorf("deleting a non-existing SyncGroup ID did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ err = sg.delSyncGroupByName(sgname)
+ if err == nil {
+ t.Errorf("deleting a non-existing SyncGroup name did not fail in SyncGroup Table file %s", sgfile)
+ }
+
+ checkSGStats(t, "del-1", 0, 0)
+
+ // Create the SyncGroup to delete later.
+ sgData := &syncGroupData{
+ SrvInfo: SyncGroupInfo{
+ Id: sgid,
+ GroupName: sgname,
+ Joiners: map[string]JoinerMetaData{
+ "phone": JoinerMetaData{SyncPriority: 10},
+ "tablet": JoinerMetaData{SyncPriority: 25},
+ "cloud": JoinerMetaData{SyncPriority: 1},
+ },
+ },
+ LocalPath: "/foo/bar",
+ }
+
+ err = sg.addSyncGroup(sgData)
+ if err != nil {
+ t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
+ }
+
+ checkSGStats(t, "del-2", 1, 3)
+
+ // Delete it by ID.
+ err = sg.delSyncGroupByID(sgid)
+ if err != nil {
+ t.Errorf("deleting SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
+ }
+
+ checkSGStats(t, "del-3", 0, 0)
+
+ // Create it again then delete it by name.
+ err = sg.addSyncGroup(sgData)
+ if err != nil {
+ t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid, sgfile, err)
+ }
+
+ checkSGStats(t, "del-4", 1, 3)
+
+ err = sg.delSyncGroupByName(sgname)
+ if err != nil {
+ t.Errorf("deleting SyncGroup name %s failed in SyncGroup Table file %s: %v", sgname, sgfile, err)
+ }
+
+ checkSGStats(t, "del-5", 0, 0)
+
+ sg.dump()
+ sg.close()
+}
+
+// TestMultiSyncGroups tests creating multiple SyncGroups.
+func TestMultiSyncGroups(t *testing.T) {
+ sgfile := getFileName()
+ defer os.Remove(sgfile)
+
+ sg, err := openSyncGroupTable(sgfile)
+ if err != nil {
+ t.Fatalf("cannot open new SyncGroup Table file %s", sgfile)
+ }
+
+ sgname1, sgname2 := "foo", "bar"
+ sgid1, err := strToGroupId("1234")
+ if err != nil {
+ t.Fatal(err)
+ }
+ sgid2, err := strToGroupId("8888")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Add two SyncGroups.
+ sgData1 := &syncGroupData{
+ SrvInfo: SyncGroupInfo{
+ Id: sgid1,
+ GroupName: sgname1,
+ Joiners: map[string]JoinerMetaData{
+ "phone": JoinerMetaData{SyncPriority: 10},
+ "tablet": JoinerMetaData{SyncPriority: 25},
+ "cloud": JoinerMetaData{SyncPriority: 1},
+ },
+ },
+ LocalPath: "/foo/bar",
+ }
+
+ sgData2 := &syncGroupData{
+ SrvInfo: SyncGroupInfo{
+ Id: sgid2,
+ GroupName: sgname2,
+ Joiners: map[string]JoinerMetaData{
+ "tablet": JoinerMetaData{SyncPriority: 111},
+ "door": JoinerMetaData{SyncPriority: 33},
+ "lamp": JoinerMetaData{SyncPriority: 9},
+ },
+ },
+ LocalPath: "/foo/bar",
+ }
+
+ err = sg.addSyncGroup(sgData1)
+ if err != nil {
+ t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid1, sgfile, err)
+ }
+
+ checkSGStats(t, "multi-1", 1, 3)
+
+ err = sg.addSyncGroup(sgData2)
+ if err != nil {
+ t.Errorf("creating SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid2, sgfile, err)
+ }
+
+ checkSGStats(t, "multi-2", 2, 5)
+
+ // Verify SyncGroup names.
+ sgNames, err := sg.getAllSyncGroupNames()
+ if err != nil {
+ t.Errorf("cannot get all SyncGroup names: %v", err)
+ }
+ if len(sgNames) != 2 {
+ t.Errorf("wrong number of SyncGroup names: %d instead of 2", len(sgNames))
+ }
+ expNames := map[string]struct{}{sgname1: struct{}{}, sgname2: struct{}{}}
+ for _, name := range sgNames {
+ if _, ok := expNames[name]; !ok {
+ t.Errorf("unknown SyncGroup name returned: %s", name)
+ } else {
+ delete(expNames, name)
+ }
+ }
+
+ if len(expNames) > 0 {
+ t.Errorf("SyncGroup names missing, not returned: %v", expNames)
+ }
+
+ // Verify SyncGroup membership data.
+ members, err := sg.getMembers()
+ if err != nil {
+ t.Errorf("cannot get all SyncGroup members: %v", err)
+ }
+
+ expMembers := map[string]uint32{"phone": 1, "tablet": 2, "cloud": 1, "door": 1, "lamp": 1}
+ if !reflect.DeepEqual(members, expMembers) {
+ t.Errorf("invalid SyncGroup members: got %v instead of %v", members, expMembers)
+ }
+
+ expMemberInfo := map[string]*memberInfo{
+ "phone": &memberInfo{
+ gids: map[GroupId]*memberMetaData{
+ sgid1: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 10}},
+ },
+ },
+ "tablet": &memberInfo{
+ gids: map[GroupId]*memberMetaData{
+ sgid1: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 25}},
+ sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 111}},
+ },
+ },
+ "cloud": &memberInfo{
+ gids: map[GroupId]*memberMetaData{
+ sgid1: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 1}},
+ },
+ },
+ "door": &memberInfo{
+ gids: map[GroupId]*memberMetaData{
+ sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 33}},
+ },
+ },
+ "lamp": &memberInfo{
+ gids: map[GroupId]*memberMetaData{
+ sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 9}},
+ },
+ },
+ }
+
+ for mm := range members {
+ info, err := sg.getMemberInfo(mm)
+ if err != nil || info == nil {
+ t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
+ }
+ expInfo := expMemberInfo[mm]
+ if !reflect.DeepEqual(info, expInfo) {
+ t.Errorf("invalid info for SyncGroup member %s: got %v instead of %v", mm, info, expInfo)
+ }
+ }
+
+ // Delete the 1st SyncGroup.
+ err = sg.delSyncGroupByID(sgid1)
+ if err != nil {
+ t.Errorf("deleting SyncGroup ID %d failed in SyncGroup Table file %s: %v", sgid1, sgfile, err)
+ }
+
+ checkSGStats(t, "multi-3", 1, 3)
+
+ // Verify SyncGroup membership data.
+ members, err = sg.getMembers()
+ if err != nil {
+ t.Errorf("cannot get all SyncGroup members: %v", err)
+ }
+
+ expMembers = map[string]uint32{"tablet": 1, "door": 1, "lamp": 1}
+ if !reflect.DeepEqual(members, expMembers) {
+ t.Errorf("invalid SyncGroup members: got %v instead of %v", members, expMembers)
+ }
+
+ expMemberInfo = map[string]*memberInfo{
+ "tablet": &memberInfo{
+ gids: map[GroupId]*memberMetaData{
+ sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 111}},
+ },
+ },
+ "door": &memberInfo{
+ gids: map[GroupId]*memberMetaData{
+ sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 33}},
+ },
+ },
+ "lamp": &memberInfo{
+ gids: map[GroupId]*memberMetaData{
+ sgid2: &memberMetaData{metaData: JoinerMetaData{SyncPriority: 9}},
+ },
+ },
+ }
+
+ for mm := range members {
+ info, err := sg.getMemberInfo(mm)
+ if err != nil || info == nil {
+ t.Errorf("cannot get info for SyncGroup member %s: info: %v, err: %v", mm, info, err)
+ }
+ expInfo := expMemberInfo[mm]
+ if !reflect.DeepEqual(info, expInfo) {
+ t.Errorf("invalid info for SyncGroup member %s: got %v instead of %v", mm, info, expInfo)
+ }
+ }
+
+ sg.dump()
+ sg.close()
+}
diff --git a/services/syncbase/sync/testdata/local-init-00.log.sync b/services/syncbase/sync/testdata/local-init-00.log.sync
new file mode 100644
index 0000000..46b3502
--- /dev/null
+++ b/services/syncbase/sync/testdata/local-init-00.log.sync
@@ -0,0 +1,6 @@
+# Create an object locally and update it twice (linked-list).
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addl|1234|1|||logrec-00|0|1|false
+addl|1234|2|1||logrec-01|0|1|false
+addl|1234|3|2||logrec-02|0|1|false
diff --git a/services/syncbase/sync/testdata/local-init-00.sync b/services/syncbase/sync/testdata/local-init-00.sync
new file mode 100644
index 0000000..9ab99dc
--- /dev/null
+++ b/services/syncbase/sync/testdata/local-init-00.sync
@@ -0,0 +1,6 @@
+# Create an object locally and update it twice (linked-list).
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addl|1234|0|||logrec-00|0|1|false
+addl|1234|1|0||logrec-01|0|1|false
+addl|1234|2|1||logrec-02|0|1|false
diff --git a/services/syncbase/sync/testdata/local-init-01.log.sync b/services/syncbase/sync/testdata/local-init-01.log.sync
new file mode 100644
index 0000000..2f5930c
--- /dev/null
+++ b/services/syncbase/sync/testdata/local-init-01.log.sync
@@ -0,0 +1,9 @@
+# Create objects locally and update one and delete another.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addl|12|1|||logrec-00|0|1|false
+addl|12|2|1||logrec-01|0|1|false
+addl|12|3|2||logrec-02|0|1|false
+
+addl|45|1|||logrec-00|0|1|false
+addl|45|0|1||logrec-00|0|1|true
\ No newline at end of file
diff --git a/services/syncbase/sync/testdata/local-init-01.sync b/services/syncbase/sync/testdata/local-init-01.sync
new file mode 100644
index 0000000..1178c62
--- /dev/null
+++ b/services/syncbase/sync/testdata/local-init-01.sync
@@ -0,0 +1,12 @@
+# Create an object DAG locally with branches and resolved conflicts.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addl|1234|0|||logrec-00|0|1|false
+addl|1234|1|0||logrec-01|0|1|false
+addl|1234|2|1||logrec-02|0|1|false
+addl|1234|3|1||logrec-03|0|1|false
+addl|1234|4|2|3|logrec-04|0|1|false
+addl|1234|5|4||logrec-05|0|1|false
+addl|1234|6|1||logrec-06|0|1|false
+addl|1234|7|5|6|logrec-07|0|1|false
+addl|1234|8|7||logrec-08|0|1|false
diff --git a/services/syncbase/sync/testdata/local-init-02.log.sync b/services/syncbase/sync/testdata/local-init-02.log.sync
new file mode 100644
index 0000000..7f9f6a7
--- /dev/null
+++ b/services/syncbase/sync/testdata/local-init-02.log.sync
@@ -0,0 +1,9 @@
+# Create objects locally and update one and delete another.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addl|12|1|||logrec-00|0|1|false
+addl|12|2|1||logrec-01|0|1|false
+addl|12|3|2||logrec-02|0|1|false
+
+addl|45|10|||logrec-00|0|1|false
+addl|45|20|10||logrec-00|0|1|false
\ No newline at end of file
diff --git a/services/syncbase/sync/testdata/local-init-02.sync b/services/syncbase/sync/testdata/local-init-02.sync
new file mode 100644
index 0000000..cb60a79
--- /dev/null
+++ b/services/syncbase/sync/testdata/local-init-02.sync
@@ -0,0 +1,10 @@
+# Create DAGs for 3 objects locally.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addl|1234|1|||logrec-a-01|0|1|false
+addl|1234|2|1||logrec-a-02|0|1|false
+
+addl|6789|1|||logrec-b-01|0|1|false
+addl|6789|2|1||logrec-b-02|0|1|false
+
+addl|2222|1|||logrec-c-01|0|1|false
diff --git a/services/syncbase/sync/testdata/local-init-03.sync b/services/syncbase/sync/testdata/local-init-03.sync
new file mode 100644
index 0000000..202a752
--- /dev/null
+++ b/services/syncbase/sync/testdata/local-init-03.sync
@@ -0,0 +1,10 @@
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addl|1234|1|||logrec-01|0|1|false
+addl|1234|2|1||logrec-02|0|1|false
+addl|1234|3|1||logrec-03|0|1|false
+addl|1234|4|2||logrec-04|0|1|false
+addl|1234|5|2||logrec-05|0|1|true
+addl|1234|6|4|5|logrec-06|0|1|false
+addl|1234|7|3|5|logrec-07|0|1|false
+addl|1234|8|6|7|logrec-08|0|1|false
diff --git a/services/syncbase/sync/testdata/local-init-watch.log.sync b/services/syncbase/sync/testdata/local-init-watch.log.sync
new file mode 100644
index 0000000..3e895c8
--- /dev/null
+++ b/services/syncbase/sync/testdata/local-init-watch.log.sync
@@ -0,0 +1,3 @@
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addl|082bc42e15af4fcf611d7f19a8d7831f|4|||logrec-00|0|1|false
diff --git a/services/syncbase/sync/testdata/local-resolve-00.sync b/services/syncbase/sync/testdata/local-resolve-00.sync
new file mode 100644
index 0000000..7026060
--- /dev/null
+++ b/services/syncbase/sync/testdata/local-resolve-00.sync
@@ -0,0 +1,4 @@
+# Create an object locally and update it twice (linked-list).
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addl|1234|6|2|5|logrec-06|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-2obj-del.log.sync b/services/syncbase/sync/testdata/remote-2obj-del.log.sync
new file mode 100644
index 0000000..5274ecf
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-2obj-del.log.sync
@@ -0,0 +1,7 @@
+# Update one object and delete another object remotely.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|12|4|3||VeyronPhone:10:1:0|0|1|false
+addr|12|5|4||VeyronPhone:10:1:1|0|1|false
+addr|12|6|5||VeyronPhone:10:1:2|0|1|true
+addr|45|2|1||VeyronPhone:10:1:3|0|1|false
\ No newline at end of file
diff --git a/services/syncbase/sync/testdata/remote-conf-00.log.sync b/services/syncbase/sync/testdata/remote-conf-00.log.sync
new file mode 100644
index 0000000..1f9bb5b
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-conf-00.log.sync
@@ -0,0 +1,8 @@
+# Update an object remotely three times triggering one conflict after
+# it was created locally up to v3 (i.e. assume the remote sync received
+# it from the local sync at v2, then updated separately).
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|4|2||VeyronPhone:10:1:0|0|1|false
+addr|1234|5|4||VeyronPhone:10:1:1|0|1|false
+addr|1234|6|5||VeyronPhone:10:1:2|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-conf-00.sync b/services/syncbase/sync/testdata/remote-conf-00.sync
new file mode 100644
index 0000000..8fae794
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-conf-00.sync
@@ -0,0 +1,8 @@
+# Update an object remotely three times triggering one conflict after
+# it was created locally up to v2 (i.e. assume the remote sync received
+# it from the local sync at v1, then updated separately).
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|3|1||logrec-03|0|1|false
+addr|1234|4|3||logrec-04|0|1|false
+addr|1234|5|4||logrec-05|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-conf-01.log.sync b/services/syncbase/sync/testdata/remote-conf-01.log.sync
new file mode 100644
index 0000000..9581f69
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-conf-01.log.sync
@@ -0,0 +1,10 @@
+# Update an object remotely three times triggering a conflict with
+# 2 graft points: v1 and v4. This assumes that the remote sync got
+# v1, made its own conflicting v4 that it resolved into v5 (against v2)
+# then made a v6 change. When the local sync gets back this info it
+# sees 2 graft points: v1-v4 and v2-v5.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|4|1||VeyronLaptop:10:1:0|0|1|false
+addr|1234|5|2|4|VeyronPhone:10:1:0|0|1|false
+addr|1234|6|5||VeyronPhone:10:1:1|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-conf-01.sync b/services/syncbase/sync/testdata/remote-conf-01.sync
new file mode 100644
index 0000000..7485aeb
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-conf-01.sync
@@ -0,0 +1,10 @@
+# Update an object remotely three times triggering a conflict with
+# 2 graft points: v0 and v2. This assumes that the remote sync got
+# v0, made its own conflicting v3 that it resolved into v4 (against v1)
+# then made a v5 change. When the local sync gets back this info it
+# sees 2 graft points: v0-v3 and v1-v4.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|3|0||logrec-03|0|1|false
+addr|1234|4|1|3|logrec-04|0|1|false
+addr|1234|5|4||logrec-05|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-conf-02.log.sync b/services/syncbase/sync/testdata/remote-conf-02.log.sync
new file mode 100644
index 0000000..3c36277
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-conf-02.log.sync
@@ -0,0 +1,15 @@
+# Update an object remotely three times triggering one conflict after
+# it was created locally up to v3 (i.e. assume the remote sync received
+# it from the local sync at v2, then updated separately).
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|4|2||VeyronPhone:10:1:0|0|1|false
+addr|1234|5|4||VeyronPhone:10:1:1|0|1|false
+addr|1234|6|5||VeyronPhone:10:1:2|0|1|false
+
+addr|12|4|2||VeyronPhone:99:1:0|0|1|false
+addr|12|5|4||VeyronPhone:99:1:1|0|1|false
+addr|12|6|5||VeyronPhone:99:1:2|0|1|false
+
+addr|45|30|20||VeyronPhone:99:2:0|0|1|false
+addr|45|40|30||VeyronPhone:99:2:1|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-conf-link.log.sync b/services/syncbase/sync/testdata/remote-conf-link.log.sync
new file mode 100644
index 0000000..a324e4f
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-conf-link.log.sync
@@ -0,0 +1,5 @@
+# Update an object remotely, detect conflict, and bless the local version.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|4|1||VeyronPhone:10:1:0|0|1|false
+linkr|1234|4|2||VeyronPhone:10:1:1
diff --git a/services/syncbase/sync/testdata/remote-init-00.log.sync b/services/syncbase/sync/testdata/remote-init-00.log.sync
new file mode 100644
index 0000000..9795d53
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-init-00.log.sync
@@ -0,0 +1,6 @@
+# Create an object remotely and update it twice (linked-list).
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|1|||VeyronPhone:10:1:0|0|1|false
+addr|1234|2|1||VeyronPhone:10:1:1|0|1|false
+addr|1234|3|2||VeyronPhone:10:1:2|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-init-00.sync b/services/syncbase/sync/testdata/remote-init-00.sync
new file mode 100644
index 0000000..18d288e
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-init-00.sync
@@ -0,0 +1,6 @@
+# Create an object remotely and update it twice (linked-list).
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|0|||logrec-00|0|1|false
+addr|1234|1|0||logrec-01|0|1|false
+addr|1234|2|1||logrec-02|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-init-01.log.sync b/services/syncbase/sync/testdata/remote-init-01.log.sync
new file mode 100644
index 0000000..1db4c50
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-init-01.log.sync
@@ -0,0 +1,6 @@
+# Create an object remotely and update it twice (linked-list).
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|1|||VeyronPhone:10:5:0|0|1|false
+addr|1234|2|1||VeyronPhone:10:5:1|0|1|false
+addr|1234|3|2||VeyronPhone:10:5:2|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-init-02.log.sync b/services/syncbase/sync/testdata/remote-init-02.log.sync
new file mode 100644
index 0000000..1274b7d
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-init-02.log.sync
@@ -0,0 +1,17 @@
+# Create objects and transactions remotely.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|12|1|||VeyronPhone:10:1:0|0|1|false
+
+addr|12|2|1||VeyronPhone:10:1:1|100|3|false
+addr|45|1|||VeyronPhone:10:1:2|100|3|false
+addr|78|1|||VeyronPhone:10:1:3|100|3|false
+
+addr|78|2|1||VeyronPhone:10:1:4|0|1|false
+
+addr|78|3|1||VeyronLaptop:10:1:0|0|1|false
+
+addr|78|4|2|3|VeyronPhone:10:2:0|0|1|false
+
+addr|12|3|2||VeyronPhone:10:2:1|101|2|false
+addr|45|2|1||VeyronPhone:10:2:2|101|2|false
diff --git a/services/syncbase/sync/testdata/remote-init-03.log.sync b/services/syncbase/sync/testdata/remote-init-03.log.sync
new file mode 100644
index 0000000..1ccac35
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-init-03.log.sync
@@ -0,0 +1,6 @@
+# Create an object remotely and delete it.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|1|||VeyronPhone:10:1:0|0|1|false
+addr|1234|2|1||VeyronPhone:10:1:1|0|1|false
+addr|1234|3|2||VeyronPhone:10:1:2|0|1|true
diff --git a/services/syncbase/sync/testdata/remote-noconf-00.log.sync b/services/syncbase/sync/testdata/remote-noconf-00.log.sync
new file mode 100644
index 0000000..e2e2afa
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-noconf-00.log.sync
@@ -0,0 +1,8 @@
+# Update an object remotely three times without triggering a conflict
+# after it was created locally up to v3 (i.e. assume the remote sync
+# received it from the local sync first, then updated it).
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|4|3||VeyronPhone:10:1:0|0|1|false
+addr|1234|5|4||VeyronPhone:10:1:1|0|1|false
+addr|1234|6|5||VeyronPhone:10:1:2|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-noconf-00.sync b/services/syncbase/sync/testdata/remote-noconf-00.sync
new file mode 100644
index 0000000..d44b6ac
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-noconf-00.sync
@@ -0,0 +1,8 @@
+# Update an object remotely three times without triggering a conflict
+# after it was created locally up to v2 (i.e. assume the remote sync
+# received it from the local sync first, then updated it).
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|3|2||logrec-03|0|1|false
+addr|1234|4|3||logrec-04|0|1|false
+addr|1234|5|4||logrec-05|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-noconf-link-00.log.sync b/services/syncbase/sync/testdata/remote-noconf-link-00.log.sync
new file mode 100644
index 0000000..6945bb2
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-noconf-link-00.log.sync
@@ -0,0 +1,5 @@
+# Update an object remotely, detect conflict, and bless the remote version.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|4|1||VeyronPhone:10:1:0|0|1|false
+linkr|1234|2|4||VeyronPhone:10:1:1
diff --git a/services/syncbase/sync/testdata/remote-noconf-link-01.log.sync b/services/syncbase/sync/testdata/remote-noconf-link-01.log.sync
new file mode 100644
index 0000000..0c6969e
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-noconf-link-01.log.sync
@@ -0,0 +1,5 @@
+# Update an object remotely, detect conflict, and bless the local version.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|4|1||VeyronPhone:10:1:0|0|1|false
+linkr|1234|4|3||VeyronPhone:10:1:1
diff --git a/services/syncbase/sync/testdata/remote-noconf-link-02.log.sync b/services/syncbase/sync/testdata/remote-noconf-link-02.log.sync
new file mode 100644
index 0000000..df9e128
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-noconf-link-02.log.sync
@@ -0,0 +1,6 @@
+# Update an object remotely, detect conflict, and bless the remote version, and continue updating.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+addr|1234|4|1||VeyronPhone:10:1:0|0|1|false
+linkr|1234|3|4||VeyronPhone:10:1:1
+addr|1234|5|3||VeyronPhone:10:2:0|0|1|false
diff --git a/services/syncbase/sync/testdata/remote-noconf-link-repeat.log.sync b/services/syncbase/sync/testdata/remote-noconf-link-repeat.log.sync
new file mode 100644
index 0000000..82e11c6
--- /dev/null
+++ b/services/syncbase/sync/testdata/remote-noconf-link-repeat.log.sync
@@ -0,0 +1,4 @@
+# Resolve the same conflict on two different devices.
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+
+linkr|1234|3|4||VeyronLaptop:10:1:0
diff --git a/services/syncbase/sync/testdata/test-1obj.gc.sync b/services/syncbase/sync/testdata/test-1obj.gc.sync
new file mode 100644
index 0000000..4d1a8d0
--- /dev/null
+++ b/services/syncbase/sync/testdata/test-1obj.gc.sync
@@ -0,0 +1,14 @@
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+# Local node is A. Remote nodes are B and C.
+### NOT UP-TO-DATE
+addr|12345|0|||C:1:0|false|false
+addr|12345|1|0||B:1:0|false|false
+addl|12345|2|0||A:1:0|false|false
+addl|12345|3|1|2|A:2:0|false|false
+addr|12345|4|3||C:2:0|false|false
+addr|12345|5|3||B:2:0|false|false
+addr|12345|6|4|5|B:3:0|false|false
+# Devtable state
+setdev|A|A:2,B:3,C:2
+setdev|B|A:2,B:3,C:2
+setdev|C|A:2,B:1,C:2
diff --git a/services/syncbase/sync/testdata/test-3obj.gc.sync b/services/syncbase/sync/testdata/test-3obj.gc.sync
new file mode 100644
index 0000000..ce0656c
--- /dev/null
+++ b/services/syncbase/sync/testdata/test-3obj.gc.sync
@@ -0,0 +1,45 @@
+# The format is: <cmd>|<objid>|<version>|<parent1>|<parent2>|<logrec>|<txid>|<txcount>|<deleted>
+# Local node is A. Remote nodes are B and C.
+### NOT UP-TO-DATE
+addl|123|1|||A:1:0|false|false
+
+addr|456|1|||B:1:0|false|false
+
+addr|456|2|1||B:2:0|false|false
+addr|123|2|1||B:2:1|false|false
+
+addl|456|3|2||A:2:0|false|false
+addl|123|4|2||A:2:1|false|false
+
+addr|789|1|||C:1:0|false|false
+
+addr|789|2|1||C:2:0|false|false
+
+addr|123|3|1||C:3:0|false|false
+addr|789|3|2||C:3:1|false|false
+
+addr|123|5|3|2|C:4:0|false|false
+
+addl|123|6|4|5|A:3:0|false|false
+addl|456|4|3||A:3:1|false|false
+addl|789|4|3||A:3:2|false|false
+
+addr|456|5|2||B:3:0|false|false
+
+addl|456|7|4|5|A:4:0|false|false
+
+addr|456|6|2||C:5:0|false|false
+addr|123|7|5||C:5:1|false|false
+addr|123|8|7||C:5:2|false|false
+addr|789|5|3||C:5:3|false|false
+
+addl|123|9|6|8|A:5:0|false|false
+addl|456|8|6|7|A:5:1|false|false
+addl|789|6|4|5|A:5:2|false|false
+
+addl|123|10|9||A:6:0|false|false
+
+# Devtable state
+setdev|A|A:6,B:3,C:5
+setdev|B|A:4,B:3,C:4
+setdev|C|A:4,B:3,C:4
diff --git a/services/syncbase/sync/util_test.go b/services/syncbase/sync/util_test.go
new file mode 100644
index 0000000..4c4bb88
--- /dev/null
+++ b/services/syncbase/sync/util_test.go
@@ -0,0 +1,250 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Utilities for testing.
+import (
+ "container/list"
+ "fmt"
+ "os"
+ "time"
+)
+
+// getFileName generates a filename for a temporary (per unit test) kvdb file.
+func getFileName() string {
+ return fmt.Sprintf("%s/sync_test_%d_%d", os.TempDir(), os.Getpid(), time.Now().UnixNano())
+}
+
+// createTempDir creates a unique temporary directory to store kvdb files.
+func createTempDir() (string, error) {
+ dir := fmt.Sprintf("%s/sync_test_%d_%d/", os.TempDir(), os.Getpid(), time.Now().UnixNano())
+ if err := os.MkdirAll(dir, 0700); err != nil {
+ return "", err
+ }
+ return dir, nil
+}
+
+// getFileSize returns the size of a file.
+func getFileSize(fname string) int64 {
+ finfo, err := os.Stat(fname)
+ if err != nil {
+ return -1
+ }
+ return finfo.Size()
+}
+
+// dummyStream struct emulates stream of log records received from RPC.
+type dummyStream struct {
+ l *list.List
+ value LogRec
+}
+
+func newStream() *dummyStream {
+ ds := &dummyStream{
+ l: list.New(),
+ }
+ return ds
+}
+
+func (ds *dummyStream) Advance() bool {
+ if ds.l.Len() > 0 {
+ ds.value = ds.l.Remove(ds.l.Front()).(LogRec)
+ return true
+ }
+ return false
+}
+
+func (ds *dummyStream) Value() LogRec {
+ return ds.value
+}
+
+func (ds *dummyStream) RecvStream() interface {
+ Advance() bool
+ Value() LogRec
+ Err() error
+} {
+ return ds
+}
+
+func (*dummyStream) Err() error { return nil }
+
+func (ds *dummyStream) Finish() (map[ObjId]GenVector, error) {
+ return nil, nil
+}
+
+func (ds *dummyStream) Cancel() {
+}
+
+func (ds *dummyStream) add(rec LogRec) {
+ ds.l.PushBack(rec)
+}
+
+// logReplayCommands replays local log records parsed from the input file.
+func logReplayCommands(log *iLog, syncfile string, srid ObjId) error {
+ cmds, err := parseSyncCommands(syncfile)
+ if err != nil {
+ return err
+ }
+
+ for _, cmd := range cmds {
+ switch cmd.cmd {
+ case addLocal:
+ parent := NoVersion
+ if cmd.parents != nil {
+ parent = cmd.parents[0]
+ }
+
+ val := &LogValue{
+ //Mutation: raw.Mutation{Version: cmd.version},
+ Delete: cmd.deleted,
+ TxId: cmd.txID,
+ TxCount: cmd.txCount,
+ }
+ err = log.processWatchRecord(cmd.objID, cmd.version, parent, val, srid)
+ if err != nil {
+ return fmt.Errorf("cannot replay local log records %d:%v err %v",
+ cmd.objID, cmd.version, err)
+ }
+ default:
+ return fmt.Errorf("unknown cmd %v", cmd.cmd)
+ }
+ }
+
+ return nil
+}
+
+// createReplayStream creates a dummy stream of log records parsed from the input file.
+func createReplayStream(syncfile string) (*dummyStream, error) {
+ cmds, err := parseSyncCommands(syncfile)
+ if err != nil {
+ return nil, err
+ }
+
+ stream := newStream()
+ for _, cmd := range cmds {
+ id, srid, gnum, lsn, err := splitLogRecKey(cmd.logrec)
+ if err != nil {
+ return nil, err
+ }
+ rec := LogRec{
+ DevId: id,
+ SyncRootId: srid,
+ GenNum: gnum,
+ SeqNum: lsn,
+ ObjId: cmd.objID,
+ CurVers: cmd.version,
+ Parents: cmd.parents,
+ Value: LogValue{
+ //Mutation: raw.Mutation{Version: cmd.version},
+ Delete: cmd.deleted,
+ TxId: cmd.txID,
+ TxCount: cmd.txCount,
+ },
+ }
+
+ switch cmd.cmd {
+ case addRemote:
+ rec.RecType = NodeRec
+ case linkRemote:
+ rec.RecType = LinkRec
+ default:
+ return nil, err
+ }
+ stream.add(rec)
+ }
+
+ return stream, nil
+}
+
+//
+// // populates the log and dag state as part of state initialization.
+// func populateLogAndDAG(s *syncd, rec *LogRec) error {
+// logKey, err := s.log.putLogRec(rec)
+// if err != nil {
+// return err
+// }
+//
+// if err := s.dag.addNode(rec.ObjId, rec.CurVers, false, rec.Value.Delete, rec.Parents, logKey, NoTxId); err != nil {
+// return err
+// }
+// if err := s.dag.moveHead(rec.ObjId, rec.CurVers); err != nil {
+// return err
+// }
+// return nil
+// }
+//
+//
+// // vsyncInitState initializes log, dag and devtable state obtained from an input trace-like file.
+// func vsyncInitState(s *syncd, syncfile string) error {
+// cmds, err := parseSyncCommands(syncfile)
+// if err != nil {
+// return err
+// }
+//
+// var curGen GenId
+// genMap := make(map[string]*genMetadata)
+//
+// for _, cmd := range cmds {
+// switch cmd.cmd {
+// case addLocal, addRemote:
+// id, sgid, gnum, lsn, err := splitLogRecKey(cmd.logrec)
+// if err != nil {
+// return err
+// }
+// rec := &LogRec{
+// DevId: id,
+// SGrpID: sgid,
+// GenNum: gnum,
+// SeqNum: lsn,
+// ObjId: cmd.objID,
+// CurVers: cmd.version,
+// Parents: cmd.parents,
+// Value: LogValue{Continued: cmd.continued, Delete: cmd.deleted},
+// }
+// if err := populateLogAndDAG(s, rec); err != nil {
+// return err
+// }
+// key := generationKey(id, sgid, gnum)
+// if m, ok := genMap[key]; !ok {
+// genMap[key] = &genMetadata{
+// Pos: s.log.head.Curorder,
+// Count: 1,
+// MaxSeqNum: rec.SeqNum,
+// }
+// s.log.head.Curorder++
+// } else {
+// m.Count++
+// if rec.SeqNum > m.MaxSeqNum {
+// m.MaxSeqNum = rec.SeqNum
+// }
+// }
+// if cmd.cmd == addLocal {
+// curGen = gnum
+// }
+//
+// case setDevTable:
+// if err := s.devtab.putGenVec(cmd.devID, cmd.genVec); err != nil {
+// return err
+// }
+// }
+// }
+//
+// // Initializing genMetadata.
+// for key, gen := range genMap {
+// dev, gnum, err := splitGenerationKey(key)
+// if err != nil {
+// return err
+// }
+// if err := s.log.putGenMetadata(dev, gnum, gen); err != nil {
+// return err
+// }
+// }
+//
+// // Initializing generation in log header.
+// s.log.head.Curgen = curGen + 1
+//
+// return nil
+// }
+//
diff --git a/services/syncbase/sync/vsync.vdl b/services/syncbase/sync/vsync.vdl
new file mode 100644
index 0000000..8ad9d30
--- /dev/null
+++ b/services/syncbase/sync/vsync.vdl
@@ -0,0 +1,182 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+import (
+ "v.io/v23/security/access"
+)
+
+// temporary types
+type ObjId string
+type Version uint64
+type GroupId uint64
+
+// DeviceId is the globally unique Id of a device.
+type DeviceId string
+// GenId is the unique Id per generation per device.
+type GenId uint64
+// SeqNum is the log sequence number.
+type SeqNum uint64
+// GenVector is the generation vector.
+type GenVector map[DeviceId]GenId
+// TxId is the unique Id per transaction.
+type TxId uint64
+// GroupIdSet is the list of SyncGroup Ids.
+type GroupIdSet []GroupId
+
+const (
+ // NodeRec type log record adds a new node in the dag.
+ NodeRec = byte(0)
+ // LinkRec type log record adds a new link in the dag.
+ LinkRec = byte(1)
+
+ // Sync interface has Object name "global/vsync/<devid>/sync".
+ SyncSuffix = "sync"
+
+ // temporary nil values
+ NoObjId = ObjId("")
+ NoVersion = Version(0)
+ NoGroupId = GroupId(0)
+)
+
+// LogRec represents a single log record that is exchanged between two
+// peers.
+//
+// It contains log related metadata: DevId is the id of the device
+// that created the log record, SyncRootId is the id of a SyncRoot this log
+// record was created under, GNum is the Id of the generation that the
+// log record is part of, SeqNum is the log sequence number of the log
+// record in the generation GNum, and RecType is the type of log
+// record.
+//
+// It also contains information relevant to the updates to an object
+// in the store: ObjId is the id of the object that was
+// updated. CurVers is the current version number of the
+// object. Parents can contain 0, 1 or 2 parent versions that the
+// current version is derived from, and Value is the actual value of
+// the object mutation.
+type LogRec struct {
+ // Log related information.
+ DevId DeviceId
+ SyncRootId ObjId
+ GenNum GenId
+ SeqNum SeqNum
+ RecType byte
+
+ // Object related information.
+ ObjId ObjId
+ CurVers Version
+ Parents []Version
+ Value LogValue
+}
+
+// LogValue represents an object mutation within a transaction.
+type LogValue struct {
+ // Mutation is the store mutation representing the change in the object.
+ //Mutation raw.Mutation
+ // SyncTime is the timestamp of the mutation when it arrives at the Sync server.
+ SyncTime int64
+ // Delete indicates whether the mutation resulted in the object being
+ // deleted from the store.
+ Delete bool
+ // TxId is the unique Id of the transaction this mutation belongs to.
+ TxId TxId
+ // TxCount is the number of mutations in the transaction TxId.
+ TxCount uint32
+}
+
+// DeviceStats contains high-level information on a device participating in
+// peer-to-peer synchronization.
+type DeviceStats struct {
+ DevId DeviceId // Device Id.
+ LastSync int64 // Timestamp of last sync from the device.
+ GenVectors map[ObjId]GenVector // Generation vectors per SyncRoot.
+ IsSelf bool // True if the responder is on this device.
+}
+
+// SyncGroupStats contains high-level information on a SyncGroup.
+type SyncGroupStats struct {
+ Name string // Global name of the SyncGroup.
+ Id GroupId // Global Id of the SyncGroup.
+ Path string // Local store path for the root of the SyncGroup.
+ RootObjId ObjId // Id of the store object at the root path.
+ NumJoiners uint32 // Number of members currently in the SyncGroup.
+}
+
+// SyncGroupMember contains information on a SyncGroup member.
+type SyncGroupMember struct {
+ Name string // Name of SyncGroup member.
+ Id GroupId // Global Id of the SyncGroup.
+ Metadata JoinerMetaData // Extra member metadata.
+}
+
+// A SyncGroupInfo is the conceptual state of a SyncGroup object.
+type SyncGroupInfo struct {
+ Id GroupId // Globally unique SyncGroup Id.
+ ServerName string // Global Vanadium name of SyncGroupServer.
+ GroupName string // Relative name of group; global name is ServerName/GroupName.
+ Config SyncGroupConfig // Configuration parameters of this SyncGroup.
+ ETag string // Version Id for concurrency control.
+
+ // A map from joiner names to the associated metaData for devices that
+ // have called Join() or Create() and not subsequently called Leave()
+ // or had Eject() called on them. The map returned by the calls below
+ // may contain only a subset of joiners if the number is large.
+ Joiners map[string]JoinerMetaData
+
+ // Blessings for joiners of this SyncGroup will be self-signed by the
+ // SyncGroupServer, and will have names matching
+ // JoinerBlessingPrefix/Name/...
+ JoinerBlessingPrefix string
+}
+
+// A SyncGroupConfig contains some fields of SyncGroupInfo that
+// are passed at create time, but which can be changed later.
+type SyncGroupConfig struct {
+ Desc string // Human readable description.
+ Options map[string]any // Options for future evolution.
+ Permissions access.Permissions // The object's Permissions.
+
+ // Mount tables used to advertise for synchronization.
+ // Typically, we will have only one entry. However, an array allows
+ // mount tables to be changed over time.
+ MountTables []string
+
+ BlessingsDurationNanos int64 // Duration of blessings, in nanoseconds. 0 => use server default.
+}
+
+// A JoinerMetaData contains the non-name information stored per joiner.
+type JoinerMetaData struct {
+ // SyncPriority is a hint to bias the choice of syncing partners.
+ // Members of the SyncGroup should choose to synchronize more often
+ // with partners with lower values.
+ SyncPriority int32
+}
+
+// Sync allows a device to GetDeltas from another device.
+type Sync interface {
+ // GetDeltas returns a device's current generation vector and all
+ // the missing log records when compared to the incoming generation vector.
+ GetDeltas(in map[ObjId]GenVector, sgs map[ObjId]GroupIdSet, clientId DeviceId) stream<_, LogRec> (map[ObjId]GenVector | error) {access.Write}
+
+ // GetObjectHistory returns the mutation history of a store object.
+ GetObjectHistory(oid ObjId) stream<_, LogRec> (Version | error)
+
+ // GetDeviceStats returns information on devices participating in
+ // peer-to-peer synchronization.
+ GetDeviceStats() stream<_, DeviceStats> error {access.Admin}
+
+ // GetSyncGroupMembers returns information on SyncGroup members.
+ // If SyncGroup names are specified, only members of these SyncGroups
+ // are returned. Otherwise, if the slice of names is nil or empty,
+ // members of all SyncGroups are returned.
+ GetSyncGroupMembers(sgNames []string) stream<_, SyncGroupMember> error {access.Admin}
+
+ // GetSyncGroupStats returns high-level information on all SyncGroups.
+ GetSyncGroupStats() stream<_, SyncGroupStats> error {access.Admin}
+
+ // Dump writes to the Sync log internal information used for debugging.
+ Dump() error {access.Admin}
+}
diff --git a/services/syncbase/sync/vsync.vdl.go b/services/syncbase/sync/vsync.vdl.go
new file mode 100644
index 0000000..e202947
--- /dev/null
+++ b/services/syncbase/sync/vsync.vdl.go
@@ -0,0 +1,1095 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file was auto-generated by the vanadium vdl tool.
+// Source: vsync.vdl
+
+package vsync
+
+import (
+ // VDL system imports
+ "io"
+ "v.io/v23"
+ "v.io/v23/context"
+ "v.io/v23/rpc"
+ "v.io/v23/vdl"
+
+ // VDL user imports
+ "v.io/v23/security/access"
+)
+
+// temporary types
+type ObjId string
+
+func (ObjId) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.ObjId"
+}) {
+}
+
+type Version uint64
+
+func (Version) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.Version"
+}) {
+}
+
+type GroupId uint64
+
+func (GroupId) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.GroupId"
+}) {
+}
+
+// DeviceId is the globally unique Id of a device.
+type DeviceId string
+
+func (DeviceId) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.DeviceId"
+}) {
+}
+
+// GenId is the unique Id per generation per device.
+type GenId uint64
+
+func (GenId) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.GenId"
+}) {
+}
+
+// SeqNum is the log sequence number.
+type SeqNum uint64
+
+func (SeqNum) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.SeqNum"
+}) {
+}
+
+// GenVector is the generation vector.
+type GenVector map[DeviceId]GenId
+
+func (GenVector) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.GenVector"
+}) {
+}
+
+// TxId is the unique Id per transaction.
+type TxId uint64
+
+func (TxId) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.TxId"
+}) {
+}
+
+// GroupIdSet is the list of SyncGroup Ids.
+type GroupIdSet []GroupId
+
+func (GroupIdSet) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.GroupIdSet"
+}) {
+}
+
+// LogRec represents a single log record that is exchanged between two
+// peers.
+//
+// It contains log related metadata: DevId is the id of the device
+// that created the log record, SyncRootId is the id of a SyncRoot this log
+// record was created under, GNum is the Id of the generation that the
+// log record is part of, SeqNum is the log sequence number of the log
+// record in the generation GNum, and RecType is the type of log
+// record.
+//
+// It also contains information relevant to the updates to an object
+// in the store: ObjId is the id of the object that was
+// updated. CurVers is the current version number of the
+// object. Parents can contain 0, 1 or 2 parent versions that the
+// current version is derived from, and Value is the actual value of
+// the object mutation.
+type LogRec struct {
+ // Log related information.
+ DevId DeviceId
+ SyncRootId ObjId
+ GenNum GenId
+ SeqNum SeqNum
+ RecType byte
+ // Object related information.
+ ObjId ObjId
+ CurVers Version
+ Parents []Version
+ Value LogValue
+}
+
+func (LogRec) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.LogRec"
+}) {
+}
+
+// LogValue represents an object mutation within a transaction.
+type LogValue struct {
+ // Mutation is the store mutation representing the change in the object.
+ //Mutation raw.Mutation
+ // SyncTime is the timestamp of the mutation when it arrives at the Sync server.
+ SyncTime int64
+ // Delete indicates whether the mutation resulted in the object being
+ // deleted from the store.
+ Delete bool
+ // TxId is the unique Id of the transaction this mutation belongs to.
+ TxId TxId
+ // TxCount is the number of mutations in the transaction TxId.
+ TxCount uint32
+}
+
+func (LogValue) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.LogValue"
+}) {
+}
+
+// DeviceStats contains high-level information on a device participating in
+// peer-to-peer synchronization.
+type DeviceStats struct {
+ DevId DeviceId // Device Id.
+ LastSync int64 // Timestamp of last sync from the device.
+ GenVectors map[ObjId]GenVector // Generation vectors per SyncRoot.
+ IsSelf bool // True if the responder is on this device.
+}
+
+func (DeviceStats) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.DeviceStats"
+}) {
+}
+
+// SyncGroupStats contains high-level information on a SyncGroup.
+type SyncGroupStats struct {
+ Name string // Global name of the SyncGroup.
+ Id GroupId // Global Id of the SyncGroup.
+ Path string // Local store path for the root of the SyncGroup.
+ RootObjId ObjId // Id of the store object at the root path.
+ NumJoiners uint32 // Number of members currently in the SyncGroup.
+}
+
+func (SyncGroupStats) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.SyncGroupStats"
+}) {
+}
+
+// SyncGroupMember contains information on a SyncGroup member.
+type SyncGroupMember struct {
+ Name string // Name of SyncGroup member.
+ Id GroupId // Global Id of the SyncGroup.
+ Metadata JoinerMetaData // Extra member metadata.
+}
+
+func (SyncGroupMember) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.SyncGroupMember"
+}) {
+}
+
+// A SyncGroupInfo is the conceptual state of a SyncGroup object.
+type SyncGroupInfo struct {
+ Id GroupId // Globally unique SyncGroup Id.
+ ServerName string // Global Vanadium name of SyncGroupServer.
+ GroupName string // Relative name of group; global name is ServerName/GroupName.
+ Config SyncGroupConfig // Configuration parameters of this SyncGroup.
+ ETag string // Version Id for concurrency control.
+ // A map from joiner names to the associated metaData for devices that
+ // have called Join() or Create() and not subsequently called Leave()
+ // or had Eject() called on them. The map returned by the calls below
+ // may contain only a subset of joiners if the number is large.
+ Joiners map[string]JoinerMetaData
+ // Blessings for joiners of this SyncGroup will be self-signed by the
+ // SyncGroupServer, and will have names matching
+ // JoinerBlessingPrefix/Name/...
+ JoinerBlessingPrefix string
+}
+
+func (SyncGroupInfo) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.SyncGroupInfo"
+}) {
+}
+
+// A SyncGroupConfig contains some fields of SyncGroupInfo that
+// are passed at create time, but which can be changed later.
+type SyncGroupConfig struct {
+ Desc string // Human readable description.
+ Options map[string]*vdl.Value // Options for future evolution.
+ Permissions access.Permissions // The object's Permissions.
+ // Mount tables used to advertise for synchronization.
+ // Typically, we will have only one entry. However, an array allows
+ // mount tables to be changed over time.
+ MountTables []string
+ BlessingsDurationNanos int64 // Duration of blessings, in nanoseconds. 0 => use server default.
+}
+
+func (SyncGroupConfig) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.SyncGroupConfig"
+}) {
+}
+
+// A JoinerMetaData contains the non-name information stored per joiner.
+type JoinerMetaData struct {
+ // SyncPriority is a hint to bias the choice of syncing partners.
+ // Members of the SyncGroup should choose to synchronize more often
+ // with partners with lower values.
+ SyncPriority int32
+}
+
+func (JoinerMetaData) __VDLReflect(struct {
+ Name string "v.io/syncbase/x/ref/services/syncbase/sync.JoinerMetaData"
+}) {
+}
+
+func init() {
+ vdl.Register((*ObjId)(nil))
+ vdl.Register((*Version)(nil))
+ vdl.Register((*GroupId)(nil))
+ vdl.Register((*DeviceId)(nil))
+ vdl.Register((*GenId)(nil))
+ vdl.Register((*SeqNum)(nil))
+ vdl.Register((*GenVector)(nil))
+ vdl.Register((*TxId)(nil))
+ vdl.Register((*GroupIdSet)(nil))
+ vdl.Register((*LogRec)(nil))
+ vdl.Register((*LogValue)(nil))
+ vdl.Register((*DeviceStats)(nil))
+ vdl.Register((*SyncGroupStats)(nil))
+ vdl.Register((*SyncGroupMember)(nil))
+ vdl.Register((*SyncGroupInfo)(nil))
+ vdl.Register((*SyncGroupConfig)(nil))
+ vdl.Register((*JoinerMetaData)(nil))
+}
+
+// NodeRec type log record adds a new node in the dag.
+const NodeRec = byte(0)
+
+// LinkRec type log record adds a new link in the dag.
+const LinkRec = byte(1)
+
+// Sync interface has Object name "global/vsync/<devid>/sync".
+const SyncSuffix = "sync"
+
+// temporary nil values
+const NoObjId = ObjId("")
+
+const NoVersion = Version(0)
+
+const NoGroupId = GroupId(0)
+
+// SyncClientMethods is the client interface
+// containing Sync methods.
+//
+// Sync allows a device to GetDeltas from another device.
+type SyncClientMethods interface {
+ // GetDeltas returns a device's current generation vector and all
+ // the missing log records when compared to the incoming generation vector.
+ GetDeltas(ctx *context.T, in map[ObjId]GenVector, sgs map[ObjId]GroupIdSet, clientId DeviceId, opts ...rpc.CallOpt) (SyncGetDeltasClientCall, error)
+ // GetObjectHistory returns the mutation history of a store object.
+ GetObjectHistory(ctx *context.T, oid ObjId, opts ...rpc.CallOpt) (SyncGetObjectHistoryClientCall, error)
+ // GetDeviceStats returns information on devices participating in
+ // peer-to-peer synchronization.
+ GetDeviceStats(*context.T, ...rpc.CallOpt) (SyncGetDeviceStatsClientCall, error)
+ // GetSyncGroupMembers returns information on SyncGroup members.
+ // If SyncGroup names are specified, only members of these SyncGroups
+ // are returned. Otherwise, if the slice of names is nil or empty,
+ // members of all SyncGroups are returned.
+ GetSyncGroupMembers(ctx *context.T, sgNames []string, opts ...rpc.CallOpt) (SyncGetSyncGroupMembersClientCall, error)
+ // GetSyncGroupStats returns high-level information on all SyncGroups.
+ GetSyncGroupStats(*context.T, ...rpc.CallOpt) (SyncGetSyncGroupStatsClientCall, error)
+ // Dump writes to the Sync log internal information used for debugging.
+ Dump(*context.T, ...rpc.CallOpt) error
+}
+
+// SyncClientStub adds universal methods to SyncClientMethods.
+type SyncClientStub interface {
+ SyncClientMethods
+ rpc.UniversalServiceMethods
+}
+
+// SyncClient returns a client stub for Sync.
+func SyncClient(name string) SyncClientStub {
+ return implSyncClientStub{name}
+}
+
+type implSyncClientStub struct {
+ name string
+}
+
+func (c implSyncClientStub) GetDeltas(ctx *context.T, i0 map[ObjId]GenVector, i1 map[ObjId]GroupIdSet, i2 DeviceId, opts ...rpc.CallOpt) (ocall SyncGetDeltasClientCall, err error) {
+ var call rpc.ClientCall
+ if call, err = v23.GetClient(ctx).StartCall(ctx, c.name, "GetDeltas", []interface{}{i0, i1, i2}, opts...); err != nil {
+ return
+ }
+ ocall = &implSyncGetDeltasClientCall{ClientCall: call}
+ return
+}
+
+func (c implSyncClientStub) GetObjectHistory(ctx *context.T, i0 ObjId, opts ...rpc.CallOpt) (ocall SyncGetObjectHistoryClientCall, err error) {
+ var call rpc.ClientCall
+ if call, err = v23.GetClient(ctx).StartCall(ctx, c.name, "GetObjectHistory", []interface{}{i0}, opts...); err != nil {
+ return
+ }
+ ocall = &implSyncGetObjectHistoryClientCall{ClientCall: call}
+ return
+}
+
+func (c implSyncClientStub) GetDeviceStats(ctx *context.T, opts ...rpc.CallOpt) (ocall SyncGetDeviceStatsClientCall, err error) {
+ var call rpc.ClientCall
+ if call, err = v23.GetClient(ctx).StartCall(ctx, c.name, "GetDeviceStats", nil, opts...); err != nil {
+ return
+ }
+ ocall = &implSyncGetDeviceStatsClientCall{ClientCall: call}
+ return
+}
+
+func (c implSyncClientStub) GetSyncGroupMembers(ctx *context.T, i0 []string, opts ...rpc.CallOpt) (ocall SyncGetSyncGroupMembersClientCall, err error) {
+ var call rpc.ClientCall
+ if call, err = v23.GetClient(ctx).StartCall(ctx, c.name, "GetSyncGroupMembers", []interface{}{i0}, opts...); err != nil {
+ return
+ }
+ ocall = &implSyncGetSyncGroupMembersClientCall{ClientCall: call}
+ return
+}
+
+func (c implSyncClientStub) GetSyncGroupStats(ctx *context.T, opts ...rpc.CallOpt) (ocall SyncGetSyncGroupStatsClientCall, err error) {
+ var call rpc.ClientCall
+ if call, err = v23.GetClient(ctx).StartCall(ctx, c.name, "GetSyncGroupStats", nil, opts...); err != nil {
+ return
+ }
+ ocall = &implSyncGetSyncGroupStatsClientCall{ClientCall: call}
+ return
+}
+
+func (c implSyncClientStub) Dump(ctx *context.T, opts ...rpc.CallOpt) (err error) {
+ var call rpc.ClientCall
+ if call, err = v23.GetClient(ctx).StartCall(ctx, c.name, "Dump", nil, opts...); err != nil {
+ return
+ }
+ err = call.Finish()
+ return
+}
+
+// SyncGetDeltasClientStream is the client stream for Sync.GetDeltas.
+type SyncGetDeltasClientStream interface {
+ // RecvStream returns the receiver side of the Sync.GetDeltas client stream.
+ RecvStream() interface {
+ // Advance stages an item so that it may be retrieved via Value. Returns
+ // true iff there is an item to retrieve. Advance must be called before
+ // Value is called. May block if an item is not available.
+ Advance() bool
+ // Value returns the item that was staged by Advance. May panic if Advance
+ // returned false or was not called. Never blocks.
+ Value() LogRec
+ // Err returns any error encountered by Advance. Never blocks.
+ Err() error
+ }
+}
+
+// SyncGetDeltasClientCall represents the call returned from Sync.GetDeltas.
+type SyncGetDeltasClientCall interface {
+ SyncGetDeltasClientStream
+ // Finish blocks until the server is done, and returns the positional return
+ // values for call.
+ //
+ // Finish returns immediately if the call has been canceled; depending on the
+ // timing the output could either be an error signaling cancelation, or the
+ // valid positional return values from the server.
+ //
+ // Calling Finish is mandatory for releasing stream resources, unless the call
+ // has been canceled or any of the other methods return an error. Finish should
+ // be called at most once.
+ Finish() (map[ObjId]GenVector, error)
+}
+
+type implSyncGetDeltasClientCall struct {
+ rpc.ClientCall
+ valRecv LogRec
+ errRecv error
+}
+
+func (c *implSyncGetDeltasClientCall) RecvStream() interface {
+ Advance() bool
+ Value() LogRec
+ Err() error
+} {
+ return implSyncGetDeltasClientCallRecv{c}
+}
+
+type implSyncGetDeltasClientCallRecv struct {
+ c *implSyncGetDeltasClientCall
+}
+
+func (c implSyncGetDeltasClientCallRecv) Advance() bool {
+ c.c.valRecv = LogRec{}
+ c.c.errRecv = c.c.Recv(&c.c.valRecv)
+ return c.c.errRecv == nil
+}
+func (c implSyncGetDeltasClientCallRecv) Value() LogRec {
+ return c.c.valRecv
+}
+func (c implSyncGetDeltasClientCallRecv) Err() error {
+ if c.c.errRecv == io.EOF {
+ return nil
+ }
+ return c.c.errRecv
+}
+func (c *implSyncGetDeltasClientCall) Finish() (o0 map[ObjId]GenVector, err error) {
+ err = c.ClientCall.Finish(&o0)
+ return
+}
+
+// SyncGetObjectHistoryClientStream is the client stream for Sync.GetObjectHistory.
+type SyncGetObjectHistoryClientStream interface {
+ // RecvStream returns the receiver side of the Sync.GetObjectHistory client stream.
+ RecvStream() interface {
+ // Advance stages an item so that it may be retrieved via Value. Returns
+ // true iff there is an item to retrieve. Advance must be called before
+ // Value is called. May block if an item is not available.
+ Advance() bool
+ // Value returns the item that was staged by Advance. May panic if Advance
+ // returned false or was not called. Never blocks.
+ Value() LogRec
+ // Err returns any error encountered by Advance. Never blocks.
+ Err() error
+ }
+}
+
+// SyncGetObjectHistoryClientCall represents the call returned from Sync.GetObjectHistory.
+type SyncGetObjectHistoryClientCall interface {
+ SyncGetObjectHistoryClientStream
+ // Finish blocks until the server is done, and returns the positional return
+ // values for call.
+ //
+ // Finish returns immediately if the call has been canceled; depending on the
+ // timing the output could either be an error signaling cancelation, or the
+ // valid positional return values from the server.
+ //
+ // Calling Finish is mandatory for releasing stream resources, unless the call
+ // has been canceled or any of the other methods return an error. Finish should
+ // be called at most once.
+ Finish() (Version, error)
+}
+
+type implSyncGetObjectHistoryClientCall struct {
+ rpc.ClientCall
+ valRecv LogRec
+ errRecv error
+}
+
+func (c *implSyncGetObjectHistoryClientCall) RecvStream() interface {
+ Advance() bool
+ Value() LogRec
+ Err() error
+} {
+ return implSyncGetObjectHistoryClientCallRecv{c}
+}
+
+type implSyncGetObjectHistoryClientCallRecv struct {
+ c *implSyncGetObjectHistoryClientCall
+}
+
+func (c implSyncGetObjectHistoryClientCallRecv) Advance() bool {
+ c.c.valRecv = LogRec{}
+ c.c.errRecv = c.c.Recv(&c.c.valRecv)
+ return c.c.errRecv == nil
+}
+func (c implSyncGetObjectHistoryClientCallRecv) Value() LogRec {
+ return c.c.valRecv
+}
+func (c implSyncGetObjectHistoryClientCallRecv) Err() error {
+ if c.c.errRecv == io.EOF {
+ return nil
+ }
+ return c.c.errRecv
+}
+func (c *implSyncGetObjectHistoryClientCall) Finish() (o0 Version, err error) {
+ err = c.ClientCall.Finish(&o0)
+ return
+}
+
+// SyncGetDeviceStatsClientStream is the client stream for Sync.GetDeviceStats.
+type SyncGetDeviceStatsClientStream interface {
+ // RecvStream returns the receiver side of the Sync.GetDeviceStats client stream.
+ RecvStream() interface {
+ // Advance stages an item so that it may be retrieved via Value. Returns
+ // true iff there is an item to retrieve. Advance must be called before
+ // Value is called. May block if an item is not available.
+ Advance() bool
+ // Value returns the item that was staged by Advance. May panic if Advance
+ // returned false or was not called. Never blocks.
+ Value() DeviceStats
+ // Err returns any error encountered by Advance. Never blocks.
+ Err() error
+ }
+}
+
+// SyncGetDeviceStatsClientCall represents the call returned from Sync.GetDeviceStats.
+type SyncGetDeviceStatsClientCall interface {
+ SyncGetDeviceStatsClientStream
+ // Finish blocks until the server is done, and returns the positional return
+ // values for call.
+ //
+ // Finish returns immediately if the call has been canceled; depending on the
+ // timing the output could either be an error signaling cancelation, or the
+ // valid positional return values from the server.
+ //
+ // Calling Finish is mandatory for releasing stream resources, unless the call
+ // has been canceled or any of the other methods return an error. Finish should
+ // be called at most once.
+ Finish() error
+}
+
+type implSyncGetDeviceStatsClientCall struct {
+ rpc.ClientCall
+ valRecv DeviceStats
+ errRecv error
+}
+
+func (c *implSyncGetDeviceStatsClientCall) RecvStream() interface {
+ Advance() bool
+ Value() DeviceStats
+ Err() error
+} {
+ return implSyncGetDeviceStatsClientCallRecv{c}
+}
+
+type implSyncGetDeviceStatsClientCallRecv struct {
+ c *implSyncGetDeviceStatsClientCall
+}
+
+func (c implSyncGetDeviceStatsClientCallRecv) Advance() bool {
+ c.c.valRecv = DeviceStats{}
+ c.c.errRecv = c.c.Recv(&c.c.valRecv)
+ return c.c.errRecv == nil
+}
+func (c implSyncGetDeviceStatsClientCallRecv) Value() DeviceStats {
+ return c.c.valRecv
+}
+func (c implSyncGetDeviceStatsClientCallRecv) Err() error {
+ if c.c.errRecv == io.EOF {
+ return nil
+ }
+ return c.c.errRecv
+}
+func (c *implSyncGetDeviceStatsClientCall) Finish() (err error) {
+ err = c.ClientCall.Finish()
+ return
+}
+
+// SyncGetSyncGroupMembersClientStream is the client stream for Sync.GetSyncGroupMembers.
+type SyncGetSyncGroupMembersClientStream interface {
+ // RecvStream returns the receiver side of the Sync.GetSyncGroupMembers client stream.
+ RecvStream() interface {
+ // Advance stages an item so that it may be retrieved via Value. Returns
+ // true iff there is an item to retrieve. Advance must be called before
+ // Value is called. May block if an item is not available.
+ Advance() bool
+ // Value returns the item that was staged by Advance. May panic if Advance
+ // returned false or was not called. Never blocks.
+ Value() SyncGroupMember
+ // Err returns any error encountered by Advance. Never blocks.
+ Err() error
+ }
+}
+
+// SyncGetSyncGroupMembersClientCall represents the call returned from Sync.GetSyncGroupMembers.
+type SyncGetSyncGroupMembersClientCall interface {
+ SyncGetSyncGroupMembersClientStream
+ // Finish blocks until the server is done, and returns the positional return
+ // values for call.
+ //
+ // Finish returns immediately if the call has been canceled; depending on the
+ // timing the output could either be an error signaling cancelation, or the
+ // valid positional return values from the server.
+ //
+ // Calling Finish is mandatory for releasing stream resources, unless the call
+ // has been canceled or any of the other methods return an error. Finish should
+ // be called at most once.
+ Finish() error
+}
+
+type implSyncGetSyncGroupMembersClientCall struct {
+ rpc.ClientCall
+ valRecv SyncGroupMember
+ errRecv error
+}
+
+func (c *implSyncGetSyncGroupMembersClientCall) RecvStream() interface {
+ Advance() bool
+ Value() SyncGroupMember
+ Err() error
+} {
+ return implSyncGetSyncGroupMembersClientCallRecv{c}
+}
+
+type implSyncGetSyncGroupMembersClientCallRecv struct {
+ c *implSyncGetSyncGroupMembersClientCall
+}
+
+func (c implSyncGetSyncGroupMembersClientCallRecv) Advance() bool {
+ c.c.valRecv = SyncGroupMember{}
+ c.c.errRecv = c.c.Recv(&c.c.valRecv)
+ return c.c.errRecv == nil
+}
+func (c implSyncGetSyncGroupMembersClientCallRecv) Value() SyncGroupMember {
+ return c.c.valRecv
+}
+func (c implSyncGetSyncGroupMembersClientCallRecv) Err() error {
+ if c.c.errRecv == io.EOF {
+ return nil
+ }
+ return c.c.errRecv
+}
+func (c *implSyncGetSyncGroupMembersClientCall) Finish() (err error) {
+ err = c.ClientCall.Finish()
+ return
+}
+
+// SyncGetSyncGroupStatsClientStream is the client stream for Sync.GetSyncGroupStats.
+type SyncGetSyncGroupStatsClientStream interface {
+ // RecvStream returns the receiver side of the Sync.GetSyncGroupStats client stream.
+ RecvStream() interface {
+ // Advance stages an item so that it may be retrieved via Value. Returns
+ // true iff there is an item to retrieve. Advance must be called before
+ // Value is called. May block if an item is not available.
+ Advance() bool
+ // Value returns the item that was staged by Advance. May panic if Advance
+ // returned false or was not called. Never blocks.
+ Value() SyncGroupStats
+ // Err returns any error encountered by Advance. Never blocks.
+ Err() error
+ }
+}
+
+// SyncGetSyncGroupStatsClientCall represents the call returned from Sync.GetSyncGroupStats.
+type SyncGetSyncGroupStatsClientCall interface {
+ SyncGetSyncGroupStatsClientStream
+ // Finish blocks until the server is done, and returns the positional return
+ // values for call.
+ //
+ // Finish returns immediately if the call has been canceled; depending on the
+ // timing the output could either be an error signaling cancelation, or the
+ // valid positional return values from the server.
+ //
+ // Calling Finish is mandatory for releasing stream resources, unless the call
+ // has been canceled or any of the other methods return an error. Finish should
+ // be called at most once.
+ Finish() error
+}
+
+type implSyncGetSyncGroupStatsClientCall struct {
+ rpc.ClientCall
+ valRecv SyncGroupStats
+ errRecv error
+}
+
+func (c *implSyncGetSyncGroupStatsClientCall) RecvStream() interface {
+ Advance() bool
+ Value() SyncGroupStats
+ Err() error
+} {
+ return implSyncGetSyncGroupStatsClientCallRecv{c}
+}
+
+type implSyncGetSyncGroupStatsClientCallRecv struct {
+ c *implSyncGetSyncGroupStatsClientCall
+}
+
+func (c implSyncGetSyncGroupStatsClientCallRecv) Advance() bool {
+ c.c.valRecv = SyncGroupStats{}
+ c.c.errRecv = c.c.Recv(&c.c.valRecv)
+ return c.c.errRecv == nil
+}
+func (c implSyncGetSyncGroupStatsClientCallRecv) Value() SyncGroupStats {
+ return c.c.valRecv
+}
+func (c implSyncGetSyncGroupStatsClientCallRecv) Err() error {
+ if c.c.errRecv == io.EOF {
+ return nil
+ }
+ return c.c.errRecv
+}
+func (c *implSyncGetSyncGroupStatsClientCall) Finish() (err error) {
+ err = c.ClientCall.Finish()
+ return
+}
+
+// SyncServerMethods is the interface a server writer
+// implements for Sync.
+//
+// Sync allows a device to GetDeltas from another device.
+type SyncServerMethods interface {
+ // GetDeltas returns a device's current generation vector and all
+ // the missing log records when compared to the incoming generation vector.
+ GetDeltas(call SyncGetDeltasServerCall, in map[ObjId]GenVector, sgs map[ObjId]GroupIdSet, clientId DeviceId) (map[ObjId]GenVector, error)
+ // GetObjectHistory returns the mutation history of a store object.
+ GetObjectHistory(call SyncGetObjectHistoryServerCall, oid ObjId) (Version, error)
+ // GetDeviceStats returns information on devices participating in
+ // peer-to-peer synchronization.
+ GetDeviceStats(SyncGetDeviceStatsServerCall) error
+ // GetSyncGroupMembers returns information on SyncGroup members.
+ // If SyncGroup names are specified, only members of these SyncGroups
+ // are returned. Otherwise, if the slice of names is nil or empty,
+ // members of all SyncGroups are returned.
+ GetSyncGroupMembers(call SyncGetSyncGroupMembersServerCall, sgNames []string) error
+ // GetSyncGroupStats returns high-level information on all SyncGroups.
+ GetSyncGroupStats(SyncGetSyncGroupStatsServerCall) error
+ // Dump writes to the Sync log internal information used for debugging.
+ Dump(rpc.ServerCall) error
+}
+
+// SyncServerStubMethods is the server interface containing
+// Sync methods, as expected by rpc.Server.
+// The only difference between this interface and SyncServerMethods
+// is the streaming methods.
+type SyncServerStubMethods interface {
+ // GetDeltas returns a device's current generation vector and all
+ // the missing log records when compared to the incoming generation vector.
+ GetDeltas(call *SyncGetDeltasServerCallStub, in map[ObjId]GenVector, sgs map[ObjId]GroupIdSet, clientId DeviceId) (map[ObjId]GenVector, error)
+ // GetObjectHistory returns the mutation history of a store object.
+ GetObjectHistory(call *SyncGetObjectHistoryServerCallStub, oid ObjId) (Version, error)
+ // GetDeviceStats returns information on devices participating in
+ // peer-to-peer synchronization.
+ GetDeviceStats(*SyncGetDeviceStatsServerCallStub) error
+ // GetSyncGroupMembers returns information on SyncGroup members.
+ // If SyncGroup names are specified, only members of these SyncGroups
+ // are returned. Otherwise, if the slice of names is nil or empty,
+ // members of all SyncGroups are returned.
+ GetSyncGroupMembers(call *SyncGetSyncGroupMembersServerCallStub, sgNames []string) error
+ // GetSyncGroupStats returns high-level information on all SyncGroups.
+ GetSyncGroupStats(*SyncGetSyncGroupStatsServerCallStub) error
+ // Dump writes to the Sync log internal information used for debugging.
+ Dump(rpc.ServerCall) error
+}
+
+// SyncServerStub adds universal methods to SyncServerStubMethods.
+type SyncServerStub interface {
+ SyncServerStubMethods
+ // Describe the Sync interfaces.
+ Describe__() []rpc.InterfaceDesc
+}
+
+// SyncServer returns a server stub for Sync.
+// It converts an implementation of SyncServerMethods into
+// an object that may be used by rpc.Server.
+func SyncServer(impl SyncServerMethods) SyncServerStub {
+ stub := implSyncServerStub{
+ impl: impl,
+ }
+ // Initialize GlobState; always check the stub itself first, to handle the
+ // case where the user has the Glob method defined in their VDL source.
+ if gs := rpc.NewGlobState(stub); gs != nil {
+ stub.gs = gs
+ } else if gs := rpc.NewGlobState(impl); gs != nil {
+ stub.gs = gs
+ }
+ return stub
+}
+
+type implSyncServerStub struct {
+ impl SyncServerMethods
+ gs *rpc.GlobState
+}
+
+func (s implSyncServerStub) GetDeltas(call *SyncGetDeltasServerCallStub, i0 map[ObjId]GenVector, i1 map[ObjId]GroupIdSet, i2 DeviceId) (map[ObjId]GenVector, error) {
+ return s.impl.GetDeltas(call, i0, i1, i2)
+}
+
+func (s implSyncServerStub) GetObjectHistory(call *SyncGetObjectHistoryServerCallStub, i0 ObjId) (Version, error) {
+ return s.impl.GetObjectHistory(call, i0)
+}
+
+func (s implSyncServerStub) GetDeviceStats(call *SyncGetDeviceStatsServerCallStub) error {
+ return s.impl.GetDeviceStats(call)
+}
+
+func (s implSyncServerStub) GetSyncGroupMembers(call *SyncGetSyncGroupMembersServerCallStub, i0 []string) error {
+ return s.impl.GetSyncGroupMembers(call, i0)
+}
+
+func (s implSyncServerStub) GetSyncGroupStats(call *SyncGetSyncGroupStatsServerCallStub) error {
+ return s.impl.GetSyncGroupStats(call)
+}
+
+func (s implSyncServerStub) Dump(call rpc.ServerCall) error {
+ return s.impl.Dump(call)
+}
+
+func (s implSyncServerStub) Globber() *rpc.GlobState {
+ return s.gs
+}
+
+func (s implSyncServerStub) Describe__() []rpc.InterfaceDesc {
+ return []rpc.InterfaceDesc{SyncDesc}
+}
+
+// SyncDesc describes the Sync interface.
+var SyncDesc rpc.InterfaceDesc = descSync
+
+// descSync hides the desc to keep godoc clean.
+var descSync = rpc.InterfaceDesc{
+ Name: "Sync",
+ PkgPath: "v.io/syncbase/x/ref/services/syncbase/sync",
+ Doc: "// Sync allows a device to GetDeltas from another device.",
+ Methods: []rpc.MethodDesc{
+ {
+ Name: "GetDeltas",
+ Doc: "// GetDeltas returns a device's current generation vector and all\n// the missing log records when compared to the incoming generation vector.",
+ InArgs: []rpc.ArgDesc{
+ {"in", ``}, // map[ObjId]GenVector
+ {"sgs", ``}, // map[ObjId]GroupIdSet
+ {"clientId", ``}, // DeviceId
+ },
+ OutArgs: []rpc.ArgDesc{
+ {"", ``}, // map[ObjId]GenVector
+ },
+ Tags: []*vdl.Value{vdl.ValueOf(access.Tag("Write"))},
+ },
+ {
+ Name: "GetObjectHistory",
+ Doc: "// GetObjectHistory returns the mutation history of a store object.",
+ InArgs: []rpc.ArgDesc{
+ {"oid", ``}, // ObjId
+ },
+ OutArgs: []rpc.ArgDesc{
+ {"", ``}, // Version
+ },
+ },
+ {
+ Name: "GetDeviceStats",
+ Doc: "// GetDeviceStats returns information on devices participating in\n// peer-to-peer synchronization.",
+ Tags: []*vdl.Value{vdl.ValueOf(access.Tag("Admin"))},
+ },
+ {
+ Name: "GetSyncGroupMembers",
+ Doc: "// GetSyncGroupMembers returns information on SyncGroup members.\n// If SyncGroup names are specified, only members of these SyncGroups\n// are returned. Otherwise, if the slice of names is nil or empty,\n// members of all SyncGroups are returned.",
+ InArgs: []rpc.ArgDesc{
+ {"sgNames", ``}, // []string
+ },
+ Tags: []*vdl.Value{vdl.ValueOf(access.Tag("Admin"))},
+ },
+ {
+ Name: "GetSyncGroupStats",
+ Doc: "// GetSyncGroupStats returns high-level information on all SyncGroups.",
+ Tags: []*vdl.Value{vdl.ValueOf(access.Tag("Admin"))},
+ },
+ {
+ Name: "Dump",
+ Doc: "// Dump writes to the Sync log internal information used for debugging.",
+ Tags: []*vdl.Value{vdl.ValueOf(access.Tag("Admin"))},
+ },
+ },
+}
+
+// SyncGetDeltasServerStream is the server stream for Sync.GetDeltas.
+type SyncGetDeltasServerStream interface {
+ // SendStream returns the send side of the Sync.GetDeltas server stream.
+ SendStream() interface {
+ // Send places the item onto the output stream. Returns errors encountered
+ // while sending. Blocks if there is no buffer space; will unblock when
+ // buffer space is available.
+ Send(item LogRec) error
+ }
+}
+
+// SyncGetDeltasServerCall represents the context passed to Sync.GetDeltas.
+type SyncGetDeltasServerCall interface {
+ rpc.ServerCall
+ SyncGetDeltasServerStream
+}
+
+// SyncGetDeltasServerCallStub is a wrapper that converts rpc.StreamServerCall into
+// a typesafe stub that implements SyncGetDeltasServerCall.
+type SyncGetDeltasServerCallStub struct {
+ rpc.StreamServerCall
+}
+
+// Init initializes SyncGetDeltasServerCallStub from rpc.StreamServerCall.
+func (s *SyncGetDeltasServerCallStub) Init(call rpc.StreamServerCall) {
+ s.StreamServerCall = call
+}
+
+// SendStream returns the send side of the Sync.GetDeltas server stream.
+func (s *SyncGetDeltasServerCallStub) SendStream() interface {
+ Send(item LogRec) error
+} {
+ return implSyncGetDeltasServerCallSend{s}
+}
+
+type implSyncGetDeltasServerCallSend struct {
+ s *SyncGetDeltasServerCallStub
+}
+
+func (s implSyncGetDeltasServerCallSend) Send(item LogRec) error {
+ return s.s.Send(item)
+}
+
+// SyncGetObjectHistoryServerStream is the server stream for Sync.GetObjectHistory.
+type SyncGetObjectHistoryServerStream interface {
+ // SendStream returns the send side of the Sync.GetObjectHistory server stream.
+ SendStream() interface {
+ // Send places the item onto the output stream. Returns errors encountered
+ // while sending. Blocks if there is no buffer space; will unblock when
+ // buffer space is available.
+ Send(item LogRec) error
+ }
+}
+
+// SyncGetObjectHistoryServerCall represents the context passed to Sync.GetObjectHistory.
+type SyncGetObjectHistoryServerCall interface {
+ rpc.ServerCall
+ SyncGetObjectHistoryServerStream
+}
+
+// SyncGetObjectHistoryServerCallStub is a wrapper that converts rpc.StreamServerCall into
+// a typesafe stub that implements SyncGetObjectHistoryServerCall.
+type SyncGetObjectHistoryServerCallStub struct {
+ rpc.StreamServerCall
+}
+
+// Init initializes SyncGetObjectHistoryServerCallStub from rpc.StreamServerCall.
+func (s *SyncGetObjectHistoryServerCallStub) Init(call rpc.StreamServerCall) {
+ s.StreamServerCall = call
+}
+
+// SendStream returns the send side of the Sync.GetObjectHistory server stream.
+func (s *SyncGetObjectHistoryServerCallStub) SendStream() interface {
+ Send(item LogRec) error
+} {
+ return implSyncGetObjectHistoryServerCallSend{s}
+}
+
+type implSyncGetObjectHistoryServerCallSend struct {
+ s *SyncGetObjectHistoryServerCallStub
+}
+
+func (s implSyncGetObjectHistoryServerCallSend) Send(item LogRec) error {
+ return s.s.Send(item)
+}
+
+// SyncGetDeviceStatsServerStream is the server stream for Sync.GetDeviceStats.
+type SyncGetDeviceStatsServerStream interface {
+ // SendStream returns the send side of the Sync.GetDeviceStats server stream.
+ SendStream() interface {
+ // Send places the item onto the output stream. Returns errors encountered
+ // while sending. Blocks if there is no buffer space; will unblock when
+ // buffer space is available.
+ Send(item DeviceStats) error
+ }
+}
+
+// SyncGetDeviceStatsServerCall represents the context passed to Sync.GetDeviceStats.
+type SyncGetDeviceStatsServerCall interface {
+ rpc.ServerCall
+ SyncGetDeviceStatsServerStream
+}
+
+// SyncGetDeviceStatsServerCallStub is a wrapper that converts rpc.StreamServerCall into
+// a typesafe stub that implements SyncGetDeviceStatsServerCall.
+type SyncGetDeviceStatsServerCallStub struct {
+ rpc.StreamServerCall
+}
+
+// Init initializes SyncGetDeviceStatsServerCallStub from rpc.StreamServerCall.
+func (s *SyncGetDeviceStatsServerCallStub) Init(call rpc.StreamServerCall) {
+ s.StreamServerCall = call
+}
+
+// SendStream returns the send side of the Sync.GetDeviceStats server stream.
+func (s *SyncGetDeviceStatsServerCallStub) SendStream() interface {
+ Send(item DeviceStats) error
+} {
+ return implSyncGetDeviceStatsServerCallSend{s}
+}
+
+type implSyncGetDeviceStatsServerCallSend struct {
+ s *SyncGetDeviceStatsServerCallStub
+}
+
+func (s implSyncGetDeviceStatsServerCallSend) Send(item DeviceStats) error {
+ return s.s.Send(item)
+}
+
+// SyncGetSyncGroupMembersServerStream is the server stream for Sync.GetSyncGroupMembers.
+type SyncGetSyncGroupMembersServerStream interface {
+ // SendStream returns the send side of the Sync.GetSyncGroupMembers server stream.
+ SendStream() interface {
+ // Send places the item onto the output stream. Returns errors encountered
+ // while sending. Blocks if there is no buffer space; will unblock when
+ // buffer space is available.
+ Send(item SyncGroupMember) error
+ }
+}
+
+// SyncGetSyncGroupMembersServerCall represents the context passed to Sync.GetSyncGroupMembers.
+type SyncGetSyncGroupMembersServerCall interface {
+ rpc.ServerCall
+ SyncGetSyncGroupMembersServerStream
+}
+
+// SyncGetSyncGroupMembersServerCallStub is a wrapper that converts rpc.StreamServerCall into
+// a typesafe stub that implements SyncGetSyncGroupMembersServerCall.
+type SyncGetSyncGroupMembersServerCallStub struct {
+ rpc.StreamServerCall
+}
+
+// Init initializes SyncGetSyncGroupMembersServerCallStub from rpc.StreamServerCall.
+func (s *SyncGetSyncGroupMembersServerCallStub) Init(call rpc.StreamServerCall) {
+ s.StreamServerCall = call
+}
+
+// SendStream returns the send side of the Sync.GetSyncGroupMembers server stream.
+func (s *SyncGetSyncGroupMembersServerCallStub) SendStream() interface {
+ Send(item SyncGroupMember) error
+} {
+ return implSyncGetSyncGroupMembersServerCallSend{s}
+}
+
+type implSyncGetSyncGroupMembersServerCallSend struct {
+ s *SyncGetSyncGroupMembersServerCallStub
+}
+
+func (s implSyncGetSyncGroupMembersServerCallSend) Send(item SyncGroupMember) error {
+ return s.s.Send(item)
+}
+
+// SyncGetSyncGroupStatsServerStream is the server stream for Sync.GetSyncGroupStats.
+type SyncGetSyncGroupStatsServerStream interface {
+ // SendStream returns the send side of the Sync.GetSyncGroupStats server stream.
+ SendStream() interface {
+ // Send places the item onto the output stream. Returns errors encountered
+ // while sending. Blocks if there is no buffer space; will unblock when
+ // buffer space is available.
+ Send(item SyncGroupStats) error
+ }
+}
+
+// SyncGetSyncGroupStatsServerCall represents the context passed to Sync.GetSyncGroupStats.
+type SyncGetSyncGroupStatsServerCall interface {
+ rpc.ServerCall
+ SyncGetSyncGroupStatsServerStream
+}
+
+// SyncGetSyncGroupStatsServerCallStub is a wrapper that converts rpc.StreamServerCall into
+// a typesafe stub that implements SyncGetSyncGroupStatsServerCall.
+type SyncGetSyncGroupStatsServerCallStub struct {
+ rpc.StreamServerCall
+}
+
+// Init initializes SyncGetSyncGroupStatsServerCallStub from rpc.StreamServerCall.
+func (s *SyncGetSyncGroupStatsServerCallStub) Init(call rpc.StreamServerCall) {
+ s.StreamServerCall = call
+}
+
+// SendStream returns the send side of the Sync.GetSyncGroupStats server stream.
+func (s *SyncGetSyncGroupStatsServerCallStub) SendStream() interface {
+ Send(item SyncGroupStats) error
+} {
+ return implSyncGetSyncGroupStatsServerCallSend{s}
+}
+
+type implSyncGetSyncGroupStatsServerCallSend struct {
+ s *SyncGetSyncGroupStatsServerCallStub
+}
+
+func (s implSyncGetSyncGroupStatsServerCallSend) Send(item SyncGroupStats) error {
+ return s.s.Send(item)
+}
diff --git a/services/syncbase/sync/vsyncd.go b/services/syncbase/sync/vsyncd.go
new file mode 100644
index 0000000..ac25c29
--- /dev/null
+++ b/services/syncbase/sync/vsyncd.go
@@ -0,0 +1,647 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Package vsync provides veyron sync daemon utility functions. Sync
+// daemon serves incoming GetDeltas requests and contacts other peers
+// to get deltas from them. When it receives a GetDeltas request, the
+// incoming generation vector is diffed with the local generation
+// vector, and missing generations are sent back. When it receives
+// log records in response to a GetDeltas request, it replays those
+// log records to get in sync with the sender.
+import (
+ "fmt"
+ "math/rand"
+ "strings"
+ "sync"
+ "time"
+
+ "v.io/v23/context"
+ "v.io/v23/naming"
+ "v.io/v23/rpc"
+ "v.io/v23/security"
+ "v.io/x/lib/vlog"
+ "v.io/x/ref/lib/stats"
+)
+
+var (
+ rng *rand.Rand
+)
+
+// Names of Sync stats entries.
+const (
+ statsKvdbPath = "vsync/kvdb-path"
+ statsDevId = "vsync/dev-id"
+ statsNumDagObj = "vsync/num-dag-objects"
+ statsNumDagNode = "vsync/num-dag-nodes"
+ statsNumDagTx = "vsync/num-dag-tx"
+ statsNumDagPrivNode = "vsync/num-dag-privnodes"
+ statsNumSyncGroup = "vsync/num-syncgroups"
+ statsNumMember = "vsync/num-members"
+)
+
+// syncd contains the metadata for the sync daemon.
+type syncd struct {
+ // Pointers to metadata structures.
+ log *iLog
+ devtab *devTable
+ dag *dag
+ sgtab *syncGroupTable
+
+ // Filesystem path to store metadata.
+ kvdbPath string
+
+ // Local device id.
+ id DeviceId
+ // Handle to the server to publish names to mount tables during syncgroup create/join.
+ server rpc.Server
+ // Map to track the set of names already published for this sync service.
+ names map[string]struct{}
+ // Relative name of the sync service to be advertised to the syncgroup server.
+ // This is built off the local device id.
+ relName string
+
+ // Authorizer for sync service.
+ auth security.Authorizer
+
+ // RWlock to concurrently access log and device table data structures.
+ lock sync.RWMutex
+
+ // Mutex to serialize syncgroup operations and an going
+ // initiation round. If both "lock" and "sgOp" need to be
+ // acquired, sgOp must be acquired first.
+ sgOp sync.Mutex
+
+ // Channel used by Sync responders to notify the SyncGroup refresher
+ // to fetch from SyncGroupServer updated info for some SyncGroups.
+ // sgRefresh chan *refreshRequest
+
+ // State to coordinate shutting down all spawned goroutines.
+ pending sync.WaitGroup
+ closed chan struct{}
+
+ // Handlers for goroutine procedures.
+ // hdlGC *syncGC
+ hdlWatcher *syncWatcher
+ hdlInitiator *syncInitiator
+
+ // The context used for background activities.
+ ctx *context.T
+}
+
+func (o ObjId) String() string {
+ return string(o)
+}
+
+func (o ObjId) IsValid() bool {
+ return o != NoObjId
+}
+
+func init() {
+ rng = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
+}
+
+func NewVersion() Version {
+ for {
+ if v := Version(rng.Int63()); v != NoVersion {
+ return v
+ }
+ }
+}
+
+// NewSyncd creates a new syncd instance.
+//
+// Syncd concurrency: syncd initializes three goroutines at
+// startup. The "watcher" thread is responsible for watching the store
+// for changes to its objects. The "initiator" thread is responsible
+// for periodically checking the neighborhood and contacting a peer to
+// obtain changes from that peer. The "gc" thread is responsible for
+// periodically checking if any log records and dag state can be
+// pruned. All these 3 threads perform write operations to the data
+// structures, and synchronize by acquiring a write lock on s.lock. In
+// addition, when syncd receives an incoming RPC, it responds to the
+// request by acquiring a read lock on s.lock. Thus, at any instant in
+// time, either one of the watcher, initiator or gc threads is active,
+// or any number of responders can be active, serving incoming
+// requests. Fairness between these threads follows from
+// sync.RWMutex. The spec says that the writers cannot be starved by
+// the readers but it does not guarantee FIFO. We may have to revisit
+// this in the future.
+func NewSyncd(devid string, syncTick time.Duration, server rpc.Server, ctx *context.T) *syncd {
+ return newSyncdCore(devid, syncTick, server, ctx)
+}
+
+// newSyncdCore is the internal function that creates the Syncd
+// structure and initilizes its thread (goroutines). It takes a
+// Veyron Store parameter to separate the core of Syncd setup from the
+// external dependency on Veyron Store.
+func newSyncdCore(devid string, syncTick time.Duration,
+ server rpc.Server, ctx *context.T) *syncd {
+ // TODO(kash): Get this to compile
+ return nil
+ // s := &syncd{ctx: ctx}
+
+ // storePath := "/tmp"
+
+ // // If no authorizer is specified, allow access to all Principals.
+ // if s.auth = vflag.NewAuthorizerOrDie(); s.auth == nil {
+ // s.auth = allowEveryone{}
+ // }
+
+ // // Bootstrap my own DeviceId.
+ // s.id = DeviceId(devid)
+
+ // // Cache the server to publish names if required.
+ // s.server = server
+ // s.relName = devIDToName(s.id)
+ // s.names = make(map[string]struct{})
+
+ // var err error
+ // // Log init.
+ // if s.log, err = openILog(storePath+"/vsync_ilog.db", s); err != nil {
+ // vlog.Fatalf("newSyncd: openILog failed: err %v", err)
+ // }
+
+ // // DevTable init.
+ // if s.devtab, err = openDevTable(storePath+"/vsync_dtab.db", s); err != nil {
+ // vlog.Fatalf("newSyncd: openDevTable failed: err %v", err)
+ // }
+
+ // // Dag init.
+ // if s.dag, err = openDAG(storePath + "/vsync_dag.db"); err != nil {
+ // vlog.Fatalf("newSyncd: OpenDag failed: err %v", err)
+ // }
+
+ // // SyncGroupTable init.
+ // if s.sgtab, err = openSyncGroupTable(storePath + "/vsync_sgtab.db"); err != nil {
+ // vlog.Fatalf("newSyncd: openSyncGroupTable failed: err %v", err)
+ // }
+
+ // // Channel for responders to notify the SyncGroup refresher.
+ // // Give the channel some buffering to ease a burst of responders
+ // // while the refresher is still finishing a previous request.
+ // // s.sgRefresh = make(chan *refreshRequest, 4096)
+
+ // // Channel to propagate close event to all threads.
+ // s.closed = make(chan struct{})
+
+ // s.pending.Add(2)
+
+ // // Get deltas every peerSyncInterval.
+ // s.hdlInitiator = newInitiator(s, syncTick)
+ // go s.hdlInitiator.contactPeers()
+
+ // // // Garbage collect every garbageCollectInterval.
+ // // s.hdlGC = newGC(s)
+ // // go s.hdlGC.garbageCollect()
+
+ // // Start a watcher thread that will get updates from local store.
+ // s.hdlWatcher = newWatcher(s)
+ // go s.hdlWatcher.watchStore()
+
+ // // Start a thread to refresh SyncGroup info from SyncGroupServer.
+ // // go s.syncGroupRefresher()
+
+ // // Server stats.
+ // stats.NewString(statsKvdbPath).Set(s.kvdbPath)
+ // stats.NewString(statsDevId).Set(string(s.id))
+
+ // return s
+}
+
+// devIDToName converts a device ID to its relative name.
+func devIDToName(id DeviceId) string {
+ return naming.Join([]string{"global", "vsync", string(id)}...)
+}
+
+// nameToDevId extracts the DeviceId from a name.
+func (s *syncd) nameToDevId(name string) DeviceId {
+ parts := strings.Split(name, "/")
+ return DeviceId(parts[len(parts)-1])
+}
+
+// Close cleans up syncd state.
+func (s *syncd) Close() {
+ close(s.closed)
+ s.pending.Wait()
+
+ stats.Delete(statsKvdbPath)
+ stats.Delete(statsDevId)
+
+ // TODO(hpucha): close without flushing.
+}
+
+// isSyncClosing returns true if Close() was called i.e. the "closed" channel is closed.
+func (s *syncd) isSyncClosing() bool {
+ select {
+ case <-s.closed:
+ return true
+ default:
+ return false
+ }
+}
+
+// GetDeltas responds to the incoming request from a client by sending missing generations to the client.
+func (s *syncd) GetDeltas(call SyncGetDeltasServerCall, in map[ObjId]GenVector,
+ syncGroups map[ObjId]GroupIdSet, clientID DeviceId) (map[ObjId]GenVector, error) {
+
+ vlog.VI(1).Infof("GetDeltas:: Received vector %v from client %s", in, clientID)
+
+ // Handle misconfiguration: the client cannot have the same ID as this node.
+ if clientID == s.id {
+ vlog.VI(1).Infof("GetDeltas:: impostor alert: client ID %s is the same as mine %s", clientID, s.id)
+ return nil, fmt.Errorf("impostor: cannot be %s, for this node is %s", clientID, s.id)
+ }
+
+ out := make(map[ObjId]GenVector)
+ for sr, gvIn := range in {
+ gvOut, gens, gensInfo, err := s.prepareGensToReply(call, sr, syncGroups[sr], gvIn)
+ if err != nil {
+ vlog.Errorf("GetDeltas:: prepareGensToReply for sr %s failed with err %v",
+ sr.String(), err)
+ continue
+ }
+
+ for pos, v := range gens {
+ gen := gensInfo[pos]
+ var count uint64
+ for i := SeqNum(0); i <= gen.MaxSeqNum; i++ {
+ count++
+ rec, err := s.getLogRec(v.devID, v.srID, v.genID, i)
+ if err != nil {
+ vlog.Fatalf("GetDeltas:: Couldn't get log record %s %s %d %d, err %v",
+ v.devID, v.srID.String(), v.genID, i, err)
+ }
+ vlog.VI(1).Infof("Sending log record %v", rec)
+ if err := call.SendStream().Send(*rec); err != nil {
+ vlog.Errorf("GetDeltas:: Couldn't send stream err: %v", err)
+ return nil, err
+ }
+ }
+ if count != gen.Count {
+ vlog.Fatalf("GetDeltas:: GenMetadata has incorrect log records for generation %s %s %d %v",
+ v.devID, v.srID.String(), v.genID, gen)
+ }
+ }
+ out[sr] = gvOut
+ }
+
+ // Notify the SyncGroup Refresher about the SyncGroup IDs associated
+ // with the SyncRoots in the incoming request from the client.
+ // Note: the initial slice capacity is only a lower-bound.
+ // Note: by using the SyncRoots from the incoming "in" map and all
+ // all SyncGroup IDs for each SyncRoot, the refresh is eager because
+ // it does not filter out SyncRoots and SyncGroup IDs by the Permissions.
+ // sgIDs := make([]syncgroup.Id, 0, len(in))
+ // for sr := range in {
+ // for _, id := range syncGroups[sr] {
+ // sgIDs = append(sgIDs, id)
+ // }
+ // }
+ // s.notifySyncGroupRefresher(devIDToName(clientID), sgIDs)
+
+ return out, nil
+}
+
+// // updateDeviceInfo updates the remote device's information based on
+// // the incoming GetDeltas request.
+// func (s *syncd) updateDeviceInfo(ClientID DeviceId, In GenVector) error {
+// s.lock.Lock()
+// defer s.lock.Unlock()
+
+// // Note that the incoming client generation vector cannot be
+// // used for garbage collection. We can only garbage collect
+// // based on the generations we receive from other
+// // devices. Receiving a set of generations assures that all
+// // updates branching from those generations are also received
+// // and hence generations present on all devices can be
+// // GC'ed. This function sends generations to other devices and
+// // hence does not use the generation vector for GC.
+// //
+// // TODO(hpucha): Cache the client's incoming generation vector
+// // to assist in tracking missing generations and hence next
+// // peer to contact.
+// if !s.devtab.hasDevInfo(ClientID) {
+// if err := s.devtab.addDevice(ClientID); err != nil {
+// return err
+// }
+// }
+// return nil
+// }
+
+// prepareGensToReply verifies if the requestor has access to the
+// requested syncroot. If allowed, it processes the incoming
+// generation vector and returns the metadata of all the missing
+// generations between the incoming and the local generation vector
+// for that syncroot.
+func (s *syncd) prepareGensToReply(call rpc.ServerCall, srid ObjId, sgVec GroupIdSet, In GenVector) (GenVector, []*genOrder, []*genMetadata, error) {
+ s.lock.RLock()
+ defer s.lock.RUnlock()
+
+ // Respond to the caller if any of the syncgroups
+ // corresponding to the syncroot allow access.
+ var allowedErr error
+ // for _, sg := range sgVec {
+ // // Check permissions for the syncgroup.
+ // sgData, err := s.sgtab.getSyncGroupByID(sg)
+ // if err != nil {
+ // return GenVector{}, nil, nil, err
+ // }
+ // if vlog.V(2) {
+ // b, _ := ctx.RemoteBlessings().ForContext(ctx)
+ // vlog.Infof("prepareGensToReply:: matching acl %v, remote names %v", sgData.SrvInfo.Config.Permissions, b)
+ // }
+ // // TODO(ashankar): Consider changing TaggedPermissionsAuthorizer so that it doesn't return an error -
+ // // instead of an error, it just locks down access completely. The error condition is really
+ // // really rare and doing so will make for shorter code?
+ // auth, err := access.TaggedPermissionsAuthorizer(sgData.SrvInfo.Config.Permissions, access.TypicalTagType())
+ // if err != nil {
+ // vlog.Errorf("unable to create authorizer: %v", err)
+ // allowedErr = fmt.Errorf("server error: authorization policy misconfigured")
+ // } else if allowedErr = auth.Authorize(ctx); allowedErr == nil {
+ // break
+ // }
+ // }
+ if allowedErr != nil {
+ return GenVector{}, nil, nil, allowedErr
+ }
+
+ // // TODO(hpucha): Need only for GC.
+ // if err := s.updateDeviceInfo(ClientID, gvIn); err != nil {
+ // vlog.Fatalf("GetDeltas:: updateDeviceInfo failed with err %v", err)
+ // }
+
+ // Get local generation vector.
+ out, err := s.devtab.getGenVec(s.id, srid)
+ if err != nil {
+ return GenVector{}, nil, nil, err
+ }
+
+ // Diff the two generation vectors.
+ gens, err := s.devtab.diffGenVectors(out, In, srid)
+ if err != nil {
+ return GenVector{}, nil, nil, err
+ }
+
+ // Get the metadata for all the generations in the reply.
+ gensInfo := make([]*genMetadata, len(gens))
+ for pos, v := range gens {
+ gen, err := s.log.getGenMetadata(v.devID, v.srID, v.genID)
+ if err != nil || gen.Count <= 0 {
+ return GenVector{}, nil, nil, err
+ }
+ gensInfo[pos] = gen
+ }
+
+ return out, gens, gensInfo, nil
+}
+
+// getLogRec gets the log record for a given generation and lsn.
+func (s *syncd) getLogRec(dev DeviceId, srid ObjId, gen GenId, lsn SeqNum) (*LogRec, error) {
+ s.lock.RLock()
+ defer s.lock.RUnlock()
+ return s.log.getLogRec(dev, srid, gen, lsn)
+}
+
+// // GetSyncGroupStats gives the client high-level information on all SyncGroups.
+// // The streamed information is returned sorted by SyncGroup name.
+// func (s *syncd) GetSyncGroupStats(ctx SyncGetSyncGroupStatsContext) error {
+// // Get a snapshot of the SyncGroup names.
+// // The names are a copy, OK to unlock here.
+// s.lock.RLock()
+// names, err := s.sgtab.getAllSyncGroupNames()
+// s.lock.RUnlock()
+
+// if err != nil {
+// vlog.Errorf("GetSyncGroupStats:: cannot get SyncGroup names: %v", err)
+// return err
+// }
+// if len(names) == 0 {
+// return nil
+// }
+
+// // Send the SyncGroup stats in the stream in sorted order.
+// // Skip SyncGroups that got deleted in the meanwhile.
+// sort.Strings(names)
+// for _, name := range names {
+// // The SyncGroup object is a copy, OK to unlock here.
+// s.lock.RLock()
+// sg, err := s.sgtab.getSyncGroupByName(name)
+// s.lock.RUnlock()
+
+// if err != nil {
+// continue // skip deleted SyncGroup
+// }
+
+// stats := SyncGroupStats{
+// Name: name,
+// SGObjId: sg.SrvInfo.SGObjId,
+// Path: sg.LocalPath,
+// RootObjId: ObjId(sg.SrvInfo.RootObjId),
+// NumJoiners: uint32(len(sg.SrvInfo.Joiners)),
+// }
+
+// if err = ctx.SendStream().Send(stats); err != nil {
+// vlog.Errorf("GetSyncGroupStats:: cannot send stream data: %v", err)
+// return err
+// }
+// }
+
+// return nil
+// }
+
+// // GetSyncGroupMembers gives the client information on SyncGroup members.
+// // If SyncGroup names are specified, only their member information is sent.
+// // Otherwise information on members from all SyncGroups is sent.
+// func (s *syncd) GetSyncGroupMembers(ctx SyncGetSyncGroupMembersContext, sgNames []string) error {
+
+// // If SyncGroup names are given, fetch their IDs.
+// // Also get a snapshot of the SyncGroup members.
+// wantedGroupIds := make(map[syncgroup.Id]bool)
+
+// s.lock.RLock()
+// for _, name := range sgNames {
+// sgid, err := s.sgtab.getSyncGroupID(name)
+// if err != nil {
+// s.lock.RUnlock()
+// vlog.Errorf("GetSyncGroupMembers:: invalid SyncGroup %s: %v", name, err)
+// return err
+// }
+// wantedGroupIds[sgid] = true
+// }
+
+// memberMap, err := s.sgtab.getMembers() // The map is a copy, OK to unlock here.
+// s.lock.RUnlock()
+
+// if err != nil {
+// vlog.Errorf("GetSyncGroupMembers:: cannot get members: %v", err)
+// return err
+// }
+// if len(memberMap) == 0 {
+// return nil
+// }
+
+// // Send the member information in the stream in sorted order.
+// // Skip members that left in the meanwhile.
+// // If SyncGroup names are given, only return members in these groups.
+// members := make([]string, 0, len(memberMap))
+// for member := range memberMap {
+// members = append(members, member)
+// }
+// sort.Strings(members)
+
+// for _, member := range members {
+// // The member info points to in-memory data; cannot unlock
+// // while accessing that info.
+// s.lock.RLock()
+// info, err := s.sgtab.getMemberInfo(member)
+// if err != nil {
+// s.lock.RUnlock()
+// continue // skip member no longer there
+// }
+
+// replyData := make([]SyncGroupMember, 0, len(info.gids))
+// for sgid, meta := range info.gids {
+// if len(wantedGroupIds) > 0 && !wantedGroupIds[sgid] {
+// continue // skip unwanted SyncGroups
+// }
+
+// data := SyncGroupMember{
+// Name: member,
+// SGObjId: sgid,
+// Metadata: meta.metaData,
+// }
+// replyData = append(replyData, data)
+// }
+
+// s.lock.RUnlock()
+
+// for _, data := range replyData {
+// if err = ctx.SendStream().Send(data); err != nil {
+// vlog.Errorf("GetSyncGroupMembers:: cannot send stream data: %v", err)
+// return err
+// }
+// }
+// }
+
+// return nil
+// }
+
+// Dump writes to the Sync log internal information used for debugging.
+func (s *syncd) Dump(call rpc.ServerCall) error {
+ s.lock.RLock()
+ defer s.lock.RUnlock()
+
+ vlog.VI(1).Infof("Dump: ==== BEGIN ====")
+ s.devtab.dump()
+ // s.sgtab.dump()
+ s.log.dump()
+ s.dag.dump()
+ vlog.VI(1).Infof("Dump: ==== END ====")
+ return nil
+}
+
+// GetDeviceStats gives the client information on devices that have synchronized
+// with the server.
+func (s *syncd) GetDeviceStats(call SyncGetDeviceStatsServerCall) error {
+ // Get a snapshot of the device IDs.
+ // The device IDs are copies, OK to unlock here.
+ s.lock.RLock()
+ myID := s.id
+ devIDs, err := s.devtab.getDeviceIds()
+ s.lock.RUnlock()
+
+ if err != nil {
+ vlog.Errorf("GetDeviceStats:: cannot get device IDs: %v", err)
+ return err
+ }
+
+ // Send the device stats in the stream.
+ for _, id := range devIDs {
+ // The device info is a copy, OK to unlock here.
+ s.lock.RLock()
+ info, err := s.devtab.getDevInfo(id)
+ s.lock.RUnlock()
+
+ if err != nil {
+ continue // skip unknown devices
+ }
+
+ stats := DeviceStats{
+ DevId: id,
+ LastSync: info.Ts.Unix(),
+ GenVectors: info.Vectors,
+ IsSelf: id == myID,
+ }
+
+ if err = call.SendStream().Send(stats); err != nil {
+ vlog.Errorf("GetDeviceStats:: cannot send stream data: %v", err)
+ return err
+ }
+ }
+
+ return nil
+}
+
+// GetObjectHistory gives the client the mutation history of a store object.
+// It also returns the object version that is at the "head" of the graph (DAG).
+func (s *syncd) GetObjectHistory(call SyncGetObjectHistoryServerCall, oid ObjId) (Version, error) {
+ // Get the object version numbers in reverse chronological order,
+ // starting from the "head" version back through its ancestors.
+ // Also get the DAG parents of each object version.
+ var versions []Version
+ parents := make(map[Version][]Version)
+
+ s.lock.RLock()
+ head, err := s.dag.getHead(oid)
+ if err != nil {
+ // For now ignore objects that do not yet have a "head" version.
+ // They are either not part of any SyncGroup, or this Sync server
+ // has not finished catching up (Sync or Watch operations).
+ vlog.Errorf("GetObjectHistory:: unknown object %v: %v", oid, err)
+ s.lock.RUnlock()
+ return NoVersion, err
+ }
+
+ s.dag.ancestorIter(oid, []Version{head},
+ func(oid ObjId, v Version, node *dagNode) error {
+ versions = append(versions, v)
+ parents[v] = node.Parents
+ return nil
+ })
+ s.lock.RUnlock()
+
+ // Stream the object log records in the given order.
+ for _, v := range versions {
+ // The log record is a copy, OK to unlock here.
+ s.lock.RLock()
+ rec, err := s.hdlInitiator.getLogRec(oid, v)
+ s.lock.RUnlock()
+
+ if err != nil {
+ vlog.Errorf("GetObjectHistory:: cannot get log record for %v:%v: %v", oid, v, err)
+ return NoVersion, err
+ }
+
+ // To give the client a full DAG, send the parents as seen in
+ // the DAG node instead of the ones in the log record.
+ // Note: DAG nodes contain the full sets of parents whereas
+ // log records may have subsets due to the different types of
+ // records: NodeRec-Parents + LinkRec-Parent == DAG-Parents.
+ rec.Parents = parents[v]
+
+ if err = call.SendStream().Send(*rec); err != nil {
+ vlog.Errorf("GetObjectHistory:: cannot send stream data: %v", err)
+ return NoVersion, err
+ }
+ }
+
+ return head, nil
+}
+
+// allowEveryone implements the authorization policy of ... allowing every principal.
+type allowEveryone struct{}
+
+func (allowEveryone) Authorize(security.Call) error { return nil }
diff --git a/services/syncbase/sync/watcher.go b/services/syncbase/sync/watcher.go
new file mode 100644
index 0000000..f9799ae
--- /dev/null
+++ b/services/syncbase/sync/watcher.go
@@ -0,0 +1,257 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vsync
+
+// Veyron Sync hook to the Store Watch API. When applications update objects
+// in the local Veyron Store, Sync learns about these via the Watch API stream
+// of object mutations. In turn, this Sync watcher thread updates the DAG and
+// ILog records to track the object change histories.
+
+import (
+ "time"
+
+ "v.io/v23/context"
+ "v.io/x/lib/vlog"
+)
+
+var (
+ // watchRetryDelay is how long the watcher waits before calling the Watch() RPC again
+ // after the previous call fails.
+ watchRetryDelay = 2 * time.Second
+ // streamRetryDelay is how long the watcher waits before creating a new Watch stream
+ // after the previous stream ends.
+ streamRetryDelay = 1 * time.Second
+)
+
+// syncWatcher contains the metadata for the Sync Watcher thread.
+type syncWatcher struct {
+ // syncd is a pointer to the Syncd instance owning this Watcher.
+ syncd *syncd
+}
+
+// newWatcher creates a new Sync Watcher instance attached to the given Syncd instance.
+func newWatcher(syncd *syncd) *syncWatcher {
+ return &syncWatcher{syncd: syncd}
+}
+
+func (w *syncWatcher) watchStreamCanceler(cancel context.CancelFunc, done <-chan struct{}) {
+ select {
+ case <-w.syncd.closed:
+ vlog.VI(1).Info("watchStreamCanceler: Syncd channel closed, cancel stream and exit")
+ cancel()
+ case <-done:
+ vlog.VI(1).Info("watchStreamCanceler: done, exit")
+ }
+}
+
+// watchStore consumes change records obtained by watching the store
+// for updates and applies them to the sync DBs.
+func (w *syncWatcher) watchStore() {
+ defer w.syncd.pending.Done()
+
+ // // If no Veyron store is configured, there is nothing to watch.
+ // if w.syncd.store == nil {
+ // vlog.VI(1).Info("watchStore: Veyron store not configured; skipping the watcher")
+ // return
+ // }
+
+ // // Get a Watch stream, process it, repeat till end-of-life.
+ // for {
+ // ctx, _ := vtrace.SetNewTrace(w.syncd.ctx)
+ // ctx, cancel := context.WithCancel(ctx)
+ // stream := w.getWatchStream(ctx)
+ // if stream == nil {
+ // return // Syncd is exiting.
+ // }
+
+ // // Spawn a goroutine to detect the Syncd "closed" channel and cancel the RPC stream
+ // // to unblock the watcher. The "done" channel lets the watcher terminate the goroutine.
+ // go w.watchStreamCanceler(cancel, ctx.Done())
+
+ // // Process the stream of Watch updates until it closes (similar to "tail -f").
+ // w.processWatchStream(stream)
+
+ // if w.syncd.isSyncClosing() {
+ // return // Syncd is exiting.
+ // }
+
+ // stream.Finish()
+ // cancel()
+
+ // // TODO(rdaoud): we need a rate-limiter here in case the stream closes too quickly.
+ // // If the stream stays up long enough, no need to sleep before getting a new one.
+ // time.Sleep(streamRetryDelay)
+ // }
+}
+
+// // getWatchStream() returns a Watch API stream and handles retries if the Watch() call fails.
+// // If the stream is nil, it means Syncd is exiting cleanly and the caller should terminate.
+// func (w *syncWatcher) getWatchStream(ctx *context.T) watch.GlobWatcherWatchGlobCall {
+// for {
+// w.syncd.lock.RLock()
+// resmark := w.syncd.devtab.head.Resmark
+// w.syncd.lock.RUnlock()
+
+// req := raw.Request{}
+// if resmark != nil {
+// req.ResumeMarker = resmark
+// }
+
+// stream, err := w.syncd.store.Watch(ctx, req)
+// if err == nil {
+// return stream
+// }
+
+// if w.syncd.isSyncClosing() {
+// vlog.VI(1).Info("getWatchStream: exiting, Syncd closed its channel: ", err)
+// return nil
+// }
+
+// vlog.VI(1).Info("getWatchStream: call to Watch() failed, retrying a bit later: ", err)
+// time.Sleep(watchRetryDelay)
+// }
+// }
+
+// // processWatchStream reads the stream of Watch updates and applies the object mutations.
+// // Ideally this call does not return as the stream should be un-ending (like "tail -f").
+// // If the stream is closed, distinguish between the cases of end-of-stream vs Syncd canceling
+// // the stream to trigger a clean exit.
+// func (w *syncWatcher) processWatchStream(call watch.GlobWatcherWatchGlobCall) {
+// var txChanges []*types.Change = nil
+// var syncTime int64
+
+// stream := call.RecvStream()
+// for stream.Advance() {
+// change := stream.Value()
+
+// // Timestamp of the start of a batch of changes arriving at the Sync server.
+// if txChanges == nil {
+// syncTime = time.Now().UnixNano()
+// }
+// txChanges = append(txChanges, &change)
+
+// // Process the transaction when its last change record is received.
+// if !change.Continued {
+// if err := w.processTransaction(txChanges, syncTime); err != nil {
+// // TODO(rdaoud): don't crash, instead add retry policies to attempt some degree of
+// // self-healing from a data corruption where feasible, otherwise quarantine this device
+// // from the cluster and stop Syncd to avoid propagating data corruptions.
+// vlog.Fatal("processWatchStream:", err)
+// }
+// txChanges = nil
+// }
+// }
+
+// err := stream.Err()
+// if err == nil {
+// err = io.EOF
+// }
+
+// if w.syncd.isSyncClosing() {
+// vlog.VI(1).Info("processWatchStream: exiting, Syncd closed its channel: ", err)
+// } else {
+// vlog.VI(1).Info("processWatchStream: RPC stream error, re-issue Watch(): ", err)
+// }
+// }
+
+// // matchSyncRoot returns the SyncRoot (SyncGroup root ObjId) that matches the given object path.
+// // It traverses the object path looking if any of the IDs along the way are SyncRoots.
+// // If no match is found, it returns the invalid ID (zero).
+// // Note: the path IDs given by Watch are ordered: object, parent, grand-parent, etc.., root.
+// // As a result, this function returns the narrowest matching SyncGroup root ObjId.
+// func matchSyncRoot(objPathIDs []ObjId, sgObjIds map[ObjId]struct{}) ObjId {
+// for _, oid := range objPathIDs {
+// if _, ok := sgObjIds[oid]; ok {
+// return oid
+// }
+// }
+// return ObjId{}
+// }
+
+// // processTransaction applies a batch of changes (object mutations) that form a single transaction
+// // received from the Watch API. The function grabs the write-lock to access the Log and DAG DBs.
+// func (w *syncWatcher) processTransaction(txBatch []*types.Change, syncTime int64) error {
+// w.syncd.lock.Lock()
+// defer w.syncd.lock.Unlock()
+
+// count := uint32(len(txBatch))
+// if count == 0 {
+// return nil
+// }
+
+// // Get the current set of SyncGroup Root ObjIds and use it to determine for each object
+// // the SyncGroup it belongs to, if any.
+// sgRootObjIds, err := w.syncd.sgtab.getAllSyncRoots()
+// if err != nil {
+// return err
+// }
+
+// // If the batch has more than one mutation, start a DAG transaction for it.
+// txID := NoTxId
+// if count > 1 {
+// txID = w.syncd.dag.addNodeTxStart(txID)
+// if txID == NoTxId {
+// return fmt.Errorf("failed to generate transaction ID")
+// }
+// }
+
+// vlog.VI(1).Infof("processTransaction: ready to process a batch of %d changes for transaction %v", count, txID)
+
+// for _, ch := range txBatch {
+// rc, ok := ch.Value.(*raw.RawChange)
+// if !ok {
+// return fmt.Errorf("invalid change value, not a raw change: %#v", ch)
+// }
+
+// mu := &rc.Mutation
+// isDeleted := ch.State == types.DoesNotExist
+
+// // Determine the SyncGroup root ObjId that matches this object, if any.
+// rootObjId := matchSyncRoot(rc.PathIDs, sgRootObjIds)
+
+// if rootObjId.IsValid() {
+// // The object matches a SyncGroup: process its watch record to track it in the Sync logs.
+// // All LogValues belonging to the same transaction get the same timestamp.
+// val := &LogValue{
+// Mutation: *mu,
+// SyncTime: syncTime,
+// Delete: isDeleted,
+// TxId: txID,
+// TxCount: count,
+// }
+// vlog.VI(2).Infof("processTransaction: processing record %v, Tx %v", val, txID)
+
+// err = w.syncd.log.processWatchRecord(mu.Id, mu.Version, mu.PriorVersion, val, rootObjId)
+// } else {
+// // The object does not match any SyncGroup: stash it in the "private" table to save its info
+// // in case it becomes shared in the future, at which time it would be added to the Sync logs.
+// if isDeleted {
+// err = w.syncd.dag.delPrivNode(mu.Id)
+// } else {
+// priv := &privNode{Mutation: mu, PathIDs: rc.PathIDs, SyncTime: syncTime, TxId: txID, TxCount: count}
+// err = w.syncd.dag.setPrivNode(mu.Id, priv)
+// }
+// }
+
+// if err != nil {
+// return fmt.Errorf("cannot process mutation: %#v, mu %#v: %s", ch, mu, err)
+// }
+// }
+
+// // End the DAG transaction if any.
+// if txID != NoTxId {
+// // Note: if there are no shared objects in the transaction, the addNodeTxEnd() call
+// // does not persist the Tx state, it only cleans up the in-memory scaffolding.
+// // This is the desired behavior, Sync only needs to track the state of transactions
+// // that have at least one shared object.
+// if err := w.syncd.dag.addNodeTxEnd(txID, count); err != nil {
+// return err
+// }
+// }
+
+// // Update the device table with the new resume marker of the last record.
+// w.syncd.devtab.head.Resmark = txBatch[count-1].ResumeMarker
+// return nil
+// }
diff --git a/services/syncbase/syncbased/main.go b/services/syncbase/syncbased/main.go
new file mode 100644
index 0000000..e4f9c18
--- /dev/null
+++ b/services/syncbase/syncbased/main.go
@@ -0,0 +1,56 @@
+// Copyright 2015 The Vanadium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// syncbased is a syncbase daemon.
+package main
+
+// Example invocation:
+// syncbased --veyron.tcp.address="127.0.0.1:0" --name=syncbased
+
+import (
+ "flag"
+
+ "v.io/v23"
+ "v.io/v23/security/access"
+ "v.io/x/lib/vlog"
+
+ "v.io/syncbase/x/ref/services/syncbase/server"
+ "v.io/syncbase/x/ref/services/syncbase/store/memstore"
+ "v.io/x/ref/lib/signals"
+ _ "v.io/x/ref/profiles/roaming"
+)
+
+// TODO(sadovsky): Perhaps this should be one of the standard Veyron flags.
+var (
+ name = flag.String("name", "", "Name to mount at.")
+)
+
+func main() {
+ ctx, shutdown := v23.Init()
+ defer shutdown()
+
+ s, err := v23.NewServer(ctx)
+ if err != nil {
+ vlog.Fatal("v23.NewServer() failed: ", err)
+ }
+ if _, err := s.Listen(v23.GetListenSpec(ctx)); err != nil {
+ vlog.Fatal("s.Listen() failed: ", err)
+ }
+
+ // TODO(sadovsky): Use a real Permissions.
+ service := server.NewService(memstore.New())
+ if err := service.Create(access.Permissions{}); err != nil {
+ vlog.Fatal("service.Create() failed: ", err)
+ }
+ d := server.NewDispatcher(service)
+
+ // Publish the service in the mount table.
+ if err := s.ServeDispatcher(*name, d); err != nil {
+ vlog.Fatal("s.ServeDispatcher() failed: ", err)
+ }
+ vlog.Info("Mounted at: ", *name)
+
+ // Wait forever.
+ <-signals.ShutdownOnSignals(ctx)
+}
