-Create a file backed persistent SerializerReaderWriter
-Move Persisting BlessingRoots/Stores over to this new implementation.

Change-Id: I13a0c5888013d9d37bded0083df97c96ec5cdb09
diff --git a/security/blessingroots.go b/security/blessingroots.go
index 5f314d8..9883122 100644
--- a/security/blessingroots.go
+++ b/security/blessingroots.go
@@ -11,17 +11,12 @@
 	"veyron.io/veyron/veyron2/security"
 )
 
-const (
-	blessingRootsDataFile = "blessingroots.data"
-	blessingRootsSigFile  = "blessingroots.sig"
-)
-
 // blessingRoots implements security.BlessingRoots.
 type blessingRoots struct {
-	dir    string
-	signer serialization.Signer
-	mu     sync.RWMutex
-	store  map[string][]security.BlessingPattern // GUARDED_BY(mu)
+	persistedData SerializerReaderWriter
+	signer        serialization.Signer
+	mu            sync.RWMutex
+	store         map[string][]security.BlessingPattern // GUARDED_BY(mu)
 }
 
 func storeMapKey(root security.PublicKey) (string, error) {
@@ -93,10 +88,14 @@
 }
 
 func (br *blessingRoots) save() error {
-	if (br.signer == nil) && (br.dir == "") {
+	if (br.signer == nil) && (br.persistedData == nil) {
 		return nil
 	}
-	return encodeAndStore(br.store, br.dir, blessingRootsDataFile, blessingRootsSigFile, br.signer)
+	data, signature, err := br.persistedData.Writers()
+	if err != nil {
+		return err
+	}
+	return encodeAndStore(br.store, data, signature, br.signer)
 }
 
 // newInMemoryBlessingRoots returns an in-memory security.BlessingRoots.
@@ -108,27 +107,26 @@
 	}
 }
 
-// newPersistingBlessingRoots returns a security.BlessingRoots that signs
-// and persists all updates to the provided directory. Signing is carried
-// out using the provided signer.
-//
-// The returned BlessingRoots is initialized from the existing data present
-// in the directory. The data is verified to have been written by a persisting
-// BlessingRoots object constructed from the same signer.
-//
-// Any errors obtained in reading or verifying the data are returned.
-func newPersistingBlessingRoots(directory string, signer serialization.Signer) (security.BlessingRoots, error) {
-	if directory == "" || signer == nil {
-		return nil, errors.New("directory or signer is not specified")
+// newPersistingBlessingRoots returns a security.BlessingRoots for a principal
+// that is initialized with the persisted data. The returned security.BlessingRoots
+// also persists any updates to its state.
+func newPersistingBlessingRoots(persistedData SerializerReaderWriter, signer serialization.Signer) (security.BlessingRoots, error) {
+	if persistedData == nil || signer == nil {
+		return nil, errors.New("persisted data or signer is not specified")
 	}
 	br := &blessingRoots{
-		store:  make(map[string][]security.BlessingPattern),
-		dir:    directory,
-		signer: signer,
+		store:         make(map[string][]security.BlessingPattern),
+		persistedData: persistedData,
+		signer:        signer,
 	}
-
-	if err := decodeFromStorage(&br.store, br.dir, blessingRootsDataFile, blessingRootsSigFile, br.signer.PublicKey()); err != nil {
+	data, signature, err := br.persistedData.Readers()
+	if err != nil {
 		return nil, err
 	}
+	if (data != nil) && (signature != nil) {
+		if err := decodeFromStorage(&br.store, data, signature, br.signer.PublicKey()); err != nil {
+			return nil, err
+		}
+	}
 	return br, nil
 }
diff --git a/security/blessingstore.go b/security/blessingstore.go
index b135580..2a03456 100644
--- a/security/blessingstore.go
+++ b/security/blessingstore.go
@@ -13,11 +13,6 @@
 	"veyron.io/veyron/veyron2/vlog"
 )
 
-const (
-	blessingStoreDataFile = "blessingstore.data"
-	blessingStoreSigFile  = "blessingstore.sig"
-)
-
 var errStoreAddMismatch = errors.New("blessing's public key does not match store's public key")
 
 type persistentState struct {
@@ -33,45 +28,45 @@
 
 // blessingStore implements security.BlessingStore.
 type blessingStore struct {
-	publicKey security.PublicKey
-	dir       string
-	signer    serialization.Signer
-	mu        sync.RWMutex
-	state     persistentState // GUARDED_BY(mu)
+	publicKey     security.PublicKey
+	persistedData SerializerReaderWriter
+	signer        serialization.Signer
+	mu            sync.RWMutex
+	state         persistentState // GUARDED_BY(mu)
 }
 
-func (s *blessingStore) Set(blessings security.Blessings, forPeers security.BlessingPattern) (security.Blessings, error) {
+func (bs *blessingStore) Set(blessings security.Blessings, forPeers security.BlessingPattern) (security.Blessings, error) {
 	if !forPeers.IsValid() {
 		return nil, fmt.Errorf("%q is an invalid BlessingPattern", forPeers)
 	}
-	if blessings != nil && !reflect.DeepEqual(blessings.PublicKey(), s.publicKey) {
+	if blessings != nil && !reflect.DeepEqual(blessings.PublicKey(), bs.publicKey) {
 		return nil, errStoreAddMismatch
 	}
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	old, hadold := s.state.Store[forPeers]
+	bs.mu.Lock()
+	defer bs.mu.Unlock()
+	old, hadold := bs.state.Store[forPeers]
 	if blessings != nil {
-		s.state.Store[forPeers] = blessings
+		bs.state.Store[forPeers] = blessings
 	} else {
-		delete(s.state.Store, forPeers)
+		delete(bs.state.Store, forPeers)
 	}
-	if err := s.save(); err != nil {
+	if err := bs.save(); err != nil {
 		if hadold {
-			s.state.Store[forPeers] = old
+			bs.state.Store[forPeers] = old
 		} else {
-			delete(s.state.Store, forPeers)
+			delete(bs.state.Store, forPeers)
 		}
 		return nil, err
 	}
 	return old, nil
 }
 
-func (s *blessingStore) ForPeer(peerBlessings ...string) security.Blessings {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
+func (bs *blessingStore) ForPeer(peerBlessings ...string) security.Blessings {
+	bs.mu.RLock()
+	defer bs.mu.RUnlock()
 
 	var ret security.Blessings
-	for pattern, blessings := range s.state.Store {
+	for pattern, blessings := range bs.state.Store {
 		if pattern.MatchedBy(peerBlessings...) {
 			if union, err := security.UnionOfBlessings(ret, blessings); err != nil {
 				vlog.Errorf("UnionOfBlessings(%v, %v) failed: %v, dropping the latter from BlessingStore.ForPeers(%v)", ret, blessings, err, peerBlessings)
@@ -83,35 +78,35 @@
 	return ret
 }
 
-func (s *blessingStore) Default() security.Blessings {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	if s.state.Default != nil {
-		return s.state.Default
+func (bs *blessingStore) Default() security.Blessings {
+	bs.mu.RLock()
+	defer bs.mu.RUnlock()
+	if bs.state.Default != nil {
+		return bs.state.Default
 	}
-	return s.ForPeer()
+	return bs.ForPeer()
 }
 
-func (s *blessingStore) SetDefault(blessings security.Blessings) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if !reflect.DeepEqual(blessings.PublicKey(), s.publicKey) {
+func (bs *blessingStore) SetDefault(blessings security.Blessings) error {
+	bs.mu.Lock()
+	defer bs.mu.Unlock()
+	if !reflect.DeepEqual(blessings.PublicKey(), bs.publicKey) {
 		return errStoreAddMismatch
 	}
-	oldDefault := s.state.Default
-	s.state.Default = blessings
-	if err := s.save(); err != nil {
-		s.state.Default = oldDefault
+	oldDefault := bs.state.Default
+	bs.state.Default = blessings
+	if err := bs.save(); err != nil {
+		bs.state.Default = oldDefault
 	}
 	return nil
 }
 
-func (s *blessingStore) PublicKey() security.PublicKey {
-	return s.publicKey
+func (bs *blessingStore) PublicKey() security.PublicKey {
+	return bs.publicKey
 }
 
-func (s *blessingStore) String() string {
-	return fmt.Sprintf("{state: %v, publicKey: %v, dir: %v}", s.state, s.publicKey, s.dir)
+func (bs *blessingStore) String() string {
+	return fmt.Sprintf("{state: %v, publicKey: %v}", bs.state, bs.publicKey)
 }
 
 // DebugString return a human-readable string encoding of the store
@@ -122,22 +117,26 @@
 // <pattern>    : <blessings>
 // ...
 // <pattern>    : <blessings>
-func (br *blessingStore) DebugString() string {
+func (bs *blessingStore) DebugString() string {
 	const format = "%-30s : %s\n"
-	b := bytes.NewBufferString(fmt.Sprintf("Default blessings: %v\n", br.state.Default))
+	b := bytes.NewBufferString(fmt.Sprintf("Default blessings: %v\n", bs.state.Default))
 
 	b.WriteString(fmt.Sprintf(format, "Peer pattern", "Blessings"))
-	for pattern, blessings := range br.state.Store {
+	for pattern, blessings := range bs.state.Store {
 		b.WriteString(fmt.Sprintf(format, pattern, blessings))
 	}
 	return b.String()
 }
 
-func (s *blessingStore) save() error {
-	if (s.signer == nil) && (s.dir == "") {
+func (bs *blessingStore) save() error {
+	if (bs.signer == nil) && (bs.persistedData == nil) {
 		return nil
 	}
-	return encodeAndStore(s.state, s.dir, blessingStoreDataFile, blessingStoreSigFile, s.signer)
+	data, signature, err := bs.persistedData.Writers()
+	if err != nil {
+		return err
+	}
+	return encodeAndStore(bs.state, data, signature, bs.signer)
 }
 
 // newInMemoryBlessingStore returns an in-memory security.BlessingStore for a
@@ -152,33 +151,31 @@
 }
 
 // newPersistingBlessingStore returns a security.BlessingStore for a principal
-// that persists all updates to the specified directory and uses the provided
-// signer to ensure integrity of data read from the filesystem.
-//
-// The returned BlessingStore is initialized from the existing data present in
-// the directory. The data is verified to have been written by a persisting
-// BlessingStore object constructed from the same signer.
-//
-// Any errors obtained in reading or verifying the data are returned.
-func newPersistingBlessingStore(directory string, signer serialization.Signer) (security.BlessingStore, error) {
-	if directory == "" || signer == nil {
-		return nil, errors.New("directory or signer is not specified")
+// that is initialized with the persisted data. The returned security.BlessingStore
+// also persists any updates to its state.
+func newPersistingBlessingStore(persistedData SerializerReaderWriter, signer serialization.Signer) (security.BlessingStore, error) {
+	if persistedData == nil || signer == nil {
+		return nil, errors.New("persisted data or signer is not specified")
 	}
-	s := &blessingStore{
-		publicKey: signer.PublicKey(),
-		state:     persistentState{Store: make(map[security.BlessingPattern]security.Blessings)},
-		dir:       directory,
-		signer:    signer,
+	bs := &blessingStore{
+		publicKey:     signer.PublicKey(),
+		state:         persistentState{Store: make(map[security.BlessingPattern]security.Blessings)},
+		persistedData: persistedData,
+		signer:        signer,
 	}
-
-	if err := decodeFromStorage(&s.state, s.dir, blessingStoreDataFile, blessingStoreSigFile, s.signer.PublicKey()); err != nil {
+	data, signature, err := bs.persistedData.Readers()
+	if err != nil {
 		return nil, err
 	}
-
-	for _, b := range s.state.Store {
-		if !reflect.DeepEqual(b.PublicKey(), s.publicKey) {
-			return nil, fmt.Errorf("directory contains Blessings: %v that are not for the provided PublicKey: %v", b, s.publicKey)
+	if data != nil && signature != nil {
+		if err := decodeFromStorage(&bs.state, data, signature, bs.signer.PublicKey()); err != nil {
+			return nil, err
 		}
 	}
-	return s, nil
+	for _, b := range bs.state.Store {
+		if !reflect.DeepEqual(b.PublicKey(), bs.publicKey) {
+			return nil, fmt.Errorf("directory contains Blessings: %v that are not for the provided PublicKey: %v", b, bs.publicKey)
+		}
+	}
+	return bs, nil
 }
diff --git a/security/principal.go b/security/principal.go
index f0dba02..7247434 100644
--- a/security/principal.go
+++ b/security/principal.go
@@ -9,7 +9,15 @@
 	"veyron.io/veyron/veyron2/security"
 )
 
-const privateKeyFile = "privatekey.pem"
+const (
+	blessingStoreDataFile = "blessingstore.data"
+	blessingStoreSigFile  = "blessingstore.sig"
+
+	blessingRootsDataFile = "blessingroots.data"
+	blessingRootsSigFile  = "blessingroots.sig"
+
+	privateKeyFile = "privatekey.pem"
+)
 
 // NewPrincipal mints a new private key and generates a principal based on
 // this key, storing its BlessingRoots and BlessingStore in memory.
@@ -27,7 +35,7 @@
 	return security.CreatePrincipal(signer, newInMemoryBlessingStore(signer.PublicKey()), newInMemoryBlessingRoots())
 }
 
-// NewPersistentPrincipalForSigner creates a new principal using the provided Signer and a
+// NewPersistentPrincipalFromSigner creates a new principal using the provided Signer and a
 // partial state (i.e., BlessingRoots, BlessingStore) that is read from the provided directory 'dir'.
 // Changes to the partial state are persisted and commited to the same directory; the provided
 // signer isn't persisted: the caller is expected to persist it separately or use the
@@ -100,11 +108,22 @@
 	if err != nil {
 		return nil, fmt.Errorf("failed to create serialization.Signer: %v", err)
 	}
-	roots, err := newPersistingBlessingRoots(dir, serializationSigner)
+	dataFile := path.Join(dir, blessingRootsDataFile)
+	signatureFile := path.Join(dir, blessingRootsSigFile)
+	fs, err := NewFileSerializer(dataFile, signatureFile)
+	if err != nil {
+		return nil, err
+	}
+	roots, err := newPersistingBlessingRoots(fs, serializationSigner)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load BlessingRoots from %q: %v", dir, err)
 	}
-	store, err := newPersistingBlessingStore(dir, serializationSigner)
+	dataFile = path.Join(dir, blessingStoreDataFile)
+	signatureFile = path.Join(dir, blessingStoreSigFile)
+	if fs, err = NewFileSerializer(dataFile, signatureFile); err != nil {
+		return nil, err
+	}
+	store, err := newPersistingBlessingStore(fs, serializationSigner)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load BlessingStore from %q: %v", dir, err)
 	}
diff --git a/security/serializer_reader_writer.go b/security/serializer_reader_writer.go
index 55248e4..633456d 100644
--- a/security/serializer_reader_writer.go
+++ b/security/serializer_reader_writer.go
@@ -2,6 +2,7 @@
 
 import (
 	"io"
+	"os"
 )
 
 // SerializerReaderWriter is a factory for managing the readers and writers used for
@@ -14,3 +15,51 @@
 	// integrity signature.
 	Writers() (data io.WriteCloser, signature io.WriteCloser, err error)
 }
+
+// FileSerializer implements SerializerReaderWriter that persists state to files.
+type FileSerializer struct {
+	data      *os.File
+	signature *os.File
+
+	dataFilePath      string
+	signatureFilePath string
+}
+
+// NewFileSerializer creates a FileSerializer with the given data and signature files.
+func NewFileSerializer(dataFilePath, signatureFilePath string) (*FileSerializer, error) {
+	data, err := os.Open(dataFilePath)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	signature, err := os.Open(signatureFilePath)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	return &FileSerializer{
+		data:              data,
+		signature:         signature,
+		dataFilePath:      dataFilePath,
+		signatureFilePath: signatureFilePath,
+	}, nil
+}
+
+func (fs *FileSerializer) Readers() (io.ReadCloser, io.ReadCloser, error) {
+	if fs.data == nil || fs.signature == nil {
+		return nil, nil, nil
+	}
+	return fs.data, fs.signature, nil
+}
+
+func (fs *FileSerializer) Writers() (io.WriteCloser, io.WriteCloser, error) {
+	// Remove previous version of the files
+	os.Remove(fs.dataFilePath)
+	os.Remove(fs.signatureFilePath)
+	var err error
+	if fs.data, err = os.Create(fs.dataFilePath); err != nil {
+		return nil, nil, err
+	}
+	if fs.signature, err = os.Create(fs.signatureFilePath); err != nil {
+		return nil, nil, err
+	}
+	return fs.data, fs.signature, nil
+}
diff --git a/security/storage.go b/security/storage.go
index 0dd755b..2fdd8cd 100644
--- a/security/storage.go
+++ b/security/storage.go
@@ -1,32 +1,19 @@
 package security
 
 import (
-	"io/ioutil"
-	"os"
-	"path"
+	"fmt"
+	"io"
 
 	"veyron.io/veyron/veyron/security/serialization"
-
 	"veyron.io/veyron/veyron2/security"
 	"veyron.io/veyron/veyron2/vom"
 )
 
-func encodeAndStore(obj interface{}, dir, dataFile, sigFile string, signer serialization.Signer) error {
-	// Save the object to temporary data and signature files, and then move
-	// those files to the actual data and signature file. This reduces the
-	// risk of loosing all saved data on disk in the event of a Write failure.
-	data, err := ioutil.TempFile(dir, "data")
-	if err != nil {
-		return err
+func encodeAndStore(obj interface{}, data, signature io.WriteCloser, signer serialization.Signer) error {
+	if data == nil || signature == nil {
+		return fmt.Errorf("invalid data/signature handles data:%v sig:%v", data, signature)
 	}
-	defer os.Remove(data.Name())
-	sig, err := ioutil.TempFile(dir, "sig")
-	if err != nil {
-		return err
-	}
-	defer os.Remove(sig.Name())
-
-	swc, err := serialization.NewSigningWriteCloser(data, sig, signer, nil)
+	swc, err := serialization.NewSigningWriteCloser(data, signature, signer, nil)
 	if err != nil {
 		return err
 	}
@@ -34,32 +21,16 @@
 		swc.Close()
 		return err
 	}
-	if err := swc.Close(); err != nil {
-		return err
-	}
-
-	if err := os.Rename(data.Name(), path.Join(dir, dataFile)); err != nil {
-		return err
-	}
-	return os.Rename(sig.Name(), path.Join(dir, sigFile))
+	return swc.Close()
 }
 
-func decodeFromStorage(obj interface{}, dir, dataFile, sigFile string, publicKey security.PublicKey) error {
-	data, dataErr := os.Open(path.Join(dir, dataFile))
-	defer data.Close()
-	sig, sigErr := os.Open(path.Join(dir, sigFile))
-	defer sig.Close()
-
-	switch {
-	case os.IsNotExist(dataErr) && os.IsNotExist(sigErr):
-		return nil
-	case dataErr != nil:
-		return dataErr
-	case sigErr != nil:
-		return sigErr
+func decodeFromStorage(obj interface{}, data, signature io.ReadCloser, publicKey security.PublicKey) error {
+	if data == nil || signature == nil {
+		return fmt.Errorf("invalid data/signature handles data:%v sig:%v", data, signature)
 	}
-
-	vr, err := serialization.NewVerifyingReader(data, sig, publicKey)
+	defer data.Close()
+	defer signature.Close()
+	vr, err := serialization.NewVerifyingReader(data, signature, publicKey)
 	if err != nil {
 		return err
 	}