veyron/lib/unixfd, veyron/lib/security/agent/server: Try to fix the flaky agent tests.

Change-Id: I932ad10651cf66bdf8d4ba232f66dc08552c1f35
diff --git a/lib/unixfd/unixfd.go b/lib/unixfd/unixfd.go
index 132a6c9..2cdcc36 100644
--- a/lib/unixfd/unixfd.go
+++ b/lib/unixfd/unixfd.go
@@ -202,16 +202,37 @@
 	}
 	defer local.maybeClose()
 	rfile := remote.releaseFile()
-	defer rfile.Close()
 
 	rights := syscall.UnixRights(int(rfile.Fd()))
 	n, oobn, err := conn.WriteMsgUnix(data, rights, nil)
 	if err != nil {
+		rfile.Close()
 		return nil, err
 	} else if n != len(data) || oobn != len(rights) {
+		rfile.Close()
 		return nil, fmt.Errorf("expected to send %d, %d bytes,  sent %d, %d", len(data), len(rights), n, oobn)
 	}
-	return local.releaseAddr(), nil
+	// Wait for the other side to acknowledge.
+	// This is to work around a race on OS X where it appears we can close
+	// the file descriptor before it gets transfered over the socket.
+	f := local.releaseFile()
+	fd, err := syscall.Dup(int(f.Fd()))
+	if err != nil {
+		f.Close()
+		rfile.Close()
+		return nil, err
+	}
+	newConn, err := net.FileConn(f)
+	f.Close()
+	if err != nil {
+		rfile.Close()
+		return nil, err
+	}
+	newConn.Read(make([]byte, 1))
+	newConn.Close()
+	rfile.Close()
+
+	return Addr(uintptr(fd)), nil
 }
 
 const cmsgDataLength = int(unsafe.Sizeof(int(1)))
@@ -232,22 +253,40 @@
 		return nil, n, err
 	}
 	fd := -1
+	// Loop through any file descriptors we are sent, and close
+	// all extras.
 	for _, scm := range scms {
 		fds, err := syscall.ParseUnixRights(&scm)
 		if err != nil {
 			return nil, n, err
 		}
 		for _, f := range fds {
-			if fd != -1 {
-				syscall.Close(fd)
+			if fd == -1 {
+				fd = f
+			} else if f != -1 {
+				syscall.Close(f)
 			}
-			fd = f
 		}
 	}
 	if fd == -1 {
 		return nil, n, nil
 	}
-	return Addr(uintptr(fd)), n, nil
+	result := Addr(uintptr(fd))
+	fd, err = syscall.Dup(fd)
+	if err != nil {
+		CloseUnixAddr(result)
+		return nil, n, err
+	}
+	file := os.NewFile(uintptr(fd), "newconn")
+	newconn, err := net.FileConn(file)
+	file.Close()
+	if err != nil {
+		CloseUnixAddr(result)
+		return nil, n, err
+	}
+	newconn.Write(make([]byte, 1))
+	newconn.Close()
+	return result, n, nil
 }
 
 func CloseUnixAddr(addr net.Addr) error {
diff --git a/lib/unixfd/unixfd_test.go b/lib/unixfd/unixfd_test.go
index 6dca48f..152efa8 100644
--- a/lib/unixfd/unixfd_test.go
+++ b/lib/unixfd/unixfd_test.go
@@ -136,15 +136,25 @@
 	if err != nil {
 		t.Fatalf("FileConn: %v", err)
 	}
+	var readErr error
+	var n int
+	var saddr net.Addr
+	done := make(chan struct{})
+	buf := make([]byte, 10)
+	go func() {
+		saddr, n, err = ReadConnection(server, buf)
+		close(done)
+	}()
 	caddr, err := SendConnection(uclient.(*net.UnixConn), []byte("hello"), true)
 	if err != nil {
 		t.Fatalf("SendConnection: %v", err)
 	}
-
-	buf := make([]byte, 10)
-	saddr, n, err := ReadConnection(server, buf)
-	if err != nil {
-		t.Fatalf("ReadConnection: %v", err)
+	<-done
+	if readErr != nil {
+		t.Fatalf("ReadConnection: %v", readErr)
+	}
+	if saddr == nil {
+		t.Fatalf("ReadConnection returned nil, %d", n)
 	}
 	data := buf[0:n]
 	if !bytes.Equal([]byte("hello"), data) {