runtimes/google/rt: bring back tests for runtime shutdown.
Change-Id: Idc01807d0beed6785d4916fb2c49d76655185359
diff --git a/lib/expect/expect.go b/lib/expect/expect.go
index 957f244..b9b8311 100644
--- a/lib/expect/expect.go
+++ b/lib/expect/expect.go
@@ -282,46 +282,75 @@
if s.Failed() {
return
}
+ if err := s.expectSetRE(len(expected), expected...); err != nil {
+ s.error(err)
+ }
+}
+
+// ExpectSetEventuallyRE is like ExpectSetRE except that it reads
+// all remaining output rather than just the next n lines and thus
+// can be used to look for a set of patterns that occur within that
+// output.
+func (s *Session) ExpectSetEventuallyRE(expected ...string) {
+ if s.Failed() {
+ return
+ }
+ if err := s.expectSetRE(-1, expected...); err != nil {
+ s.error(err)
+ }
+}
+
+// expectSetRE will look for the expected set of patterns in the next
+// numLines of output or in all remaining output.
+func (s *Session) expectSetRE(numLines int, expected ...string) error {
+
regexps := make([]*regexp.Regexp, len(expected))
for i, expRE := range expected {
re, err := regexp.Compile(expRE)
if err != nil {
- s.error(err)
- return
+ return err
}
regexps[i] = re
}
- actual := make([]string, len(expected))
- for i := 0; i < len(expected); i++ {
+ actual := []string{}
+ i := 0
+ for {
line, err := s.read(readLine)
line = strings.TrimRight(line, "\n")
s.log(err, "ExpectSetRE: %s", line)
if err != nil {
- s.error(err)
- return
+ if numLines >= 0 {
+ return err
+ }
+ break
}
- actual[i] = line
+ actual = append(actual, line)
+ i++
+ if numLines > 0 && i >= numLines {
+ break
+ }
}
+
// Match each line against all regexp's and remove each regexp
// that matches.
for _, l := range actual {
- found := false
for i, re := range regexps {
if re == nil {
continue
}
if re.MatchString(l) {
- // We remove each RE that matches.
- found = true
regexps[i] = nil
break
}
}
- if !found {
- s.error(fmt.Errorf("found no match for %q", l))
- return
+ }
+ // It's an error if there are any unmatched regexps.
+ for _, re := range regexps {
+ if re != nil {
+ return fmt.Errorf("found no match for %q", re)
}
}
+ return nil
}
// ReadLine reads the next line, if any, from the input stream. It will set
diff --git a/lib/expect/expect_test.go b/lib/expect/expect_test.go
index 8fa62da..8e8424d 100644
--- a/lib/expect/expect_test.go
+++ b/lib/expect/expect_test.go
@@ -102,12 +102,42 @@
buffer.WriteString("ooh\n")
buffer.WriteString("aah\n")
s.ExpectSetRE("bar=.*", "def")
- if got, want := s.Error(), "expect_test.go:104: found no match for \"ooh\""; got == nil || got.Error() != want {
+ if got, want := s.Error(), "expect_test.go:104: found no match for \"bar=.*\""; got == nil || got.Error() != want {
t.Errorf("got %v, want %q", got, want)
}
s.ExpectEOF()
}
+func TestExpectSetEventuallyRE(t *testing.T) {
+ buf := []byte{}
+ buffer := bytes.NewBuffer(buf)
+ buffer.WriteString("bar=baz\n")
+ buffer.WriteString("abc\n")
+ buffer.WriteString("def\n")
+ buffer.WriteString("abc\n")
+ s := expect.NewSession(nil, bufio.NewReader(buffer), time.Minute)
+ s.SetVerbosity(testing.Verbose())
+ s.ExpectSetEventuallyRE("^bar=.*$", "def")
+ if s.Error() != nil {
+ t.Errorf("unexpected error: %s", s.Error())
+ }
+ s.ExpectSetEventuallyRE("abc")
+ if got, want := s.Error(), "expect_test.go:124: found no match for \"abc\""; got == nil || got.Error() != want {
+ t.Errorf("got %q, want %q", got, want)
+ }
+ // Need to clear the EOF from the previous ExpectSetEventuallyRE call
+ buf = []byte{}
+ buffer = bytes.NewBuffer(buf)
+ s = expect.NewSession(nil, bufio.NewReader(buffer), time.Minute)
+ buffer.WriteString("ooh\n")
+ buffer.WriteString("aah\n")
+ s.ExpectSetEventuallyRE("zzz")
+ if got, want := s.Error(), "expect_test.go:134: found no match for \"zzz\""; got == nil || got.Error() != want {
+ t.Errorf("got %q, want %q", got, want)
+ }
+ s.ExpectEOF()
+}
+
func TestRead(t *testing.T) {
buf := []byte{}
buffer := bytes.NewBuffer(buf)
diff --git a/runtimes/google/rt/shutdown_servers_test.go b/runtimes/google/rt/shutdown_servers_test.go
new file mode 100644
index 0000000..3e3f730
--- /dev/null
+++ b/runtimes/google/rt/shutdown_servers_test.go
@@ -0,0 +1,271 @@
+package rt_test
+
+import (
+ "bufio"
+ "fmt"
+ "io"
+ "os"
+ "os/signal"
+ "sync"
+ "syscall"
+
+ "veyron.io/veyron/veyron2/ipc"
+ "veyron.io/veyron/veyron2/rt"
+ "veyron.io/veyron/veyron2/vlog"
+
+ "veyron.io/veyron/veyron/lib/modules"
+ "veyron.io/veyron/veyron/lib/signals"
+ "veyron.io/veyron/veyron/profiles"
+)
+
+func init() {
+ modules.RegisterChild("simpleServerProgram", "", simpleServerProgram)
+ modules.RegisterChild("complexServerProgram", "", complexServerProgram)
+}
+
+type dummy struct{}
+
+func (*dummy) Echo(ipc.ServerContext) error { return nil }
+
+// makeServer sets up a simple dummy server.
+func makeServer() ipc.Server {
+ server, err := rt.R().NewServer()
+ if err != nil {
+ vlog.Fatalf("r.NewServer error: %s", err)
+ }
+ if _, err := server.Listen(profiles.LocalListenSpec); err != nil {
+ vlog.Fatalf("server.Listen error: %s", err)
+ }
+ if err := server.Serve("", &dummy{}, nil); err != nil {
+ vlog.Fatalf("server.Serve error: %s", err)
+ }
+ return server
+}
+
+// remoteCmdLoop listens on stdin and interprets commands sent over stdin (from
+// the parent process).
+func remoteCmdLoop(stdin io.Reader) func() {
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ scanner := bufio.NewScanner(stdin)
+ for scanner.Scan() {
+ switch scanner.Text() {
+ case "stop":
+ rt.R().AppCycle().Stop()
+ case "forcestop":
+ fmt.Println("straight exit")
+ rt.R().AppCycle().ForceStop()
+ case "close":
+ return
+ }
+ }
+ }()
+ return func() { <-done }
+}
+
+// complexServerProgram demonstrates the recommended way to write a more
+// complex server application (with several servers, a mix of interruptible
+// and blocking cleanup, and parallel and sequential cleanup execution).
+// For a more typical server, see simpleServerProgram.
+func complexServerProgram(stdin io.Reader, stdout, stderr io.Writer, env map[string]string, args ...string) error {
+ // Initialize the runtime. This is boilerplate.
+ r := rt.Init()
+
+ // This is part of the test setup -- we need a way to accept
+ // commands from the parent process to simulate Stop and
+ // RemoteStop commands that would normally be issued from
+ // application code.
+ go remoteCmdLoop(stdin)()
+
+ // r.Cleanup is optional, but it's a good idea to clean up, especially
+ // since it takes care of flushing the logs before exiting.
+ defer r.Cleanup()
+
+ // Create a couple servers, and start serving.
+ server1 := makeServer()
+ server2 := makeServer()
+
+ // This is how to wait for a shutdown. In this example, a shutdown
+ // comes from a signal or a stop command.
+ var done sync.WaitGroup
+ done.Add(1)
+
+ // This is how to configure signal handling to allow clean shutdown.
+ sigChan := make(chan os.Signal, 2)
+ signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT)
+
+ // This is how to configure handling of stop commands to allow clean
+ // shutdown.
+ stopChan := make(chan string, 2)
+ r.AppCycle().WaitForStop(stopChan)
+
+ // Blocking is used to prevent the process from exiting upon receiving a
+ // second signal or stop command while critical cleanup code is
+ // executing.
+ var blocking sync.WaitGroup
+ blockingCh := make(chan struct{})
+
+ // This is how to wait for a signal or stop command and initiate the
+ // clean shutdown steps.
+ go func() {
+ // First signal received.
+ select {
+ case sig := <-sigChan:
+ // If the developer wants to take different actions
+ // depending on the type of signal, they can do it here.
+ fmt.Fprintln(stdout, "Received signal", sig)
+ case stop := <-stopChan:
+ fmt.Fprintln(stdout, "Stop", stop)
+ }
+ // This commences the cleanup stage.
+ done.Done()
+ // Wait for a second signal or stop command, and force an exit,
+ // but only once all blocking cleanup code (if any) has
+ // completed.
+ select {
+ case <-sigChan:
+ case <-stopChan:
+ }
+ <-blockingCh
+ os.Exit(1)
+ }()
+
+ // This communicates to the parent test driver process in our unit test
+ // that this server is ready and waiting on signals or stop commands.
+ // It's purely an artifact of our test setup.
+ fmt.Fprintln(stdout, "Ready")
+
+ // Wait for shutdown.
+ done.Wait()
+
+ // Stop the servers. In this example we stop them in goroutines to
+ // parallelize the wait, but if there was a dependency between the
+ // servers, the developer can simply stop them sequentially.
+ var waitServerStop sync.WaitGroup
+ waitServerStop.Add(2)
+ go func() {
+ server1.Stop()
+ waitServerStop.Done()
+ }()
+ go func() {
+ server2.Stop()
+ waitServerStop.Done()
+ }()
+ waitServerStop.Wait()
+
+ // This is where all cleanup code should go. By placing it at the end,
+ // we make its purpose and order of execution clear.
+
+ // This is an example of how to mix parallel and sequential cleanup
+ // steps. Most real-world servers will likely be simpler, with either
+ // just sequential or just parallel cleanup stages.
+
+ // parallelCleanup is used to wait for all goroutines executing cleanup
+ // code in parallel to finish.
+ var parallelCleanup sync.WaitGroup
+
+ // Simulate four parallel cleanup steps, two blocking and two
+ // interruptible.
+ parallelCleanup.Add(1)
+ blocking.Add(1)
+ go func() {
+ fmt.Fprintln(stdout, "Parallel blocking cleanup1")
+ blocking.Done()
+ parallelCleanup.Done()
+ }()
+
+ parallelCleanup.Add(1)
+ blocking.Add(1)
+ go func() {
+ fmt.Fprintln(stdout, "Parallel blocking cleanup2")
+ blocking.Done()
+ parallelCleanup.Done()
+ }()
+
+ parallelCleanup.Add(1)
+ go func() {
+ fmt.Fprintln(stdout, "Parallel interruptible cleanup1")
+ parallelCleanup.Done()
+ }()
+
+ parallelCleanup.Add(1)
+ go func() {
+ fmt.Fprintln(stdout, "Parallel interruptible cleanup2")
+ parallelCleanup.Done()
+ }()
+
+ // Simulate two sequential cleanup steps, one blocking and one
+ // interruptible.
+ fmt.Fprintln(stdout, "Sequential blocking cleanup")
+ blocking.Wait()
+ close(blockingCh)
+
+ fmt.Fprintln(stdout, "Sequential interruptible cleanup")
+
+ parallelCleanup.Wait()
+ return nil
+}
+
+// simpleServerProgram demonstrates the recommended way to write a typical
+// simple server application (with one server and a clean shutdown triggered by
+// a signal or a stop command). For an example of something more involved, see
+// complexServerProgram.
+func simpleServerProgram(stdin io.Reader, stdout, stderr io.Writer, env map[string]string, args ...string) error {
+ // Initialize the runtime. This is boilerplate.
+ r := rt.Init()
+
+ // This is part of the test setup -- we need a way to accept
+ // commands from the parent process to simulate Stop and
+ // RemoteStop commands that would normally be issued from
+ // application code.
+ go remoteCmdLoop(stdin)()
+
+ // r.Cleanup is optional, but it's a good idea to clean up, especially
+ // since it takes care of flushing the logs before exiting.
+ //
+ // We use defer to ensure this is the last thing in the program (to
+ // avoid shutting down the runtime while it may still be in use), and to
+ // allow it to execute even if a panic occurs down the road.
+ defer r.Cleanup()
+
+ // Create a server, and start serving.
+ server := makeServer()
+
+ // This is how to wait for a shutdown. In this example, a shutdown
+ // comes from a signal or a stop command.
+ //
+ // Note, if the developer wants to exit immediately upon receiving a
+ // signal or stop command, they can skip this, in which case the default
+ // behavior is for the process to exit.
+ waiter := signals.ShutdownOnSignals()
+
+ // This communicates to the parent test driver process in our unit test
+ // that this server is ready and waiting on signals or stop commands.
+ // It's purely an artifact of our test setup.
+ fmt.Fprintln(stdout, "Ready")
+
+ // Use defer for anything that should still execute even if a panic
+ // occurs.
+ defer fmt.Fprintln(stdout, "Deferred cleanup")
+
+ // Wait for shutdown.
+ sig := <-waiter
+ // The developer could take different actions depending on the type of
+ // signal.
+ fmt.Fprintln(stdout, "Received signal", sig)
+
+ // Cleanup code starts here. Alternatively, these steps could be
+ // invoked through defer, but we list them here to make the order of
+ // operations obvious.
+
+ // Stop the server.
+ server.Stop()
+
+ // Note, this will not execute in cases of forced shutdown
+ // (e.g. SIGSTOP), when the process calls os.Exit (e.g. via log.Fatal),
+ // or when a panic occurs.
+ fmt.Fprintln(stdout, "Interruptible cleanup")
+
+ return nil
+}
diff --git a/runtimes/google/rt/shutdown_test.go b/runtimes/google/rt/shutdown_test.go
new file mode 100644
index 0000000..f0a57aa
--- /dev/null
+++ b/runtimes/google/rt/shutdown_test.go
@@ -0,0 +1,228 @@
+package rt_test
+
+import (
+ "fmt"
+ "io"
+ "os"
+ "syscall"
+ "testing"
+ "time"
+
+ "veyron.io/veyron/veyron2"
+
+ "veyron.io/veyron/veyron/lib/expect"
+ "veyron.io/veyron/veyron/lib/modules"
+ "veyron.io/veyron/veyron/lib/signals"
+ "veyron.io/veyron/veyron/lib/testutil"
+)
+
+var cstderr io.Writer
+
+func init() {
+ testutil.Init()
+ if testing.Verbose() {
+ cstderr = os.Stderr
+ }
+}
+
+// TestSimpleServerSignal verifies that sending a signal to the simple server
+// causes it to exit cleanly.
+func TestSimpleServerSignal(t *testing.T) {
+ sh := modules.NewShell()
+ defer sh.Cleanup(os.Stdout, cstderr)
+ h, _ := sh.Start("simpleServerProgram", nil)
+ s := expect.NewSession(t, h.Stdout(), time.Minute)
+ s.Expect("Ready")
+ syscall.Kill(h.Pid(), syscall.SIGINT)
+ s.Expect("Received signal interrupt")
+ s.Expect("Interruptible cleanup")
+ s.Expect("Deferred cleanup")
+ fmt.Fprintln(h.Stdin(), "close")
+ s.ExpectEOF()
+}
+
+// TestSimpleServerLocalStop verifies that sending a local stop command to the
+// simple server causes it to exit cleanly.
+func TestSimpleServerLocalStop(t *testing.T) {
+ sh := modules.NewShell()
+ defer sh.Cleanup(os.Stdout, cstderr)
+ h, _ := sh.Start("simpleServerProgram", nil)
+ s := expect.NewSession(t, h.Stdout(), time.Minute)
+ s.Expect("Ready")
+ fmt.Fprintln(h.Stdin(), "stop")
+ s.Expect(fmt.Sprintf("Received signal %s", veyron2.LocalStop))
+ s.Expect("Interruptible cleanup")
+ s.Expect("Deferred cleanup")
+ fmt.Fprintln(h.Stdin(), "close")
+ s.ExpectEOF()
+}
+
+// TestSimpleServerDoubleSignal verifies that sending a succession of two
+// signals to the simple server causes it to initiate the cleanup sequence on
+// the first signal and then exit immediately on the second signal.
+func TestSimpleServerDoubleSignal(t *testing.T) {
+ sh := modules.NewShell()
+ defer sh.Cleanup(os.Stdout, cstderr)
+ h, _ := sh.Start("simpleServerProgram", nil)
+ s := expect.NewSession(t, h.Stdout(), time.Minute)
+ s.Expect("Ready")
+ syscall.Kill(h.Pid(), syscall.SIGINT)
+ s.Expect("Received signal interrupt")
+ syscall.Kill(h.Pid(), syscall.SIGINT)
+ err := h.Shutdown(os.Stdout, cstderr)
+ if err == nil {
+ t.Fatalf("expected an error")
+ }
+ if got, want := err.Error(), fmt.Sprintf("exit status %d", signals.DoubleStopExitCode); got != want {
+ t.Errorf("got %q, want %q", got, want)
+ }
+}
+
+// TestSimpleServerLocalForceStop verifies that sending a local ForceStop
+// command to the simple server causes it to exit immediately.
+func TestSimpleServerLocalForceStop(t *testing.T) {
+ sh := modules.NewShell()
+ defer sh.Cleanup(os.Stdout, cstderr)
+ h, _ := sh.Start("simpleServerProgram", nil)
+ s := expect.NewSession(t, h.Stdout(), time.Minute)
+ s.Expect("Ready")
+ fmt.Fprintln(h.Stdin(), "forcestop")
+ s.Expect("straight exit")
+ err := h.Shutdown(os.Stdout, cstderr)
+ if err == nil {
+ t.Fatalf("expected an error")
+ }
+ if got, want := err.Error(), fmt.Sprintf("exit status %d", veyron2.ForceStopExitCode); got != want {
+ t.Errorf("got %q, want %q", got, want)
+ }
+}
+
+// TestSimpleServerKill demonstrates that a SIGKILL still forces the server
+// to exit regardless of our signal handling.
+func TestSimpleServerKill(t *testing.T) {
+ sh := modules.NewShell()
+ defer sh.Cleanup(os.Stdout, cstderr)
+ h, _ := sh.Start("simpleServerProgram", nil)
+ s := expect.NewSession(t, h.Stdout(), time.Minute)
+ s.Expect("Ready")
+ syscall.Kill(h.Pid(), syscall.SIGKILL)
+ err := h.Shutdown(os.Stdout, cstderr)
+ if err == nil {
+ t.Fatalf("expected an error")
+ }
+ if got, want := err.Error(), "signal: killed"; got != want {
+ t.Errorf("got %q, want %q", got, want)
+ }
+}
+
+// TestComplexServerSignal verifies that sending a signal to the complex server
+// initiates the cleanup sequence in that server (we observe the printouts
+// corresponding to all the simulated sequential/parallel and
+// blocking/interruptible shutdown steps), and then exits cleanly.
+func TestComplexServerSignal(t *testing.T) {
+ sh := modules.NewShell()
+ defer sh.Cleanup(os.Stdout, cstderr)
+ h, _ := sh.Start("complexServerProgram", nil)
+ s := expect.NewSession(t, h.Stdout(), time.Minute)
+ s.Expect("Ready")
+ syscall.Kill(h.Pid(), syscall.SIGINT)
+ s.Expect("Received signal interrupt")
+ s.ExpectSetRE("Sequential blocking cleanup",
+ "Sequential interruptible cleanup",
+ "Parallel blocking cleanup1",
+ "Parallel blocking cleanup2",
+ "Parallel interruptible cleanup1",
+ "Parallel interruptible cleanup2")
+ fmt.Fprintln(h.Stdin(), "close")
+ s.ExpectEOF()
+}
+
+// TestComplexServerLocalStop verifies that sending a local stop command to the
+// complex server initiates the cleanup sequence in that server (we observe the
+// printouts corresponding to all the simulated sequential/parallel and
+// blocking/interruptible shutdown steps), and then exits cleanly.
+func TestComplexServerLocalStop(t *testing.T) {
+ sh := modules.NewShell()
+ defer sh.Cleanup(os.Stdout, cstderr)
+ h, _ := sh.Start("complexServerProgram", nil)
+ s := expect.NewSession(t, h.Stdout(), time.Minute)
+ s.Expect("Ready")
+
+ fmt.Fprintln(h.Stdin(), "stop")
+ s.Expect(fmt.Sprintf("Stop %s", veyron2.LocalStop))
+ s.ExpectSetRE(
+ "Sequential blocking cleanup",
+ "Sequential interruptible cleanup",
+ "Parallel blocking cleanup1",
+ "Parallel blocking cleanup2",
+ "Parallel interruptible cleanup1",
+ "Parallel interruptible cleanup2",
+ )
+ fmt.Fprintln(h.Stdin(), "close")
+ s.ExpectEOF()
+}
+
+// TestComplexServerDoubleSignal verifies that sending a succession of two
+// signals to the complex server has the expected effect: the first signal
+// initiates the cleanup steps and the second signal kills the process, but only
+// after the blocking shutdown steps were allowed to complete (as observed by
+// the corresponding printouts from the server). Note that we have no
+// expectations on whether or not the interruptible shutdown steps execute.
+func TestComplexServerDoubleSignal(t *testing.T) {
+ sh := modules.NewShell()
+ defer sh.Cleanup(os.Stdout, cstderr)
+ h, _ := sh.Start("complexServerProgram", nil)
+ s := expect.NewSession(t, h.Stdout(), time.Minute)
+ s.Expect("Ready")
+ syscall.Kill(h.Pid(), syscall.SIGINT)
+ s.Expect("Received signal interrupt")
+ syscall.Kill(h.Pid(), syscall.SIGINT)
+ s.ExpectSetEventuallyRE(
+ "Sequential blocking cleanup",
+ "Parallel blocking cleanup1",
+ "Parallel blocking cleanup2")
+ err := h.Shutdown(os.Stdout, cstderr)
+ if err == nil {
+ t.Fatalf("expected an error")
+ }
+ if got, want := err.Error(), fmt.Sprintf("exit status %d", signals.DoubleStopExitCode); got != want {
+ t.Errorf("got %q, want %q", got, want)
+ }
+}
+
+// TestComplexServerLocalForceStop verifies that sending a local ForceStop
+// command to the complex server forces it to exit immediately.
+func TestComplexServerLocalForceStop(t *testing.T) {
+ sh := modules.NewShell()
+ defer sh.Cleanup(os.Stdout, cstderr)
+ h, _ := sh.Start("complexServerProgram", nil)
+ s := expect.NewSession(t, h.Stdout(), time.Minute)
+ s.Expect("Ready")
+ fmt.Fprintln(h.Stdin(), "forcestop")
+ s.Expect("straight exit")
+ err := h.Shutdown(os.Stdout, cstderr)
+ if err == nil {
+ t.Fatalf("expected an error")
+ }
+ if got, want := err.Error(), fmt.Sprintf("exit status %d", veyron2.ForceStopExitCode); got != want {
+ t.Errorf("got %q, want %q", got, want)
+ }
+}
+
+// TestComplexServerKill demonstrates that a SIGKILL still forces the server to
+// exit regardless of our signal handling.
+func TestComplexServerKill(t *testing.T) {
+ sh := modules.NewShell()
+ defer sh.Cleanup(os.Stdout, cstderr)
+ h, _ := sh.Start("complexServerProgram", nil)
+ s := expect.NewSession(t, h.Stdout(), time.Minute)
+ s.Expect("Ready")
+ syscall.Kill(h.Pid(), syscall.SIGKILL)
+ err := h.Shutdown(os.Stdout, cstderr)
+ if err == nil {
+ t.Fatalf("expected an error")
+ }
+ if got, want := err.Error(), "signal: killed"; got != want {
+ t.Errorf("got %q, want %q", got, want)
+ }
+}
diff --git a/runtimes/google/rt/signal.go b/runtimes/google/rt/signal.go
index fc13847..0e5d607 100644
--- a/runtimes/google/rt/signal.go
+++ b/runtimes/google/rt/signal.go
@@ -4,6 +4,7 @@
"os"
"os/signal"
"syscall"
+
"veyron.io/veyron/veyron2/vlog"
)
diff --git a/security/testdata/blessingstore.sig b/security/testdata/blessingstore.sig
index 2f2a57c..1040fb8 100644
--- a/security/testdata/blessingstore.sig
+++ b/security/testdata/blessingstore.sig
Binary files differ
