Merge "veyron/tools/gclogs: A tool that safely deletes old log files."
diff --git a/tools/gclogs/doc.go b/tools/gclogs/doc.go
new file mode 100644
index 0000000..302fe98
--- /dev/null
+++ b/tools/gclogs/doc.go
@@ -0,0 +1,31 @@
+// This file was auto-generated via go generate.
+// DO NOT UPDATE MANUALLY
+
+/*
+gclogs is a utility that safely deletes old log files.
+
+It looks for file names that match the format of files produced by the vlog
+package, and deletes the ones that have not changed in the amount of time
+specified by the --cutoff flag.
+
+Only files produced by the same user as the one running the gclogs command are
+considered for deletion.
+
+Usage:
+ gclogs [flags] <dir> ...
+
+<dir> ... A list of directories where to look for log files.
+
+The gclogs flags are:
+ -cutoff=24h0m0s
+ The age cut-off for a log file to be considered for garbage collection.
+ -n=false
+ If true, log files that would be deleted are shown on stdout, but not
+ actually deleted.
+ -program=.*
+ A regular expression to apply to the program part of the log file name, e.g
+ ".*test".
+ -verbose=false
+ If true, each deleted file is shown on stdout.
+*/
+package main
diff --git a/tools/gclogs/format.go b/tools/gclogs/format.go
new file mode 100644
index 0000000..502db25
--- /dev/null
+++ b/tools/gclogs/format.go
@@ -0,0 +1,69 @@
+package main
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "syscall"
+ "time"
+)
+
+var (
+ // The format of the log file names is:
+ // program.host.user.log.logger.tag.YYYYMMDD-HHmmss.pid
+ logFileRE = regexp.MustCompile(`^(.*)\.([^.]*)\.([^.]*)\.log\.([^.]*)\.([^.]*)\.(........-......)\.([0-9]*)$`)
+)
+
+type logFile struct {
+ // TODO(rthellend): Some of these fields are not used by anything yet,
+ // but they will be eventually when we need to sort the log files
+ // associated with a given instance.
+ symlink bool
+ program, host, user, logger, tag string
+ time time.Time
+ pid int
+}
+
+func parseFileInfo(dir string, fileInfo os.FileInfo) (*logFile, error) {
+ fileName := fileInfo.Name()
+ if fileInfo.Mode()&os.ModeSymlink != 0 {
+ buf := make([]byte, syscall.NAME_MAX)
+ n, err := syscall.Readlink(filepath.Join(dir, fileName), buf)
+ if err != nil {
+ return nil, err
+ }
+ linkName := string(buf[:n])
+ lf, err := parseFileName(linkName)
+ if err != nil {
+ return nil, err
+ }
+ lf.symlink = true
+ return lf, nil
+ }
+ return parseFileName(fileName)
+}
+
+func parseFileName(fileName string) (*logFile, error) {
+ if m := logFileRE.FindStringSubmatch(fileName); len(m) != 0 {
+ t, err := time.ParseInLocation("20060102-150405", m[6], time.Local)
+ if err != nil {
+ return nil, err
+ }
+ pid, err := strconv.ParseInt(m[7], 10, 32)
+ if err != nil {
+ return nil, err
+ }
+ return &logFile{
+ program: m[1],
+ host: m[2],
+ user: m[3],
+ logger: m[4],
+ tag: m[5],
+ time: t,
+ pid: int(pid),
+ }, nil
+ }
+ return nil, errors.New("not a recognized log file name")
+}
diff --git a/tools/gclogs/format_test.go b/tools/gclogs/format_test.go
new file mode 100644
index 0000000..5396894
--- /dev/null
+++ b/tools/gclogs/format_test.go
@@ -0,0 +1,97 @@
+package main
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+ "time"
+)
+
+func TestParseFileNameNoError(t *testing.T) {
+ testcases := []struct {
+ filename string
+ lf *logFile
+ }{
+ {
+ "program.host.user.log.veyron.INFO.20141204-131502.12345",
+ &logFile{false, "program", "host", "user", "veyron", "INFO", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345},
+ },
+ {
+ "prog.test.host-name.user.log.veyron.ERROR.20141204-131502.12345",
+ &logFile{false, "prog.test", "host-name", "user", "veyron", "ERROR", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345},
+ },
+ }
+ for _, tc := range testcases {
+ lf, err := parseFileName(tc.filename)
+ if err != nil {
+ t.Errorf("unexpected error for %q: %v", tc.filename, err)
+ }
+ if !reflect.DeepEqual(tc.lf, lf) {
+ t.Errorf("unexpected result: got %+v, expected %+v", lf, tc.lf)
+ }
+ }
+}
+
+func TestParseFileNameError(t *testing.T) {
+ testcases := []string{
+ "program.host.user.log.veyron.INFO.20141204-131502",
+ "prog.test.host-name.user.log.veyron.20141204-131502.12345",
+ "foo.txt",
+ }
+ for _, tc := range testcases {
+ if _, err := parseFileName(tc); err == nil {
+ t.Errorf("unexpected success for %q", tc)
+ }
+ }
+}
+
+func TestParseFileInfo(t *testing.T) {
+ tmpdir, err := ioutil.TempDir("", "parse-file-info-")
+ if err != nil {
+ t.Fatalf("ioutil.TempDir failed: %v", err)
+ }
+ defer os.RemoveAll(tmpdir)
+
+ name := "program.host.user.log.veyron.INFO.20141204-131502.12345"
+ if err := ioutil.WriteFile(filepath.Join(tmpdir, name), []byte{}, 0644); err != nil {
+ t.Fatalf("ioutil.WriteFile failed: %v", err)
+ }
+ link := "program.INFO"
+ if err := os.Symlink(name, filepath.Join(tmpdir, link)); err != nil {
+ t.Fatalf("os.Symlink failed: %v", err)
+ }
+
+ // Test regular file.
+ {
+ fi, err := os.Lstat(filepath.Join(tmpdir, name))
+ if err != nil {
+ t.Fatalf("os.Lstat failed: %v", err)
+ }
+ lf, err := parseFileInfo(tmpdir, fi)
+ if err != nil {
+ t.Errorf("parseFileInfo(%v, %v) failed: %v", tmpdir, fi, err)
+ }
+ expected := &logFile{false, "program", "host", "user", "veyron", "INFO", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345}
+ if !reflect.DeepEqual(lf, expected) {
+ t.Errorf("unexpected result: got %+v, expected %+v", lf, expected)
+ }
+ }
+
+ // Test symlink.
+ {
+ fi, err := os.Lstat(filepath.Join(tmpdir, link))
+ if err != nil {
+ t.Fatalf("os.Lstat failed: %v", err)
+ }
+ lf, err := parseFileInfo(tmpdir, fi)
+ if err != nil {
+ t.Errorf("parseFileInfo(%v, %v) failed: %v", tmpdir, fi, err)
+ }
+ expected := &logFile{true, "program", "host", "user", "veyron", "INFO", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345}
+ if !reflect.DeepEqual(lf, expected) {
+ t.Errorf("unexpected result: got %+v, expected %+v", lf, expected)
+ }
+ }
+}
diff --git a/tools/gclogs/gclogs.go b/tools/gclogs/gclogs.go
new file mode 100644
index 0000000..6fde8af
--- /dev/null
+++ b/tools/gclogs/gclogs.go
@@ -0,0 +1,157 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "os"
+ "os/user"
+ "path/filepath"
+ "regexp"
+ "time"
+
+ "veyron.io/lib/cmdline"
+)
+
+var (
+ flagCutoff time.Duration
+ flagProgname string
+ flagVerbose bool
+ flagDryrun bool
+
+ cmdGCLogs = &cmdline.Command{
+ Run: garbageCollectLogs,
+ Name: "gclogs",
+ Short: "gclogs is a utility that safely deletes old log files.",
+ Long: `
+gclogs is a utility that safely deletes old log files.
+
+It looks for file names that match the format of files produced by the vlog
+package, and deletes the ones that have not changed in the amount of time
+specified by the --cutoff flag.
+
+Only files produced by the same user as the one running the gclogs command
+are considered for deletion.
+`,
+ ArgsName: "<dir> ...",
+ ArgsLong: "<dir> ... A list of directories where to look for log files.",
+ }
+)
+
+func init() {
+ cmdGCLogs.Flags.DurationVar(&flagCutoff, "cutoff", 24*time.Hour, "The age cut-off for a log file to be considered for garbage collection.")
+ cmdGCLogs.Flags.StringVar(&flagProgname, "program", ".*", `A regular expression to apply to the program part of the log file name, e.g ".*test".`)
+ cmdGCLogs.Flags.BoolVar(&flagVerbose, "verbose", false, "If true, each deleted file is shown on stdout.")
+ cmdGCLogs.Flags.BoolVar(&flagDryrun, "n", false, "If true, log files that would be deleted are shown on stdout, but not actually deleted.")
+}
+
+func garbageCollectLogs(cmd *cmdline.Command, args []string) error {
+ if len(args) == 0 {
+ cmd.UsageErrorf("gclogs requires at least one argument")
+ }
+ timeCutoff := time.Now().Add(-flagCutoff)
+ currentUser, err := user.Current()
+ if err != nil {
+ return err
+ }
+ programRE, err := regexp.Compile(flagProgname)
+ if err != nil {
+ return err
+ }
+ var lastErr error
+ for _, logdir := range args {
+ if err := processDirectory(cmd, logdir, timeCutoff, programRE, currentUser.Username); err != nil {
+ lastErr = err
+ }
+ }
+ return lastErr
+}
+
+func processDirectory(cmd *cmdline.Command, logdir string, timeCutoff time.Time, programRE *regexp.Regexp, username string) error {
+ fmt.Fprintf(cmd.Stdout(), "Processing: %q\n", logdir)
+
+ f, err := os.Open(logdir)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ var lastErr error
+ deleted := 0
+ symlinks := []string{}
+ for {
+ fi, err := f.Readdir(100)
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ lastErr = err
+ break
+ }
+ for _, file := range fi {
+ fullname := filepath.Join(logdir, file.Name())
+ if file.IsDir() {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Skipped directory: %q\n", fullname)
+ }
+ continue
+ }
+ lf, err := parseFileInfo(logdir, file)
+ if err != nil {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Not a log file: %q\n", fullname)
+ }
+ continue
+ }
+ if lf.user != username {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Skipped log file created by other user: %q\n", fullname)
+ }
+ continue
+ }
+ if !programRE.MatchString(lf.program) {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Skipped log file doesn't match %q: %q\n", flagProgname, fullname)
+ }
+ continue
+ }
+ if lf.symlink {
+ symlinks = append(symlinks, fullname)
+ continue
+ }
+ if file.ModTime().Before(timeCutoff) {
+ if flagDryrun {
+ fmt.Fprintf(cmd.Stdout(), "Would delete %q\n", fullname)
+ continue
+ }
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Deleting %q\n", fullname)
+ }
+ if err := os.Remove(fullname); err != nil {
+ lastErr = err
+ } else {
+ deleted++
+ }
+ }
+ }
+ }
+ // Delete broken links.
+ for _, sl := range symlinks {
+ if _, err := os.Stat(sl); err != nil && os.IsNotExist(err) {
+ if flagDryrun {
+ fmt.Fprintf(cmd.Stdout(), "Would delete symlink %q\n", sl)
+ continue
+ }
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Deleting symlink %q\n", sl)
+ }
+ if err := os.Remove(sl); err != nil {
+ lastErr = err
+ } else {
+ deleted++
+ }
+ }
+
+ }
+ fmt.Fprintf(cmd.Stdout(), "Number of files deleted: %d\n", deleted)
+ return lastErr
+}
diff --git a/tools/gclogs/gclogs_test.go b/tools/gclogs/gclogs_test.go
new file mode 100644
index 0000000..f9168fe
--- /dev/null
+++ b/tools/gclogs/gclogs_test.go
@@ -0,0 +1,177 @@
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/user"
+ "path/filepath"
+ "sort"
+ "strings"
+ "testing"
+ "time"
+)
+
+func setup(t *testing.T, workdir, username string) (tmpdir string) {
+ var err error
+ tmpdir, err = ioutil.TempDir(workdir, "parse-file-info-")
+ if err != nil {
+ t.Fatalf("ioutil.TempDir failed: %v", err)
+ }
+ logfiles := []struct {
+ name string
+ link string
+ age time.Duration
+ }{
+ {"prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345", "", 4 * time.Hour},
+ {"prog1.host.%USER%.log.veyron.INFO.20141204-145040.23456", "prog1.INFO", 1 * time.Hour},
+ {"prog1.host.%USER%.log.veyron.ERROR.20141204-145040.23456", "prog1.ERROR", 1 * time.Hour},
+ {"prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456", "prog2.INFO", 4 * time.Hour},
+ {"prog3.host.otheruser.log.veyron.INFO.20141204-135040.23456", "prog3.INFO", 1 * time.Hour},
+ {"foo.txt", "", 1 * time.Hour},
+ {"bar.txt", "", 1 * time.Hour},
+ }
+ for _, l := range logfiles {
+ l.name = strings.Replace(l.name, "%USER%", username, -1)
+ filename := filepath.Join(tmpdir, l.name)
+ if err := ioutil.WriteFile(filename, []byte{}, 0644); err != nil {
+ t.Fatalf("ioutil.WriteFile failed: %v", err)
+ }
+ mtime := time.Now().Add(-l.age)
+ if err := os.Chtimes(filename, mtime, mtime); err != nil {
+ t.Fatalf("os.Chtimes failed: %v", err)
+ }
+ if l.link != "" {
+ if err := os.Symlink(l.name, filepath.Join(tmpdir, l.link)); err != nil {
+ t.Fatalf("os.Symlink failed: %v", err)
+ }
+ }
+ }
+ if err := os.Mkdir(filepath.Join(tmpdir, "subdir"), 0700); err != nil {
+ t.Fatalf("os.Mkdir failed: %v", err)
+ }
+ return
+}
+
+func TestGCLogs(t *testing.T) {
+ workdir, err := ioutil.TempDir("", "parse-file-info-")
+ if err != nil {
+ t.Fatalf("ioutil.TempDir failed: %v", err)
+ }
+ defer os.RemoveAll(workdir)
+
+ u, err := user.Current()
+ if err != nil {
+ t.Fatalf("user.Current failed: %v", err)
+ }
+
+ cmd := cmdGCLogs
+ var stdout, stderr bytes.Buffer
+ cmd.Init(nil, &stdout, &stderr)
+
+ testcases := []struct {
+ cutoff time.Duration
+ verbose bool
+ dryrun bool
+ expected []string
+ }{
+ {
+ cutoff: 6 * time.Hour,
+ verbose: false,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Number of files deleted: 0`,
+ },
+ },
+ {
+ cutoff: 2 * time.Hour,
+ verbose: false,
+ dryrun: true,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Would delete "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345"`,
+ `Would delete "%TESTDIR%/prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456"`,
+ `Number of files deleted: 0`,
+ },
+ },
+ {
+ cutoff: 2 * time.Hour,
+ verbose: false,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Number of files deleted: 3`,
+ },
+ },
+ {
+ cutoff: 2 * time.Hour,
+ verbose: true,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345"`,
+ `Deleting "%TESTDIR%/prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456"`,
+ `Deleting symlink "%TESTDIR%/prog2.INFO"`,
+ `Not a log file: "%TESTDIR%/bar.txt"`,
+ `Not a log file: "%TESTDIR%/foo.txt"`,
+ `Skipped directory: "%TESTDIR%/subdir"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.INFO"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.host.otheruser.log.veyron.INFO.20141204-135040.23456"`,
+ `Number of files deleted: 3`,
+ },
+ },
+ {
+ cutoff: time.Minute,
+ verbose: true,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.ERROR.20141204-145040.23456"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-145040.23456"`,
+ `Deleting "%TESTDIR%/prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456"`,
+ `Deleting symlink "%TESTDIR%/prog1.ERROR"`,
+ `Deleting symlink "%TESTDIR%/prog1.INFO"`,
+ `Deleting symlink "%TESTDIR%/prog2.INFO"`,
+ `Not a log file: "%TESTDIR%/bar.txt"`,
+ `Not a log file: "%TESTDIR%/foo.txt"`,
+ `Skipped directory: "%TESTDIR%/subdir"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.INFO"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.host.otheruser.log.veyron.INFO.20141204-135040.23456"`,
+ `Number of files deleted: 7`,
+ },
+ },
+ {
+ cutoff: time.Minute,
+ verbose: false,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Number of files deleted: 7`,
+ },
+ },
+ }
+ for _, tc := range testcases {
+ testdir := setup(t, workdir, u.Username)
+ cutoff := fmt.Sprintf("--cutoff=%s", tc.cutoff)
+ verbose := fmt.Sprintf("--verbose=%v", tc.verbose)
+ dryrun := fmt.Sprintf("--n=%v", tc.dryrun)
+ if err := cmd.Execute([]string{cutoff, verbose, dryrun, testdir}); err != nil {
+ t.Fatalf("%v: %v", stderr.String(), err)
+ }
+ gotsl := strings.Split(stdout.String(), "\n")
+ if len(gotsl) >= 2 {
+ sort.Strings(gotsl[1 : len(gotsl)-2])
+ }
+ got := strings.Join(gotsl, "\n")
+ expected := strings.Join(tc.expected, "\n") + "\n"
+ expected = strings.Replace(expected, "%TESTDIR%", testdir, -1)
+ expected = strings.Replace(expected, "%USER%", u.Username, -1)
+ if got != expected {
+ t.Errorf("Unexpected result for (%v, %v): got %q, expected %q", tc.cutoff, tc.verbose, got, expected)
+ }
+ stdout.Reset()
+ }
+}
diff --git a/tools/gclogs/main.go b/tools/gclogs/main.go
new file mode 100644
index 0000000..5cdbead
--- /dev/null
+++ b/tools/gclogs/main.go
@@ -0,0 +1,8 @@
+// The following enables go generate to generate the doc.go file.
+//go:generate go run $VEYRON_ROOT/veyron/go/src/veyron.io/lib/cmdline/testdata/gendoc.go . -help
+
+package main
+
+func main() {
+ cmdGCLogs.Main()
+}
