veyron/tools/gclogs: A tool that safely deletes old log files.
This new tool can be used to safely delete old log files from log
directories.
$ gclogs --help
gclogs is a utility that safely deletes old log files.
It looks for file names that match the format of files produced by the vlog
package, and deletes the ones that have not changed in the amount of time
specified by the --cutoff flag.
Only files produced by the same user as the one running the gclogs command are
considered for deletion.
Usage:
gclogs [flags] <dir> ...
<dir> ... A list of directories where to look for log files.
The gclogs flags are:
-cutoff=24h0m0s
The age cut-off for a log file to be considered for garbage collection.
-n=false
If true, log files that would be deleted are shown on stdout, but not
actually deleted.
-program=.*
A regular expression to apply to the program part of the log file name, e.g
".*test".
-verbose=false
If true, each deleted file is shown on stdout.
Change-Id: I65f8abaf7a03c6d99d9411003d9746127d284107
diff --git a/tools/gclogs/doc.go b/tools/gclogs/doc.go
new file mode 100644
index 0000000..302fe98
--- /dev/null
+++ b/tools/gclogs/doc.go
@@ -0,0 +1,31 @@
+// This file was auto-generated via go generate.
+// DO NOT UPDATE MANUALLY
+
+/*
+gclogs is a utility that safely deletes old log files.
+
+It looks for file names that match the format of files produced by the vlog
+package, and deletes the ones that have not changed in the amount of time
+specified by the --cutoff flag.
+
+Only files produced by the same user as the one running the gclogs command are
+considered for deletion.
+
+Usage:
+ gclogs [flags] <dir> ...
+
+<dir> ... A list of directories where to look for log files.
+
+The gclogs flags are:
+ -cutoff=24h0m0s
+ The age cut-off for a log file to be considered for garbage collection.
+ -n=false
+ If true, log files that would be deleted are shown on stdout, but not
+ actually deleted.
+ -program=.*
+ A regular expression to apply to the program part of the log file name, e.g
+ ".*test".
+ -verbose=false
+ If true, each deleted file is shown on stdout.
+*/
+package main
diff --git a/tools/gclogs/format.go b/tools/gclogs/format.go
new file mode 100644
index 0000000..502db25
--- /dev/null
+++ b/tools/gclogs/format.go
@@ -0,0 +1,69 @@
+package main
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "syscall"
+ "time"
+)
+
+var (
+ // The format of the log file names is:
+ // program.host.user.log.logger.tag.YYYYMMDD-HHmmss.pid
+ logFileRE = regexp.MustCompile(`^(.*)\.([^.]*)\.([^.]*)\.log\.([^.]*)\.([^.]*)\.(........-......)\.([0-9]*)$`)
+)
+
+type logFile struct {
+ // TODO(rthellend): Some of these fields are not used by anything yet,
+ // but they will be eventually when we need to sort the log files
+ // associated with a given instance.
+ symlink bool
+ program, host, user, logger, tag string
+ time time.Time
+ pid int
+}
+
+func parseFileInfo(dir string, fileInfo os.FileInfo) (*logFile, error) {
+ fileName := fileInfo.Name()
+ if fileInfo.Mode()&os.ModeSymlink != 0 {
+ buf := make([]byte, syscall.NAME_MAX)
+ n, err := syscall.Readlink(filepath.Join(dir, fileName), buf)
+ if err != nil {
+ return nil, err
+ }
+ linkName := string(buf[:n])
+ lf, err := parseFileName(linkName)
+ if err != nil {
+ return nil, err
+ }
+ lf.symlink = true
+ return lf, nil
+ }
+ return parseFileName(fileName)
+}
+
+func parseFileName(fileName string) (*logFile, error) {
+ if m := logFileRE.FindStringSubmatch(fileName); len(m) != 0 {
+ t, err := time.ParseInLocation("20060102-150405", m[6], time.Local)
+ if err != nil {
+ return nil, err
+ }
+ pid, err := strconv.ParseInt(m[7], 10, 32)
+ if err != nil {
+ return nil, err
+ }
+ return &logFile{
+ program: m[1],
+ host: m[2],
+ user: m[3],
+ logger: m[4],
+ tag: m[5],
+ time: t,
+ pid: int(pid),
+ }, nil
+ }
+ return nil, errors.New("not a recognized log file name")
+}
diff --git a/tools/gclogs/format_test.go b/tools/gclogs/format_test.go
new file mode 100644
index 0000000..5396894
--- /dev/null
+++ b/tools/gclogs/format_test.go
@@ -0,0 +1,97 @@
+package main
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+ "time"
+)
+
+func TestParseFileNameNoError(t *testing.T) {
+ testcases := []struct {
+ filename string
+ lf *logFile
+ }{
+ {
+ "program.host.user.log.veyron.INFO.20141204-131502.12345",
+ &logFile{false, "program", "host", "user", "veyron", "INFO", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345},
+ },
+ {
+ "prog.test.host-name.user.log.veyron.ERROR.20141204-131502.12345",
+ &logFile{false, "prog.test", "host-name", "user", "veyron", "ERROR", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345},
+ },
+ }
+ for _, tc := range testcases {
+ lf, err := parseFileName(tc.filename)
+ if err != nil {
+ t.Errorf("unexpected error for %q: %v", tc.filename, err)
+ }
+ if !reflect.DeepEqual(tc.lf, lf) {
+ t.Errorf("unexpected result: got %+v, expected %+v", lf, tc.lf)
+ }
+ }
+}
+
+func TestParseFileNameError(t *testing.T) {
+ testcases := []string{
+ "program.host.user.log.veyron.INFO.20141204-131502",
+ "prog.test.host-name.user.log.veyron.20141204-131502.12345",
+ "foo.txt",
+ }
+ for _, tc := range testcases {
+ if _, err := parseFileName(tc); err == nil {
+ t.Errorf("unexpected success for %q", tc)
+ }
+ }
+}
+
+func TestParseFileInfo(t *testing.T) {
+ tmpdir, err := ioutil.TempDir("", "parse-file-info-")
+ if err != nil {
+ t.Fatalf("ioutil.TempDir failed: %v", err)
+ }
+ defer os.RemoveAll(tmpdir)
+
+ name := "program.host.user.log.veyron.INFO.20141204-131502.12345"
+ if err := ioutil.WriteFile(filepath.Join(tmpdir, name), []byte{}, 0644); err != nil {
+ t.Fatalf("ioutil.WriteFile failed: %v", err)
+ }
+ link := "program.INFO"
+ if err := os.Symlink(name, filepath.Join(tmpdir, link)); err != nil {
+ t.Fatalf("os.Symlink failed: %v", err)
+ }
+
+ // Test regular file.
+ {
+ fi, err := os.Lstat(filepath.Join(tmpdir, name))
+ if err != nil {
+ t.Fatalf("os.Lstat failed: %v", err)
+ }
+ lf, err := parseFileInfo(tmpdir, fi)
+ if err != nil {
+ t.Errorf("parseFileInfo(%v, %v) failed: %v", tmpdir, fi, err)
+ }
+ expected := &logFile{false, "program", "host", "user", "veyron", "INFO", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345}
+ if !reflect.DeepEqual(lf, expected) {
+ t.Errorf("unexpected result: got %+v, expected %+v", lf, expected)
+ }
+ }
+
+ // Test symlink.
+ {
+ fi, err := os.Lstat(filepath.Join(tmpdir, link))
+ if err != nil {
+ t.Fatalf("os.Lstat failed: %v", err)
+ }
+ lf, err := parseFileInfo(tmpdir, fi)
+ if err != nil {
+ t.Errorf("parseFileInfo(%v, %v) failed: %v", tmpdir, fi, err)
+ }
+ expected := &logFile{true, "program", "host", "user", "veyron", "INFO", time.Date(2014, 12, 4, 13, 15, 2, 0, time.Local), 12345}
+ if !reflect.DeepEqual(lf, expected) {
+ t.Errorf("unexpected result: got %+v, expected %+v", lf, expected)
+ }
+ }
+}
diff --git a/tools/gclogs/gclogs.go b/tools/gclogs/gclogs.go
new file mode 100644
index 0000000..6fde8af
--- /dev/null
+++ b/tools/gclogs/gclogs.go
@@ -0,0 +1,157 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "os"
+ "os/user"
+ "path/filepath"
+ "regexp"
+ "time"
+
+ "veyron.io/lib/cmdline"
+)
+
+var (
+ flagCutoff time.Duration
+ flagProgname string
+ flagVerbose bool
+ flagDryrun bool
+
+ cmdGCLogs = &cmdline.Command{
+ Run: garbageCollectLogs,
+ Name: "gclogs",
+ Short: "gclogs is a utility that safely deletes old log files.",
+ Long: `
+gclogs is a utility that safely deletes old log files.
+
+It looks for file names that match the format of files produced by the vlog
+package, and deletes the ones that have not changed in the amount of time
+specified by the --cutoff flag.
+
+Only files produced by the same user as the one running the gclogs command
+are considered for deletion.
+`,
+ ArgsName: "<dir> ...",
+ ArgsLong: "<dir> ... A list of directories where to look for log files.",
+ }
+)
+
+func init() {
+ cmdGCLogs.Flags.DurationVar(&flagCutoff, "cutoff", 24*time.Hour, "The age cut-off for a log file to be considered for garbage collection.")
+ cmdGCLogs.Flags.StringVar(&flagProgname, "program", ".*", `A regular expression to apply to the program part of the log file name, e.g ".*test".`)
+ cmdGCLogs.Flags.BoolVar(&flagVerbose, "verbose", false, "If true, each deleted file is shown on stdout.")
+ cmdGCLogs.Flags.BoolVar(&flagDryrun, "n", false, "If true, log files that would be deleted are shown on stdout, but not actually deleted.")
+}
+
+func garbageCollectLogs(cmd *cmdline.Command, args []string) error {
+ if len(args) == 0 {
+ cmd.UsageErrorf("gclogs requires at least one argument")
+ }
+ timeCutoff := time.Now().Add(-flagCutoff)
+ currentUser, err := user.Current()
+ if err != nil {
+ return err
+ }
+ programRE, err := regexp.Compile(flagProgname)
+ if err != nil {
+ return err
+ }
+ var lastErr error
+ for _, logdir := range args {
+ if err := processDirectory(cmd, logdir, timeCutoff, programRE, currentUser.Username); err != nil {
+ lastErr = err
+ }
+ }
+ return lastErr
+}
+
+func processDirectory(cmd *cmdline.Command, logdir string, timeCutoff time.Time, programRE *regexp.Regexp, username string) error {
+ fmt.Fprintf(cmd.Stdout(), "Processing: %q\n", logdir)
+
+ f, err := os.Open(logdir)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ var lastErr error
+ deleted := 0
+ symlinks := []string{}
+ for {
+ fi, err := f.Readdir(100)
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ lastErr = err
+ break
+ }
+ for _, file := range fi {
+ fullname := filepath.Join(logdir, file.Name())
+ if file.IsDir() {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Skipped directory: %q\n", fullname)
+ }
+ continue
+ }
+ lf, err := parseFileInfo(logdir, file)
+ if err != nil {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Not a log file: %q\n", fullname)
+ }
+ continue
+ }
+ if lf.user != username {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Skipped log file created by other user: %q\n", fullname)
+ }
+ continue
+ }
+ if !programRE.MatchString(lf.program) {
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Skipped log file doesn't match %q: %q\n", flagProgname, fullname)
+ }
+ continue
+ }
+ if lf.symlink {
+ symlinks = append(symlinks, fullname)
+ continue
+ }
+ if file.ModTime().Before(timeCutoff) {
+ if flagDryrun {
+ fmt.Fprintf(cmd.Stdout(), "Would delete %q\n", fullname)
+ continue
+ }
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Deleting %q\n", fullname)
+ }
+ if err := os.Remove(fullname); err != nil {
+ lastErr = err
+ } else {
+ deleted++
+ }
+ }
+ }
+ }
+ // Delete broken links.
+ for _, sl := range symlinks {
+ if _, err := os.Stat(sl); err != nil && os.IsNotExist(err) {
+ if flagDryrun {
+ fmt.Fprintf(cmd.Stdout(), "Would delete symlink %q\n", sl)
+ continue
+ }
+ if flagVerbose {
+ fmt.Fprintf(cmd.Stdout(), "Deleting symlink %q\n", sl)
+ }
+ if err := os.Remove(sl); err != nil {
+ lastErr = err
+ } else {
+ deleted++
+ }
+ }
+
+ }
+ fmt.Fprintf(cmd.Stdout(), "Number of files deleted: %d\n", deleted)
+ return lastErr
+}
diff --git a/tools/gclogs/gclogs_test.go b/tools/gclogs/gclogs_test.go
new file mode 100644
index 0000000..f9168fe
--- /dev/null
+++ b/tools/gclogs/gclogs_test.go
@@ -0,0 +1,177 @@
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/user"
+ "path/filepath"
+ "sort"
+ "strings"
+ "testing"
+ "time"
+)
+
+func setup(t *testing.T, workdir, username string) (tmpdir string) {
+ var err error
+ tmpdir, err = ioutil.TempDir(workdir, "parse-file-info-")
+ if err != nil {
+ t.Fatalf("ioutil.TempDir failed: %v", err)
+ }
+ logfiles := []struct {
+ name string
+ link string
+ age time.Duration
+ }{
+ {"prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345", "", 4 * time.Hour},
+ {"prog1.host.%USER%.log.veyron.INFO.20141204-145040.23456", "prog1.INFO", 1 * time.Hour},
+ {"prog1.host.%USER%.log.veyron.ERROR.20141204-145040.23456", "prog1.ERROR", 1 * time.Hour},
+ {"prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456", "prog2.INFO", 4 * time.Hour},
+ {"prog3.host.otheruser.log.veyron.INFO.20141204-135040.23456", "prog3.INFO", 1 * time.Hour},
+ {"foo.txt", "", 1 * time.Hour},
+ {"bar.txt", "", 1 * time.Hour},
+ }
+ for _, l := range logfiles {
+ l.name = strings.Replace(l.name, "%USER%", username, -1)
+ filename := filepath.Join(tmpdir, l.name)
+ if err := ioutil.WriteFile(filename, []byte{}, 0644); err != nil {
+ t.Fatalf("ioutil.WriteFile failed: %v", err)
+ }
+ mtime := time.Now().Add(-l.age)
+ if err := os.Chtimes(filename, mtime, mtime); err != nil {
+ t.Fatalf("os.Chtimes failed: %v", err)
+ }
+ if l.link != "" {
+ if err := os.Symlink(l.name, filepath.Join(tmpdir, l.link)); err != nil {
+ t.Fatalf("os.Symlink failed: %v", err)
+ }
+ }
+ }
+ if err := os.Mkdir(filepath.Join(tmpdir, "subdir"), 0700); err != nil {
+ t.Fatalf("os.Mkdir failed: %v", err)
+ }
+ return
+}
+
+func TestGCLogs(t *testing.T) {
+ workdir, err := ioutil.TempDir("", "parse-file-info-")
+ if err != nil {
+ t.Fatalf("ioutil.TempDir failed: %v", err)
+ }
+ defer os.RemoveAll(workdir)
+
+ u, err := user.Current()
+ if err != nil {
+ t.Fatalf("user.Current failed: %v", err)
+ }
+
+ cmd := cmdGCLogs
+ var stdout, stderr bytes.Buffer
+ cmd.Init(nil, &stdout, &stderr)
+
+ testcases := []struct {
+ cutoff time.Duration
+ verbose bool
+ dryrun bool
+ expected []string
+ }{
+ {
+ cutoff: 6 * time.Hour,
+ verbose: false,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Number of files deleted: 0`,
+ },
+ },
+ {
+ cutoff: 2 * time.Hour,
+ verbose: false,
+ dryrun: true,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Would delete "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345"`,
+ `Would delete "%TESTDIR%/prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456"`,
+ `Number of files deleted: 0`,
+ },
+ },
+ {
+ cutoff: 2 * time.Hour,
+ verbose: false,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Number of files deleted: 3`,
+ },
+ },
+ {
+ cutoff: 2 * time.Hour,
+ verbose: true,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345"`,
+ `Deleting "%TESTDIR%/prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456"`,
+ `Deleting symlink "%TESTDIR%/prog2.INFO"`,
+ `Not a log file: "%TESTDIR%/bar.txt"`,
+ `Not a log file: "%TESTDIR%/foo.txt"`,
+ `Skipped directory: "%TESTDIR%/subdir"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.INFO"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.host.otheruser.log.veyron.INFO.20141204-135040.23456"`,
+ `Number of files deleted: 3`,
+ },
+ },
+ {
+ cutoff: time.Minute,
+ verbose: true,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.ERROR.20141204-145040.23456"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-131502.12345"`,
+ `Deleting "%TESTDIR%/prog1.host.%USER%.log.veyron.INFO.20141204-145040.23456"`,
+ `Deleting "%TESTDIR%/prog2.host.%USER%.log.veyron.INFO.20141204-135040.23456"`,
+ `Deleting symlink "%TESTDIR%/prog1.ERROR"`,
+ `Deleting symlink "%TESTDIR%/prog1.INFO"`,
+ `Deleting symlink "%TESTDIR%/prog2.INFO"`,
+ `Not a log file: "%TESTDIR%/bar.txt"`,
+ `Not a log file: "%TESTDIR%/foo.txt"`,
+ `Skipped directory: "%TESTDIR%/subdir"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.INFO"`,
+ `Skipped log file created by other user: "%TESTDIR%/prog3.host.otheruser.log.veyron.INFO.20141204-135040.23456"`,
+ `Number of files deleted: 7`,
+ },
+ },
+ {
+ cutoff: time.Minute,
+ verbose: false,
+ dryrun: false,
+ expected: []string{
+ `Processing: "%TESTDIR%"`,
+ `Number of files deleted: 7`,
+ },
+ },
+ }
+ for _, tc := range testcases {
+ testdir := setup(t, workdir, u.Username)
+ cutoff := fmt.Sprintf("--cutoff=%s", tc.cutoff)
+ verbose := fmt.Sprintf("--verbose=%v", tc.verbose)
+ dryrun := fmt.Sprintf("--n=%v", tc.dryrun)
+ if err := cmd.Execute([]string{cutoff, verbose, dryrun, testdir}); err != nil {
+ t.Fatalf("%v: %v", stderr.String(), err)
+ }
+ gotsl := strings.Split(stdout.String(), "\n")
+ if len(gotsl) >= 2 {
+ sort.Strings(gotsl[1 : len(gotsl)-2])
+ }
+ got := strings.Join(gotsl, "\n")
+ expected := strings.Join(tc.expected, "\n") + "\n"
+ expected = strings.Replace(expected, "%TESTDIR%", testdir, -1)
+ expected = strings.Replace(expected, "%USER%", u.Username, -1)
+ if got != expected {
+ t.Errorf("Unexpected result for (%v, %v): got %q, expected %q", tc.cutoff, tc.verbose, got, expected)
+ }
+ stdout.Reset()
+ }
+}
diff --git a/tools/gclogs/main.go b/tools/gclogs/main.go
new file mode 100644
index 0000000..5cdbead
--- /dev/null
+++ b/tools/gclogs/main.go
@@ -0,0 +1,8 @@
+// The following enables go generate to generate the doc.go file.
+//go:generate go run $VEYRON_ROOT/veyron/go/src/veyron.io/lib/cmdline/testdata/gendoc.go . -help
+
+package main
+
+func main() {
+ cmdGCLogs.Main()
+}
