go/src/github.com/steveyen/gkvlite/tools/slab/slab.go - third_party - Git at Google

 // ignore this package, not needed by veyron yet. avoid dependency on go-slab
 // +build ignore

 package main

 // Test integration of gkvlite with go-slab, using gkvlite's optional
 // ItemAddRef/ItemDecRef() callbacks to integrate with a slab memory
 // allocator.

 import (
 	"bytes"
 	"flag"
 	"fmt"
 	"io"
 	"log"
 	"math/rand"
 	"os"
 	"sort"
 	"testing"
 	"unsafe"

 	"github.com/steveyen/gkvlite"
 	"github.com/steveyen/go-slab"
 )

 var maxOps = flag.Int("ops", 0,
 	"max number of ops; 0 means run forever")
 var maxItems = flag.Int("n", 10000,
 	"max number of items")
 var maxItemBytes = flag.Int("maxItemBytes", 20000,
 	"max bytes for an item")
 var pctSets = flag.Int("sets", 50,
 	"percentage of sets; 50 means 50% sets")
 var pctDeletes = flag.Int("deletes", 5,
 	"percentage of deletes; 5 means 5% deletes")
 var pctEvicts = flag.Int("evicts", 3,
 	"percentage of evicts; 3 means 3% evicts")
 var pctReopens = flag.Int("reopens", 0,
 	"percentage of reopens; 0 means 0% reopens")
 var useSlab = flag.Bool("useSlab", true,
 	"whether to use slab allocator")
 var flushEvery = flag.Int("flushEvery", 0,
 	"flush every N ops; 0 means no flushing")
 var slabStats = flag.Bool("slabStats", false,
 	"whether to emit slab stats")

 func usage() {
 	fmt.Fprintf(os.Stderr, "gkvlite slab testing tool\n")
 	fmt.Fprintf(os.Stderr, "\nusage: %s [flags] dbFileName\n", os.Args[0])
 	fmt.Fprintf(os.Stderr, "  dbFileName - file name used during test\n")
 	fmt.Fprintf(os.Stderr, "\nexamples:\n")
 	fmt.Fprintf(os.Stderr, "  ./slab -n 1000 -sets 30 test.tmp\n")
 	fmt.Fprintf(os.Stderr, "\nflags:\n")
 	flag.PrintDefaults()
 	fmt.Fprintf(os.Stderr, "  -h: print this usage/help message\n")
 }

 func main() {
 	flag.Usage = usage
 	flag.Parse()
 	args := flag.Args()
 	if len(args) < 1 {
 		log.Fatalf("error: missing dbFileName")
 	}
 	if *pctSets + *pctDeletes + *pctEvicts + *pctReopens > 100 {
 		log.Fatalf("error: percentages are > 100%% (%d%%+%d%%+%d%%+%d%%)",
 			*pctSets, *pctDeletes, *pctEvicts + *pctReopens)
 	}
 	run(args[0], *useSlab, *flushEvery, *maxItemBytes,
 		*maxOps, *maxItems, *pctSets, *pctDeletes, *pctEvicts, *pctReopens)
 }

 // Helper function to read a contiguous byte sequence, splitting
 // it up into chained buf's of maxBufSize.  The last buf in the
 // chain can have length <= maxBufSize.
 func readBufChain(arena *slab.Arena, maxBufSize int, r io.ReaderAt, offset int64,
 	valLength uint32) ([]byte, error) {
 	n := int(valLength)
 	if n > maxBufSize {
 		n = maxBufSize
 	}
 	b := arena.Alloc(n)
 	_, err := r.ReadAt(b, offset)
 	if err != nil {
 		arena.DecRef(b)
 		return nil, err
 	}
 	remaining := valLength - uint32(n)
 	if remaining > 0 {
 		next, err := readBufChain(arena, maxBufSize, r, offset + int64(n), remaining)
 		if err != nil {
 			arena.DecRef(b)
 			return nil, err
 		}
 		arena.SetNext(b, next)
 	}
 	return b, nil
 }

 type ItemNode struct {
 	refs int
 	next *ItemNode
 	item gkvlite.Item
 }

 var freeItemNodes *ItemNode

 func setupStoreArena(t *testing.T, maxBufSize int) (
 	*slab.Arena, gkvlite.StoreCallbacks) {
 	arena := slab.NewArena(48, // The smallest slab class "chunk size" is 48 bytes.
 		1024*1024, // Each slab will be 1MB in size.
 		2,         // Power of 2 growth in "chunk sizes".
 		nil)       // Use default make([]byte) for slab memory.
 	if arena == nil {
 		t.Errorf("expected arena")
 	}

 	itemValLength := func(c *gkvlite.Collection, i *gkvlite.Item) int {
 		if i.Val == nil {
 			if t != nil {
 				t.Fatalf("itemValLength on nil i.Val, i: %#v", i)
 			} else {
 				panic(fmt.Sprintf("itemValLength on nil i.Val, i: %#v", i))
 			}
 		}
 		s := 0
 		b := i.Val
 		for b != nil {
 			s = s + len(b)
 			n := arena.GetNext(b)
 			if s > len(b) {
 				arena.DecRef(b)
 			}
 			b = n
 		}
 		return s
 	}
 	itemValWrite := func(c *gkvlite.Collection,
 		i *gkvlite.Item, w io.WriterAt, offset int64) error {
 		s := 0
 		b := i.Val
 		for b != nil {
 			_, err := w.WriteAt(b, offset + int64(s))
 			if err != nil {
 				return err
 			}
 			s = s + len(b)
 			n := arena.GetNext(b)
 			if s > len(b) {
 				arena.DecRef(b)
 			}
 			b = n
 		}
 		return nil
 	}
 	itemValRead := func(c *gkvlite.Collection,
 		i *gkvlite.Item, r io.ReaderAt, offset int64, valLength uint32) error {
 		b, err := readBufChain(arena, maxBufSize, r, offset, valLength)
 		if err != nil {
 			return err
 		}
 		i.Val = b
 		return nil
 	}
 	itemAlloc := func(c *gkvlite.Collection, keyLength uint16) *gkvlite.Item {
 		var n *ItemNode
 		if freeItemNodes != nil {
 			n = freeItemNodes
 			freeItemNodes = freeItemNodes.next
 			n.next = nil
 		} else {
 			n = &ItemNode{}
 			n.item.Transient = unsafe.Pointer(n)
 		}
 		if n.refs != 0 ||
 			n.item.Key != nil || n.item.Val != nil || n.item.Priority != 0 ||
 			n.item.Transient != unsafe.Pointer(n) {
 			panic("unexpected ItemNode refs or item fields")
 		}
 		n.refs = 1
 		n.next = nil
 		n.item.Key = arena.Alloc(int(keyLength))
 		return &n.item
 	}
 	itemAddRef := func(c *gkvlite.Collection, i *gkvlite.Item) {
 		n := (*ItemNode)(i.Transient)
 		n.refs++
 	}
 	itemDecRef := func(c *gkvlite.Collection, i *gkvlite.Item) {
 		n := (*ItemNode)(i.Transient)
 		n.refs--
 		if n.refs == 0 {
 			if i.Key == nil {
 				panic("expected a Key")
 			}
 			arena.DecRef(i.Key)
 			i.Key = nil
 			if i.Val != nil {
 				arena.DecRef(i.Val)
 				i.Val = nil
 			}
 			i.Priority = 0
 			n.next = freeItemNodes
 			freeItemNodes = n
 		}
 	}

 	scb := gkvlite.StoreCallbacks{
 		ItemValLength: itemValLength,
 		ItemValWrite:  itemValWrite,
 		ItemValRead:   itemValRead,
 		ItemAlloc:     itemAlloc,
 		ItemAddRef:    itemAddRef,
 		ItemDecRef:    itemDecRef,
 	}
 	return arena, scb
 }

 func run(fname string, useSlab bool, flushEvery int, maxItemBytes int,
 	maxOps, maxItems, pctSets, pctDeletes, pctEvicts, pctReopens int) {
 	fmt.Printf("fname: %s, useSlab: %v, flushEvery: %d" +
 		", maxItemBytes: %d, maxOps: %d, maxItems: %d" +
 		", pctSets: %d, pctDeletes; %d, pctEvicts: %d, pctReopens: %d\n",
 		fname, useSlab, flushEvery, maxItemBytes,
 		maxOps, maxItems, pctSets, pctDeletes, pctEvicts, pctReopens)

 	os.Remove(fname)
 	f, err := os.Create(fname)
 	if err != nil {
 		panic(fmt.Sprintf("error: could not create file: %s, err: %v", fname, err))
 	}

 	arenaStats := map[string]int64{}
 	arena, scb := setupStoreArena(nil, 256)
 	if !useSlab {
 		arena = nil
 		scb = gkvlite.StoreCallbacks{
 			ItemValLength: func(c *gkvlite.Collection, i *gkvlite.Item) int {
 				return len(i.Val)
 			},
 		}
 	}

 	start := func(f *os.File) (*gkvlite.Store, *gkvlite.Collection) {
 		s, err := gkvlite.NewStoreEx(f, scb)
 		if err != nil || s == nil {
 			panic(fmt.Sprintf("error: expected NewStoreEx to work, err: %v", err))
 		}
 		x := s.SetCollection("x", bytes.Compare)
 		if x == nil {
 			panic(fmt.Sprintf("error: expected SetColl/GetColl to work"))
 		}
 		return s, x
 	}

 	s, x := start(f)

 	stop := func() {
 		s.Flush()
 		s.Close()
 		f.Close()
 		f = nil
 	}

 	numGets := 0
 	numSets := 0
 	numDeletes := 0
 	numEvicts := 0
 	numReopens := 0
 	numFlushes := 0

 	psd := pctSets + pctDeletes
 	psde := pctSets + pctDeletes + pctEvicts
 	psder := pctSets + pctDeletes + pctEvicts + pctReopens

 	for i := 0; maxOps <= 0 || i < maxOps; i++ {
 		kr := rand.Int() % maxItems
 		kr4 := kr * kr * kr * kr
 		if kr4 > maxItemBytes {
 			kr4 = maxItemBytes
 		}
 		ks := fmt.Sprintf("%03d", kr)
 		k := []byte(ks)
 		r := rand.Int() % 100
 		if r < pctSets {
 			numSets++
 			var b []byte
 			if arena != nil {
 				b = arena.Alloc(kr4)
 			} else {
 				b = make([]byte, kr4)
 			}
 			pri := rand.Int31()
 			var it *gkvlite.Item
 			if scb.ItemAlloc != nil {
 				it = scb.ItemAlloc(x, uint16(len(k)))
 			} else {
 				it = &gkvlite.Item{Key: make([]byte, len(k))}
 			}
 			copy(it.Key, k)
 			it.Val = b
 			it.Priority = pri
 			err := x.SetItem(it)
 			if err != nil {
 				panic(fmt.Sprintf("error: expected nil error, got: %v", err))
 			}
 			if scb.ItemDecRef != nil {
 				scb.ItemDecRef(x, it)
 			}
 		} else if r < psd {
 			numDeletes++
 			_, err := x.Delete(k)
 			if err != nil {
 				panic(fmt.Sprintf("error: expected nil error, got: %v", err))
 			}
 		} else if r < psde {
 			numEvicts++
 			x.EvictSomeItems()
 		} else if r < psder {
 			numReopens++
 			stop()
 			f, _ = os.OpenFile(fname, os.O_RDWR, 0666)
 			s, x = start(f)
 		} else {
 			numGets++
 			i, err := x.GetItem(k, true)
 			if err != nil {
 				panic(fmt.Sprintf("error: expected nil error, got: %v", err))
 			}
 			if i != nil {
 				if scb.ItemValLength(x, i) != kr4 {
 					panic(fmt.Sprintf("error: expected len: %d, got %d",
 						kr4, scb.ItemValLength(x, i)))
 				}
 				if scb.ItemDecRef != nil {
 					scb.ItemDecRef(x, i)
 				}
 			}
 		}

 		if flushEvery > 0 && i % flushEvery == 0 {
 			numFlushes++
 			s.Flush()
 		}

 		if i % 10000 == 0 {
 			if arena != nil && *slabStats == true {
 				arena.Stats(arenaStats)
 				mk := []string{}
 				for k, _ := range arenaStats {
 					mk = append(mk, k)
 				}
 				sort.Strings(mk)
 				for _, k := range mk {
 					log.Printf("%s = %d", k, arenaStats[k])
 				}
 			}
 			log.Printf("i: %d, numGets: %d, numSets: %d, numDeletes: %d" +
 				", numEvicts: %d, numReopens: %d, numFlushes: %d\n",
 				i, numGets, numSets, numDeletes, numEvicts, numReopens, numFlushes)
 		}
 	}
 }
	// ignore this package, not needed by veyron yet. avoid dependency on go-slab
	// +build ignore

	package main

	// Test integration of gkvlite with go-slab, using gkvlite's optional
	// ItemAddRef/ItemDecRef() callbacks to integrate with a slab memory
	// allocator.

	import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"os"
	"sort"
	"testing"
	"unsafe"

	"github.com/steveyen/gkvlite"
	"github.com/steveyen/go-slab"
	)

	var maxOps = flag.Int("ops", 0,
	"max number of ops; 0 means run forever")
	var maxItems = flag.Int("n", 10000,
	"max number of items")
	var maxItemBytes = flag.Int("maxItemBytes", 20000,
	"max bytes for an item")
	var pctSets = flag.Int("sets", 50,
	"percentage of sets; 50 means 50% sets")
	var pctDeletes = flag.Int("deletes", 5,
	"percentage of deletes; 5 means 5% deletes")
	var pctEvicts = flag.Int("evicts", 3,
	"percentage of evicts; 3 means 3% evicts")
	var pctReopens = flag.Int("reopens", 0,
	"percentage of reopens; 0 means 0% reopens")
	var useSlab = flag.Bool("useSlab", true,
	"whether to use slab allocator")
	var flushEvery = flag.Int("flushEvery", 0,
	"flush every N ops; 0 means no flushing")
	var slabStats = flag.Bool("slabStats", false,
	"whether to emit slab stats")

	func usage() {
	fmt.Fprintf(os.Stderr, "gkvlite slab testing tool\n")
	fmt.Fprintf(os.Stderr, "\nusage: %s [flags] dbFileName\n", os.Args[0])
	fmt.Fprintf(os.Stderr, " dbFileName - file name used during test\n")
	fmt.Fprintf(os.Stderr, "\nexamples:\n")
	fmt.Fprintf(os.Stderr, " ./slab -n 1000 -sets 30 test.tmp\n")
	fmt.Fprintf(os.Stderr, "\nflags:\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, " -h: print this usage/help message\n")
	}

	func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()
	if len(args) < 1 {
	log.Fatalf("error: missing dbFileName")
	}
	if pctSets + pctDeletes + pctEvicts + pctReopens > 100 {
	log.Fatalf("error: percentages are > 100%% (%d%%+%d%%+%d%%+%d%%)",
	pctSets, pctDeletes, pctEvicts + pctReopens)
	}
	run(args[0], useSlab, flushEvery, *maxItemBytes,
	maxOps, maxItems, pctSets, pctDeletes, pctEvicts, pctReopens)
	}

	// Helper function to read a contiguous byte sequence, splitting
	// it up into chained buf's of maxBufSize. The last buf in the
	// chain can have length <= maxBufSize.
	func readBufChain(arena *slab.Arena, maxBufSize int, r io.ReaderAt, offset int64,
	valLength uint32) ([]byte, error) {
	n := int(valLength)
	if n > maxBufSize {
	n = maxBufSize
	}
	b := arena.Alloc(n)
	_, err := r.ReadAt(b, offset)
	if err != nil {
	arena.DecRef(b)
	return nil, err
	}
	remaining := valLength - uint32(n)
	if remaining > 0 {
	next, err := readBufChain(arena, maxBufSize, r, offset + int64(n), remaining)
	if err != nil {
	arena.DecRef(b)
	return nil, err
	}
	arena.SetNext(b, next)
	}
	return b, nil
	}

	type ItemNode struct {
	refs int
	next *ItemNode
	item gkvlite.Item
	}

	var freeItemNodes *ItemNode

	func setupStoreArena(t *testing.T, maxBufSize int) (
	*slab.Arena, gkvlite.StoreCallbacks) {
	arena := slab.NewArena(48, // The smallest slab class "chunk size" is 48 bytes.
	1024*1024, // Each slab will be 1MB in size.
	2, // Power of 2 growth in "chunk sizes".
	nil) // Use default make([]byte) for slab memory.
	if arena == nil {
	t.Errorf("expected arena")
	}

	itemValLength := func(c gkvlite.Collection, i gkvlite.Item) int {
	if i.Val == nil {
	if t != nil {
	t.Fatalf("itemValLength on nil i.Val, i: %#v", i)
	} else {
	panic(fmt.Sprintf("itemValLength on nil i.Val, i: %#v", i))
	}
	}
	s := 0
	b := i.Val
	for b != nil {
	s = s + len(b)
	n := arena.GetNext(b)
	if s > len(b) {
	arena.DecRef(b)
	}
	b = n
	}
	return s
	}
	itemValWrite := func(c *gkvlite.Collection,
	i *gkvlite.Item, w io.WriterAt, offset int64) error {
	s := 0
	b := i.Val
	for b != nil {
	_, err := w.WriteAt(b, offset + int64(s))
	if err != nil {
	return err
	}
	s = s + len(b)
	n := arena.GetNext(b)
	if s > len(b) {
	arena.DecRef(b)
	}
	b = n
	}
	return nil
	}
	itemValRead := func(c *gkvlite.Collection,
	i *gkvlite.Item, r io.ReaderAt, offset int64, valLength uint32) error {
	b, err := readBufChain(arena, maxBufSize, r, offset, valLength)
	if err != nil {
	return err
	}
	i.Val = b
	return nil
	}
	itemAlloc := func(c gkvlite.Collection, keyLength uint16) gkvlite.Item {
	var n *ItemNode
	if freeItemNodes != nil {
	n = freeItemNodes
	freeItemNodes = freeItemNodes.next
	n.next = nil
	} else {
	n = &ItemNode{}
	n.item.Transient = unsafe.Pointer(n)
	}
	if n.refs != 0 \|\|
	n.item.Key != nil \|\| n.item.Val != nil \|\| n.item.Priority != 0 \|\|
	n.item.Transient != unsafe.Pointer(n) {
	panic("unexpected ItemNode refs or item fields")
	}
	n.refs = 1
	n.next = nil
	n.item.Key = arena.Alloc(int(keyLength))
	return &n.item
	}
	itemAddRef := func(c gkvlite.Collection, i gkvlite.Item) {
	n := (*ItemNode)(i.Transient)
	n.refs++
	}
	itemDecRef := func(c gkvlite.Collection, i gkvlite.Item) {
	n := (*ItemNode)(i.Transient)
	n.refs--
	if n.refs == 0 {
	if i.Key == nil {
	panic("expected a Key")
	}
	arena.DecRef(i.Key)
	i.Key = nil
	if i.Val != nil {
	arena.DecRef(i.Val)
	i.Val = nil
	}
	i.Priority = 0
	n.next = freeItemNodes
	freeItemNodes = n
	}
	}

	scb := gkvlite.StoreCallbacks{
	ItemValLength: itemValLength,
	ItemValWrite: itemValWrite,
	ItemValRead: itemValRead,
	ItemAlloc: itemAlloc,
	ItemAddRef: itemAddRef,
	ItemDecRef: itemDecRef,
	}
	return arena, scb
	}

	func run(fname string, useSlab bool, flushEvery int, maxItemBytes int,
	maxOps, maxItems, pctSets, pctDeletes, pctEvicts, pctReopens int) {
	fmt.Printf("fname: %s, useSlab: %v, flushEvery: %d" +
	", maxItemBytes: %d, maxOps: %d, maxItems: %d" +
	", pctSets: %d, pctDeletes; %d, pctEvicts: %d, pctReopens: %d\n",
	fname, useSlab, flushEvery, maxItemBytes,
	maxOps, maxItems, pctSets, pctDeletes, pctEvicts, pctReopens)

	os.Remove(fname)
	f, err := os.Create(fname)
	if err != nil {
	panic(fmt.Sprintf("error: could not create file: %s, err: %v", fname, err))
	}

	arenaStats := map[string]int64{}
	arena, scb := setupStoreArena(nil, 256)
	if !useSlab {
	arena = nil
	scb = gkvlite.StoreCallbacks{
	ItemValLength: func(c gkvlite.Collection, i gkvlite.Item) int {
	return len(i.Val)
	},
	}
	}

	start := func(f os.File) (gkvlite.Store, *gkvlite.Collection) {
	s, err := gkvlite.NewStoreEx(f, scb)
	if err != nil \|\| s == nil {
	panic(fmt.Sprintf("error: expected NewStoreEx to work, err: %v", err))
	}
	x := s.SetCollection("x", bytes.Compare)
	if x == nil {
	panic(fmt.Sprintf("error: expected SetColl/GetColl to work"))
	}
	return s, x
	}

	s, x := start(f)

	stop := func() {
	s.Flush()
	s.Close()
	f.Close()
	f = nil
	}

	numGets := 0
	numSets := 0
	numDeletes := 0
	numEvicts := 0
	numReopens := 0
	numFlushes := 0

	psd := pctSets + pctDeletes
	psde := pctSets + pctDeletes + pctEvicts
	psder := pctSets + pctDeletes + pctEvicts + pctReopens

	for i := 0; maxOps <= 0 \|\| i < maxOps; i++ {
	kr := rand.Int() % maxItems
	kr4 := kr * kr * kr * kr
	if kr4 > maxItemBytes {
	kr4 = maxItemBytes
	}
	ks := fmt.Sprintf("%03d", kr)
	k := []byte(ks)
	r := rand.Int() % 100
	if r < pctSets {
	numSets++
	var b []byte
	if arena != nil {
	b = arena.Alloc(kr4)
	} else {
	b = make([]byte, kr4)
	}
	pri := rand.Int31()
	var it *gkvlite.Item
	if scb.ItemAlloc != nil {
	it = scb.ItemAlloc(x, uint16(len(k)))
	} else {
	it = &gkvlite.Item{Key: make([]byte, len(k))}
	}
	copy(it.Key, k)
	it.Val = b
	it.Priority = pri
	err := x.SetItem(it)
	if err != nil {
	panic(fmt.Sprintf("error: expected nil error, got: %v", err))
	}
	if scb.ItemDecRef != nil {
	scb.ItemDecRef(x, it)
	}
	} else if r < psd {
	numDeletes++
	_, err := x.Delete(k)
	if err != nil {
	panic(fmt.Sprintf("error: expected nil error, got: %v", err))
	}
	} else if r < psde {
	numEvicts++
	x.EvictSomeItems()
	} else if r < psder {
	numReopens++
	stop()
	f, _ = os.OpenFile(fname, os.O_RDWR, 0666)
	s, x = start(f)
	} else {
	numGets++
	i, err := x.GetItem(k, true)
	if err != nil {
	panic(fmt.Sprintf("error: expected nil error, got: %v", err))
	}
	if i != nil {
	if scb.ItemValLength(x, i) != kr4 {
	panic(fmt.Sprintf("error: expected len: %d, got %d",
	kr4, scb.ItemValLength(x, i)))
	}
	if scb.ItemDecRef != nil {
	scb.ItemDecRef(x, i)
	}
	}
	}

	if flushEvery > 0 && i % flushEvery == 0 {
	numFlushes++
	s.Flush()
	}

	if i % 10000 == 0 {
	if arena != nil && *slabStats == true {
	arena.Stats(arenaStats)
	mk := []string{}
	for k, _ := range arenaStats {
	mk = append(mk, k)
	}
	sort.Strings(mk)
	for _, k := range mk {
	log.Printf("%s = %d", k, arenaStats[k])
	}
	}
	log.Printf("i: %d, numGets: %d, numSets: %d, numDeletes: %d" +
	", numEvicts: %d, numReopens: %d, numFlushes: %d\n",
	i, numGets, numSets, numDeletes, numEvicts, numReopens, numFlushes)
	}
	}
	}