vmon,internal/monitoring: move some common functions from vmon to internal/monitoring.
I also removed "oncall collect" related files. See reasons below.
In this CL I only moved stuff from vmon/servicecommon.go to
internal/monitoring/monitoring.go and updated all the references.
I didn't change any code logic.
This is the first step of making the oncall-dashboard better.
Now, there are two components to make the oncall-dashboard work.
1. "oncall collect" command talks to GCM, collects all data needed in
dashboard (all the metrics from all our services), and write the result to
a (large) file in json in storage bucket. This runs every minute.
2. "oncall serve" serves the data file above and static resources
to the frontend UI.
I don't remember exactly why I chose this approach, but it has many
flaws:
- The data shown in the dashboard is not in "real time". It depends on
how often the "oncall collect" command runs. It basically just shows the data
from the last "oncall collect" run.
- The data shown in the dashboard is kind of "static". For example,
people cannot change data duration because the duration is hard-coded
in the "oncall collect" code (1h by default).
- It is pretty wasteful. For example, the instance-level data is only needed
when people are looking at the instance-level view.
- Hard to maintain. Too many moving parts.
I plan to remove the "oncall collect" part, and move data retrieval functions
to the "oncall serve" command so it becomes a single backend server for
the dashboard. It serves static resources, and gets requests from the frontend
UI and talks to GCM.
Change-Id: I1ca294fbfbcd7c25b192660e1a3f075a8b80d380
diff --git a/internal/monitoring/monitoring.go b/internal/monitoring/monitoring.go
index a5e096c..1d68f7f 100644
--- a/internal/monitoring/monitoring.go
+++ b/internal/monitoring/monitoring.go
@@ -6,9 +6,21 @@
import (
"fmt"
+ "io"
"io/ioutil"
"net/http"
"sort"
+ "strings"
+ "time"
+
+ "v.io/jiri/tool"
+ "v.io/v23"
+ "v.io/v23/context"
+ "v.io/v23/naming"
+ "v.io/v23/options"
+ "v.io/v23/rpc"
+ "v.io/v23/services/stats"
+ "v.io/v23/vdl"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
@@ -17,8 +29,56 @@
const (
customMetricPrefix = "custom.googleapis.com"
+ defaultTimeout = 20 * time.Second
)
+// Human-readable service names.
+const (
+ SNMounttable = "mounttable"
+ SNIdentity = "identity service"
+ SNMacaroon = "macaroon service"
+ SNGoogleIdentity = "google identity service"
+ SNBinaryDischarger = "binary discharger"
+ SNRole = "role service"
+ SNProxy = "proxy service"
+
+ hostnameStatSuffix = "__debug/stats/system/hostname"
+ zoneStatSuffix = "__debug/stats/system/gce/zone"
+)
+
+// serviceMountedNames is a map from human-readable service names to their
+// relative mounted names in the global mounttable.
+var serviceMountedNames = map[string]string{
+ SNMounttable: "",
+ SNIdentity: "identity/dev.v.io:u",
+ SNMacaroon: "identity/dev.v.io:u/macaroon",
+ SNGoogleIdentity: "identity/dev.v.io:u/google",
+ SNBinaryDischarger: "identity/dev.v.io:u/discharger",
+ SNRole: "identity/role",
+ SNProxy: "proxy-mon",
+}
+
+// StatValue stores the name and the value returned from the GetStat function.
+type StatValue struct {
+ Name string
+ Value interface{}
+}
+
+func (sv *StatValue) GetStringValue() string {
+ return fmt.Sprint(sv.Value)
+}
+
+func (sv *StatValue) GetFloat64Value() (float64, error) {
+ switch i := sv.Value.(type) {
+ case float64:
+ return i, nil
+ case int64:
+ return float64(i), nil
+ default:
+ return 0, fmt.Errorf("invalid value: %v", sv.Value)
+ }
+}
+
type ServiceLocation struct {
Instance string
Zone string
@@ -189,6 +249,153 @@
return names
}
+// GetServiceMountedName gets the full mounted name for the given service.
+func GetServiceMountedName(namespaceRoot, serviceName string) (string, error) {
+ relativeName, ok := serviceMountedNames[serviceName]
+ if !ok {
+ return "", fmt.Errorf("service %q doesn't exist", serviceName)
+ }
+ return fmt.Sprintf("%s/%s", namespaceRoot, relativeName), nil
+}
+
+// ResolveAndProcessServiceName resolves the given service name and groups the
+// result entries by their routing ids.
+func ResolveAndProcessServiceName(v23ctx *context.T, ctx *tool.Context, serviceName, serviceMountedName string) (map[string]naming.MountEntry, error) {
+ // Resolve the name.
+ v23ctx, cancel := context.WithTimeout(v23ctx, defaultTimeout)
+ defer cancel()
+
+ ns := v23.GetNamespace(v23ctx)
+ entry, err := ns.ShallowResolve(v23ctx, serviceMountedName)
+ if err != nil {
+ return nil, err
+ }
+ resolvedNames := []string{}
+ for _, server := range entry.Servers {
+ fullName := naming.JoinAddressName(server.Server, entry.Name)
+ resolvedNames = append(resolvedNames, fullName)
+ }
+
+ // Group resolved names by their routing ids.
+ groups := map[string]naming.MountEntry{}
+ if serviceName == SNMounttable {
+ // Mounttable resolves to itself, so we just use a dummy routing id with
+ // its original mounted name.
+ groups["-"] = naming.MountEntry{
+ Servers: []naming.MountedServer{naming.MountedServer{Server: serviceMountedName}},
+ }
+ } else {
+ for _, resolvedName := range resolvedNames {
+ serverName, relativeName := naming.SplitAddressName(resolvedName)
+ ep, err := v23.NewEndpoint(serverName)
+ if err != nil {
+ return nil, err
+ }
+ routingId := ep.RoutingID().String()
+ if _, ok := groups[routingId]; !ok {
+ groups[routingId] = naming.MountEntry{}
+ }
+ curMountEntry := groups[routingId]
+ curMountEntry.Servers = append(curMountEntry.Servers, naming.MountedServer{Server: serverName})
+ // resolvedNames are resolved from the same service so they should have
+ // the same relative name.
+ curMountEntry.Name = relativeName
+ groups[routingId] = curMountEntry
+ }
+ }
+
+ return groups, nil
+}
+
+// GetServiceLocation returns the given service's location (instance and zone).
+// If the service is replicated, the instance name is the pod name.
+//
+// To make it simpler and faster, we look up service's location in hard-coded "zone maps"
+// for both non-replicated and replicated services.
+func GetServiceLocation(v23ctx *context.T, ctx *tool.Context, me naming.MountEntry) (*ServiceLocation, error) {
+ // Check "__debug/stats/system/metadata/hostname" stat to get service's
+ // host name.
+ me.Name = ""
+ hostnameResult, err := GetStat(v23ctx, ctx, me, hostnameStatSuffix)
+ if err != nil {
+ return nil, err
+ }
+ hostname := hostnameResult[0].GetStringValue()
+
+ // Check "__debug/stats/system/gce/zone" stat to get service's
+ // zone name.
+ zoneResult, err := GetStat(v23ctx, ctx, me, zoneStatSuffix)
+ if err != nil {
+ return nil, err
+ }
+ zone := zoneResult[0].GetStringValue()
+ // The zone stat exported by services is in the form of:
+ // projects/632758215260/zones/us-central1-c
+ // We only need the last part.
+ parts := strings.Split(zone, "/")
+ zone = parts[len(parts)-1]
+
+ return &ServiceLocation{
+ Instance: hostname,
+ Zone: zone,
+ }, nil
+}
+
+// GetStat gets the given stat using rpc.
+func GetStat(v23ctx *context.T, ctx *tool.Context, me naming.MountEntry, pattern string) ([]*StatValue, error) {
+ v23ctx, cancel := context.WithTimeout(v23ctx, defaultTimeout)
+ defer cancel()
+
+ call, err := v23.GetClient(v23ctx).StartCall(v23ctx, "", rpc.GlobMethod, []interface{}{pattern}, options.Preresolved{&me})
+ if err != nil {
+ return nil, err
+ }
+ hasErrors := false
+ ret := []*StatValue{}
+ mountEntryName := me.Name
+ for {
+ var gr naming.GlobReply
+ err := call.Recv(&gr)
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ switch v := gr.(type) {
+ case naming.GlobReplyEntry:
+ me.Name = naming.Join(mountEntryName, v.Value.Name)
+ value, err := stats.StatsClient("").Value(v23ctx, options.Preresolved{&me})
+ if err != nil {
+ fmt.Fprintf(ctx.Stderr(), "Failed to get stat (pattern: %q, entry: %#v): %v\n%v\n", pattern, me, v.Value.Name, err)
+ hasErrors = true
+ continue
+ }
+ var convertedValue interface{}
+ if err := vdl.Convert(&convertedValue, value); err != nil {
+ fmt.Fprintf(ctx.Stderr(), "Failed to convert value for %v (pattern: %q, entry: %#v): %v\n", pattern, me, v.Value.Name, err)
+ hasErrors = true
+ continue
+ }
+ ret = append(ret, &StatValue{
+ Name: v.Value.Name,
+ Value: convertedValue,
+ })
+ case naming.GlobReplyError:
+ fmt.Fprintf(ctx.Stderr(), "Glob failed at %q: %v", v.Value.Name, v.Value.Error)
+ }
+ }
+ if hasErrors || len(ret) == 0 {
+ return nil, fmt.Errorf("failed to get stat (pattern: %q, entry: %#v)", pattern, me)
+ }
+ if err := call.Finish(); err != nil {
+ return nil, err
+ }
+
+ return ret, nil
+}
+
func createClient(keyFilePath string) (*http.Client, error) {
if len(keyFilePath) > 0 {
data, err := ioutil.ReadFile(keyFilePath)
diff --git a/oncall/cmd.go b/oncall/cmd.go
index 1d25c10..006d1ba 100644
--- a/oncall/cmd.go
+++ b/oncall/cmd.go
@@ -30,5 +30,5 @@
Name: "oncall",
Short: "Command oncall implements oncall specific utilities used by Vanadium team",
Long: "Command oncall implements oncall specific utilities used by Vanadium team.",
- Children: []*cmdline.Command{cmdCollect, cmdServe},
+ Children: []*cmdline.Command{cmdServe},
}
diff --git a/oncall/collect.go b/oncall/collect.go
deleted file mode 100644
index 6af7299..0000000
--- a/oncall/collect.go
+++ /dev/null
@@ -1,918 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "math"
- "os"
- "path/filepath"
- "regexp"
- "sort"
- "strconv"
- "strings"
- "time"
-
- cloudmonitoring "google.golang.org/api/monitoring/v3"
-
- "v.io/jiri/tool"
- "v.io/x/devtools/internal/monitoring"
- "v.io/x/lib/cmdline"
-)
-
-const (
- cloudServiceLatencyMetric = "custom.cloudmonitoring.googleapis.com/vanadium/service/latency"
- cloudServiceCountersMetric = "custom.cloudmonitoring.googleapis.com/vanadium/service/counters"
- cloudServiceQPSMetric = "custom.cloudmonitoring.googleapis.com/vanadium/service/qps/total"
- nginxStatsMetric = "custom.cloudmonitoring.googleapis.com/vanadium/nginx/stats"
- gceStatsMetric = "custom.cloudmonitoring.googleapis.com/vanadium/gce-instance/stats"
- metricNameLabelKey = "custom.cloudmonitoring.googleapis.com/metric-name"
- gceInstanceLabelKey = "custom.cloudmonitoring.googleapis.com/gce-instance"
- gceZoneLabelKey = "custom.cloudmonitoring.googleapis.com/gce-zone"
- historyDuration = time.Hour
- serviceStatusOK = "serviceStatusOK"
- serviceStatusWarning = "serviceStatusWarning"
- serviceStatusDown = "serviceStatusDown"
- warningLatency = 2000
- criticalLatency = 5000
-)
-
-const (
- thresholdHoldMinutes = 5
-
- thresholdCPU = 90
- thresholdDisk = 85
- thresholdMounttableQPS = 150
- thresholdPing = 500
- thresholdRam = 90
- thresholdServiceLatency = 2000.0
- thresholdTCPConn = 200
-
- buildInfoEndpointPrefix = "devmgr/apps/*/*/*/stats/system/metadata"
- namespaceRoot = "/ns.dev.v.io:8151"
-)
-
-var (
- binDirFlag string
- credentialsFlag string
- keyFileFlag string
- projectFlag string
- serviceAccountFlag string
- // Running debug stats read
- // devmgr/apps/*/*/*/stats/system/metadata/build.[TPUM]* takes > 1
- // minute on the jenkins corp nodes with the RPC backward compatibility
- // retry change.
- debugCommandTimeout = 2 * time.Minute
- debugRPCTimeout = 90 * time.Second
- buildInfoRE = regexp.MustCompile(`devmgr/apps/([^/]*)/.*/stats/system/metadata/build.(Pristine|Time|User|Manifest):\s*(.*)`)
- manifestRE = regexp.MustCompile(`.*label="(.*)">`)
-)
-
-type oncallData struct {
- CollectionTimestamp int64
- Zones map[string]*zoneData // Indexed by zone names.
- OncallIDs string // IDs separated by ",".
-}
-
-type zoneData struct {
- Instances map[string]*allMetricsData // Indexed by instance names.
- Max *allMetricsData
- Average *allMetricsData
-}
-
-type allMetricsData struct {
- CloudServiceLatency map[string]*metricData // Indexed by metric names. Same below.
- CloudServiceStats map[string]*metricData
- CloudServiceGCE map[string]*metricData
- CloudServiceBuildInfo map[string]*buildInfoData
- NginxLoad map[string]*metricData
- NginxGCE map[string]*metricData
- GCEInfo *gceInfoData
- Range *rangeData
-}
-
-type metricData struct {
- ZoneName string
- InstanceName string
- Name string
- CurrentValue float64
- MinTime int64
- MaxTime int64
- MinValue float64
- MaxValue float64
- HistoryTimestamps []int64
- HistoryValues []float64
- Threshold float64
- Healthy bool
-}
-
-// metricDataMap is a map with the following structure:
-// {zoneName, {instanceName, {metricName, *metricData}}}.
-type metricDataMap map[string]map[string]map[string]*metricData
-
-type gceInfoData struct {
- Status string
- Id string
-}
-
-type buildInfoData struct {
- ZoneName string
- InstanceName string
- ServiceName string
- IsPristine string
- Snapshot string
- Time string
- User string
-}
-
-type rangeData struct {
- MinTime int64
- MaxTime int64
-}
-
-func (r *rangeData) update(newMinTime, newMaxTime int64) {
- if newMinTime < r.MinTime {
- r.MinTime = newMinTime
- }
- if newMaxTime > r.MaxTime {
- r.MaxTime = newMaxTime
- }
-}
-
-type aggMetricData struct {
- TimestampsToValues map[int64][]float64
-}
-type aggAllMetricsData struct {
- CloudServiceLatency map[string]*aggMetricData // Indexed by metric names. Same below.
- CloudServiceStats map[string]*aggMetricData
- CloudServiceGCE map[string]*aggMetricData
- NginxLoad map[string]*aggMetricData
- NginxGCE map[string]*aggMetricData
-}
-
-type serviceStatusData struct {
- CollectionTimestamp int64
- Status []statusData
-}
-
-type statusData struct {
- Name string
- BuildTimestamp string
- SnapshotLabel string
- CurrentStatus string
- Incidents []incidentData
-}
-
-type incidentData struct {
- Start int64
- Duration int64
- Status string
-}
-
-type int64arr []int64
-
-func (a int64arr) Len() int { return len(a) }
-func (a int64arr) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
-func (a int64arr) Less(i, j int) bool { return a[i] < a[j] }
-func (a int64arr) Sort() { sort.Sort(a) }
-
-func init() {
- cmdCollect.Flags.StringVar(&binDirFlag, "bin-dir", "", "The path where all binaries are downloaded.")
- cmdCollect.Flags.StringVar(&keyFileFlag, "key", "", "The path to the service account's JSON credentials file.")
- cmdCollect.Flags.StringVar(&projectFlag, "project", "", "The GCM's corresponding GCE project ID.")
- cmdCollect.Flags.StringVar(&credentialsFlag, "v23.credentials", "", "The path to v23 credentials.")
-}
-
-// cmdCollect represents the 'collect' command of the oncall tool.
-var cmdCollect = &cmdline.Command{
- Name: "collect",
- Short: "Collect data for oncall dashboard",
- Long: `
-This subcommand collects data from Google Cloud Monitoring and stores the
-processed data to Google Storage.
-`,
- Runner: cmdline.RunnerFunc(runCollect),
-}
-
-func runCollect(env *cmdline.Env, _ []string) error {
- ctx := tool.NewContextFromEnv(env)
- s, err := monitoring.Authenticate(keyFileFlag)
- if err != nil {
- return err
- }
- now := time.Now()
-
- // Collect oncall related data used in the internal oncall dashboard.
- zones := map[string]*zoneData{}
- oncall := &oncallData{
- CollectionTimestamp: now.Unix(),
- Zones: zones,
- }
- if err := collectCloudServicesData(ctx, s, now, zones); err != nil {
- return err
- }
- if err := collectCloudServicesBuildInfo(ctx, zones); err != nil {
- return err
- }
- if err := collectNginxData(ctx, s, now, zones); err != nil {
- return err
- }
- if err := collectGCEInstancesData(ctx, s, now, zones); err != nil {
- return err
- }
- if err := collectOncallIDsData(ctx, oncall); err != nil {
- return err
- }
-
- // Collect service status data used in the external dashboard.
- buildInfo := zones["us-central1-c"].Instances["vanadium-cell-master"].CloudServiceBuildInfo
- statusData, err := collectServiceStatusData(ctx, s, now, buildInfo)
- if err != nil {
- return err
- }
-
- if err := persistOncallData(ctx, statusData, oncall, now); err != nil {
- return err
- }
-
- return nil
-}
-
-func collectServiceStatusData(ctx *tool.Context, s *cloudmonitoring.Service, now time.Time, buildInfo map[string]*buildInfoData) (*serviceStatusData, error) {
- // Collect data for the last 8 days and aggregate data every 10 minutes.
- resp, err := s.Projects.TimeSeries.List(fmt.Sprintf("projects/%s", projectFlag)).
- Filter(fmt.Sprintf("metric.type=%s", cloudServiceLatencyMetric)).
- AggregationAlignmentPeriod(fmt.Sprintf("%ds", 10*60)).
- AggregationPerSeriesAligner("ALIGN_MAX").
- IntervalStartTime(now.AddDate(0, 0, -8).Format(time.RFC3339)).
- IntervalEndTime(now.Format(time.RFC3339)).Do()
- if err != nil {
- return nil, fmt.Errorf("List failed: %v", err)
- }
-
- status := []statusData{}
- for _, t := range resp.TimeSeries {
- serviceName := t.Metric.Labels[metricNameLabelKey]
- curStatusData := statusData{
- Name: serviceName,
- CurrentStatus: statusForLatency(t.Points[0].Value.DoubleValue), // t.Points[0] is the latest
- }
- incidents, err := calcIncidents(t.Points)
- if err != nil {
- return nil, err
- }
- curStatusData.Incidents = incidents
- buildInfoServiceName := serviceName
- switch serviceName {
- case "binary discharger", "google identity service", "macaroon service":
- buildInfoServiceName = "identityd"
- case "mounttable":
- buildInfoServiceName = "mounttabled"
- case "proxy service":
- buildInfoServiceName = "proxyd"
- case "binary repository":
- buildInfoServiceName = "binaryd"
- case "application repository":
- buildInfoServiceName = "applicationd"
- }
- curBuildInfo := buildInfo[buildInfoServiceName]
- if curBuildInfo != nil {
- ts, err := strconv.ParseInt(curBuildInfo.Time, 10, 64)
- if err != nil {
- return nil, fmt.Errorf("ParseInt(%s) failed: %v", curBuildInfo.Time, err)
- }
- curStatusData.BuildTimestamp = time.Unix(ts, 0).Format(time.RFC822)
- curStatusData.SnapshotLabel = strings.Replace(curBuildInfo.Snapshot, "snapshot/labels/", "", -1)
- }
- status = append(status, curStatusData)
- }
- return &serviceStatusData{
- CollectionTimestamp: now.Unix(),
- Status: status,
- }, nil
-}
-
-func calcIncidents(points []*cloudmonitoring.Point) ([]incidentData, error) {
- lastStatus := serviceStatusOK
- incidents := []incidentData{}
- var curIncident incidentData
- // "points" are sorted from now to past. To calculate incidents, we iterate
- // through them backwards.
- for i := len(points) - 1; i >= 0; i-- {
- point := points[i]
- value := point.Value.DoubleValue
- curStatus := statusForLatency(value)
- if curStatus != lastStatus {
- pointTime, err := time.Parse(time.RFC3339, point.Interval.StartTime)
- if err != nil {
- return nil, fmt.Errorf("time.Parse(%s) failed: %v", point.Interval.StartTime, err)
- }
-
- // Set the duration of the last incident.
- if curIncident.Status != "" {
- curIncident.Duration = pointTime.Unix() - curIncident.Start
- incidents = append(incidents, curIncident)
- curIncident.Status = ""
- }
-
- // At the start of an incident, create a new incidentData object, and
- // record the incident start time and status.
- if curStatus != serviceStatusOK {
- curIncident = incidentData{}
- curIncident.Start = pointTime.Unix()
- curIncident.Status = curStatus
- }
- lastStatus = curStatus
- }
- }
- // Process the possible last incident.
- if lastStatus != serviceStatusOK {
- strLastPointTime := points[0].Interval.StartTime
- pointTime, err := time.Parse(time.RFC3339, strLastPointTime)
- if err != nil {
- return nil, fmt.Errorf("time.Parse(%q) failed: %v", strLastPointTime, err)
- }
- curIncident.Duration = pointTime.Unix() - curIncident.Start
- incidents = append(incidents, curIncident)
- }
- return incidents, nil
-}
-
-func statusForLatency(latency float64) string {
- if latency < warningLatency {
- return serviceStatusOK
- }
- if latency < criticalLatency {
- return serviceStatusWarning
- }
- return serviceStatusDown
-}
-
-func collectCloudServicesData(ctx *tool.Context, s *cloudmonitoring.Service, now time.Time, zones map[string]*zoneData) error {
- // Collect and add latency data.
- latencyMetrics, err := getMetricData(ctx, s, cloudServiceLatencyMetric, now, "latency")
- if err != nil {
- return err
- }
- for zone, instanceMap := range latencyMetrics {
- if zones[zone] == nil {
- zones[zone] = newZoneData(zone)
- }
- zoneData := zones[zone]
- aggData := map[string]*aggMetricData{}
- for instance, metricMap := range instanceMap {
- if zoneData.Instances[instance] == nil {
- zoneData.Instances[instance] = newInstanceData()
- }
- zoneData.Instances[instance].CloudServiceLatency = metricMap
- for _, metric := range metricMap {
- metric.Threshold = thresholdServiceLatency
- if metric.Threshold != -1 {
- metric.Healthy = !overThresholdFor(metric.HistoryTimestamps, metric.HistoryValues, thresholdServiceLatency, thresholdHoldMinutes)
- }
- aggregateMetricData(aggData, metric)
- }
- }
- maxData, maxRangeData, averageData, averageRangeData := calculateMaxAndAverageData(aggData, zone)
- zoneData.Max.CloudServiceLatency, zoneData.Average.CloudServiceLatency = maxData, averageData
- zoneData.Max.Range.update(maxRangeData.MinTime, maxRangeData.MaxTime)
- zoneData.Average.Range.update(averageRangeData.MinTime, averageRangeData.MaxTime)
- }
-
- // Collect and add stats (counters + qps) data.
- counterMetrics, err := getMetricData(ctx, s, cloudServiceCountersMetric, now, "")
- if err != nil {
- return err
- }
- qpsMetrics, err := getMetricData(ctx, s, cloudServiceQPSMetric, now, "qps")
- if err != nil {
- return err
- }
- aggDataByZone := map[string]map[string]*aggMetricData{}
- addStatsFn := func(metrics metricDataMap) {
- for zone, instanceMap := range metrics {
- if zones[zone] == nil {
- zones[zone] = newZoneData(zone)
- }
- zoneData := zones[zone]
- aggData := aggDataByZone[zone]
- if aggData == nil {
- aggData = map[string]*aggMetricData{}
- }
- aggDataByZone[zone] = aggData
- for instance, metricMap := range instanceMap {
- if zoneData.Instances[instance] == nil {
- zoneData.Instances[instance] = newInstanceData()
- }
- stats := zoneData.Instances[instance].CloudServiceStats
- if stats == nil {
- stats = map[string]*metricData{}
- zoneData.Instances[instance].CloudServiceStats = stats
- }
- for metricName, metric := range metricMap {
- metric.Threshold = getThreshold(metricName)
- if metric.Threshold != -1 {
- metric.Healthy = !overThresholdFor(metric.HistoryTimestamps, metric.HistoryValues, metric.Threshold, thresholdHoldMinutes)
- }
- stats[metricName] = metric
- aggregateMetricData(aggData, metric)
- }
- }
- }
- }
- addStatsFn(counterMetrics)
- addStatsFn(qpsMetrics)
-
- for zone, aggData := range aggDataByZone {
- zoneData := zones[zone]
- maxData, maxRangeData, averageData, averageRangeData := calculateMaxAndAverageData(aggData, zone)
- zoneData.Max.CloudServiceStats, zoneData.Average.CloudServiceStats = maxData, averageData
- zoneData.Max.Range.update(maxRangeData.MinTime, maxRangeData.MaxTime)
- zoneData.Average.Range.update(averageRangeData.MinTime, averageRangeData.MaxTime)
- }
-
- return nil
-}
-
-func collectCloudServicesBuildInfo(ctx *tool.Context, zones map[string]*zoneData) error {
- serviceLocation := monitoring.ServiceLocationMap[namespaceRoot]
- if serviceLocation == nil {
- return fmt.Errorf("failed to find service location for %q", namespaceRoot)
- }
- zone := serviceLocation.Zone
- instance := serviceLocation.Instance
- if zones[zone] == nil {
- zones[zone] = newZoneData(zone)
- }
-
- // Run "debug stats read" command to query build info data.
- debug := filepath.Join(binDirFlag, "debug")
- var stdoutBuf, stderrBuf bytes.Buffer
- if err := ctx.NewSeq().
- Capture(&stdoutBuf, &stderrBuf).
- Timeout(debugCommandTimeout).
- Last(debug,
- "--timeout", debugRPCTimeout.String(),
- "--v23.namespace.root", namespaceRoot,
- "--v23.credentials", credentialsFlag, "stats", "read", fmt.Sprintf("%s/build.[TPUM]*", buildInfoEndpointPrefix)); err != nil {
- return fmt.Errorf("debug command failed: %v\nSTDERR:\n%s\nSTDOUT:\n%s\nEND\n", err, stderrBuf.String(), stdoutBuf.String())
- }
-
- // Parse output.
- lines := strings.Split(stdoutBuf.String(), "\n")
- buildInfoByServiceName := map[string]*buildInfoData{}
- for _, line := range lines {
- matches := buildInfoRE.FindStringSubmatch(line)
- if matches != nil {
- service := matches[1]
- metadataName := matches[2]
- value := matches[3]
- if _, ok := buildInfoByServiceName[service]; !ok {
- buildInfoByServiceName[service] = &buildInfoData{
- ZoneName: zone,
- InstanceName: instance,
- ServiceName: service,
- }
- }
- curBuildInfo := buildInfoByServiceName[service]
- switch metadataName {
- case "Manifest":
- manifestMatches := manifestRE.FindStringSubmatch(value)
- if manifestMatches != nil {
- curBuildInfo.Snapshot = strings.Replace(manifestMatches[1], "snapshot/labels/", "", -1)
- }
- case "Pristine":
- curBuildInfo.IsPristine = value
- case "Time":
- t, err := time.Parse(time.RFC3339, value)
- if err != nil {
- return fmt.Errorf("Parse(%s) failed: %v", value, err)
- }
- curBuildInfo.Time = fmt.Sprintf("%d", t.Unix())
- case "User":
- curBuildInfo.User = value
- }
- }
- }
-
- if zones[zone].Instances[instance] == nil {
- zones[zone].Instances[instance] = newInstanceData()
- }
- zones[zone].Instances[instance].CloudServiceBuildInfo = buildInfoByServiceName
-
- return nil
-}
-
-func collectNginxData(ctx *tool.Context, s *cloudmonitoring.Service, now time.Time, zones map[string]*zoneData) error {
- nginxMetrics, err := getMetricData(ctx, s, nginxStatsMetric, now, "")
- if err != nil {
- return err
- }
- for zone, instanceMap := range nginxMetrics {
- if zones[zone] == nil {
- zones[zone] = newZoneData(zone)
- }
- zoneData := zones[zone]
- aggData := map[string]*aggMetricData{}
- for instance, metricMap := range instanceMap {
- if !strings.HasPrefix(instance, "nginx") {
- continue
- }
- if zoneData.Instances[instance] == nil {
- zoneData.Instances[instance] = newInstanceData()
- }
- zoneData.Instances[instance].NginxLoad = metricMap
- for _, metric := range metricMap {
- aggregateMetricData(aggData, metric)
- }
- }
- maxData, maxRangeData, averageData, averageRangeData := calculateMaxAndAverageData(aggData, zone)
- zoneData.Max.NginxLoad, zoneData.Average.NginxLoad = maxData, averageData
- zoneData.Max.Range.update(maxRangeData.MinTime, maxRangeData.MaxTime)
- zoneData.Average.Range.update(averageRangeData.MinTime, averageRangeData.MaxTime)
- }
-
- return nil
-}
-
-func collectGCEInstancesData(ctx *tool.Context, s *cloudmonitoring.Service, now time.Time, zones map[string]*zoneData) error {
- // Query and add GCE stats.
- gceMetrics, err := getMetricData(ctx, s, gceStatsMetric, now, "")
- if err != nil {
- return err
- }
- for zone, instanceMap := range gceMetrics {
- if zones[zone] == nil {
- zones[zone] = newZoneData(zone)
- }
- zoneData := zones[zone]
- aggDataCloudSerivcesGCE := map[string]*aggMetricData{}
- aggDataNginxGCE := map[string]*aggMetricData{}
- for instance, metricMap := range instanceMap {
- if zoneData.Instances[instance] == nil {
- zoneData.Instances[instance] = newInstanceData()
- }
- cloudServiceGCE := zoneData.Instances[instance].CloudServiceGCE
- nginxGCE := zoneData.Instances[instance].NginxGCE
- if cloudServiceGCE == nil {
- cloudServiceGCE = map[string]*metricData{}
- zoneData.Instances[instance].CloudServiceGCE = cloudServiceGCE
- }
- if nginxGCE == nil {
- nginxGCE = map[string]*metricData{}
- zoneData.Instances[instance].NginxGCE = nginxGCE
- }
- // Set thresholds and calculate health.
- for metricName, metric := range metricMap {
- metric.Threshold = getThreshold(metricName)
- if metric.Threshold != -1 {
- metric.Healthy = !overThresholdFor(metric.HistoryTimestamps, metric.HistoryValues, metric.Threshold, thresholdHoldMinutes)
- }
- if strings.HasPrefix(instance, "vanadium") {
- cloudServiceGCE[metricName] = metric
- aggregateMetricData(aggDataCloudSerivcesGCE, metric)
- } else if strings.HasPrefix(instance, "nginx") {
- nginxGCE[metricName] = metric
- aggregateMetricData(aggDataNginxGCE, metric)
- }
- }
- }
-
- maxData, maxRangeData1, averageData, averageRangeData1 := calculateMaxAndAverageData(aggDataCloudSerivcesGCE, zone)
- zoneData.Max.CloudServiceGCE, zoneData.Average.CloudServiceGCE = maxData, averageData
- maxData, maxRangeData2, averageData, averageRangeData2 := calculateMaxAndAverageData(aggDataNginxGCE, zone)
- zoneData.Max.NginxGCE, zoneData.Average.NginxGCE = maxData, averageData
- zoneData.Max.Range.update(maxRangeData1.MinTime, maxRangeData1.MaxTime)
- zoneData.Max.Range.update(maxRangeData2.MinTime, maxRangeData2.MaxTime)
- zoneData.Average.Range.update(averageRangeData1.MinTime, averageRangeData1.MaxTime)
- zoneData.Average.Range.update(averageRangeData2.MinTime, averageRangeData2.MaxTime)
- }
-
- // Use "gcloud compute instances list" to get instances status.
- var out bytes.Buffer
- if err := ctx.NewSeq().Capture(&out, &out).Last("gcloud", "-q", "--project="+projectFlag, "compute", "instances", "list", "--format=json"); err != nil {
- return err
- }
- type instanceData struct {
- Name string
- Zone string
- Status string
- Id string
- }
- var instances []instanceData
- if err := json.Unmarshal(out.Bytes(), &instances); err != nil {
- return fmt.Errorf("Unmarshal() failed: %v", err)
- }
- instancesByZone := map[string][]instanceData{}
- for _, instance := range instances {
- if strings.HasPrefix(instance.Name, "nginx") || strings.HasPrefix(instance.Name, "vanadium") {
- instancesByZone[instance.Zone] = append(instancesByZone[instance.Zone], instance)
- }
- }
-
- // Add instance status.
- for zone, instances := range instancesByZone {
- curZone := zones[zone]
- if curZone == nil {
- continue
- }
- for _, instance := range instances {
- curZone.Instances[instance.Name].GCEInfo = &gceInfoData{
- Status: instance.Status,
- Id: instance.Id,
- }
- }
- }
-
- return nil
-}
-
-func collectOncallIDsData(ctx *tool.Context, oncall *oncallData) error {
- var out bytes.Buffer
- if err := ctx.NewSeq().Capture(&out, &out).Last("jiri", "oncall"); err != nil {
- return err
- }
- oncall.OncallIDs = strings.TrimSpace(out.String())
- return nil
-}
-
-// overThresholdFor checks whether the most recent values of the given time
-// series are over the given threshold for the the given amount of time.
-// This function assumes that the given time series data points are sorted by
-// time (oldest first).
-func overThresholdFor(timestamps []int64, values []float64, threshold float64, holdMinutes int) bool {
- numPoints := len(timestamps)
- maxTime := timestamps[numPoints-1]
- for i := numPoints - 1; i >= 0; i-- {
- t := timestamps[i]
- v := values[i]
- if v >= threshold {
- if (maxTime - t) >= int64(holdMinutes*60) {
- return true
- }
- } else {
- return false
- }
- }
- return false
-}
-
-func newZoneData(zone string) *zoneData {
- return &zoneData{
- Instances: map[string]*allMetricsData{},
- Max: newInstanceData(),
- Average: newInstanceData(),
- }
-}
-
-func newInstanceData() *allMetricsData {
- return &allMetricsData{
- CloudServiceLatency: map[string]*metricData{},
- CloudServiceStats: map[string]*metricData{},
- CloudServiceGCE: map[string]*metricData{},
- CloudServiceBuildInfo: map[string]*buildInfoData{},
- NginxLoad: map[string]*metricData{},
- NginxGCE: map[string]*metricData{},
- Range: newRangeData(),
- }
-}
-
-func newRangeData() *rangeData {
- return &rangeData{
- MinTime: math.MaxInt64,
- MaxTime: 0,
- }
-}
-
-// getMetricData queries GCM with the given metric, and returns metric items
-// (metricData) organized in metricDataMap.
-func getMetricData(ctx *tool.Context, s *cloudmonitoring.Service, metric string, now time.Time, metricNameSuffix string) (metricDataMap, error) {
- // Query the given metric.
- resp, err := s.Projects.TimeSeries.List(fmt.Sprintf("projects/%s", projectFlag)).
- Filter(fmt.Sprintf("metric.type=%s", metric)).
- IntervalStartTime(now.Add(-historyDuration).Format(time.RFC3339)).
- IntervalEndTime(now.Format(time.RFC3339)).Do()
- if err != nil {
- return nil, fmt.Errorf("List() failed: %v", err)
- }
-
- // Populate metric items and put them into a metricDataMap.
- data := metricDataMap{}
- for _, t := range resp.TimeSeries {
- zone := t.Metric.Labels[gceZoneLabelKey]
- instance := t.Metric.Labels[gceInstanceLabelKey]
- metricName := t.Metric.Labels[metricNameLabelKey]
- if metricNameSuffix != "" {
- metricName = fmt.Sprintf("%s %s", metricName, metricNameSuffix)
- }
-
- instanceMap := data[zone]
- if instanceMap == nil {
- instanceMap = map[string]map[string]*metricData{}
- data[zone] = instanceMap
- }
-
- metricMap := instanceMap[instance]
- if metricMap == nil {
- metricMap = map[string]*metricData{}
- instanceMap[instance] = metricMap
- }
-
- curMetricData := metricMap[metricName]
- if curMetricData == nil {
- curMetricData = &metricData{
- ZoneName: zone,
- InstanceName: instance,
- Name: metricName,
- CurrentValue: t.Points[0].Value.DoubleValue,
- MinTime: math.MaxInt64,
- MaxTime: 0,
- MinValue: math.MaxFloat64,
- MaxValue: 0,
- Threshold: -1,
- Healthy: true,
- }
- metricMap[metricName] = curMetricData
- }
-
- numPoints := len(t.Points)
- timestamps := []int64{}
- values := []float64{}
- // t.Points are sorted from now to past. We process them starting with the
- // latest and going back in time.
- for i := numPoints - 1; i >= 0; i-- {
- point := t.Points[i]
- epochTime, err := time.Parse(time.RFC3339, point.Interval.StartTime)
- if err != nil {
- fmt.Fprintf(ctx.Stderr(), "%v\n", err)
- continue
- }
- timestamp := epochTime.Unix()
- timestamps = append(timestamps, timestamp)
- values = append(values, point.Value.DoubleValue)
- curMetricData.MinTime = int64(math.Min(float64(curMetricData.MinTime), float64(timestamp)))
- curMetricData.MaxTime = int64(math.Max(float64(curMetricData.MaxTime), float64(timestamp)))
- curMetricData.MinValue = math.Min(curMetricData.MinValue, point.Value.DoubleValue)
- curMetricData.MaxValue = math.Max(curMetricData.MaxValue, point.Value.DoubleValue)
- }
- curMetricData.HistoryTimestamps = timestamps
- curMetricData.HistoryValues = values
- }
- return data, nil
-}
-
-// aggregateMetricData aggregates the history values of the given metric data
-// into the given aggData map indexed by metric names.
-func aggregateMetricData(aggData map[string]*aggMetricData, metric *metricData) {
- metricName := metric.Name
- curAggMetricData := aggData[metricName]
- if curAggMetricData == nil {
- curAggMetricData = &aggMetricData{
- TimestampsToValues: map[int64][]float64{},
- }
- aggData[metricName] = curAggMetricData
- }
- numPoints := len(metric.HistoryTimestamps)
- for i := 0; i < numPoints; i++ {
- t := metric.HistoryTimestamps[i]
- v := metric.HistoryValues[i]
- curAggMetricData.TimestampsToValues[t] = append(curAggMetricData.TimestampsToValues[t], v)
- }
-}
-
-// calculateMaxAndAverageData calculates and returns the max and average data
-// from the given aggregated data.
-func calculateMaxAndAverageData(aggData map[string]*aggMetricData, zone string) (map[string]*metricData, *rangeData, map[string]*metricData, *rangeData) {
- maxData := map[string]*metricData{}
- maxRangeData := newRangeData()
- averageData := map[string]*metricData{}
- averageRangeData := newRangeData()
-
- for metricName, metricAggData := range aggData {
- sortedTimestamps := int64arr{}
- for timestamp := range metricAggData.TimestampsToValues {
- sortedTimestamps = append(sortedTimestamps, timestamp)
- }
- sortedTimestamps.Sort()
- maxHistoryValues := []float64{}
- averageHistoryValues := []float64{}
- minMaxValue := math.MaxFloat64
- maxMaxValue := 0.0
- minAverageValue := math.MaxFloat64
- maxAverageValue := 0.0
- for _, timestamp := range sortedTimestamps {
- values := metricAggData.TimestampsToValues[timestamp]
- curMax := values[0]
- curSum := 0.0
- for _, v := range values {
- if v > curMax {
- curMax = v
- }
- curSum += v
- }
- curAverage := curSum / float64(len(values))
- maxHistoryValues = append(maxHistoryValues, curMax)
- averageHistoryValues = append(averageHistoryValues, curAverage)
- minMaxValue = math.Min(minMaxValue, curMax)
- maxMaxValue = math.Max(maxMaxValue, curMax)
- minAverageValue = math.Min(minAverageValue, curAverage)
- maxAverageValue = math.Max(maxAverageValue, curAverage)
- }
- minTime := sortedTimestamps[0]
- maxTime := sortedTimestamps[len(sortedTimestamps)-1]
- threshold := getThreshold(metricName)
- maxData[metricName] = &metricData{
- ZoneName: zone,
- Name: metricName,
- CurrentValue: maxHistoryValues[len(maxHistoryValues)-1],
- MinTime: minTime,
- MaxTime: maxTime,
- MinValue: minMaxValue,
- MaxValue: maxMaxValue,
- HistoryTimestamps: sortedTimestamps,
- HistoryValues: maxHistoryValues,
- Threshold: threshold,
- Healthy: true,
- }
- if threshold != -1 {
- maxData[metricName].Healthy = !overThresholdFor(sortedTimestamps, maxHistoryValues, threshold, thresholdHoldMinutes)
- }
- maxRangeData.update(minTime, maxTime)
- averageData[metricName] = &metricData{
- ZoneName: zone,
- Name: metricName,
- CurrentValue: averageHistoryValues[len(maxHistoryValues)-1],
- MinTime: minTime,
- MaxTime: maxTime,
- MinValue: minAverageValue,
- MaxValue: maxAverageValue,
- HistoryTimestamps: sortedTimestamps,
- HistoryValues: averageHistoryValues,
- Threshold: threshold,
- Healthy: true,
- }
- if threshold != -1 {
- averageData[metricName].Healthy = !overThresholdFor(sortedTimestamps, averageHistoryValues, threshold, thresholdHoldMinutes)
- }
- averageRangeData.update(minTime, maxTime)
- }
-
- return maxData, maxRangeData, averageData, averageRangeData
-}
-
-func getThreshold(metricName string) float64 {
- if strings.HasSuffix(metricName, "latency") {
- return thresholdServiceLatency
- }
- switch metricName {
- case "mounttable qps":
- return thresholdMounttableQPS
- case "cpu-usage":
- return thresholdCPU
- case "disk-usage":
- return thresholdDisk
- case "memory-usage":
- return thresholdRam
- case "ping":
- return thresholdPing
- case "tcpconn":
- return thresholdTCPConn
- }
- return -1.0
-}
-
-func persistOncallData(ctx *tool.Context, statusData *serviceStatusData, oncall *oncallData, now time.Time) error {
- // Use timestamp (down to the minute part) as the main file name.
- // We store oncall data and status data separately for efficiency.
- bytesStatus, err := json.MarshalIndent(statusData, "", " ")
- if err != nil {
- return fmt.Errorf("MarshalIndent() failed: %v", err)
- }
- bytesOncall, err := json.MarshalIndent(oncall, "", " ")
- if err != nil {
- return fmt.Errorf("MarshalIndent() failed: %v", err)
- }
-
- s := ctx.NewSeq()
- // Write data to a temporary directory.
- curTime := now.Format("200601021504")
- tmpDir, err := s.TempDir("", "")
- if err != nil {
- return err
- }
- defer ctx.NewSeq().RemoveAll(tmpDir)
- statusDataFile := filepath.Join(tmpDir, fmt.Sprintf("%s.status", curTime))
- oncallDataFile := filepath.Join(tmpDir, fmt.Sprintf("%s.oncall", curTime))
- latestFile := filepath.Join(tmpDir, "latest")
- args := []string{"-q", "cp", filepath.Join(tmpDir, "*"), bucketData + "/"}
-
- // Upload data to Google Storage.
- return s.WriteFile(statusDataFile, bytesStatus, os.FileMode(0600)).
- WriteFile(oncallDataFile, bytesOncall, os.FileMode(0600)).
- WriteFile(latestFile, []byte(curTime), os.FileMode(0600)).
- Last("gsutil", args...)
-}
diff --git a/oncall/collect_test.go b/oncall/collect_test.go
deleted file mode 100644
index 0de0e17..0000000
--- a/oncall/collect_test.go
+++ /dev/null
@@ -1,417 +0,0 @@
-// Copyright 2015 The Vanadium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "reflect"
- "testing"
- "time"
-
- cloudmonitoring "google.golang.org/api/monitoring/v3"
-)
-
-func TestCalcIncidents(t *testing.T) {
- testCases := []struct {
- points []*cloudmonitoring.Point
- expectedIncidentData []incidentData
- }{
- // No incidents.
- {
- points: []*cloudmonitoring.Point{
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896102, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 1000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896101, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 1000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896100, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 1000,
- },
- },
- },
- expectedIncidentData: []incidentData{},
- },
- // One warning incident.
- {
- points: []*cloudmonitoring.Point{
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896103, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 1000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896102, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 3000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896101, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 3000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896100, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 1000,
- },
- },
- },
- expectedIncidentData: []incidentData{
- incidentData{
- Start: 1429896101,
- Duration: 2,
- Status: serviceStatusWarning,
- },
- },
- },
- // One warning incident and one critical incident.
- {
- points: []*cloudmonitoring.Point{
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896104, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 1000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896103, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 3000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896102, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 3000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896101, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 5000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896100, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 1000,
- },
- },
- },
- expectedIncidentData: []incidentData{
- incidentData{
- Start: 1429896101,
- Duration: 1,
- Status: serviceStatusDown,
- },
- incidentData{
- Start: 1429896102,
- Duration: 2,
- Status: serviceStatusWarning,
- },
- },
- },
- // One warning incident at the beginning and one critical incident at the end.
- {
- points: []*cloudmonitoring.Point{
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896103, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 3000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896102, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 3000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896101, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 1000,
- },
- },
- &cloudmonitoring.Point{
- Interval: &cloudmonitoring.TimeInterval{
- StartTime: time.Unix(1429896100, 0).Format(time.RFC3339),
- },
- Value: &cloudmonitoring.TypedValue{
- DoubleValue: 5000,
- },
- },
- },
- expectedIncidentData: []incidentData{
- incidentData{
- Start: 1429896100,
- Duration: 1,
- Status: serviceStatusDown,
- },
- incidentData{
- Start: 1429896102,
- Duration: 1,
- Status: serviceStatusWarning,
- },
- },
- },
- }
-
- for index, test := range testCases {
- got, err := calcIncidents(test.points)
- if err != nil {
- t.Fatalf("index #%d: want no errors, got: %v", index, err)
- }
- if expected := test.expectedIncidentData; !reflect.DeepEqual(got, expected) {
- t.Fatalf("index #%d: want: %#v, got: %#v", index, expected, got)
- }
- }
-}
-
-func TestOverThresholdFor(t *testing.T) {
- testCases := []struct {
- timestamps []int64
- values []float64
- threshold float64
- holdMinutes int
- expectedResult bool
- }{
- // Not over threshold.
- {
- timestamps: []int64{0, 60, 120, 180, 240, 300, 360},
- values: []float64{0, 0, 0, 0, 0, 0, 0},
- threshold: 100,
- holdMinutes: 5,
- expectedResult: false,
- },
- // Over threshold, but not long enough.
- {
- timestamps: []int64{0, 60, 120, 180, 240, 300, 360},
- values: []float64{0, 0, 0, 0, 0, 200, 200},
- threshold: 100,
- holdMinutes: 5,
- expectedResult: false,
- },
- // Over threshold and long enough.
- {
- timestamps: []int64{0, 60, 120, 180, 240, 300, 360},
- values: []float64{0, 200, 200, 200, 200, 200, 200},
- threshold: 100,
- holdMinutes: 5,
- expectedResult: true,
- },
- }
-
- for _, test := range testCases {
- got := overThresholdFor(test.timestamps, test.values, test.threshold, test.holdMinutes)
- if got != test.expectedResult {
- t.Fatalf("want %v, got %v", test.expectedResult, got)
- }
- }
-}
-
-func TestAggregateMetricData(t *testing.T) {
- aggData := map[string]*aggMetricData{}
- testSteps := []struct {
- metric *metricData
- expectedAggData map[string]*aggMetricData
- }{
- {
- metric: &metricData{
- Name: "metric1",
- HistoryTimestamps: []int64{1, 2, 3},
- HistoryValues: []float64{100.0, 200.0, 300.0},
- },
- expectedAggData: map[string]*aggMetricData{
- "metric1": &aggMetricData{
- TimestampsToValues: map[int64][]float64{
- 1: []float64{100.0},
- 2: []float64{200.0},
- 3: []float64{300.0},
- },
- },
- },
- },
- {
- metric: &metricData{
- Name: "metric1",
- HistoryTimestamps: []int64{1, 2},
- HistoryValues: []float64{101.0, 201.0},
- },
- expectedAggData: map[string]*aggMetricData{
- "metric1": &aggMetricData{
- TimestampsToValues: map[int64][]float64{
- 1: []float64{100.0, 101.0},
- 2: []float64{200.0, 201.0},
- 3: []float64{300.0},
- },
- },
- },
- },
- {
- metric: &metricData{
- Name: "metric2",
- HistoryTimestamps: []int64{4, 5, 6},
- HistoryValues: []float64{10.0, 20.0, 30.0},
- },
- expectedAggData: map[string]*aggMetricData{
- "metric1": &aggMetricData{
- TimestampsToValues: map[int64][]float64{
- 1: []float64{100.0, 101.0},
- 2: []float64{200.0, 201.0},
- 3: []float64{300.0},
- },
- },
- "metric2": &aggMetricData{
- TimestampsToValues: map[int64][]float64{
- 4: []float64{10.0},
- 5: []float64{20.0},
- 6: []float64{30.0},
- },
- },
- },
- },
- }
- for _, test := range testSteps {
- aggregateMetricData(aggData, test.metric)
- if got, want := aggData, test.expectedAggData; !reflect.DeepEqual(got, want) {
- t.Fatalf("want %v, got %v", want, got)
- }
- }
-}
-
-func TestCalculateMaxAndAverageData(t *testing.T) {
- aggData := map[string]*aggMetricData{
- "metric1": &aggMetricData{
- TimestampsToValues: map[int64][]float64{
- 1: []float64{100.0, 200.0},
- 2: []float64{200.0, 300.0},
- 3: []float64{300.0},
- },
- },
- "metric2": &aggMetricData{
- TimestampsToValues: map[int64][]float64{
- 4: []float64{10.0},
- },
- },
- }
- gotMaxData, gotMaxRangeData, gotAverageData, gotAverageRangeData := calculateMaxAndAverageData(aggData, "zone1")
- expectedMaxData := map[string]*metricData{
- "metric1": &metricData{
- ZoneName: "zone1",
- Name: "metric1",
- CurrentValue: 300.0,
- MinTime: 1,
- MaxTime: 3,
- MinValue: 200.0,
- MaxValue: 300.0,
- HistoryTimestamps: []int64{1, 2, 3},
- HistoryValues: []float64{200.0, 300.0, 300.0},
- Threshold: -1,
- Healthy: true,
- },
- "metric2": &metricData{
- ZoneName: "zone1",
- Name: "metric2",
- CurrentValue: 10.0,
- MinTime: 4,
- MaxTime: 4,
- MinValue: 10.0,
- MaxValue: 10.0,
- HistoryTimestamps: []int64{4},
- HistoryValues: []float64{10.0},
- Threshold: -1,
- Healthy: true,
- },
- }
- expectedMaxRangeData := &rangeData{
- MinTime: 1,
- MaxTime: 4,
- }
- expectedAverageData := map[string]*metricData{
- "metric1": &metricData{
- ZoneName: "zone1",
- Name: "metric1",
- CurrentValue: 300.0,
- MinTime: 1,
- MaxTime: 3,
- MinValue: 150.0,
- MaxValue: 300.0,
- HistoryTimestamps: []int64{1, 2, 3},
- HistoryValues: []float64{150.0, 250.0, 300.0},
- Threshold: -1,
- Healthy: true,
- },
- "metric2": &metricData{
- ZoneName: "zone1",
- Name: "metric2",
- CurrentValue: 10.0,
- MinTime: 4,
- MaxTime: 4,
- MinValue: 10.0,
- MaxValue: 10.0,
- HistoryTimestamps: []int64{4},
- HistoryValues: []float64{10.0},
- Threshold: -1,
- Healthy: true,
- },
- }
- expectedAverageRangeData := &rangeData{
- MinTime: 1,
- MaxTime: 4,
- }
- if !reflect.DeepEqual(gotMaxData, expectedMaxData) {
- t.Fatalf("want %#v, got %#v", expectedMaxData, gotMaxData)
- }
- if !reflect.DeepEqual(gotMaxRangeData, expectedMaxRangeData) {
- t.Fatalf("want %#v, got %#v", expectedMaxRangeData, gotMaxRangeData)
- }
- if !reflect.DeepEqual(gotAverageData, expectedAverageData) {
- t.Fatalf("want %#v, got %#v", expectedAverageData, gotAverageData)
- }
- if !reflect.DeepEqual(gotAverageRangeData, expectedAverageRangeData) {
- t.Fatalf("want %#v, got %#v", expectedAverageRangeData, gotAverageRangeData)
- }
-}
diff --git a/oncall/doc.go b/oncall/doc.go
index 6c53f5f..5e9725e 100644
--- a/oncall/doc.go
+++ b/oncall/doc.go
@@ -12,7 +12,6 @@
oncall [flags] <command>
The oncall commands are:
- collect Collect data for oncall dashboard
serve Serve oncall dashboard data from Google Storage
help Display help for commands or topics
@@ -28,29 +27,6 @@
-time=false
Dump timing information to stderr before exiting the program.
-Oncall collect - Collect data for oncall dashboard
-
-This subcommand collects data from Google Cloud Monitoring and stores the
-processed data to Google Storage.
-
-Usage:
- oncall collect [flags]
-
-The oncall collect flags are:
- -bin-dir=
- The path where all binaries are downloaded.
- -key=
- The path to the service account's JSON credentials file.
- -project=
- The GCM's corresponding GCE project ID.
- -v23.credentials=
- The path to v23 credentials.
-
- -color=true
- Use color to format output.
- -v=false
- Print verbose output.
-
Oncall serve - Serve oncall dashboard data from Google Storage
Serve oncall dashboard data from Google Storage.
diff --git a/vmon/servicecommon.go b/vmon/servicecommon.go
index b7153a9..b27a403 100644
--- a/vmon/servicecommon.go
+++ b/vmon/servicecommon.go
@@ -6,50 +6,15 @@
import (
"fmt"
- "io"
"math"
"strings"
cloudmonitoring "google.golang.org/api/monitoring/v3"
"v.io/jiri/tool"
- "v.io/v23"
- "v.io/v23/context"
- "v.io/v23/naming"
- "v.io/v23/options"
- "v.io/v23/rpc"
- "v.io/v23/services/stats"
- "v.io/v23/vdl"
- "v.io/x/devtools/internal/monitoring"
"v.io/x/devtools/internal/test"
)
-// Human-readable service names.
-const (
- snMounttable = "mounttable"
- snIdentity = "identity service"
- snMacaroon = "macaroon service"
- snGoogleIdentity = "google identity service"
- snBinaryDischarger = "binary discharger"
- snRole = "role service"
- snProxy = "proxy service"
-
- hostnameStatSuffix = "__debug/stats/system/hostname"
- zoneStatSuffix = "__debug/stats/system/gce/zone"
-)
-
-// serviceMountedNames is a map from human-readable service names to their
-// relative mounted names in the global mounttable.
-var serviceMountedNames = map[string]string{
- snMounttable: "",
- snIdentity: "identity/dev.v.io:u",
- snMacaroon: "identity/dev.v.io:u/macaroon",
- snGoogleIdentity: "identity/dev.v.io:u/google",
- snBinaryDischarger: "identity/dev.v.io:u/discharger",
- snRole: "identity/role",
- snProxy: "proxy-mon",
-}
-
type aggregator struct {
data []float64
min float64
@@ -103,152 +68,6 @@
}
}
-func getMountedName(serviceName string) (string, error) {
- relativeName, ok := serviceMountedNames[serviceName]
- if !ok {
- return "", fmt.Errorf("service name %q not found", serviceName)
- }
- return fmt.Sprintf("%s/%s", namespaceRootFlag, relativeName), nil
-}
-
-// getStat gets the given stat using rpc.
-func getStat(v23ctx *context.T, ctx *tool.Context, me naming.MountEntry, pattern string) ([]*statValue, error) {
- v23ctx, cancel := context.WithTimeout(v23ctx, timeout)
- defer cancel()
-
- call, err := v23.GetClient(v23ctx).StartCall(v23ctx, "", rpc.GlobMethod, []interface{}{pattern}, options.Preresolved{&me})
- if err != nil {
- return nil, err
- }
- hasErrors := false
- ret := []*statValue{}
- mountEntryName := me.Name
- for {
- var gr naming.GlobReply
- err := call.Recv(&gr)
- if err == io.EOF {
- break
- }
- if err != nil {
- return nil, err
- }
-
- switch v := gr.(type) {
- case naming.GlobReplyEntry:
- me.Name = naming.Join(mountEntryName, v.Value.Name)
- value, err := stats.StatsClient("").Value(v23ctx, options.Preresolved{&me})
- if err != nil {
- fmt.Fprintf(ctx.Stderr(), "Failed to get stat (pattern: %q, entry: %#v): %v\n%v\n", pattern, me, v.Value.Name, err)
- hasErrors = true
- continue
- }
- var convertedValue interface{}
- if err := vdl.Convert(&convertedValue, value); err != nil {
- fmt.Fprintf(ctx.Stderr(), "Failed to convert value for %v (pattern: %q, entry: %#v): %v\n", pattern, me, v.Value.Name, err)
- hasErrors = true
- continue
- }
- ret = append(ret, &statValue{
- name: v.Value.Name,
- value: convertedValue,
- })
- case naming.GlobReplyError:
- fmt.Fprintf(ctx.Stderr(), "Glob failed at %q: %v", v.Value.Name, v.Value.Error)
- }
- }
- if hasErrors || len(ret) == 0 {
- return nil, fmt.Errorf("failed to get stat (pattern: %q, entry: %#v)", pattern, me)
- }
- if err := call.Finish(); err != nil {
- return nil, err
- }
-
- return ret, nil
-}
-
-// resolveAndProcessServiceName resolves the given service name and groups the
-// result entries by their routing ids.
-func resolveAndProcessServiceName(v23ctx *context.T, ctx *tool.Context, serviceName, serviceMountedName string) (map[string]naming.MountEntry, error) {
- // Resolve the name.
- v23ctx, cancel := context.WithTimeout(v23ctx, timeout)
- defer cancel()
-
- ns := v23.GetNamespace(v23ctx)
- entry, err := ns.ShallowResolve(v23ctx, serviceMountedName)
- if err != nil {
- return nil, err
- }
- resolvedNames := []string{}
- for _, server := range entry.Servers {
- fullName := naming.JoinAddressName(server.Server, entry.Name)
- resolvedNames = append(resolvedNames, fullName)
- }
-
- // Group resolved names by their routing ids.
- groups := map[string]naming.MountEntry{}
- if serviceName == snMounttable {
- // Mounttable resolves to itself, so we just use a dummy routing id with
- // its original mounted name.
- groups["-"] = naming.MountEntry{
- Servers: []naming.MountedServer{naming.MountedServer{Server: serviceMountedName}},
- }
- } else {
- for _, resolvedName := range resolvedNames {
- serverName, relativeName := naming.SplitAddressName(resolvedName)
- ep, err := v23.NewEndpoint(serverName)
- if err != nil {
- return nil, err
- }
- routingId := ep.RoutingID().String()
- if _, ok := groups[routingId]; !ok {
- groups[routingId] = naming.MountEntry{}
- }
- curMountEntry := groups[routingId]
- curMountEntry.Servers = append(curMountEntry.Servers, naming.MountedServer{Server: serverName})
- // resolvedNames are resolved from the same service so they should have
- // the same relative name.
- curMountEntry.Name = relativeName
- groups[routingId] = curMountEntry
- }
- }
-
- return groups, nil
-}
-
-// getServiceLocation returns the given service's location (instance and zone).
-// If the service is replicated, the instance name is the pod name.
-//
-// To make it simpler and faster, we look up service's location in hard-coded "zone maps"
-// for both non-replicated and replicated services.
-func getServiceLocation(v23ctx *context.T, ctx *tool.Context, me naming.MountEntry) (*monitoring.ServiceLocation, error) {
- // Check "__debug/stats/system/metadata/hostname" stat to get service's
- // host name.
- me.Name = ""
- hostnameResult, err := getStat(v23ctx, ctx, me, hostnameStatSuffix)
- if err != nil {
- return nil, err
- }
- hostname := hostnameResult[0].getStringValue()
-
- // Check "__debug/stats/system/gce/zone" stat to get service's
- // zone name.
- zoneResult, err := getStat(v23ctx, ctx, me, zoneStatSuffix)
- if err != nil {
- return nil, err
- }
- zone := zoneResult[0].getStringValue()
- // The zone stat exported by services is in the form of:
- // projects/632758215260/zones/us-central1-c
- // We only need the last part.
- parts := strings.Split(zone, "/")
- zone = parts[len(parts)-1]
-
- return &monitoring.ServiceLocation{
- Instance: hostname,
- Zone: zone,
- }, nil
-}
-
// sendDataToGCM sends the given metric to Google Cloud Monitoring.
func sendDataToGCM(s *cloudmonitoring.Service, md *cloudmonitoring.MetricDescriptor, value float64, now, instance, zone string, extraLabelKeys ...string) error {
// Sending value 0 will cause error.
diff --git a/vmon/servicecounters.go b/vmon/servicecounters.go
index deb8636..1e43f04 100644
--- a/vmon/servicecounters.go
+++ b/vmon/servicecounters.go
@@ -29,7 +29,7 @@
// checkServiceCounters checks all service counters and adds the results to GCM.
func checkServiceCounters(v23ctx *context.T, ctx *tool.Context, s *cloudmonitoring.Service) error {
counters := map[string][]prodServiceCounter{
- snMounttable: []prodServiceCounter{
+ monitoring.SNMounttable: []prodServiceCounter{
prodServiceCounter{
name: "mounttable nodes",
statSuffix: "__debug/stats/mounttable/num-nodes",
@@ -88,13 +88,13 @@
}
func checkSingleCounter(v23ctx *context.T, ctx *tool.Context, serviceName string, counter prodServiceCounter) ([]counterData, error) {
- mountedName, err := getMountedName(serviceName)
+ mountedName, err := monitoring.GetServiceMountedName(namespaceRootFlag, serviceName)
if err != nil {
return nil, err
}
// Resolve name and group results by routing ids.
- groups, err := resolveAndProcessServiceName(v23ctx, ctx, serviceName, mountedName)
+ groups, err := monitoring.ResolveAndProcessServiceName(v23ctx, ctx, serviceName, mountedName)
if err != nil {
return nil, err
}
@@ -103,17 +103,17 @@
counters := []counterData{}
errors := []error{}
for _, group := range groups {
- counterResult, err := getStat(v23ctx, ctx, group, counter.statSuffix)
+ counterResult, err := monitoring.GetStat(v23ctx, ctx, group, counter.statSuffix)
if err != nil {
errors = append(errors, err)
continue
}
- value, err := counterResult[0].getFloat64Value()
+ value, err := counterResult[0].GetFloat64Value()
if err != nil {
errors = append(errors, err)
continue
}
- location, err := getServiceLocation(v23ctx, ctx, group)
+ location, err := monitoring.GetServiceLocation(v23ctx, ctx, group)
if err != nil {
errors = append(errors, err)
continue
diff --git a/vmon/servicelatency.go b/vmon/servicelatency.go
index 1eb3fef..8856a80 100644
--- a/vmon/servicelatency.go
+++ b/vmon/servicelatency.go
@@ -35,12 +35,12 @@
// checkServiceLatency checks all services and adds their check latency to GCM.
func checkServiceLatency(v23ctx *context.T, ctx *tool.Context, s *cloudmonitoring.Service) error {
serviceNames := []string{
- snMounttable,
- snMacaroon,
- snGoogleIdentity,
- snBinaryDischarger,
- snRole,
- snProxy,
+ monitoring.SNMounttable,
+ monitoring.SNMacaroon,
+ monitoring.SNGoogleIdentity,
+ monitoring.SNBinaryDischarger,
+ monitoring.SNRole,
+ monitoring.SNProxy,
}
hasError := false
@@ -94,16 +94,16 @@
func checkSingleServiceLatency(v23ctx *context.T, ctx *tool.Context, serviceName string) ([]latencyData, error) {
// Get service's mounted name.
- serviceMountedName, err := getMountedName(serviceName)
+ serviceMountedName, err := monitoring.GetServiceMountedName(namespaceRootFlag, serviceName)
if err != nil {
return nil, err
}
// For proxy, we send "signature" RPC to "proxy-mon/__debug" endpoint.
- if serviceName == snProxy {
+ if serviceName == monitoring.SNProxy {
serviceMountedName = fmt.Sprintf("%s/__debug", serviceMountedName)
}
// Resolve name and group results by routing ids.
- groups, err := resolveAndProcessServiceName(v23ctx, ctx, serviceName, serviceMountedName)
+ groups, err := monitoring.ResolveAndProcessServiceName(v23ctx, ctx, serviceName, serviceMountedName)
if err != nil {
return nil, err
}
@@ -123,7 +123,7 @@
} else {
latency = time.Now().Sub(start)
}
- location, err := getServiceLocation(v23ctx, ctx, group)
+ location, err := monitoring.GetServiceLocation(v23ctx, ctx, group)
if err != nil {
return nil, err
}
diff --git a/vmon/servicemetadata.go b/vmon/servicemetadata.go
index e6f1506..cda3aee 100644
--- a/vmon/servicemetadata.go
+++ b/vmon/servicemetadata.go
@@ -28,10 +28,10 @@
// checkServiceMetadata checks all service metadata and adds the results to GCM.
func checkServiceMetadata(v23ctx *context.T, ctx *tool.Context, s *cloudmonitoring.Service) error {
serviceNames := []string{
- snMounttable,
- snIdentity,
- snRole,
- snProxy,
+ monitoring.SNMounttable,
+ monitoring.SNIdentity,
+ monitoring.SNRole,
+ monitoring.SNProxy,
}
hasError := false
@@ -92,13 +92,13 @@
}
func checkSingleServiceMetadata(v23ctx *context.T, ctx *tool.Context, serviceName string) ([]metadataData, error) {
- mountedName, err := getMountedName(serviceName)
+ mountedName, err := monitoring.GetServiceMountedName(namespaceRootFlag, serviceName)
if err != nil {
return nil, err
}
// Resolve name and group results by routing ids.
- groups, err := resolveAndProcessServiceName(v23ctx, ctx, serviceName, mountedName)
+ groups, err := monitoring.ResolveAndProcessServiceName(v23ctx, ctx, serviceName, mountedName)
if err != nil {
return nil, err
}
@@ -108,18 +108,18 @@
errors := []error{}
for _, group := range groups {
// Query build time.
- timeResult, err := getStat(v23ctx, ctx, group, buildTimeStatSuffix)
+ timeResult, err := monitoring.GetStat(v23ctx, ctx, group, buildTimeStatSuffix)
if err != nil {
errors = append(errors, err)
continue
}
- strTime := timeResult[0].getStringValue()
+ strTime := timeResult[0].GetStringValue()
buildTime, err := time.Parse("2006-01-02T15:04:05Z", strTime)
if err != nil {
errors = append(errors, fmt.Errorf("Parse(%v) failed: %v", strTime, err))
continue
}
- location, err := getServiceLocation(v23ctx, ctx, group)
+ location, err := monitoring.GetServiceLocation(v23ctx, ctx, group)
if err != nil {
errors = append(errors, err)
continue
diff --git a/vmon/servicemethodlatency.go b/vmon/servicemethodlatency.go
index 34ff880..fb713d1 100644
--- a/vmon/servicemethodlatency.go
+++ b/vmon/servicemethodlatency.go
@@ -34,10 +34,10 @@
// adds the results to GCM.
func checkServicePerMethodLatency(v23ctx *context.T, ctx *tool.Context, s *cloudmonitoring.Service) error {
serviceNames := []string{
- snMounttable,
- snIdentity,
- snRole,
- snProxy,
+ monitoring.SNMounttable,
+ monitoring.SNIdentity,
+ monitoring.SNRole,
+ monitoring.SNProxy,
}
hasError := false
@@ -99,13 +99,13 @@
}
func checkSingleServicePerMethodLatency(v23ctx *context.T, ctx *tool.Context, serviceName string) ([]perMethodLatencyData, error) {
- mountedName, err := getMountedName(serviceName)
+ mountedName, err := monitoring.GetServiceMountedName(namespaceRootFlag, serviceName)
if err != nil {
return nil, err
}
// Resolve name and group results by routing ids.
- groups, err := resolveAndProcessServiceName(v23ctx, ctx, serviceName, mountedName)
+ groups, err := monitoring.ResolveAndProcessServiceName(v23ctx, ctx, serviceName, mountedName)
if err != nil {
return nil, err
}
@@ -116,7 +116,7 @@
for _, group := range groups {
latency := map[string]float64{}
// Run "debug stats read" for the corresponding object.
- statsResult, err := getStat(v23ctx, ctx, group, statsSuffix)
+ statsResult, err := monitoring.GetStat(v23ctx, ctx, group, statsSuffix)
if err != nil {
errors = append(errors, err)
continue
@@ -124,11 +124,11 @@
// Parse output.
latPerMethod := map[string]float64{}
for _, r := range statsResult {
- data, ok := r.value.(stats.HistogramValue)
+ data, ok := r.Value.(stats.HistogramValue)
if !ok {
return nil, fmt.Errorf("invalid latency data: %v", r)
}
- matches := latMethodRE.FindStringSubmatch(r.name)
+ matches := latMethodRE.FindStringSubmatch(r.Name)
if matches == nil {
continue
}
@@ -144,7 +144,7 @@
errors = append(errors, fmt.Errorf("failed to check latency for service %q", serviceName))
continue
}
- location, err := getServiceLocation(v23ctx, ctx, group)
+ location, err := monitoring.GetServiceLocation(v23ctx, ctx, group)
if err != nil {
errors = append(errors, err)
continue
diff --git a/vmon/serviceqps.go b/vmon/serviceqps.go
index 4fa0c9c..ceae0db 100644
--- a/vmon/serviceqps.go
+++ b/vmon/serviceqps.go
@@ -37,10 +37,10 @@
// the results to GCM.
func checkServiceQPS(v23ctx *context.T, ctx *tool.Context, s *cloudmonitoring.Service) error {
serviceNames := []string{
- snMounttable,
- snIdentity,
- snRole,
- snProxy,
+ monitoring.SNMounttable,
+ monitoring.SNIdentity,
+ monitoring.SNRole,
+ monitoring.SNProxy,
}
hasError := false
@@ -124,13 +124,13 @@
}
func checkSingleServiceQPS(v23ctx *context.T, ctx *tool.Context, serviceName string) ([]qpsData, error) {
- mountedName, err := getMountedName(serviceName)
+ mountedName, err := monitoring.GetServiceMountedName(namespaceRootFlag, serviceName)
if err != nil {
return nil, err
}
// Resolve name and group results by routing ids.
- groups, err := resolveAndProcessServiceName(v23ctx, ctx, serviceName, mountedName)
+ groups, err := monitoring.ResolveAndProcessServiceName(v23ctx, ctx, serviceName, mountedName)
if err != nil {
return nil, err
}
@@ -141,7 +141,7 @@
for _, group := range groups {
perMethodQPS := map[string]float64{}
totalQPS := 0.0
- qpsResults, err := getStat(v23ctx, ctx, group, qpsSuffix)
+ qpsResults, err := monitoring.GetStat(v23ctx, ctx, group, qpsSuffix)
if err != nil {
errors = append(errors, err)
continue
@@ -149,11 +149,11 @@
curPerMethodQPS := map[string]float64{}
curTotalQPS := 0.0
for _, r := range qpsResults {
- data, ok := r.value.(stats.HistogramValue)
+ data, ok := r.Value.(stats.HistogramValue)
if !ok {
return nil, fmt.Errorf("invalid qps data: %v", r)
}
- matches := qpsRE.FindStringSubmatch(r.name)
+ matches := qpsRE.FindStringSubmatch(r.Name)
if matches == nil {
continue
}
@@ -168,7 +168,7 @@
errors = append(errors, fmt.Errorf("failed to check qps for service %q", serviceName))
continue
}
- location, err := getServiceLocation(v23ctx, ctx, group)
+ location, err := monitoring.GetServiceLocation(v23ctx, ctx, group)
if err != nil {
errors = append(errors, err)
continue
