Skip to content

Commit

Permalink
godev/cmd/worker: include all configured versions in the Version chart
Browse files Browse the repository at this point in the history 
A common use case for telemetry data is to see the saturation of
versions among users. For this reason, it is helpful not to project
versions to their MajorMinor, but rather to preserve the exact version
value. For example, we want to see how many users have adopted the most
recent patch of gopls (v0.16.2, as of writing).

However, this could lead to far too many data points, since the upload
config includes every version ever released. Address this by excluding
versions that have no reports.

For now, make this change just for the Version chart. We should consider
it fo the GoVersion chart as well.

Change-Id: I7d30c0e363caba9af9b2960c6e76b975e20f7202
Reviewed-on: https://go-review.googlesource.com/c/telemetry/+/613078
Reviewed-by: Hyang-Ah Hana Kim <[email protected]>
Commit-Queue: Robert Findley <[email protected]>
Auto-Submit: Robert Findley <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>
  • Loading branch information
@findleyr @gopherbot
findleyr authored and gopherbot committed Sep 13, 2024
1 parent 9621737 commit c4c42c5
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 50 deletions.
75 changes: 44 additions & 31 deletions godev/cmd/worker/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -401,21 +401,23 @@ func charts(cfg *tconfig.Config, start, end string, d data, xs []float64) *chart
var charts []*chart
program := programName(p.Name)
if !telemetry.IsToolchainProgram(p.Name) {
charts = append(charts, d.partition(program, "Version", toSliceOf[bucketName](p.Versions), func(b bucketName) bucketName {
if b == "devel" {
return b
}
// map v1.2.3 -> v1.2
return bucketName(semver.MajorMinor(string(b)))
}, compareSemver))
charts = append(charts, d.partition(program, versionCounter, toSliceOf[bucketName](p.Versions), partitionOptions{
ignoreEmptyBuckets: true,
// Don't normalize buckets: we want to see counts for all versions.
compareBuckets: compareSemver,
}))
}
charts = append(charts,
d.partition(program, "GOOS", toSliceOf[bucketName](cfg.GOOS), nil, nil),
d.partition(program, "GOARCH", toSliceOf[bucketName](cfg.GOARCH), nil, nil),
d.partition(program, "GoVersion", toSliceOf[bucketName](cfg.GoVersion), func(b bucketName) bucketName {
// map go1.2.3 -> go1.2
return bucketName(goMajorMinor(string(b)))
}, version.Compare))
d.partition(program, goosCounter, toSliceOf[bucketName](cfg.GOOS), partitionOptions{}),
d.partition(program, goarchCounter, toSliceOf[bucketName](cfg.GOARCH), partitionOptions{}),
d.partition(program, goversionCounter, toSliceOf[bucketName](cfg.GoVersion), partitionOptions{
ignoreEmptyBuckets: true,
normalizeBucket: func(b bucketName) bucketName {
// map go1.2.3 -> go1.2
return bucketName(goMajorMinor(string(b)))
},
compareBuckets: version.Compare,
}))
for _, c := range p.Counters {
// TODO: add support for histogram counters by getting the counter type
// from the chart config.
Expand All @@ -425,7 +427,7 @@ func charts(cfg *tconfig.Config, start, end string, d data, xs []float64) *chart
_, bucket := splitCounterName(counter)
buckets = append(buckets, bucket)
}
charts = append(charts, d.partition(program, chart, buckets, nil, nil))
charts = append(charts, d.partition(program, chart, buckets, partitionOptions{}))
}
for _, p := range charts {
if p != nil {
Expand Down Expand Up @@ -466,16 +468,24 @@ func compareLexically(x, y string) int {
}
}

// partitionOptions controls the behavior of partition charts
type partitionOptions struct {
// If ignoreEmptyBuckets is set, don't include data points with 0 count.
ignoreEmptyBuckets bool

// If normalizeBuckets is provided, it is used to map bucket names to new
// values. Buckets that map to the same value will be merged.
normalizeBucket func(bucketName) bucketName // if set, used to

// If compareBuckets is provided, it is used to sort the buckets, where
// compareBuckets returns -1, 0, or +1 if x < y, x == y, or x > y.
// Otherwise, buckets are sorted lexically.
compareBuckets func(x, y string) int
}

// partition builds a chart for the program and the counter. It can return nil
// if there is no data for the counter in d.
//
// If normalizeBuckets is provided, it is used to map bucket names to new
// values. Buckets that map to the same value will be merged.
//
// If compareBuckets is provided, it is used to sort the buckets, where
// compareBuckets returns -1, 0, or +1 if x < y, x == y, or x > y.
// Otherwise, buckets are sorted lexically.
func (d data) partition(program programName, chartName graphName, buckets []bucketName, normalizeBucket func(bucketName) bucketName, compareBuckets func(x, y string) int) *chart {
func (d data) partition(program programName, chartName graphName, buckets []bucketName, opts partitionOptions) *chart {
chart := &chart{
ID: fmt.Sprintf("charts:%s:%s", program, chartName),
Name: string(chartName),
Expand All @@ -501,8 +511,8 @@ func (d data) partition(program programName, chartName graphName, buckets []buck
}
seen[bucket] = true
key := bucket
if normalizeBucket != nil {
key = normalizeBucket(bucket)
if opts.normalizeBucket != nil {
key = opts.normalizeBucket(bucket)
}
if _, ok := merged[key]; !ok {
merged[key] = make(map[reportID]struct{})
Expand All @@ -520,15 +530,18 @@ func (d data) partition(program programName, chartName graphName, buckets []buck

// datum.Week always points to the end date
for bucket, v := range merged {
d := &datum{
Week: string(end),
Key: string(bucket),
Value: float64(len(v)),
if len(v) > 0 || !opts.ignoreEmptyBuckets {
d := &datum{
Week: string(end),
Key: string(bucket),
Value: float64(len(v)),
}
chart.Data = append(chart.Data, d)
}
chart.Data = append(chart.Data, d)
}
if compareBuckets == nil {
compareBuckets = compareLexically
compareBuckets := compareLexically
if opts.compareBuckets != nil {
compareBuckets = opts.compareBuckets
}
// Sort the data based on bucket name to ensure deterministic output.
sort.Slice(chart.Data, func(i, j int) bool {
Expand Down
57 changes: 38 additions & 19 deletions godev/cmd/worker/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,34 +235,54 @@ func TestPartition(t *testing.T) {
buckets []bucketName
}
tests := []struct {
name string
data data
args args
normalize func(bucketName) bucketName
want *chart
name string
data data
args args
opts partitionOptions
want *chart
}{
{
name: "major.minor.patch version counter",
data: exampleData,
args: args{
program: "example.com/mod/pkg",
name: "Version",
buckets: []bucketName{"v1.2.3", "v2.3.4"},
buckets: []bucketName{"v1.2.3", "v2.3.4", "v4.5.6"},
},
opts: partitionOptions{normalizeBucket: normalVersion},
want: &chart{
ID: "charts:example.com/mod/pkg:Version",
Name: "Version",
Type: "partition",
Data: []*datum{
{Week: "2999-01-01", Key: "v1.2", Value: 2},
{Week: "2999-01-01", Key: "v2.3", Value: 2},
{Week: "2999-01-01", Key: "v4.5", Value: 0},
},
},
},
{
name: "all nonempty versions",
data: exampleData,
args: args{
program: "example.com/mod/pkg",
name: "Version",
buckets: []bucketName{"v1.2.3", "v2.3.4", "v4.5.6"},
},
normalize: normalVersion,
opts: partitionOptions{ignoreEmptyBuckets: true},
want: &chart{
ID: "charts:example.com/mod/pkg:Version",
Name: "Version",
Type: "partition",
Data: []*datum{
{
Week: "2999-01-01",
Key: "v1.2",
Key: "v1.2.3",
Value: 2,
},
{
Week: "2999-01-01",
Key: "v2.3",
Key: "v2.3.4",
Value: 2,
},
},
Expand All @@ -276,7 +296,7 @@ func TestPartition(t *testing.T) {
name: "Version",
buckets: []bucketName{"v1.2.3", "v2.3.4"},
},
normalize: normalVersion,
opts: partitionOptions{normalizeBucket: normalVersion},
want: &chart{
ID: "charts:example.com/mod/pkg:Version",
Name: "Version",
Expand All @@ -303,7 +323,7 @@ func TestPartition(t *testing.T) {
name: "Version",
buckets: []bucketName{"v1.2.3", "v2.3.4", "v1.2.3"},
},
normalize: normalVersion,
opts: partitionOptions{normalizeBucket: normalVersion},
want: &chart{
ID: "charts:example.com/mod/pkg:Version",
Name: "Version",
Expand Down Expand Up @@ -356,7 +376,7 @@ func TestPartition(t *testing.T) {
name: "GoVersion",
buckets: []bucketName{"go1.2.3", "go2.3.4"},
},
normalize: normalGoVersion,
opts: partitionOptions{normalizeBucket: normalGoVersion},
want: &chart{
ID: "charts:example.com/mod/pkg:GoVersion",
Name: "GoVersion",
Expand Down Expand Up @@ -397,7 +417,7 @@ func TestPartition(t *testing.T) {
name: "Version",
buckets: []bucketName{"v1.2.3", "v2.3.4"},
},
normalize: normalVersion,
opts: partitionOptions{normalizeBucket: normalVersion},
want: &chart{
ID: "charts:example.com/mod/pkg:Version",
Name: "Version",
Expand Down Expand Up @@ -517,13 +537,13 @@ func TestPartition(t *testing.T) {
name: "Version",
buckets: []bucketName{"v1.2.3", "v2.3.4"},
},
normalize: normalVersion,
want: nil,
opts: partitionOptions{normalizeBucket: normalVersion},
want: nil,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := tc.data.partition(tc.args.program, tc.args.name, tc.args.buckets, tc.normalize, nil)
got := tc.data.partition(tc.args.program, tc.args.name, tc.args.buckets, tc.opts)
if diff := cmp.Diff(tc.want, got); diff != "" {
t.Errorf("partition() mismatch (-want +got):\n%s", diff)
}
Expand Down Expand Up @@ -592,7 +612,6 @@ func TestCharts(t *testing.T) {
Type: "partition",
Data: []*datum{
{Week: "2999-01-01", Key: "go1.2", Value: 1},
{Week: "2999-01-01", Key: "go1.19"},
},
},
{
Expand All @@ -618,8 +637,8 @@ func TestCharts(t *testing.T) {
Name: "Version",
Type: "partition",
Data: []*datum{
{Week: "2999-01-01", Key: "v0.15", Value: 0},
{Week: "2999-01-01", Key: "v2.3", Value: 2},
{Week: "2999-01-01", Key: "v2.3.4-pre.1", Value: 1},
{Week: "2999-01-01", Key: "v2.3.4", Value: 2},
},
},
{
Expand Down

0 comments on commit c4c42c5

Please sign in to comment.