diff --git a/asap-precompute-go/config.go b/asap-precompute-go/config.go index fb42f07ba..d9b3cde38 100644 --- a/asap-precompute-go/config.go +++ b/asap-precompute-go/config.go @@ -124,6 +124,11 @@ type PrecomputeConfig struct { // SketchType determines which Sketch implementation handles // observations for this AggId. SketchType SketchType + // AggKind is the umbrella aggregation kind stamped onto every emitted + // envelope (Sketch vs Sum). Unset (AggKindUnspecified) is resolved to + // AggKindSketch for any SketchType-bearing config, so existing sketch + // configs are unaffected; a Sum config sets AggKindSum. + AggKind AggregationKind // Mode picks the windowing strategy. Mode AggregationMode // Window configures size / slide / lateness. diff --git a/asap-precompute-go/envelope.go b/asap-precompute-go/envelope.go index 6408b7b37..5979777c4 100644 --- a/asap-precompute-go/envelope.go +++ b/asap-precompute-go/envelope.go @@ -48,6 +48,39 @@ func (s SketchType) String() string { return "Unspecified" } +// AggregationKind is the umbrella over WHAT kind of aggregate an envelope +// carries: a Sketch (whose SketchType sub-tag names the algorithm — +// DDSketch/KLL/HLL/CountSketch/CountMinSketch) or a scalar Sum. This mirrors +// the ASAPQuery backend's AggKind { Sketch | ExactAgg } split: "sketch" is +// ONE aggregation kind and "sum" is a sibling, NOT a SketchType. +// +// Backward compatibility: every producer that predates this field emits the +// zero value (AggKindUnspecified). On decode, an Unspecified AggKind paired +// with a real SketchType is read as AggKindSketch (see EffectiveAggKind), so +// existing sketch envelopes stay byte-identical and decode unchanged. +type AggregationKind uint8 + +const ( + // AggKindUnspecified is the proto3 zero value; resolved via + // EffectiveAggKind (Unspecified + a real SketchType => Sketch). + AggKindUnspecified AggregationKind = iota + // AggKindSketch: the payload is a sketch; SketchType names which. + AggKindSketch + // AggKindSum: the payload is a scalar Sum aggregate ({sum,count}). + AggKindSum +) + +// String returns the canonical aggregation-kind name. +func (a AggregationKind) String() string { + switch a { + case AggKindSketch: + return "Sketch" + case AggKindSum: + return "Sum" + } + return "Unspecified" +} + // Encoding describes how the bytes in SketchEnvelope.Payload are // encoded. Mirrors the design-doc §5.1 SketchEnvelope.encoding enum. type Encoding uint8 @@ -111,6 +144,11 @@ type SketchEnvelope struct { SchemaVersion uint32 // SketchType identifies which sketch algorithm produced Payload. SketchType SketchType + // AggKind is the umbrella aggregation kind (Sketch vs Sum). The zero + // value (AggKindUnspecified) is resolved by EffectiveAggKind to + // AggKindSketch whenever SketchType is set, so every pre-existing + // (sketch-only) producer is byte-for-byte unaffected. + AggKind AggregationKind // AggID is the controller-plan join key. Pairs the envelope to // a specific PrecomputeConfig. AggID AggId @@ -162,4 +200,25 @@ type SketchEnvelope struct { // adapter encode-side reads this to set // Sum.SetAggregationTemporality(...). In-process only. AggregationTemporality int32 + // RelativeAccuracy is the DDSketch alpha (relative accuracy) the + // producing sketch was built with — non-zero only for + // SketchType==DDSketch. In-process only (NOT a proto wire field): the + // OTel adapter's Encode stamps it onto the output pmetric.DDSketch + // container's relative_accuracy so the backend registers a non-zero ε. + // A 0.0 here leaves the container at its zero value, which the backend + // treats as a degenerate (exact, no-bucket) DDSketch — quantile queries + // then capability-miss to the archive and return empty. + RelativeAccuracy float64 +} + +// EffectiveAggKind resolves the envelope's aggregation kind, applying the +// backward-compat default: an unset AggKind on an envelope that carries a +// real SketchType is treated as AggKindSketch (every producer emitted before +// AggKind existed predates the field and only ever produced sketches). A Sum +// producer sets AggKind = AggKindSum explicitly. +func (e *SketchEnvelope) EffectiveAggKind() AggregationKind { + if e.AggKind == AggKindUnspecified && e.SketchType != SketchTypeUnspecified { + return AggKindSketch + } + return e.AggKind } diff --git a/asap-precompute-go/otel/decode.go b/asap-precompute-go/otel/decode.go index 526e47fd8..867afa35d 100644 --- a/asap-precompute-go/otel/decode.go +++ b/asap-precompute-go/otel/decode.go @@ -109,6 +109,28 @@ func decodeMetric( Value: precompute.EnvelopeValue(env), }) } + case pmetric.MetricTypeSumAgg: + dps := m.SumAgg().DataPoints() + for i := 0; i < dps.Len(); i++ { + dp := dps.At(i) + env := &precompute.SketchEnvelope{ + SchemaVersion: 1, + AggKind: precompute.AggKindSum, + ResourceLabels: resourceLabels, + Labels: AttributesToKeyValues(dp.Attributes()), + WindowStartMs: timestampMs(dp.StartTimestamp()), + WindowEndMs: timestampMs(dp.Timestamp()), + Encoding: sumAggEncodingToHostNeutral(dp.Encoding()), + Payload: copyBytes(dp.Sketch()), + } + *out = append(*out, precompute.Observation{ + TimestampMs: timestampMs(dp.Timestamp()), + Metric: name, + ResourceLabels: resourceLabels, + Labels: env.Labels, + Value: precompute.EnvelopeValue(env), + }) + } case pmetric.MetricTypeKLLSketch: dps := m.KLLSketch().DataPoints() for i := 0; i < dps.Len(); i++ { @@ -263,6 +285,18 @@ func ddSketchEncodingToHostNeutral(e pmetric.DDSketchEncoding) precompute.Encodi return precompute.EncodingUnspecified } +func sumAggEncodingToHostNeutral(e pmetric.SumAggEncoding) precompute.Encoding { + switch e { + case pmetric.SumAggEncodingProto: + return precompute.EncodingProtoFull + case pmetric.SumAggEncodingProtoDelta: + return precompute.EncodingProtoDelta + case pmetric.SumAggEncodingMsgpack, pmetric.SumAggEncodingMsgpackDelta: + return precompute.EncodingMsgpack + } + return precompute.EncodingUnspecified +} + // kllSketchEncodingToHostNeutral mirrors the DDSketch helper for KLL. func kllSketchEncodingToHostNeutral(e pmetric.KLLSketchEncoding) precompute.Encoding { switch e { diff --git a/asap-precompute-go/otel/decode_test.go b/asap-precompute-go/otel/decode_test.go index 71048264e..d35106a5f 100644 --- a/asap-precompute-go/otel/decode_test.go +++ b/asap-precompute-go/otel/decode_test.go @@ -76,6 +76,60 @@ func TestDecode_GaugeReadAsInt(t *testing.T) { } } +func TestRoundTrip_SumAggEnvelope(t *testing.T) { + t.Parallel() + // The fixed 16-byte Sum payload SumWrapper produces (and the backend + // cross-language golden): float64 sum (LE) || uint64 count (LE), + // here sum=100, count=4. + payload := []byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x40, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + } + in := &precompute.SketchEnvelope{ + SchemaVersion: 1, + AggKind: precompute.AggKindSum, + MetricName: "google_cluster_2019_cpu_rate", + Labels: []precompute.KeyValue{{Key: "zone", Value: "z1"}}, + WindowStartMs: 1_000, + WindowEndMs: 2_000, + Encoding: precompute.EncodingProtoFull, + Payload: payload, + } + md, err := Encode([]*precompute.SketchEnvelope{in}, &AdapterConfig{}) + if err != nil { + t.Fatalf("encode: %v", err) + } + m := md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0) + if m.Type() != pmetric.MetricTypeSumAgg { + t.Fatalf("encoded metric type: want SumAgg, got %v", m.Type()) + } + dp := m.SumAgg().DataPoints().At(0) + if string(dp.Sketch()) != string(payload) { + t.Errorf("encoded sketch payload mismatch") + } + if dp.Encoding() != pmetric.SumAggEncodingProto { + t.Errorf("encoded encoding: %v", dp.Encoding()) + } + + obs, err := Decode(md, &AdapterConfig{}) + if err != nil { + t.Fatalf("decode: %v", err) + } + if len(obs) != 1 { + t.Fatalf("obs len: want 1, got %d", len(obs)) + } + env := obs[0].Value.Envelope + if env == nil || env.EffectiveAggKind() != precompute.AggKindSum { + t.Fatalf("decoded agg kind: %+v", env) + } + if string(env.Payload) != string(payload) { + t.Errorf("payload round-trip mismatch: want %x got %x", payload, env.Payload) + } + if env.WindowStartMs != 1_000 || env.WindowEndMs != 2_000 { + t.Errorf("window round-trip: [%d,%d)", env.WindowStartMs, env.WindowEndMs) + } +} + func TestDecode_DDSketchProducesEnvelope(t *testing.T) { t.Parallel() payload := []byte{0xDE, 0xAD, 0xBE, 0xEF} diff --git a/asap-precompute-go/otel/encode.go b/asap-precompute-go/otel/encode.go index 2eac310a0..7c35dc8b8 100644 --- a/asap-precompute-go/otel/encode.go +++ b/asap-precompute-go/otel/encode.go @@ -88,9 +88,30 @@ func writeMetric(out pmetric.Metric, env *precompute.SketchEnvelope, cfg *Adapte startTs := pcommon.Timestamp(env.WindowStartMs * 1_000_000) endTs := pcommon.Timestamp(env.WindowEndMs * 1_000_000) + // Sum is a first-class AggregationType (not a sketch): emit the + // modified-OTLP SumAgg metric carrying the SumState envelope bytes. + if env.EffectiveAggKind() == precompute.AggKindSum { + dst := out.SetEmptySumAgg() + dp := dst.DataPoints().AppendEmpty() + KeyValuesToAttributes(env.Labels, dp.Attributes()) + dp.SetStartTimestamp(startTs) + dp.SetTimestamp(endTs) + dp.SetSketch(env.Payload) + dp.SetEncoding(hostNeutralToSumAggEncoding(env.Encoding)) + return nil + } + switch env.SketchType { case precompute.SketchTypeDDSketch: dst := out.SetEmptyDDSketch() + // Stamp the container's relative_accuracy (the DDSketch alpha) so the + // backend registers a non-zero ε. Omitting it left the container at + // 0.0 — a degenerate sketch that makes quantile queries + // capability-miss to the archive and return empty. Mirrors the + // standalone ddsketchprocessor (shim_helpers.go SetRelativeAccuracy). + if env.RelativeAccuracy > 0 { + dst.SetRelativeAccuracy(env.RelativeAccuracy) + } dp := dst.DataPoints().AppendEmpty() KeyValuesToAttributes(env.Labels, dp.Attributes()) dp.SetStartTimestamp(startTs) @@ -190,6 +211,18 @@ func hostNeutralToDDSketchEncoding(e precompute.Encoding) pmetric.DDSketchEncodi return pmetric.DDSketchEncodingProto } +func hostNeutralToSumAggEncoding(e precompute.Encoding) pmetric.SumAggEncoding { + switch e { + case precompute.EncodingProtoFull: + return pmetric.SumAggEncodingProto + case precompute.EncodingProtoDelta: + return pmetric.SumAggEncodingProtoDelta + case precompute.EncodingMsgpack: + return pmetric.SumAggEncodingMsgpack + } + return pmetric.SumAggEncodingProto +} + func hostNeutralToKLLSketchEncoding(e precompute.Encoding) pmetric.KLLSketchEncoding { switch e { case precompute.EncodingMsgpack: diff --git a/asap-precompute-go/otel/relative_accuracy_integration_test.go b/asap-precompute-go/otel/relative_accuracy_integration_test.go new file mode 100644 index 000000000..c3e3778e3 --- /dev/null +++ b/asap-precompute-go/otel/relative_accuracy_integration_test.go @@ -0,0 +1,95 @@ +package otel_test + +import ( + "testing" + "time" + + precompute "github.com/ProjectASAP/asap-precompute-go" + oteladapter "github.com/ProjectASAP/asap-precompute-go/otel" + "github.com/ProjectASAP/asap-precompute-go/sketches" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +// Exercises the REAL DDSketch wrapper through observe -> Drain -> Encode and +// asserts the relative_accuracy (alpha) survives onto the emitted +// pmetric.DDSketch container. Regression guard for the fused-edge bug where +// the wire frame shipped relative_accuracy=0.0 (degenerate sketch -> backend +// quantile queries capability-miss to archive). +func TestDDSketchRelativeAccuracyReachesWire(t *testing.T) { + const alpha = 0.02 + cfg := &precompute.PrecomputeConfig{ + AggID: 1, + SketchType: precompute.SketchTypeDDSketch, + Mode: precompute.Tumbling, + Window: precompute.WindowSpec{Size: 10 * time.Second}, + MetricName: "m", + } + factory := func() precompute.Sketch { return sketches.NewDDSketchWrapper(alpha) } + p := precompute.New(cfg, factory, sketches.DDSketchObserver{}) + for i := 0; i < 5; i++ { + if err := p.Observe(&precompute.Observation{ + TimestampMs: uint64(1000 * (i + 1)), + Metric: "m", + Labels: []precompute.KeyValue{{Key: "k", Value: "v"}}, + Value: precompute.FloatValue(float64(i + 1)), + }); err != nil { + t.Fatalf("observe: %v", err) + } + } + envs := p.Tick(10_000) + if len(envs) == 0 { + t.Fatal("no envelopes drained") + } + if envs[0].RelativeAccuracy != alpha { + t.Fatalf("envelope.RelativeAccuracy: want %v, got %v", alpha, envs[0].RelativeAccuracy) + } + + md, err := oteladapter.Encode(envs, &oteladapter.AdapterConfig{}) + if err != nil { + t.Fatalf("encode: %v", err) + } + var got float64 = -1 + rms := md.ResourceMetrics() + for i := 0; i < rms.Len(); i++ { + sms := rms.At(i).ScopeMetrics() + for j := 0; j < sms.Len(); j++ { + ms := sms.At(j).Metrics() + for k := 0; k < ms.Len(); k++ { + if ms.At(k).Type() == pmetric.MetricTypeDDSketch { + got = ms.At(k).DDSketch().RelativeAccuracy() + } + } + } + } + if got != alpha { + t.Fatalf("in-memory pmetric.DDSketch.RelativeAccuracy: want %v, got %v", alpha, got) + } + + // Decisive: marshal to OTLP proto bytes and back (what the gRPC exporter + // actually ships). If the pdata marshaler drops field 3, relative_accuracy + // is lost on the wire even though it was set in memory. + b, err := (&pmetric.ProtoMarshaler{}).MarshalMetrics(md) + if err != nil { + t.Fatalf("marshal: %v", err) + } + md2, err := (&pmetric.ProtoUnmarshaler{}).UnmarshalMetrics(b) + if err != nil { + t.Fatalf("unmarshal: %v", err) + } + var wire float64 = -1 + rms2 := md2.ResourceMetrics() + for i := 0; i < rms2.Len(); i++ { + sms := rms2.At(i).ScopeMetrics() + for j := 0; j < sms.Len(); j++ { + ms := sms.At(j).Metrics() + for k := 0; k < ms.Len(); k++ { + if ms.At(k).Type() == pmetric.MetricTypeDDSketch { + wire = ms.At(k).DDSketch().RelativeAccuracy() + } + } + } + } + if wire != alpha { + t.Fatalf("AFTER proto round-trip DDSketch.RelativeAccuracy: want %v, got %v (pdata marshaler drops it)", alpha, wire) + } +} diff --git a/asap-precompute-go/precompute.go b/asap-precompute-go/precompute.go index ad352cbd3..112956546 100644 --- a/asap-precompute-go/precompute.go +++ b/asap-precompute-go/precompute.go @@ -588,6 +588,7 @@ func (p *precompute) serializeSeries(entry *seriesEntry, cfg *PrecomputeConfig, return &SketchEnvelope{ SchemaVersion: 1, SketchType: cfg.SketchType, + AggKind: cfg.AggKind, AggID: cfg.AggID, ResourceLabels: entry.ResourceLabels, Labels: labels, @@ -598,9 +599,22 @@ func (p *precompute) serializeSeries(entry *seriesEntry, cfg *PrecomputeConfig, MetricName: cfg.MetricName, Count: entry.Count, AggregationTemporality: cfg.Temporality, + RelativeAccuracy: sketchRelativeAccuracy(entry.Sketch), }, nil } +// sketchRelativeAccuracy reads the DDSketch relative-accuracy alpha off a +// sketch instance when it exposes one (DDSketchWrapper); 0 for every other +// family. Stamped onto the envelope so the OTel encoder can set the output +// pmetric.DDSketch container's relative_accuracy (an ε=0 container is a +// degenerate sketch the backend can't answer quantiles from). +func sketchRelativeAccuracy(s Sketch) float64 { + if ra, ok := s.(interface{ RelativeAccuracy() float64 }); ok { + return ra.RelativeAccuracy() + } + return 0 +} + // UpdateConfig implements Precompute.UpdateConfig. // // Currently picks the FIRST config in the set whose AggID matches diff --git a/asap-precompute-go/sketches/accuracy_realdata_test.go b/asap-precompute-go/sketches/accuracy_realdata_test.go new file mode 100644 index 000000000..548923934 --- /dev/null +++ b/asap-precompute-go/sketches/accuracy_realdata_test.go @@ -0,0 +1,155 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package sketches + +import ( + "bufio" + "encoding/json" + "math" + "os" + "sort" + "testing" +) + +// TestRealDataSketchAccuracy is the "agent-accuracy" check (Phase 4): it +// replays the REAL google_cluster mapped rows through the SAME sketch +// wrappers the fused asap_edge agent uses (sketchlib-go) and asserts each +// family's estimate sits within its accuracy envelope vs exact ground truth. +// +// It reads the JSONL produced by datasets_eval/google_cluster (run.py map); +// point GCT_JSONL at it. Skips (not fails) when the dataset isn't present, +// so it is safe in CI without the ~MB sample: +// +// GCT_JSONL=/tmp/gct-otlp.jsonl go test ./sketches/ -run TestRealDataSketchAccuracy -v +type gctRow struct { + Metric string `json:"metric"` + Value float64 `json:"value"` + Attributes map[string]string `json:"attributes"` +} + +func quantileLinear(sorted []float64, q float64) float64 { + n := len(sorted) + if n == 0 { + return math.NaN() + } + if n == 1 { + return sorted[0] + } + pos := q * float64(n-1) + lo := int(math.Floor(pos)) + hi := int(math.Ceil(pos)) + if lo == hi { + return sorted[lo] + } + frac := pos - float64(lo) + return sorted[lo]*(1-frac) + sorted[hi]*frac +} + +func relErr(got, want float64) float64 { + return math.Abs(got-want) / math.Max(math.Abs(want), 1e-12) +} + +func TestRealDataSketchAccuracy(t *testing.T) { + path := os.Getenv("GCT_JSONL") + if path == "" { + path = "/tmp/gct-otlp.jsonl" + } + f, err := os.Open(path) + if err != nil { + t.Skipf("real-data accuracy: %s not present (run `run.py map`); skipping", path) + } + defer f.Close() + + const cpuMetric = "google_cluster_2019_cpu_rate" + var cpuPos []float64 // cpu_rate values > 0 (DDSketch domain) + var cpuAll []float64 // all cpu_rate values (Sum domain) + distinctSvc := map[string]struct{}{} + + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1<<20), 1<<20) + for sc.Scan() { + line := sc.Bytes() + if len(line) == 0 { + continue + } + var r gctRow + if err := json.Unmarshal(line, &r); err != nil { + t.Fatalf("bad JSONL line: %v", err) + } + if r.Metric != cpuMetric { + continue + } + cpuAll = append(cpuAll, r.Value) + if r.Value > 0 { + cpuPos = append(cpuPos, r.Value) + } + if svc := r.Attributes["service"]; svc != "" { + distinctSvc[svc] = struct{}{} + } + } + if err := sc.Err(); err != nil { + t.Fatalf("scan: %v", err) + } + if len(cpuPos) < 100 { + t.Skipf("real-data accuracy: only %d positive cpu_rate samples; too few", len(cpuPos)) + } + + // --- DDSketch quantile accuracy (alpha = 0.01) --- + dd := NewDDSketchWrapper(0.01) + for _, v := range cpuPos { + dd.Update(v) + } + sorted := append([]float64(nil), cpuPos...) + sort.Float64s(sorted) + for _, q := range []float64{0.50, 0.99} { + want := quantileLinear(sorted, q) + got := dd.Quantile(q) + if e := relErr(got, want); e > 0.05 { + t.Errorf("DDSketch p%.0f: got=%.6g want=%.6g rel_err=%.4f > 0.05", q*100, got, want, e) + } else { + t.Logf("DDSketch p%.0f rel_err=%.4f (got=%.6g want=%.6g)", q*100, e, got, want) + } + } + + // --- HLL distinct-service cardinality --- + hll := NewHLLWrapper() + for _, v := range cpuAll { // re-iterate rows is fine; cardinality is over services + _ = v + } + // Feed each distinct service once is exact; to exercise the estimator we + // feed every row's service (duplicates collapse in HLL). + f2, _ := os.Open(path) + defer f2.Close() + sc2 := bufio.NewScanner(f2) + sc2.Buffer(make([]byte, 1<<20), 1<<20) + for sc2.Scan() { + var r gctRow + if json.Unmarshal(sc2.Bytes(), &r) != nil || r.Metric != cpuMetric { + continue + } + if svc := r.Attributes["service"]; svc != "" { + hll.UpdateBytes([]byte(svc)) + } + } + wantCard := float64(len(distinctSvc)) + gotCard := hll.EstimateCardinality() + if e := relErr(gotCard, wantCard); e > 0.10 { + t.Errorf("HLL distinct(service): got=%.1f want=%.0f rel_err=%.4f > 0.10", gotCard, wantCard, e) + } else { + t.Logf("HLL distinct(service) rel_err=%.4f (got=%.1f want=%.0f)", e, gotCard, wantCard) + } + + // --- Sum (first-class aggregate) lossless --- + sw := NewSumWrapper() + var exactSum float64 + for _, v := range cpuAll { + sw.Update(v) + exactSum += v + } + if e := relErr(sw.Sum(), exactSum); e > 1e-9 { + t.Errorf("Sum: got=%.6f want=%.6f rel_err=%.3g (must be lossless)", sw.Sum(), exactSum, e) + } else { + t.Logf("Sum lossless: %.6f over %d samples", sw.Sum(), sw.Count()) + } +} diff --git a/asap-precompute-go/sketches/ddsketch.go b/asap-precompute-go/sketches/ddsketch.go index 73beee082..9eb499b9a 100644 --- a/asap-precompute-go/sketches/ddsketch.go +++ b/asap-precompute-go/sketches/ddsketch.go @@ -57,6 +57,16 @@ func NewDDSketchWrapper(alpha float64) *DDSketchWrapper { return &DDSketchWrapper{sk: ddsketch.NewDDSketch(alpha), alpha: alpha, sampleP: 1.0} } +// RelativeAccuracy returns the DDSketch alpha (relative accuracy) this +// wrapper was built with. The OTel encoder stamps it onto the emitted +// pmetric.DDSketch container's relative_accuracy field so the backend +// records a non-zero ε on registration. Without it the container defaults +// to 0.0 — a degenerate sketch the backend can't answer quantiles from, so +// `quantile_over_time(...)` capability-misses to the archive and returns +// empty. The standalone ddsketchprocessor sets this via its config; the +// fused asap_edge path lost it because the runtime envelope didn't carry it. +func (w *DDSketchWrapper) RelativeAccuracy() float64 { return w.alpha } + // WithSampleP enables NitroSketch geometric skip-sampling at probability p in // (0,1]. p>=1 (or NaN) disables sampling (exact, the default). Unlike HLL's // hash-threshold sampling, the skip decision is value-independent, so a skipped diff --git a/asap-precompute-go/sketches/sum.go b/asap-precompute-go/sketches/sum.go new file mode 100644 index 000000000..b61e2bbc4 --- /dev/null +++ b/asap-precompute-go/sketches/sum.go @@ -0,0 +1,137 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package sketches + +import ( + "encoding/binary" + "fmt" + "math" + + precompute "github.com/ProjectASAP/asap-precompute-go" +) + +// SumWrapper is the scalar Sum aggregate as a first-class precompute +// family. It is NOT a sketch — the envelope's AggregationKind is Sum — but +// it implements the same precompute.Sketch surface so a precompute.Precompute +// can own it generically, exactly like the sketch wrappers. Its state is a +// running {sum, count} that is additively mergeable (Merge folds partials, +// which is what lets cross-shard / cross-host Sum merge at flush/query time). +// +// Wire format: Sum is an aggregation, NOT a sketch, so it deliberately does +// NOT ride the sketchlib sketch-envelope proto (keeping the Sum aggregate out +// of the public sketch proto entirely). Snapshot emits a small self-contained +// payload — float64 sum (little-endian) followed by uint64 count +// (little-endian), 16 bytes — carried in the modified-OTLP SumAgg metric's +// bytes field. The backend decodes the same fixed layout (no proto, no +// sketchlib dependency). +// +// Delta: Sum is additively mergeable, but this wrapper is FULL-ONLY for now +// (like KLLWrapper) — ComputeDeltaAgainst returns the full snapshot. The +// 16-byte payload makes a delta pointless. True per-window delta is a +// documented follow-up. +type SumWrapper struct { + sum float64 + count uint64 +} + +// NewSumWrapper builds an empty Sum aggregate. +func NewSumWrapper() *SumWrapper { return &SumWrapper{} } + +// Update folds one observation into the running sum. +func (w *SumWrapper) Update(v float64) { + w.sum += v + w.count++ +} + +// sumPayloadLen is the fixed Sum payload size: float64 sum || uint64 count. +const sumPayloadLen = 16 + +// Snapshot emits the fixed 16-byte {sum,count} payload (little-endian). An +// empty window (count == 0) emits nothing (nil), matching the sketch +// wrappers' empty-window behavior. +func (w *SumWrapper) Snapshot() ([]byte, error) { + if w.count == 0 { + return nil, nil + } + b := make([]byte, sumPayloadLen) + binary.LittleEndian.PutUint64(b[0:8], math.Float64bits(w.sum)) + binary.LittleEndian.PutUint64(b[8:16], w.count) + return b, nil +} + +// ComputeDeltaAgainst returns the full snapshot (Sum is full-only for now; +// see the type doc). isFull = true so the runtime tags the frame PROTO_FULL. +func (w *SumWrapper) ComputeDeltaAgainst(_ []byte, _ uint64) ([]byte, bool, error) { + full, err := w.Snapshot() + return full, true, err +} + +// ApplyDelta loads a 16-byte {sum,count} payload and folds it into this +// aggregate (additive). The runtime's mergeFullEnvelope path builds a temp +// sketch and calls ApplyDelta before Merge-ing; for Sum "apply" == add. +func (w *SumWrapper) ApplyDelta(payload []byte) error { + if len(payload) == 0 { + return nil + } + if len(payload) < sumPayloadLen { + return fmt.Errorf("sum.ApplyDelta: payload too short (%d bytes, want %d)", len(payload), sumPayloadLen) + } + w.sum += math.Float64frombits(binary.LittleEndian.Uint64(payload[0:8])) + w.count += binary.LittleEndian.Uint64(payload[8:16]) + return nil +} + +// Merge folds another SumWrapper into this one (associative add). +func (w *SumWrapper) Merge(other precompute.Sketch) error { + if other == nil { + return nil + } + o, ok := other.(*SumWrapper) + if !ok { + return fmt.Errorf("SumWrapper: Merge with %T", other) + } + w.sum += o.sum + w.count += o.count + return nil +} + +// Reset zeros the aggregate in place. +func (w *SumWrapper) Reset() { + w.sum = 0 + w.count = 0 +} + +// Sum returns the accumulated sum (used by the otel adapter encode path to +// stamp the emitted Sum data point's value). +func (w *SumWrapper) Sum() float64 { return w.sum } + +// Count returns the accumulated observation count. +func (w *SumWrapper) Count() uint64 { return w.count } + +// SumObserver implements precompute.SketchObserver: a KindFloat observation +// is folded via Update (the numeric value is the summand). Inbound SumState +// envelopes (KindEnvelope) are routed through Precompute.ObserveEnvelope by +// the runtime and never reach this observer. +type SumObserver struct{} + +// Observe folds a precompute.ObservationValue into the wrapped Sum. +func (SumObserver) Observe(s precompute.Sketch, v precompute.ObservationValue) error { + w, ok := s.(*SumWrapper) + if !ok { + return fmt.Errorf("SumObserver: sketch is %T", s) + } + switch v.Kind { + case precompute.KindFloat: + w.Update(v.Float) + return nil + default: + return fmt.Errorf("SumObserver: unsupported value kind %s", v.Kind) + } +} + +// Compile-time assertions that SumWrapper satisfies the Sketch surface. +var ( + _ precompute.Sketch = (*SumWrapper)(nil) + _ precompute.SketchObserver = SumObserver{} +) diff --git a/asap-precompute-go/sketches/sum_test.go b/asap-precompute-go/sketches/sum_test.go new file mode 100644 index 000000000..21790e902 --- /dev/null +++ b/asap-precompute-go/sketches/sum_test.go @@ -0,0 +1,72 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package sketches + +import ( + "encoding/binary" + "math" + "testing" + + precompute "github.com/ProjectASAP/asap-precompute-go" +) + +func TestSumWrapper_RoundTripMergeReset(t *testing.T) { + w := NewSumWrapper() + for _, v := range []float64{1, 2, 3, 4} { + w.Update(v) + } + if w.Sum() != 10 || w.Count() != 4 { + t.Fatalf("after Update: sum=%v count=%v, want 10/4", w.Sum(), w.Count()) + } + + // Snapshot emits the fixed 16-byte {sum,count} payload; it must decode + // back to the same values. + b, err := w.Snapshot() + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + if len(b) != 16 { + t.Fatalf("Snapshot len = %d, want 16", len(b)) + } + gotSum := math.Float64frombits(binary.LittleEndian.Uint64(b[0:8])) + gotCount := binary.LittleEndian.Uint64(b[8:16]) + if gotSum != 10 || gotCount != 4 { + t.Fatalf("decoded payload sum=%v count=%v, want 10/4", gotSum, gotCount) + } + + // ApplyDelta folds the snapshot into a fresh wrapper (additive load). + w2 := NewSumWrapper() + if err := w2.ApplyDelta(b); err != nil { + t.Fatalf("ApplyDelta: %v", err) + } + if w2.Sum() != 10 || w2.Count() != 4 { + t.Fatalf("after ApplyDelta: sum=%v count=%v, want 10/4", w2.Sum(), w2.Count()) + } + + // Merge is associative add. + if err := w2.Merge(w); err != nil { + t.Fatalf("Merge: %v", err) + } + if w2.Sum() != 20 || w2.Count() != 8 { + t.Fatalf("after Merge: sum=%v count=%v, want 20/8", w2.Sum(), w2.Count()) + } + + // Reset zeros; an empty window emits a nil payload. + w2.Reset() + if w2.Sum() != 0 || w2.Count() != 0 { + t.Fatalf("after Reset: sum=%v count=%v, want 0/0", w2.Sum(), w2.Count()) + } + if empty, _ := w2.Snapshot(); empty != nil { + t.Fatalf("empty Snapshot = %v, want nil", empty) + } + + // Observer routes a KindFloat observation via Update. + w3 := NewSumWrapper() + if err := (SumObserver{}).Observe(w3, precompute.FloatValue(5)); err != nil { + t.Fatalf("Observe: %v", err) + } + if w3.Sum() != 5 || w3.Count() != 1 { + t.Fatalf("after Observe: sum=%v count=%v, want 5/1", w3.Sum(), w3.Count()) + } +} diff --git a/datasets_eval/google_cluster/e2e/README.md b/datasets_eval/google_cluster/e2e/README.md new file mode 100644 index 000000000..fad40eac3 --- /dev/null +++ b/datasets_eval/google_cluster/e2e/README.md @@ -0,0 +1,78 @@ +# google_cluster E2E validation (backend query path) + +Validates the **full warm path** on a real trace: + +``` +mapped OTLP JSONL --replay--> fused asap_edge agent (:4317) + --windows close + ship--> data plane +each query.metricsql --query--> data-plane asap_query (:9091) + --compare--> exact offline ground truth (gt_eval, stdlib-only) +``` + +Unlike `datasets_eval/debs/benchmark/` (which scrapes the standalone +collectors' Prometheus `/metrics`), this exercises the fused processor +and the backend query engine, and compares to **exact offline GT** over +the same replayed rows — not the archive tier. + +## Components + +| File | Role | Status | +|---|---|---| +| `gt_eval.py` | exact offline GT from the mapped JSONL (quantile/sum/count_distinct/topk/frequency) | ✅ verified on real data | +| `compare.py` | per-family pass/fail (quantile<2%, sum lossless, HLL<2%, topk overlap≥0.8 & ρ>0.7, CMS one-sided <5%) | ✅ unit-verified | +| `query_client.py` | instant query against `:9091/api/v1/query`, captures `data_source` | ✅ (needs live stack to exercise) | +| `run_e2e.py` | orchestrator: replay → wait → query → gt → compare → report | ✅ offline path verified | +| `workload-google-cluster.yaml` | controller workload (families/grouping/item_label) | ⚠️ starting point — see CONSTRAINT | +| `../queries.json` | +`id`/`metricsql`/`gt` specs, + CMS `frequency` query | ✅ `run.py validate` green | + +## Recipe + +```bash +cd datasets_eval/google_cluster +# 1. fetch + map a real subsample (egress to storage.googleapis.com required) +python3 run.py fetch --year 2019 --out-dir /tmp/gct --max-rows 100000 +python3 run.py map --year 2019 --in-dir /tmp/gct --out /tmp/gct-otlp.jsonl --cardinality-cap 1000 + +# 2. bring up the fused multinode stack with this workload (see CONSTRAINT) +# deploy/mvp-multinode/scripts/run_demo.sh, CONTROLLER_WORKLOADS=workload-google-cluster.yaml +# (data-plane FIRST, then control-plane; synthetic producer disabled so :4317 is free) + +# 3. run the E2E validation +python3 e2e/run_e2e.py all \ + --jsonl /tmp/gct-otlp.jsonl \ + --otlp-endpoint :4317 \ + --backend http://:9091 \ + --warmup-secs 70 +``` + +Pass = every family meets its threshold **and** reports a warm +`data_source` (a `thanos_archive` fall-through is a fail even if the +number matches). + +## CONSTRAINT: one family per metric + +The fused edge assigns ONE warm family per metric name, and the mapper +emits only two metrics. So a single workload can't run DDSketch + +CountSketch + HLL + CMS on the same metric at once. Either run a +per-family-variant workload (the `workload-google-cluster.yaml` comments +list the variant entries), or extend `otlp_mapper.py` to emit +family-specific metric aliases for a single-pass run. Sum is an +exact-agg, also one-per-metric in the edge config. + +## Open items (need the live multinode stack to finalize) + +- `run_demo.sh gctrace` arm: mount this workload as `CONTROLLER_WORKLOADS`, + disable the synthetic otel-app producer so `:4317` is free for replay. +- Confirm the data plane's exact query response shape + `data_source` + field/header (`query_client.py` parses both body and `X-ASAP-*` headers). +- Validate the `metricsql` spellings resolve warm (adjust per reducer). +- Orchestration gotchas (encoded in the plan): data-plane-before-control-plane, + restart control-plane after any data-plane restart (dedup), kill stale + singlenode stack on :19091/:18080, query `[30s]` matching the sealed window. + +## Verified locally (offline, real data) + +`fetch 4000 → map (cap 200) → gt_eval → compare` produced correct GT for +all 11 queries (p99/p50, by-zone, sums, distinct=173, topk led by +svc-000003, CMS freq=209) and `compare.py` correctly fails a perturbed +lossless-sum and passes a within-band CMS over-estimate. diff --git a/datasets_eval/google_cluster/e2e/compare.py b/datasets_eval/google_cluster/e2e/compare.py new file mode 100644 index 000000000..e36ca6d42 --- /dev/null +++ b/datasets_eval/google_cluster/e2e/compare.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Compare warm backend query results to exact offline ground truth. + +Consumes (a) the GT map from gt_eval.evaluate_queries and (b) a +normalized warm-result map produced by run_e2e.py from the data-plane +query responses, and applies per-family success thresholds: + + quantile rel_err < 0.02 (DDSketch / KLL) + sum rel_err <= 1e-6 (Sum envelope — lossless) + count_unique rel_err < 0.02 (HLL) + topk overlap@k >= 0.80 AND spearman > 0.70 (CountSketch) + frequency warm >= gt (one-sided) AND rel_err < 0.05 (CMS estimate) + +A warm result that fell through to the archive tier is a FAIL even if +the number matches — the harness records `data_source` per query and +flags any non-warm source. + +Normalized warm-result shapes (per query id): + scalar family (global quantile/sum/count/frequency): float + grouped family (by-zone quantile/sum): {group_key: float} + topk family: {member_key: float} +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any + + +# Per-family thresholds (rel-err unless noted). +THRESHOLDS: dict[str, dict[str, float]] = { + "quantile": {"rel_err": 0.02}, + "sum": {"rel_err": 1e-6}, + "count_unique": {"rel_err": 0.02}, + "topk": {"overlap": 0.80, "spearman": 0.70}, + "frequency": {"rel_err": 0.05}, +} + + +def _rel_err(warm: float, gt: float) -> float: + denom = max(abs(gt), 1e-12) + return abs(warm - gt) / denom + + +def _spearman(common: list[str], warm: dict[str, float], gt: dict[str, float]) -> float: + """Spearman rank correlation over the keys present in BOTH maps.""" + if len(common) < 2: + return 1.0 if common else 0.0 + def ranks(d: dict[str, float]) -> dict[str, float]: + ordered = sorted(common, key=lambda k: d[k]) + # average ranks for ties + rk: dict[str, float] = {} + i = 0 + while i < len(ordered): + j = i + while j + 1 < len(ordered) and d[ordered[j + 1]] == d[ordered[i]]: + j += 1 + avg = (i + j) / 2.0 + 1.0 + for t in range(i, j + 1): + rk[ordered[t]] = avg + i = j + 1 + return rk + rw, rg = ranks(warm), ranks(gt) + n = len(common) + d2 = sum((rw[k] - rg[k]) ** 2 for k in common) + return 1.0 - (6.0 * d2) / (n * (n * n - 1)) + + +def compare_one(kind: str, warm: Any, gt: Any) -> dict[str, Any]: + """Return a result dict {pass, metric, detail} for one query.""" + th = THRESHOLDS.get(kind, {}) + + if kind in ("quantile", "sum", "count_unique"): + if isinstance(gt, dict): # grouped + errs = {} + for g, gv in gt.items(): + wv = warm.get(g) if isinstance(warm, dict) else None + errs[g] = _rel_err(float(wv), float(gv)) if wv is not None else float("inf") + worst = max(errs.values()) if errs else float("inf") + ok = worst < th["rel_err"] + return {"pass": ok, "metric": "max_rel_err", "value": worst, + "threshold": th["rel_err"], "per_group": errs} + err = _rel_err(float(warm), float(gt)) if warm is not None else float("inf") + return {"pass": err < th["rel_err"], "metric": "rel_err", + "value": err, "threshold": th["rel_err"]} + + if kind == "topk": + if not isinstance(warm, dict): + return {"pass": False, "metric": "overlap", "value": 0.0, + "detail": "warm result not a top-k map"} + gset, wset = set(gt.keys()), set(warm.keys()) + k = max(len(gset), 1) + overlap = len(gset & wset) / k + common = sorted(gset & wset) + rho = _spearman(common, warm, gt) + ok = overlap >= th["overlap"] and rho > th["spearman"] + return {"pass": ok, "metric": "overlap/spearman", + "overlap": overlap, "spearman": rho, + "overlap_threshold": th["overlap"], "spearman_threshold": th["spearman"], + "missing": sorted(gset - wset)} + + if kind == "frequency": + if warm is None: + return {"pass": False, "metric": "rel_err", "value": float("inf"), + "detail": "no warm result (capability miss / archive fallthrough?)"} + w, g = float(warm), float(gt) + one_sided = w >= g - 1e-9 # CMS over-estimates + err = _rel_err(w, g) + return {"pass": one_sided and err < th["rel_err"], "metric": "rel_err", + "value": err, "threshold": th["rel_err"], "one_sided_ok": one_sided} + + return {"pass": False, "metric": "unknown_kind", "detail": kind} + + +def compare_all( + queries: list[dict[str, Any]], + gt: dict[str, Any], + warm: dict[str, Any], + data_source: dict[str, str] | None = None, +) -> dict[str, Any]: + data_source = data_source or {} + results: dict[str, Any] = {} + n_pass = 0 + for q in queries: + qid = q.get("id") or q.get("promql") + if qid not in gt: + continue + r = compare_one(q["kind"], warm.get(qid), gt[qid]) + src = data_source.get(qid, "unknown") + # A non-warm data source is a fail regardless of numeric match. + if src not in ("", "unknown") and "archive" in src.lower(): + r["pass"] = False + r["data_source_fail"] = src + r["kind"] = q["kind"] + r["data_source"] = src + results[qid] = r + n_pass += 1 if r["pass"] else 0 + return {"n_queries": len(results), "n_pass": n_pass, + "n_fail": len(results) - n_pass, "results": results} + + +def render_report(summary: dict[str, Any]) -> str: + lines = ["# google_cluster E2E accuracy report", ""] + lines.append(f"**{summary['n_pass']}/{summary['n_queries']} queries passed** " + f"({summary['n_fail']} failed)\n") + lines.append("| query | kind | metric | value | threshold | source | pass |") + lines.append("|---|---|---|---|---|---|---|") + for qid, r in sorted(summary["results"].items()): + if r["metric"] == "overlap/spearman": + val = f"ov={r['overlap']:.2f} ρ={r['spearman']:.2f}" + thr = f"ov≥{r['overlap_threshold']} ρ>{r['spearman_threshold']}" + else: + v = r.get("value") + val = f"{v:.3g}" if isinstance(v, (int, float)) and math.isfinite(v) else str(v) + thr = str(r.get("threshold", "")) + lines.append(f"| {qid} | {r['kind']} | {r['metric']} | {val} | {thr} " + f"| {r.get('data_source','?')} | {'✅' if r['pass'] else '❌'} |") + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Compare warm results to GT.") + ap.add_argument("--queries", type=Path, + default=Path(__file__).resolve().parent.parent / "queries.json") + ap.add_argument("--gt", type=Path, required=True, help="GT JSON from gt_eval.") + ap.add_argument("--warm", type=Path, required=True, help="Normalized warm-result JSON.") + ap.add_argument("--data-source", type=Path, default=None, + help="Optional {query_id: data_source} JSON.") + ap.add_argument("--report", type=Path, default=None) + args = ap.parse_args(argv) + + queries = json.loads(args.queries.read_text()) + gt = json.loads(args.gt.read_text()) + warm = json.loads(args.warm.read_text()) + ds = json.loads(args.data_source.read_text()) if args.data_source else {} + summary = compare_all(queries, gt, warm, ds) + report = render_report(summary) + if args.report: + args.report.write_text(report) + print(report) + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 if summary["n_fail"] == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/datasets_eval/google_cluster/e2e/gt_eval.py b/datasets_eval/google_cluster/e2e/gt_eval.py new file mode 100644 index 000000000..30aff14f9 --- /dev/null +++ b/datasets_eval/google_cluster/e2e/gt_eval.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Exact offline ground truth for the google_cluster E2E harness. + +Computes the *exact* answer to each query directly from the mapped +OTLP JSONL (the same bytes replayed into the fused asap_edge agent), +so the warm backend query result can be compared against a real +oracle — not against the archive tier. + +This is deliberately dependency-free (stdlib only): the mapped rows +are small (a bounded subsample) and the aggregates are simple, so we +avoid a pandas/numpy requirement that would not be present on every +eval host. + +Each queries.json entry carries a structured ``gt`` spec (added by +this harness) describing how to compute the exact answer: + + "gt": { + "op": "quantile" | "sum" | "count_distinct" + | "topk_sum" | "topk_count" | "frequency", + "metric": "google_cluster_2019_cpu_rate", + "q": 0.99, # quantile only + "by": ["zone"], # group-by labels ([] = global) + "k": 10, # topk only + "key_label": "host", # topk grouping / count_distinct dim + "item_label": "service", # frequency: which attr is the item + "item_value": "svc-svc-000123" # frequency: the specific item value + } + +Using a structured spec rather than parsing ``expected_ground_truth_query`` +keeps the oracle unambiguous and testable. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any, Iterable + + +# --------------------------------------------------------------------------- +# Row loading + window selection +# --------------------------------------------------------------------------- + + +def load_rows(jsonl_path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + with open(jsonl_path, "r", encoding="utf-8") as fp: + for line in fp: + line = line.strip() + if not line: + continue + rows.append(json.loads(line)) + return rows + + +def select_window( + rows: list[dict[str, Any]], + window_start_ms: int | None, + window_end_ms: int | None, +) -> list[dict[str, Any]]: + """Filter rows to ``[window_start_ms, window_end_ms)``. + + With both bounds ``None`` (the recommended deterministic mode) the + GT is computed over *all* replayed rows — pair this with the + "replay all -> one warm window -> query that window" replay mode so + the warm answer and the GT see the identical sample set. + """ + if window_start_ms is None and window_end_ms is None: + return rows + lo = window_start_ms if window_start_ms is not None else -(1 << 62) + hi = window_end_ms if window_end_ms is not None else (1 << 62) + return [r for r in rows if lo <= int(r["timestamp_ms"]) < hi] + + +# --------------------------------------------------------------------------- +# Aggregate primitives +# --------------------------------------------------------------------------- + + +def _quantile_linear(values: list[float], q: float) -> float: + """Exact φ-quantile with linear interpolation (numpy 'linear' / PromQL). + + index = q * (n - 1); interpolate between the two neighbouring + order statistics. Matches how quantile_over_time computes the + reference value, so a sketch's approximation is measured against + the same definition. + """ + if not values: + return float("nan") + s = sorted(values) + n = len(s) + if n == 1: + return s[0] + pos = q * (n - 1) + lo = math.floor(pos) + hi = math.ceil(pos) + if lo == hi: + return s[int(pos)] + frac = pos - lo + return s[lo] * (1.0 - frac) + s[hi] * frac + + +def _group_key(attrs: dict[str, str], by: list[str]) -> tuple: + return tuple(attrs.get(k, "") for k in by) + + +def _rows_for_metric(rows: Iterable[dict[str, Any]], metric: str) -> Iterable[dict[str, Any]]: + return (r for r in rows if r["metric"] == metric) + + +# --------------------------------------------------------------------------- +# Per-op evaluators +# --------------------------------------------------------------------------- + + +def gt_quantile(rows, spec) -> dict[str, float] | float: + metric = spec["metric"] + q = float(spec["q"]) + by = spec.get("by", []) or [] + if not by: + vals = [float(r["value"]) for r in _rows_for_metric(rows, metric)] + return _quantile_linear(vals, q) + groups: dict[tuple, list[float]] = defaultdict(list) + for r in _rows_for_metric(rows, metric): + groups[_group_key(r["attributes"], by)].append(float(r["value"])) + return {":".join(k): _quantile_linear(v, q) for k, v in groups.items()} + + +def gt_sum(rows, spec) -> dict[str, float] | float: + metric = spec["metric"] + by = spec.get("by", []) or [] + if not by: + return sum(float(r["value"]) for r in _rows_for_metric(rows, metric)) + groups: dict[tuple, float] = defaultdict(float) + for r in _rows_for_metric(rows, metric): + groups[_group_key(r["attributes"], by)] += float(r["value"]) + return {":".join(k): v for k, v in groups.items()} + + +def gt_count_distinct(rows, spec) -> float: + """Exact distinct count of ``key_label`` (or a tuple of labels). + + Under the mapper's --cardinality-cap the distinct alphabet is + closed at N; if ``cap_rescale`` is set the harness recovers the + true cardinality by U/N. Here we report the *observed* distinct + count (what the HLL sees), which is what the warm result is + compared against. + """ + metric = spec["metric"] + dim = spec["key_label"] + dims = dim if isinstance(dim, list) else [dim] + seen: set[tuple] = set() + for r in _rows_for_metric(rows, metric): + seen.add(tuple(r["attributes"].get(d, "") for d in dims)) + return float(len(seen)) + + +def gt_topk(rows, spec, inner: str) -> dict[str, float]: + """Top-k groups by an inner aggregate (sum or count) of ``key_label``.""" + metric = spec["metric"] + k = int(spec["k"]) + key = spec["key_label"] + agg: dict[str, float] = defaultdict(float) + if inner == "sum": + for r in _rows_for_metric(rows, metric): + agg[r["attributes"].get(key, "")] += float(r["value"]) + else: # count + c: Counter = Counter(r["attributes"].get(key, "") for r in _rows_for_metric(rows, metric)) + agg = {kk: float(vv) for kk, vv in c.items()} + top = sorted(agg.items(), key=lambda kv: (-kv[1], kv[0]))[:k] + return dict(top) + + +def gt_frequency(rows, spec) -> float: + """Exact per-item frequency: count of rows whose ``item_label`` equals + ``item_value`` for ``metric`` (optionally within a ``by`` group). + + This is the oracle for the new CMS per-item ``estimate(key)``. + A CMS is a one-sided over-estimator, so the warm answer should be + ``>= `` this value within the relative-error band. + """ + metric = spec["metric"] + item_label = spec["item_label"] + item_value = spec["item_value"] + by = spec.get("by", []) or [] + by_val = spec.get("by_value") + n = 0 + for r in _rows_for_metric(rows, metric): + if r["attributes"].get(item_label, "") != item_value: + continue + if by and by_val is not None: + if _group_key(r["attributes"], by) != tuple(by_val): + continue + n += 1 + return float(n) + + +_OPS = { + "quantile": lambda rows, spec: gt_quantile(rows, spec), + "sum": lambda rows, spec: gt_sum(rows, spec), + "count_distinct": lambda rows, spec: gt_count_distinct(rows, spec), + "topk_sum": lambda rows, spec: gt_topk(rows, spec, "sum"), + "topk_count": lambda rows, spec: gt_topk(rows, spec, "count"), + "frequency": lambda rows, spec: gt_frequency(rows, spec), +} + + +def evaluate_gt(rows: list[dict[str, Any]], spec: dict[str, Any]) -> Any: + op = spec.get("op") + if op not in _OPS: + raise ValueError(f"gt_eval: unknown op {op!r}; supported: {sorted(_OPS)}") + return _OPS[op](rows, spec) + + +def evaluate_queries( + queries: list[dict[str, Any]], + rows: list[dict[str, Any]], +) -> dict[str, Any]: + """Return {query_id_or_index: gt_result} for every query carrying a gt spec.""" + out: dict[str, Any] = {} + for i, q in enumerate(queries): + spec = q.get("gt") + if not spec: + continue + qid = q.get("id") or q.get("promql") or f"query[{i}]" + out[qid] = evaluate_gt(rows, spec) + return out + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Exact offline GT for google_cluster queries.") + ap.add_argument("--jsonl", type=Path, required=True, help="Mapped OTLP JSONL.") + ap.add_argument("--queries", type=Path, + default=Path(__file__).resolve().parent.parent / "queries.json") + ap.add_argument("--out", type=Path, default=None, help="Write GT JSON here (default stdout).") + ap.add_argument("--window-start-ms", type=int, default=None) + ap.add_argument("--window-end-ms", type=int, default=None) + args = ap.parse_args(argv) + + rows = load_rows(args.jsonl) + rows = select_window(rows, args.window_start_ms, args.window_end_ms) + queries = json.loads(args.queries.read_text()) + gt = evaluate_queries(queries, rows) + + payload = json.dumps(gt, indent=2, sort_keys=True) + if args.out: + args.out.write_text(payload + "\n") + print(f"gt_eval: wrote {len(gt)} GT results -> {args.out} " + f"({len(rows)} rows)", file=sys.stderr) + else: + print(payload) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/datasets_eval/google_cluster/e2e/query_client.py b/datasets_eval/google_cluster/e2e/query_client.py new file mode 100644 index 000000000..7e635323c --- /dev/null +++ b/datasets_eval/google_cluster/e2e/query_client.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Thin client for the ASAP data-plane query engine (asap_query). + +Issues an instant MetricsQL/PromQL query against the data plane's +`/api/v1/query` surface (default node2 `:9091`) and returns the parsed +Prometheus-style response plus the `data_source` the engine reports — +so the harness can flag any query that fell through to the archive +tier instead of being answered warm. + +Stdlib-only (urllib) so it runs without the requests dependency. +""" + +from __future__ import annotations + +import json +import urllib.parse +import urllib.request +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class QueryResult: + query: str + ok: bool + result_type: str = "" # "vector" | "scalar" | "matrix" + series: list[dict[str, Any]] = field(default_factory=list) # [{labels, value}] + data_source: str = "unknown" # "warm"/"sketch"/"precompute"/"thanos_archive"/... + fallback_used: bool = False + raw: dict[str, Any] = field(default_factory=dict) + error: str = "" + + +def query_instant( + base_url: str, + promql: str, + timeout_s: float = 20.0, + engine_header: str | None = None, +) -> QueryResult: + """Run an instant query. `engine_header` sets X-ASAP-Engine (e.g. + 'thanos_archive') for the optional archive cross-check.""" + url = base_url.rstrip("/") + "/api/v1/query?" + urllib.parse.urlencode({"query": promql}) + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + if engine_header: + req.add_header("X-ASAP-Engine", engine_header) + hdr_src = None + try: + with urllib.request.urlopen(req, timeout=timeout_s) as resp: + body = json.loads(resp.read().decode("utf-8")) + hdr_src = resp.headers.get("X-ASAP-Data-Source") or resp.headers.get("X-ASAP-Engine") + except urllib.error.HTTPError as exc: + # The data plane returns 4xx with a JSON body for some error shapes; + # parse it so callers still see data_source / error text. + try: + body = json.loads(exc.read().decode("utf-8")) + except Exception: # noqa: BLE001 + return QueryResult(query=promql, ok=False, error=str(exc)) + except Exception as exc: # noqa: BLE001 — transport failure + return QueryResult(query=promql, ok=False, error=str(exc)) + + if not isinstance(body, dict): + return QueryResult(query=promql, ok=False, error="non-object response body") + + # data_source is reported in the `infos` array as "data_source: " + # (e.g. "asap_query", "thanos_archive"); fall back to header / body field. + src = hdr_src or "unknown" + for info in body.get("infos") or []: + if isinstance(info, str) and info.strip().startswith("data_source:"): + src = info.split(":", 1)[1].strip() + break + if body.get("data_source"): + src = body["data_source"] + + # `data` is null on a "No result" response — that's a valid EMPTY result + # (the warm tier answered, just no series), NOT a transport error. + data = body.get("data") or {} + rtype = data.get("resultType", "") if isinstance(data, dict) else "" + series: list[dict[str, Any]] = [] + if isinstance(data, dict) and rtype in ("vector", "matrix"): + for s in data.get("result", []): + val = s.get("value") or (s.get("values") or [[None, None]])[-1] + series.append({"labels": s.get("metric", {}), + "value": float(val[1]) if val and val[1] is not None else None}) + elif isinstance(data, dict) and rtype == "scalar": + v = data.get("result") + if v: + series.append({"labels": {}, "value": float(v[1])}) + + err = "" if body.get("status") != "error" else str(body.get("error", "")) + return QueryResult(query=promql, ok=True, result_type=rtype, series=series, + data_source=src, fallback_used=bool(body.get("fallback_used")), + raw=body, error=err) + + +def await_ready(base_url: str, attempts: int = 30, delay_s: float = 2.0) -> bool: + """Poll until the data plane answers a trivial query (or give up).""" + import time + for _ in range(attempts): + r = query_instant(base_url, "vector(1)", timeout_s=5.0) + if r.ok: + return True + time.sleep(delay_s) + return False diff --git a/datasets_eval/google_cluster/e2e/run_e2e.py b/datasets_eval/google_cluster/e2e/run_e2e.py new file mode 100644 index 000000000..2f7e65b09 --- /dev/null +++ b/datasets_eval/google_cluster/e2e/run_e2e.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""End-to-end validation orchestrator for the google_cluster trace. + +Drives the FULL backend query path: + + mapped OTLP JSONL --replay--> fused asap_edge agent (:4317) + --windows close + ship--> data plane + each query.metricsql --query--> data plane asap_query (:9091) + --vs--> exact offline pandas-free GT (gt_eval) + +and emits a per-family pass/fail report (compare). + +Assumes the stack is already up (e.g. via +`deploy/mvp-multinode/scripts/run_demo.sh up gctrace`); use `--bring-up` +to shell that out first. The replay reuses run.py's OTLP/gRPC sender. + +Subcommands: + all replay -> wait -> query -> gt -> compare -> report + query query-only (stack already fed): query -> gt -> compare + +Determinism: default replay mode pushes ALL rows as fast as possible, +then we query the single sealed warm window and compute GT over all +replayed rows (so warm and GT see the identical sample set). The +agent/window timing gotchas from the plan are encoded in WARMUP_S. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent +DATASET_ROOT = ROOT.parent +sys.path.insert(0, str(ROOT)) + +import gt_eval # noqa: E402 +import compare as cmp # noqa: E402 +import query_client # noqa: E402 + + +def _normalize_warm(qspec: dict[str, Any], qr: "query_client.QueryResult") -> Any: + """Map a Prometheus-style response to the shape compare.py expects. + + scalar family -> float; grouped/topk family -> {key: float} keyed by + the gt spec's grouping label (`by[0]` or `key_label`). + """ + gt = qspec.get("gt", {}) + op = gt.get("op", "") + if not qr.ok or not qr.series: + return None + if op in ("quantile", "sum") and (gt.get("by")): + label = gt["by"][0] + return {s["labels"].get(label, ""): s["value"] for s in qr.series} + if op in ("topk_sum", "topk_count"): + label = gt["key_label"] + return {s["labels"].get(label, ""): s["value"] for s in qr.series} + # scalar families: global quantile/sum, count_distinct, frequency + return qr.series[0]["value"] + + +def _run_queries(base_url: str, queries: list[dict[str, Any]], + archive_cross_check: bool) -> tuple[dict, dict]: + warm: dict[str, Any] = {} + data_source: dict[str, str] = {} + for q in queries: + if "gt" not in q: + continue + qid = q.get("id") or q.get("promql") + promql = q.get("metricsql") or q["promql"] + qr = query_client.query_instant(base_url, promql) + warm[qid] = _normalize_warm(q, qr) + data_source[qid] = qr.data_source if qr.ok else f"error:{qr.error[:60]}" + status = "ok" if qr.ok else "ERR" + print(f" query {qid:24s} [{status}] src={data_source[qid]} -> {warm[qid]}", + file=sys.stderr) + return warm, data_source + + +def cmd_all(args: argparse.Namespace) -> int: + if args.bring_up: + print("run_e2e: bringing up stack (run_demo.sh up gctrace)...", file=sys.stderr) + subprocess.check_call([str(args.run_demo), "up", "gctrace"]) + + print(f"run_e2e: awaiting data plane at {args.backend} ...", file=sys.stderr) + if not query_client.await_ready(args.backend): + print("run_e2e: data plane not ready; aborting.", file=sys.stderr) + return 2 + + # Replay the mapped JSONL into the agent via run.py's OTLP/gRPC sender. + print(f"run_e2e: replaying {args.jsonl} -> {args.otlp_endpoint} ...", file=sys.stderr) + rc = subprocess.call([ + sys.executable, str(DATASET_ROOT / "run.py"), "replay", + "--jsonl", str(args.jsonl), + "--endpoint", args.otlp_endpoint, + "--pace-factor", str(args.pace_factor), + ]) + if rc != 0: + print(f"run_e2e: replay failed (rc={rc})", file=sys.stderr) + return rc + + print(f"run_e2e: waiting {args.warmup_secs}s for window close + ship ...", file=sys.stderr) + time.sleep(args.warmup_secs) + return _query_gt_compare(args) + + +def cmd_query(args: argparse.Namespace) -> int: + return _query_gt_compare(args) + + +def _query_gt_compare(args: argparse.Namespace) -> int: + queries = json.loads(args.queries.read_text()) + rows = gt_eval.load_rows(args.jsonl) + rows = gt_eval.select_window(rows, args.window_start_ms, args.window_end_ms) + + gt = gt_eval.evaluate_queries(queries, rows) + warm, data_source = _run_queries(args.backend, queries, args.archive_cross_check) + + summary = cmp.compare_all(queries, gt, warm, data_source) + report = cmp.render_report(summary) + + args.out_dir.mkdir(parents=True, exist_ok=True) + (args.out_dir / "gt.json").write_text(json.dumps(gt, indent=2, sort_keys=True) + "\n") + (args.out_dir / "warm.json").write_text(json.dumps(warm, indent=2, sort_keys=True) + "\n") + (args.out_dir / "data_source.json").write_text(json.dumps(data_source, indent=2, sort_keys=True) + "\n") + (args.out_dir / "report.md").write_text(report) + (args.out_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") + print(report) + print(f"run_e2e: {summary['n_pass']}/{summary['n_queries']} passed -> {args.out_dir}", + file=sys.stderr) + return 0 if summary["n_fail"] == 0 else 1 + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="google_cluster E2E validation orchestrator.") + ap.add_argument("command", choices=("all", "query")) + ap.add_argument("--jsonl", type=Path, required=True) + ap.add_argument("--queries", type=Path, default=DATASET_ROOT / "queries.json") + ap.add_argument("--backend", default="http://127.0.0.1:9091", + help="Data-plane query base URL.") + ap.add_argument("--otlp-endpoint", default="127.0.0.1:4317", + help="Fused agent OTLP/gRPC endpoint.") + ap.add_argument("--out-dir", type=Path, + default=Path("/tmp/gct-e2e-" + time.strftime("%Y%m%d-%H%M%S"))) + ap.add_argument("--pace-factor", type=float, default=0.0) + ap.add_argument("--warmup-secs", type=float, default=70.0) + ap.add_argument("--window-start-ms", type=int, default=None) + ap.add_argument("--window-end-ms", type=int, default=None) + ap.add_argument("--archive-cross-check", action="store_true") + ap.add_argument("--bring-up", action="store_true") + ap.add_argument("--run-demo", type=Path, + default=DATASET_ROOT.parent.parent / "deploy" / "mvp-multinode" / "scripts" / "run_demo.sh") + args = ap.parse_args(argv) + return cmd_all(args) if args.command == "all" else cmd_query(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/datasets_eval/google_cluster/e2e/workload-google-cluster.yaml b/datasets_eval/google_cluster/e2e/workload-google-cluster.yaml new file mode 100644 index 000000000..eea6c20bf --- /dev/null +++ b/datasets_eval/google_cluster/e2e/workload-google-cluster.yaml @@ -0,0 +1,82 @@ +# google_cluster E2E controller workload (CONTROLLER_WORKLOADS) +# +# Mirrors deploy/mvp-multinode/configs/asap/mvp-workload.yaml schema +# (controller/src/config/workloads.rs::WorkloadEntry). Drives the fused +# asap_edge agent for the google_cluster trace replay. +# +# IMPORTANT CONSTRAINT (one family per metric in the fused edge): +# the fused asap_edge processor assigns ONE warm family per metric name. +# The mapper (otlp_mapper.py) emits only two metrics +# (google_cluster_2019_cpu_rate, google_cluster_2019_memory_usage), so a +# single workload cannot exercise DDSketch + CountSketch + HLL + CMS on +# the SAME metric simultaneously. Two ways to get full per-family coverage +# on real data: +# (A) run the harness once per family-variant workload (this file is the +# quantile+sum variant; see the *_topk / *_card / *_cms siblings), or +# (B) extend otlp_mapper.py to emit family-specific metric aliases +# (e.g. *_cpu_rate_q, *_cpu_rate_topk, *_cpu_rate_card, *_cpu_rate_cms) +# from the same rows, then declare each alias with its own family here. +# (B) is preferred for a single-pass E2E run; it is a small mapper change. +# This file uses (A)'s quantile+sum slice as the default, and lists the +# other families as commented entries to copy into per-family variants. + +# --- DDSketch quantile, per-series (q-cpu-p99, q-cpu-p50) --- +- metric_name: google_cluster_2019_cpu_rate + query_string: "quantile_over_time(0.99, google_cluster_2019_cpu_rate[30s])" + accuracy_sla: 0.02 + assign_to_role: agent + sketch_family_override: DDSketch + # NO grouping_labels — per-series quantile preserves PromQL semantics. + +# --- DDSketch quantile, grouped by zone (q-mem-p99-by-zone) --- +- metric_name: google_cluster_2019_memory_usage + query_string: "quantile_over_time(0.99, google_cluster_2019_memory_usage[30s])" + accuracy_sla: 0.02 + assign_to_role: agent + sketch_family_override: DDSketch + grouping_labels: + - zone + +# --- Sum aggregation-type (q-sum-cpu, q-sum-mem-by-zone) --- +# Exact-agg / Sum path. With Phase 3 landed these resolve via the new +# first-class Sum *envelope*; before Phase 3 they resolve via the legacy +# ExactAgg(Sum) precompute path. Both should be lossless. +# NOTE: cpu_rate is declared as a sketch above; a metric cannot be both +# a sketch family AND a Sum entry in one fused config, so the sum-cpu +# query belongs in a Sum-variant workload (assign cpu_rate family=Sum). +- metric_name: google_cluster_2019_memory_usage_sum + query_string: "sum by (zone) (google_cluster_2019_memory_usage_sum)" + accuracy_sla: 0.0 + assign_to_role: agent + grouping_labels: + - zone + +# === Per-family variant entries (copy into a dedicated workload, or pair +# with mapper alias metrics, per the CONSTRAINT note above) =========== +# +# --- CountSketch top-K by host CPU (q-topk-host-cpu) --- +# - metric_name: google_cluster_2019_cpu_rate +# query_string: "topk(10, sum by (host) (google_cluster_2019_cpu_rate))" +# accuracy_sla: 0.05 +# assign_to_role: agent +# sketch_family_override: CountSketch +# grouping_labels: [zone] +# item_label: host # heavy-hitter dimension the heap ranks +# +# --- HLL distinct-service cardinality (q-card-service) --- +# - metric_name: google_cluster_2019_cpu_rate +# query_string: "count(google_cluster_2019_cpu_rate)" +# accuracy_sla: 0.02 +# assign_to_role: agent +# sketch_family_override: HLL +# grouping_labels: [zone] +# item_label: service # HLL counts DISTINCT service values +# +# --- CountMinSketch per-item frequency (q-cms-freq-service) --- +# - metric_name: google_cluster_2019_cpu_rate +# query_string: "count_over_time(google_cluster_2019_cpu_rate[30s])" +# accuracy_sla: 0.05 +# assign_to_role: agent +# sketch_family_override: CountMinSketch +# grouping_labels: [zone] +# item_label: service # REQUIRED for per-item estimate(key) (Phase 2) diff --git a/datasets_eval/google_cluster/queries.json b/datasets_eval/google_cluster/queries.json index 9792dc4bf..6d97dabdb 100644 --- a/datasets_eval/google_cluster/queries.json +++ b/datasets_eval/google_cluster/queries.json @@ -1,62 +1,101 @@ [ { + "id": "q-cpu-p99", "kind": "quantile", "promql": "histogram_quantile(0.99, sum by (le) (google_cluster_2019_cpu_rate))", + "metricsql": "quantile_over_time(0.99, google_cluster_2019_cpu_rate[30s])", "expected_ground_truth_query": "quantile(0.99, google_cluster_2019_cpu_rate)", + "gt": {"op": "quantile", "metric": "google_cluster_2019_cpu_rate", "q": 0.99, "by": []}, "rationale": "p99 of per-instance CPU usage rates over the cell — claim #1 (quantile accuracy). Natural for the Google trace because instance_usage is sampled at 5-min intervals across all instances in a cell." }, { + "id": "q-cpu-p50", "kind": "quantile", "promql": "histogram_quantile(0.50, sum by (le) (google_cluster_2019_cpu_rate))", + "metricsql": "quantile_over_time(0.50, google_cluster_2019_cpu_rate[30s])", "expected_ground_truth_query": "quantile(0.50, google_cluster_2019_cpu_rate)", + "gt": {"op": "quantile", "metric": "google_cluster_2019_cpu_rate", "q": 0.50, "by": []}, "rationale": "p50 of per-instance CPU usage rates — paired with p99 for the quantile accuracy claim." }, { + "id": "q-mem-p99-by-zone", "kind": "quantile", "promql": "histogram_quantile(0.99, sum by (le, zone) (google_cluster_2019_memory_usage))", + "metricsql": "quantile_over_time(0.99, google_cluster_2019_memory_usage[30s])", "expected_ground_truth_query": "quantile by (zone) (0.99, google_cluster_2019_memory_usage)", - "rationale": "Per-zone p99 memory usage — exercises group-by + quantile (claim #2 cross-key roll-up accuracy)." + "gt": {"op": "quantile", "metric": "google_cluster_2019_memory_usage", "q": 0.99, "by": ["zone"]}, + "rationale": "Per-zone p99 memory usage — exercises group-by + quantile (claim #2 cross-key roll-up accuracy). Requires the sketch's grouping_labels=[zone]." }, { + "id": "q-topk-host-cpu", "kind": "topk", "promql": "topk(10, sum by (host) (google_cluster_2019_cpu_rate))", + "metricsql": "topk(10, sum by (host) (google_cluster_2019_cpu_rate))", "expected_ground_truth_query": "topk(10, sum by (host) (google_cluster_2019_cpu_rate))", + "gt": {"op": "topk_sum", "metric": "google_cluster_2019_cpu_rate", "k": 10, "key_label": "host"}, "rationale": "Top-10 busiest hosts by CPU — claim #3 (heavy-hitter accuracy with CountSketch). Natural Google-trace question: 'which machines are most loaded?'" }, { + "id": "q-topk-service-count", "kind": "topk", "promql": "topk(10, count by (service) (google_cluster_2019_cpu_rate))", + "metricsql": "topk(10, count by (service) (google_cluster_2019_cpu_rate))", "expected_ground_truth_query": "topk(10, count by (service) (google_cluster_2019_cpu_rate))", + "gt": {"op": "topk_count", "metric": "google_cluster_2019_cpu_rate", "k": 10, "key_label": "service"}, "rationale": "Top-10 services by sample count — heavy-hitter on identity, complements the CPU-load topk." }, { + "id": "q-sum-cpu", "kind": "sum", "promql": "sum(google_cluster_2019_cpu_rate)", + "metricsql": "sum(google_cluster_2019_cpu_rate)", "expected_ground_truth_query": "sum(google_cluster_2019_cpu_rate)", - "rationale": "Cluster-wide CPU rate — claim #4 (counter exactness; Sum aggregation should be lossless)." + "gt": {"op": "sum", "metric": "google_cluster_2019_cpu_rate", "by": []}, + "rationale": "Cluster-wide CPU rate — claim #4 (counter exactness; the first-class Sum aggregation-type envelope should be lossless)." }, { + "id": "q-sum-mem-by-zone", "kind": "sum", "promql": "sum by (zone) (google_cluster_2019_memory_usage)", + "metricsql": "sum by (zone) (google_cluster_2019_memory_usage)", "expected_ground_truth_query": "sum by (zone) (google_cluster_2019_memory_usage)", - "rationale": "Per-zone memory total — exercises sum + group-by, the simplest cross-key roll-up (claim #2)." + "gt": {"op": "sum", "metric": "google_cluster_2019_memory_usage", "by": ["zone"]}, + "rationale": "Per-zone memory total — exercises the Sum envelope + group-by, the simplest cross-key roll-up (claim #2)." }, { + "id": "q-card-service", "kind": "count_unique", "promql": "count(count by (service) (google_cluster_2019_cpu_rate))", + "metricsql": "count(count by (service) (google_cluster_2019_cpu_rate))", "expected_ground_truth_query": "count_distinct(service) where metric=google_cluster_2019_cpu_rate", + "gt": {"op": "count_distinct", "metric": "google_cluster_2019_cpu_rate", "key_label": "service"}, "rationale": "Distinct-service cardinality — claim #5 (HLL accuracy). Natural Google-trace question: 'how many unique services are running in this cell?'" }, { + "id": "q-card-host", "kind": "count_unique", "promql": "count(count by (host) (google_cluster_2019_cpu_rate))", + "metricsql": "count(count by (host) (google_cluster_2019_cpu_rate))", "expected_ground_truth_query": "count_distinct(host) where metric=google_cluster_2019_cpu_rate", + "gt": {"op": "count_distinct", "metric": "google_cluster_2019_cpu_rate", "key_label": "host"}, "rationale": "Distinct-host cardinality — exercises HLL on a different attribute and confirms cross-attribute consistency." }, { + "id": "q-card-service-task", "kind": "count_unique", "promql": "count(count by (service, task) (google_cluster_2019_cpu_rate))", + "metricsql": "count(count by (service, task) (google_cluster_2019_cpu_rate))", "expected_ground_truth_query": "count_distinct((service, task)) where metric=google_cluster_2019_cpu_rate", + "gt": {"op": "count_distinct", "metric": "google_cluster_2019_cpu_rate", "key_label": ["service", "task"]}, "rationale": "Distinct-instance cardinality — saturates at --cardinality-cap when the cap is engaged; the harness scales by U/N to recover the true count (see otlp_mapper.py docstring 'Cardinality cap')." + }, + { + "id": "q-cms-freq-service", + "kind": "frequency", + "promql": "count_over_time(google_cluster_2019_cpu_rate{service=\"svc-000003\"}[30s])", + "metricsql": "count_over_time(google_cluster_2019_cpu_rate{service=\"svc-000003\"}[30s])", + "expected_ground_truth_query": "frequency(service=svc-000003) where metric=google_cluster_2019_cpu_rate", + "gt": {"op": "frequency", "metric": "google_cluster_2019_cpu_rate", "item_label": "service", "item_value": "svc-000003"}, + "rationale": "Per-item frequency of one service via CountMinSketch estimate(key) (NEW feature). CMS is a one-sided over-estimator: warm answer >= exact within the relative-error band. Requires the CMS metric's item_label=service." } ] diff --git a/datasets_eval/google_cluster/run.py b/datasets_eval/google_cluster/run.py index 3ce1595c7..d67ec3731 100644 --- a/datasets_eval/google_cluster/run.py +++ b/datasets_eval/google_cluster/run.py @@ -44,8 +44,12 @@ # google-cluster log adds `expected_ground_truth_query` so the # accuracy reducer can compare against ground truth. QUERY_REQUIRED_KEYS = {"kind", "promql"} -QUERY_OPTIONAL_KEYS = {"expected_ground_truth_query", "rationale"} -QUERY_ALLOWED_KINDS = {"quantile", "topk", "sum", "count_unique"} +# `metricsql` is the warm-tier query string the E2E harness sends to the +# data-plane query engine; `gt` is the structured ground-truth spec consumed +# by e2e/gt_eval.py; `id` is a stable handle for reports. +QUERY_OPTIONAL_KEYS = {"expected_ground_truth_query", "rationale", "id", "metricsql", "gt"} +# `frequency` covers the CountMinSketch per-item estimate(key) path. +QUERY_ALLOWED_KINDS = {"quantile", "topk", "sum", "count_unique", "frequency"} def _run_module(module_path: Path, argv: list[str]) -> int: diff --git a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/all_families_test.go b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/all_families_test.go index 2c0072e56..7ca372137 100644 --- a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/all_families_test.go +++ b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/all_families_test.go @@ -16,7 +16,7 @@ import ( // processor (gauge in → ObserveKeyed → Drain → Encode out) and asserts the // family constructs, observes, and emits without panic. func TestAllSketchFamiliesFlush(t *testing.T) { - for _, fam := range []FamilyKind{FamilyDDSketch, FamilyKLL, FamilyHLL, FamilyCountSketch, FamilyCountMinSketch} { + for _, fam := range []FamilyKind{FamilyDDSketch, FamilyKLL, FamilyHLL, FamilyCountSketch, FamilyCountMinSketch, FamilySum} { t.Run(string(fam), func(t *testing.T) { cap := &capMetrics{} cfg := &Config{ diff --git a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/fixes_test.go b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/fixes_test.go index d75ca0ed8..192e0cc2b 100644 --- a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/fixes_test.go +++ b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/fixes_test.go @@ -100,21 +100,6 @@ func TestSketchMaxSeriesBounds(t *testing.T) { } } -// TestSumMaxGroupsBounds covers P0 #2 (sum half): the group map is capped and -// overflow is counted. -func TestSumMaxGroupsBounds(t *testing.T) { - sa := newSumAggregator([]string{"zone"}, 2) - for i := 0; i < 5; i++ { - sa.observe(map[string]string{"zone": string(rune('a' + i))}, 1) - } - if len(sa.groups) != 2 { - t.Fatalf("group map size = %d, want 2 (cap not enforced)", len(sa.groups)) - } - if sa.overflowCount.Load() != 3 { - t.Fatalf("overflowCount = %d, want 3", sa.overflowCount.Load()) - } -} - // TestCountSketchCountsAttributeSet covers B6 (#9): CountSketch must count the // per-attribute-set frequency (like CMS), NOT the degenerate single metric-name // key. After observing N samples of attrs {zone=z0}, the reconstructed sketch @@ -530,105 +515,6 @@ func TestShutdownDrainsColdPartEvenWithExpiredCtx(t *testing.T) { } } -// TestSumNeverEmitsInvertedTimestamps covers P2 #7: a future-timestamped sample -// must not permanently skew maxObserved, and an idle window must never emit a -// data point whose start > end. After a window with a far-future sample, the -// watermark is reset; the next (idle) window emits start <= end. -func TestSumNeverEmitsInvertedTimestamps(t *testing.T) { - cap := &capMetrics{} - cfg := &Config{ - ShardCount: 1, - WindowDuration: time.Hour, - DropOriginal: true, - Metrics: []MetricFamily{{Metric: "reqs", Family: FamilySum, AggregateBy: []string{"zone"}}}, - Cold: ColdConfig{Enabled: false}, - } - if err := cfg.Validate(); err != nil { - t.Fatal(err) - } - p, err := newProcessor(cfg, testSettings(), cap) - if err != nil { - t.Fatal(err) - } - // Seed windowStartMs to "now" as Start would. - now := uint64(time.Now().UnixMilli()) - p.windowStartMs.Store(now) - - // A far-future sample raises maxObserved well past windowStart. - future := now + 365*24*3600*1000 // +1 year - md := pmetric.NewMetrics() - m := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() - m.SetName("reqs") - s := m.SetEmptySum() - s.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) - dp := s.DataPoints().AppendEmpty() - dp.Attributes().PutStr("zone", "z0") - dp.SetDoubleValue(1) - dp.SetTimestamp(pcommon.Timestamp(future * 1e6)) - if err := p.ConsumeMetrics(context.Background(), md); err != nil { - t.Fatal(err) - } - // Window 1 flush: emits [start, future]; watermark is then reset to future. - p.flushAll(context.Background()) - // After reset, maxObservedMs must equal windowStartMs (not stuck at future - // for a later idle window in a way that inverts it). - if p.maxObservedMs.Load() != p.windowStartMs.Load() { - t.Fatalf("maxObserved=%d windowStart=%d: watermark not reset to window boundary", - p.maxObservedMs.Load(), p.windowStartMs.Load()) - } - - // Window 2: idle (no new samples). Must emit a point with start <= end. - cap.got = nil - p.flushAll(context.Background()) - // No groups -> emitSumMetric returns early, so no batch is forwarded; assert - // no inverted point exists in whatever was forwarded. - assertNoInvertedSum(t, cap.got) - - // Window 3: one in-range sample after the future skew. - cap.got = nil - md3 := pmetric.NewMetrics() - m3 := md3.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() - m3.SetName("reqs") - s3 := m3.SetEmptySum() - s3.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) - dp3 := s3.DataPoints().AppendEmpty() - dp3.Attributes().PutStr("zone", "z0") - dp3.SetDoubleValue(2) - dp3.SetTimestamp(pcommon.Timestamp((uint64(time.Now().UnixMilli())) * 1e6)) - if err := p.ConsumeMetrics(context.Background(), md3); err != nil { - t.Fatal(err) - } - p.flushAll(context.Background()) - assertNoInvertedSum(t, cap.got) -} - -func assertNoInvertedSum(t *testing.T, batches []pmetric.Metrics) { - t.Helper() - for _, md := range batches { - rms := md.ResourceMetrics() - for i := 0; i < rms.Len(); i++ { - sms := rms.At(i).ScopeMetrics() - for j := 0; j < sms.Len(); j++ { - ms := sms.At(j).Metrics() - for k := 0; k < ms.Len(); k++ { - mm := ms.At(k) - if mm.Type() != pmetric.MetricTypeSum { - continue - } - dps := mm.Sum().DataPoints() - for d := 0; d < dps.Len(); d++ { - dp := dps.At(d) - if dp.StartTimestamp() > dp.Timestamp() { - t.Fatalf("%s dp[%d]: start %d > end %d (inverted)", - mm.Name(), d, dp.StartTimestamp(), dp.Timestamp()) - } - } - } - } - } - } -} - // componenttestHost is a minimal component.Host for Start in tests. type componenttestHost struct{} diff --git a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/flush.go b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/flush.go index 7f57c937c..da4b1b7e0 100644 --- a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/flush.go +++ b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/flush.go @@ -47,19 +47,13 @@ func (p *asapEdgeProcessor) flushLoop() { for { select { case <-p.stopCh: - // Final drain: flush every shard (cold + sketch) AND the unified - // sum so no un-flushed shard is lost on Shutdown. + // Final drain: flush every shard (cold + sketch) so no un-flushed + // shard is lost on Shutdown. p.flushAll(context.Background()) return case <-t.C: shardIdx := tick % n p.flushShardWarmCold(context.Background(), shardIdx) - // Once per full window cycle (after the last shard in a round), - // merge + emit the cross-shard sum so its output cadence and totals - // stay window-aligned and unchanged. - if shardIdx == n-1 { - p.flushSum(context.Background()) - } tick++ } } @@ -103,9 +97,9 @@ func (p *asapEdgeProcessor) flushAll(ctx context.Context) { } out := pmetric.NewMetrics() - // Warm sum: merge partials across all shards, emit one delta Sum per metric. - p.appendSumMetrics(out) - // Warm sketches: per-shard flush (each series lives in one shard). + // Warm: per-shard flush of every aggregator (sketches + Sum, which now + // flushes a SumAgg envelope through the same path; each series lives in one + // shard and the backend sums the per-window SumAgg deltas for the same sid). for _, sh := range p.shards { sh.mu.Lock() for _, sa := range sh.sketchAggs { @@ -117,9 +111,8 @@ func (p *asapEdgeProcessor) flushAll(ctx context.Context) { } // flushShardWarmCold drains ONE shard's cold fragments and flushes that shard's -// sketch aggregators, then forwards the sketch envelopes. The cross-shard sum is -// NOT touched here — it is merged + emitted on the window-aligned cadence by -// flushSum so its delta totals stay unchanged. This is the staggered per-tick +// sketch aggregators (including Sum, which now flushes a SumAgg envelope through +// the same path), then forwards the envelopes. This is the staggered per-tick // unit of work: only shard idx's state is built up and released, so the N shards' // sawtooths phase-shift instead of releasing in lockstep. func (p *asapEdgeProcessor) flushShardWarmCold(ctx context.Context, idx int) { @@ -140,56 +133,3 @@ func (p *asapEdgeProcessor) flushShardWarmCold(ctx context.Context, idx int) { sh.mu.Unlock() p.forward(ctx, out) } - -// flushSum merges the sum partials across ALL shards and emits one delta Sum -// metric per Sum metric, then resets every shard's partials. Run once per -// WindowDuration so the backend's per-group delta total per window is identical -// to the original single-flush behavior. -func (p *asapEdgeProcessor) flushSum(ctx context.Context) { - if len(p.sumMetrics) == 0 { - return - } - out := pmetric.NewMetrics() - p.appendSumMetrics(out) - p.forward(ctx, out) -} - -// appendSumMetrics merges each Sum metric's partials across all shards (resetting -// each shard) and appends one delta Sum metric per name to out. Sum is -// associative, so this is byte/semantically identical regardless of how many -// shard-ticks elapsed since the last sum flush. -func (p *asapEdgeProcessor) appendSumMetrics(out pmetric.Metrics) { - startMs := p.windowStartMs.Load() - now := uint64(time.Now().UnixMilli()) - // endMs is the window's max observed sample timestamp. Two corrections: - // 1. Idle window: no sample this window => maxObserved is still <= startMs, - // which would emit start==end or (after a stale future sample) start>end. - // Clamp endMs up to now so the emitted point spans [start, now], never - // inverted. - // 2. start>end guard: if even now < startMs (clock skew), fall back to - // startMs so we never emit an inverted (start>end) data point. - endMs := p.maxObservedMs.Load() - if endMs <= startMs { - endMs = now - } - if endMs < startMs { - endMs = startMs - } - for name := range p.sumMetrics { - merged := make(map[string]*sumGroup) - for _, sh := range p.shards { - sh.mu.Lock() - mergeSumGroups(merged, sh.sumAggs[name].groups) - sh.sumAggs[name].reset() - sh.mu.Unlock() - } - emitSumMetric(out, name, merged, startMs, endMs) - } - // Advance the window start to this window's end and reset the max-observed - // watermark down to the same boundary. Resetting (rather than letting it - // only ever rise) prevents one future-timestamped sample from permanently - // skewing every later window's endMs; the next window's max is rebuilt from - // its own samples. - p.windowStartMs.Store(endMs) - p.resetMaxObserved(endMs) -} diff --git a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/flush_stagger_test.go b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/flush_stagger_test.go index 95e49842d..1d2c65d61 100644 --- a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/flush_stagger_test.go +++ b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/flush_stagger_test.go @@ -166,101 +166,6 @@ func TestStaggeredEnabled(t *testing.T) { } } -// TestStaggeredRoundCoversAllShardsAndSum drives one full round-robin window -// cycle the way flushLoop does (flush shard k%N each tick; unified sum on the -// last shard of the round) and asserts: -// - every shard's sketches are flushed exactly once over the round, and -// - the per-zone delta sum total equals exactly what one flushAll emits -// (output unchanged — the staggering only changes WHEN, not the totals). -func TestStaggeredRoundCoversAllShardsAndSum(t *testing.T) { - const shards = 4 - zones := []string{"z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7"} - - // --- reference: a single flushAll over the same input --- - refCap := &capMetrics{} - ref := newStaggerProc(t, refCap, shards) - feedZoneRequests(t, ref, zones, 5) - ref.flushAll(context.Background()) - wantSum := sumByZone(refCap) - wantSketchDPs := countSketchDPs(refCap) - if len(wantSum) != len(zones) { - t.Fatalf("reference flush: got %d zones, want %d", len(wantSum), len(zones)) - } - - // --- staggered: one full round of N shard ticks --- - stCap := &capMetrics{} - st := newStaggerProc(t, stCap, shards) - feedZoneRequests(t, st, zones, 5) - n := len(st.shards) - for tick := 0; tick < n; tick++ { - shardIdx := tick % n - st.flushShardWarmCold(context.Background(), shardIdx) - if shardIdx == n-1 { - st.flushSum(context.Background()) - } - } - - gotSum := sumByZone(stCap) - if len(gotSum) != len(wantSum) { - t.Fatalf("staggered sum zones: got %v want %v", gotSum, wantSum) - } - for z, v := range wantSum { - if gotSum[z] != v { - t.Fatalf("zone %s: staggered sum %v != reference %v (full got=%v)", z, gotSum[z], v, gotSum) - } - } - - // Sketches: every series was observed once; over the full round each shard - // flushes once, so the total sketch dp count matches the single flushAll. - if gotSketchDPs := countSketchDPs(stCap); gotSketchDPs != wantSketchDPs { - t.Fatalf("staggered sketch dps = %d, want %d (one flush per shard per round)", gotSketchDPs, wantSketchDPs) - } -} - -// TestStaggeredSumEmittedOncePerWindow asserts the unified sum is emitted on the -// last shard tick of the round (window-aligned), not on every shard tick — so -// the sum output cadence is one batch per WindowDuration, unchanged from before. -func TestStaggeredSumEmittedOncePerWindow(t *testing.T) { - const shards = 4 - cap := &capMetrics{} - p := newStaggerProc(t, cap, shards) - feedZoneRequests(t, p, []string{"z0", "z1", "z2", "z3"}, 2) - - sumBatches := func() int { - n := 0 - for _, b := range cap.got { - rms := b.ResourceMetrics() - for i := 0; i < rms.Len(); i++ { - sms := rms.At(i).ScopeMetrics() - for j := 0; j < sms.Len(); j++ { - ms := sms.At(j).Metrics() - for k := 0; k < ms.Len(); k++ { - if ms.At(k).Name() == "http_requests_total" { - n++ - } - } - } - } - } - return n - } - - // First N-1 shard ticks: only sketches, no sum metric. - for tick := 0; tick < shards-1; tick++ { - p.flushShardWarmCold(context.Background(), tick) - } - if got := sumBatches(); got != 0 { - t.Fatalf("sum emitted before the round completed: got %d sum metrics, want 0", got) - } - - // Last shard tick of the round + the unified sum flush. - p.flushShardWarmCold(context.Background(), shards-1) - p.flushSum(context.Background()) - if got := sumBatches(); got != 1 { - t.Fatalf("sum should be emitted exactly once per window, got %d", got) - } -} - // TestStaggeredLoopShipsPerShardOverTime runs the REAL flushLoop with a short // window and cold shipping enabled, feeding many distinct cold series so they // spread across shards. It asserts the staggered ticker ships in MULTIPLE diff --git a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/ingest.go b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/ingest.go index 926aee7f6..7a8e9fc1e 100644 --- a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/ingest.go +++ b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/ingest.go @@ -93,7 +93,6 @@ func (p *asapEdgeProcessor) ConsumeMetrics(ctx context.Context, md pmetric.Metri func (p *asapEdgeProcessor) consumeMetric(m pmetric.Metric) { name := m.Name() - sumAgg := p.sumMetrics[name] // nil if not a warm Sum-family metric // coldArchive: add raw samples to the cold gorilla stream unless this // metric is configured tier=warm. Unconfigured metrics and tier∈{both,cold} // are archived as before. @@ -139,9 +138,7 @@ func (p *asapEdgeProcessor) consumeMetric(m pmetric.Metric) { Value: val, }) } - if sumAgg != nil { - sh.sumAggs[name].observe(am, val) - } else if sa := sh.sketchAggs[name]; sa != nil { + if sa := sh.sketchAggs[name]; sa != nil { sa.observe(am, val, tsMs) } sh.mu.Unlock() diff --git a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/processor.go b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/processor.go index 7adcaabfa..4636cadfe 100644 --- a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/processor.go +++ b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/processor.go @@ -24,7 +24,6 @@ type asapEdgeProcessor struct { hashSeed maphash.Seed shards []*shard - sumMetrics map[string]*MetricFamily sketchMetrics map[string]*MetricFamily // configured is the set of every metric name listed in cfg.Metrics // (regardless of tier). Used by DropOriginal so a tier=cold metric's raw @@ -79,10 +78,6 @@ type asapEdgeProcessor struct { // failure dropped the window's envelopes silently; this keeps the loss // observable across all aggregators. sketchEncodeDropCount atomic.Uint64 - // sumOverflowCount counts sum-aggregator group observations dropped because - // a shard's group map hit MaxSeries. - sumOverflowCount atomic.Uint64 - stopCh chan struct{} doneCh chan struct{} flushStarted bool @@ -106,7 +101,6 @@ func newProcessor(cfg *Config, set processor.Settings, next consumer.Metrics) (* telemetry: set.TelemetrySettings, hashSeed: maphash.MakeSeed(), shards: make([]*shard, cfg.ShardCount), - sumMetrics: make(map[string]*MetricFamily), sketchMetrics: make(map[string]*MetricFamily), configured: make(map[string]struct{}), coldSkip: make(map[string]struct{}), @@ -121,11 +115,10 @@ func newProcessor(cfg *Config, set processor.Settings, next consumer.Metrics) (* // Build the warm aggregator only when the tier includes warm // (warm|both). A tier=cold metric is cold-archived only. if m.warmEligible() { - if m.Family == FamilySum { - p.sumMetrics[m.Metric] = m - } else { - p.sketchMetrics[m.Metric] = m - } + // Sum routes through the same sketchMetrics/precompute path as the + // sketch families now (FamilySum builds a SumWrapper aggregator and + // emits a first-class SumAgg envelope); there is no separate sum path. + p.sketchMetrics[m.Metric] = m } // A tier=warm metric is excluded from the cold gorilla archive. if !m.coldEligible() { @@ -148,14 +141,8 @@ func newProcessor(cfg *Config, set processor.Settings, next consumer.Metrics) (* } for i := range p.shards { sh := &shard{ - sumAggs: make(map[string]*sumAggregator, len(p.sumMetrics)), sketchAggs: make(map[string]*sketchAggregator, len(p.sketchMetrics)), } - for name, fam := range p.sumMetrics { - sa := newSumAggregator(fam.AggregateBy, fam.MaxSeries) - sa.procOverflowCount = &p.sumOverflowCount - sh.sumAggs[name] = sa - } for name, fam := range p.sketchMetrics { opts := sketchOpts{ window: cfg.WindowDuration, diff --git a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/shard.go b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/shard.go index b3d1e7c49..83f464482 100644 --- a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/shard.go +++ b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/shard.go @@ -19,9 +19,6 @@ type shard struct { // shipped to the backend merger — the edge no longer builds TSDB blocks. // nil when the cold tier is disabled. cold *gorilla.StreamingFragmentEncoder - // sumAggs holds one sumAggregator per Sum-family metric (cross-shard - // merged at flush). - sumAggs map[string]*sumAggregator // sketchAggs holds one precompute-backed aggregator per sketch-family // metric. Series live in a single shard, so these flush independently // per shard (no cross-shard merge, unlike sum). diff --git a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/testhelpers_test.go b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/testhelpers_test.go new file mode 100644 index 000000000..8dd7d6840 --- /dev/null +++ b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/testhelpers_test.go @@ -0,0 +1,24 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package asapedgeprocessor + +import ( + "context" + + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +// capMetrics is the shared test sink: it captures every forwarded +// pmetric.Metrics batch so tests can assert on the processor's output. +// (Relocated here from the retired warm_sum_test.go so it stays available +// to the ~13 test files that use it.) +type capMetrics struct{ got []pmetric.Metrics } + +func (c *capMetrics) Capabilities() consumer.Capabilities { return consumer.Capabilities{} } + +func (c *capMetrics) ConsumeMetrics(_ context.Context, md pmetric.Metrics) error { + c.got = append(c.got, md) + return nil +} diff --git a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/warm_sketch.go b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/warm_sketch.go index bd3e65d8d..4bbd0670f 100644 --- a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/warm_sketch.go +++ b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/warm_sketch.go @@ -193,6 +193,10 @@ func newSketchAggregator(metric string, fam *MetricFamily, opts sketchOpts, logg // serializeSeries) and the otel adapter maps those to the heap-bearing // pmetric encodings the backend promotes to CountSketchWithHeap. encoding = precompute.EncodingProtoFull + // aggKind is the umbrella AggregationKind stamped on the config/envelope. + // Defaults to Sketch; the FamilySum case flips it to Sum so the otel + // adapter emits a first-class SumAgg envelope instead of a sketch metric. + aggKind = precompute.AggKindSketch // obsKindOverride, when non-zero-meaningful, replaces observeKindFor for // this family. Used by the heap CountSketch path so each sample keys the // sketch by the configured item_label value (the heavy-hitter dimension) @@ -337,12 +341,23 @@ func newSketchAggregator(metric string, fam *MetricFamily, opts sketchOpts, logg obsKindOverride = &k itemLabel = fam.ItemLabel } + case FamilySum: + // Sum is a first-class AggregationType, NOT a sketch: build a + // precompute-backed SumWrapper so it flushes a SumAgg envelope + // (AggKind=Sum) through the same windowed runtime as the sketch + // families. Per-shard partials are additive; the backend sums the + // per-window SumAgg deltas for the same sid (ExactAgg(Sum)). + st = precompute.SketchTypeUnspecified + aggKind = precompute.AggKindSum + factory = func() precompute.Sketch { return sketches.NewSumWrapper() } + observer = sketches.SumObserver{} default: return nil, false } pcfg := &precompute.PrecomputeConfig{ AggID: precompute.AggId(fnv64(metric)), SketchType: st, + AggKind: aggKind, Mode: precompute.Tumbling, Window: precompute.WindowSpec{Size: window, AllowedLateness: opts.allowedLateness}, AggregateBy: fam.AggregateBy, @@ -381,10 +396,17 @@ func newSketchAggregator(metric string, fam *MetricFamily, opts sketchOpts, logg if obsKindOverride != nil { obsKind = *obsKindOverride } + // Sketch families suffix the output metric ("_ddsketch" etc.); Sum is a + // first-class aggregate and keeps the metric's OWN name (no suffix) so a + // `sum(metric)` query resolves to the same name the backend registers. + metricSuffix := "_" + string(fam.Family) + if fam.Family == FamilySum { + metricSuffix = "" + } return &sketchAggregator{ pc: precompute.New(pcfg, factory, observer), pcfg: pcfg, - enc: &oteladapter.AdapterConfig{MetricSuffix: "_" + string(fam.Family), DropOriginal: true}, + enc: &oteladapter.AdapterConfig{MetricSuffix: metricSuffix, DropOriginal: true}, factory: factory, obsKind: obsKind, itemLabel: itemLabel, diff --git a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/warm_sum.go b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/warm_sum.go deleted file mode 100644 index a91f1f964..000000000 --- a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/warm_sum.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package asapedgeprocessor - -import ( - "sort" - "strings" - "sync/atomic" - - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" -) - -// sumAggregator is the asap-native replacement for contrib metricstransform's -// aggregate_labels/sum. It groups (delta) values by the configured -// AggregateBy label set and sums them — the per-shard half of the -// sum-by-zone edge aggregation. There is NO json.Marshal / reflection / map -// AsRaw per data point (the metricstransform path that cost ~17% of agent -// CPU): the group key is a small sorted "k=v;" byte string over only the -// AggregateBy labels. -// -// Sum is associative, so each shard keeps its own partial sums and the -// processor merges partials across shards by group key at flush, then emits -// one delta Sum data point per group — byte/semantically identical to what -// metricstransform emits today (backend unchanged). -type sumAggregator struct { - // aggregateBy is the (already config-time) label set to group by, e.g. - // [zone]. Empty groups everything into a single series. - aggregateBy []string - groups map[string]*sumGroup - // maxGroups bounds the group map per shard (0 => unlimited). A new group - // past the cap is dropped (not summed) so a cardinality explosion in the - // AggregateBy space cannot grow the map without bound. Existing groups keep - // accepting samples. - maxGroups int - // overflowCount counts observations dropped because the group map was full. - overflowCount atomic.Uint64 - // procOverflowCount, when non-nil, is the processor-wide sum-overflow - // counter this aggregator also bumps so per-shard drops roll up. - procOverflowCount *atomic.Uint64 -} - -type sumGroup struct { - labels []kv // the AggregateBy label values identifying this group - sum float64 - count uint64 -} - -// kv is a minimal label pair (avoids importing precompute just for KeyValue). -type kv struct { - k string - v string -} - -func newSumAggregator(aggregateBy []string, maxGroups int) *sumAggregator { - // Copy + sort aggregateBy for a stable group-key layout. - ab := append([]string(nil), aggregateBy...) - sort.Strings(ab) - return &sumAggregator{aggregateBy: ab, groups: make(map[string]*sumGroup), maxGroups: maxGroups} -} - -// observe adds value to the group identified by attrs filtered to -// aggregateBy. attrs is the data point's already-decoded attribute map -// (decoded ONCE by the processor and shared with the cold + sketch paths). -func (s *sumAggregator) observe(attrs map[string]string, value float64) { - // Build the group key (sorted "k=v;" over aggregateBy) into a small - // builder — only the aggregateBy labels, not all attributes. - var b strings.Builder - grp := make([]kv, 0, len(s.aggregateBy)) - for _, k := range s.aggregateBy { // aggregateBy is pre-sorted - vs, ok := attrs[k] - if !ok { - continue - } - b.WriteString(k) - b.WriteByte('=') - b.WriteString(vs) - b.WriteByte(';') - grp = append(grp, kv{k: k, v: vs}) - } - key := b.String() - g := s.groups[key] - if g == nil { - // Cap the group map: drop a NEW group once at the limit (existing groups - // still accept samples). 0 => unlimited. - if s.maxGroups > 0 && len(s.groups) >= s.maxGroups { - s.overflowCount.Add(1) - if s.procOverflowCount != nil { - s.procOverflowCount.Add(1) - } - return - } - g = &sumGroup{labels: grp} - s.groups[key] = g - } - g.sum += value - g.count++ -} - -func (s *sumAggregator) reset() { - s.groups = make(map[string]*sumGroup) -} - -// mergeSumGroups folds per-shard sum partials (same metric) into one map -// keyed by group key. Caller drains every shard's aggregator for a metric. -func mergeSumGroups(dst map[string]*sumGroup, src map[string]*sumGroup) { - for key, g := range src { - d := dst[key] - if d == nil { - d = &sumGroup{labels: g.labels} - dst[key] = d - } - d.sum += g.sum - d.count += g.count - } -} - -// emitSumMetric appends a delta Sum metric named metricName to md, with one -// data point per merged group (attributes = the group's AggregateBy labels, -// value = summed delta, count = sample count). Matches the metricstransform -// aggregate_labels/sum output the backend ingests. -func emitSumMetric(md pmetric.Metrics, metricName string, groups map[string]*sumGroup, startMs, endMs uint64) { - if len(groups) == 0 { - return - } - rm := md.ResourceMetrics().AppendEmpty() - sm := rm.ScopeMetrics().AppendEmpty() - m := sm.Metrics().AppendEmpty() - m.SetName(metricName) - sum := m.SetEmptySum() - sum.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) - sum.SetIsMonotonic(true) - dps := sum.DataPoints() - for _, g := range groups { - dp := dps.AppendEmpty() - dp.SetStartTimestamp(pcommon.Timestamp(startMs * 1e6)) - dp.SetTimestamp(pcommon.Timestamp(endMs * 1e6)) - dp.SetDoubleValue(g.sum) - attrs := dp.Attributes() - for _, l := range g.labels { - attrs.PutStr(l.k, l.v) - } - } -} diff --git a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/warm_sum_test.go b/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/warm_sum_test.go deleted file mode 100644 index b9b80d656..000000000 --- a/opentelemetry-collector-contrib-patch/processor/asapedgeprocessor/warm_sum_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package asapedgeprocessor - -import ( - "context" - "testing" - "time" - - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/consumer" - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" - "go.opentelemetry.io/collector/processor" - "go.uber.org/zap" -) - -type capMetrics struct{ got []pmetric.Metrics } - -func (c *capMetrics) Capabilities() consumer.Capabilities { return consumer.Capabilities{} } -func (c *capMetrics) ConsumeMetrics(_ context.Context, md pmetric.Metrics) error { - c.got = append(c.got, md) - return nil -} - -// TestSumPathMergesByZone feeds a delta-Sum counter with series spread across -// zones (and methods), sharded across N shards, and verifies the flushed -// output is one delta Sum metric with per-zone merged sums — i.e. the -// asap-native equivalent of metricstransform aggregate_labels label_set:[zone]. -func TestSumPathMergesByZone(t *testing.T) { - cap := &capMetrics{} - cfg := &Config{ - ShardCount: 4, - WindowDuration: time.Hour, - DropOriginal: true, - Metrics: []MetricFamily{{Metric: "http_requests_total", Family: FamilySum, AggregateBy: []string{"zone"}}}, - } - if err := cfg.Validate(); err != nil { - t.Fatal(err) - } - set := processor.Settings{TelemetrySettings: component.TelemetrySettings{Logger: zap.NewNop()}} - p, err := newProcessor(cfg, set, cap) - if err != nil { - t.Fatal(err) - } - - md := pmetric.NewMetrics() - m := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() - m.SetName("http_requests_total") - s := m.SetEmptySum() - s.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) - now := pcommon.Timestamp(uint64(time.Now().UnixMilli()) * 1e6) - add := func(zone, method string, v float64) { - dp := s.DataPoints().AppendEmpty() - dp.Attributes().PutStr("zone", zone) - dp.Attributes().PutStr("method", method) - dp.SetDoubleValue(v) - dp.SetTimestamp(now) - } - // distinct series (zone×method), summed per zone: z0=1+2=3, z1=4+8=12, z2=16 - add("z0", "GET", 1) - add("z0", "POST", 2) - add("z1", "GET", 4) - add("z1", "POST", 8) - add("z2", "GET", 16) - - if err := p.ConsumeMetrics(context.Background(), md); err != nil { - t.Fatal(err) - } - if len(cap.got) != 0 { - t.Fatalf("drop_original: expected no passthrough, got %d batches", len(cap.got)) - } - - p.flushAll(context.Background()) - if len(cap.got) != 1 { - t.Fatalf("expected 1 flushed batch, got %d", len(cap.got)) - } - got := map[string]float64{} - var temporality pmetric.AggregationTemporality - rms := cap.got[0].ResourceMetrics() - for i := 0; i < rms.Len(); i++ { - sms := rms.At(i).ScopeMetrics() - for j := 0; j < sms.Len(); j++ { - ms := sms.At(j).Metrics() - for k := 0; k < ms.Len(); k++ { - mm := ms.At(k) - if mm.Name() != "http_requests_total" || mm.Type() != pmetric.MetricTypeSum { - continue - } - temporality = mm.Sum().AggregationTemporality() - dps := mm.Sum().DataPoints() - for d := 0; d < dps.Len(); d++ { - dp := dps.At(d) - z, _ := dp.Attributes().Get("zone") - if dp.Attributes().Len() != 1 { - t.Fatalf("expected only [zone] on output dp, got %d attrs", dp.Attributes().Len()) - } - got[z.AsString()] += dp.DoubleValue() - } - } - } - } - if temporality != pmetric.AggregationTemporalityDelta { - t.Fatalf("expected delta temporality, got %v", temporality) - } - want := map[string]float64{"z0": 3, "z1": 12, "z2": 16} - if len(got) != len(want) { - t.Fatalf("zone count: got %v want %v", got, want) - } - for z, v := range want { - if got[z] != v { - t.Fatalf("zone %s: got %v want %v (full=%v)", z, got[z], v, got) - } - } -} diff --git a/opentelemetry-collector-patch/internal/cmd/pdatagen/internal/pdata/pmetric_package.go b/opentelemetry-collector-patch/internal/cmd/pdatagen/internal/pdata/pmetric_package.go index 00b744a97..acff94a2a 100644 --- a/opentelemetry-collector-patch/internal/cmd/pdatagen/internal/pdata/pmetric_package.go +++ b/opentelemetry-collector-patch/internal/cmd/pdatagen/internal/pdata/pmetric_package.go @@ -57,6 +57,7 @@ var pmetric = &Package{ hllsketch, countsketch, countminsketch, + sumAgg, summary, numberDataPointSlice, numberDataPoint, @@ -74,6 +75,8 @@ var pmetric = &Package{ countsketchDataPoint, countminsketchDataPointSlice, countminsketchDataPoint, + sumAggDataPointSlice, + sumAggDataPoint, bucketsValues, summaryDataPointSlice, summaryDataPoint, @@ -89,6 +92,7 @@ var pmetric = &Package{ hllsketchEncodingEnum, countsketchEncodingEnum, countminsketchEncodingEnum, + sumAggEncodingEnum, }, } @@ -279,6 +283,11 @@ var metric = &messageStruct{ protoID: 17, returnMessage: hllsketch, }, + &OneOfMessageValue{ + fieldName: "SumAgg", + protoID: 18, + returnMessage: sumAgg, + }, }, }, &SliceField{ @@ -396,6 +405,26 @@ var ddsketch = &messageStruct{ }, } +var sumAgg = &messageStruct{ + structName: "SumAgg", + description: "// SumAgg represents a first-class scalar Sum aggregate (AggregationKind = Sum), carried as a portable {sum,count} envelope in the Sketch bytes. It is NOT a sketch; it rides the modified-OTLP metric data oneof alongside the sketch families so the backend can decode it via the same envelope path (into the ExactAgg(Sum) accumulator).", + protoName: "SumAgg", + upstreamProto: "gootlpmetrics.SumAgg", + fields: []Field{ + &SliceField{ + fieldName: "DataPoints", + protoID: 1, + protoType: proto.TypeMessage, + returnSlice: sumAggDataPointSlice, + }, + &TypedField{ + fieldName: "AggregationTemporality", + protoID: 2, + returnType: aggregationTemporalityType, + }, + }, +} + var kllsketch = &messageStruct{ structName: "KLLSketch", description: "// KLLSketch represents the type of a metric encoded using the KLL quantile sketch algorithm.", @@ -866,6 +895,70 @@ var ddsketchDataPoint = &messageStruct{ }, } +var sumAggDataPointSlice = &messageSlice{ + structName: "SumAggDataPointSlice", + elementNullable: true, + element: sumAggDataPoint, +} + +var sumAggDataPoint = &messageStruct{ + structName: "SumAggDataPoint", + description: "// SumAggDataPoint is a single data point carrying a scalar Sum aggregate as a portable {sum,count} envelope in the Sketch bytes.", + protoName: "SumAggDataPoint", + upstreamProto: "gootlpmetrics.SumAggDataPoint", + fields: []Field{ + &SliceField{ + fieldName: "Attributes", + protoID: 9, + protoType: proto.TypeMessage, + returnSlice: mapStruct, + }, + &PrimitiveField{ + fieldName: "SeriesID", + protoID: 16, + protoType: proto.TypeUint64, + }, + &TypedField{ + fieldName: "StartTimestamp", + originFieldName: "StartTimeUnixNano", + protoID: 2, + returnType: timestampType, + }, + &TypedField{ + fieldName: "Timestamp", + originFieldName: "TimeUnixNano", + protoID: 3, + returnType: timestampType, + }, + &PrimitiveField{ + fieldName: "Sketch", + protoID: 8, + protoType: proto.TypeBytes, + }, + &TypedField{ + fieldName: "Encoding", + protoID: 10, + returnType: sumAggEncodingType, + }, + &SliceField{ + fieldName: "Exemplars", + protoID: 11, + protoType: proto.TypeMessage, + returnSlice: exemplarSlice, + }, + &TypedField{ + fieldName: "Flags", + protoID: 15, + returnType: &TypedType{ + structName: "DataPointFlags", + protoType: proto.TypeUint32, + defaultVal: "0", + testVal: "1", + }, + }, + }, +} + var kllsketchDataPointSlice = &messageSlice{ structName: "KLLSketchDataPointSlice", elementNullable: true, @@ -1285,6 +1378,26 @@ var ddsketchEncodingEnum = &proto.Enum{ }, } +var sumAggEncodingType = &TypedType{ + structName: "SumAggEncoding", + protoType: proto.TypeEnum, + messageName: "SumAggEncoding", + defaultVal: "SumAggEncoding(0)", + testVal: "SumAggEncoding(1)", +} + +var sumAggEncodingEnum = &proto.Enum{ + Name: "SumAggEncoding", + Description: "// SumAggEncoding identifies how the SumAgg payload bytes are encoded.", + Fields: []*proto.EnumField{ + {Name: "SUM_AGG_ENCODING_UNSPECIFIED", Value: 0}, + {Name: "SUM_AGG_ENCODING_PROTO", Value: 1}, + {Name: "SUM_AGG_ENCODING_PROTO_DELTA", Value: 2}, + {Name: "SUM_AGG_ENCODING_MSGPACK", Value: 3}, + {Name: "SUM_AGG_ENCODING_MSGPACK_DELTA", Value: 4}, + }, +} + var kllsketchEncodingType = &TypedType{ structName: "KLLSketchEncoding", protoType: proto.TypeEnum, diff --git a/opentelemetry-collector-patch/pdata/internal/generated_enum_sumaggencoding.go b/opentelemetry-collector-patch/pdata/internal/generated_enum_sumaggencoding.go new file mode 100644 index 000000000..a749dc544 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_enum_sumaggencoding.go @@ -0,0 +1,34 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +const ( + SumAggEncoding_SUM_AGG_ENCODING_UNSPECIFIED = SumAggEncoding(0) + SumAggEncoding_SUM_AGG_ENCODING_PROTO = SumAggEncoding(1) + SumAggEncoding_SUM_AGG_ENCODING_PROTO_DELTA = SumAggEncoding(2) + SumAggEncoding_SUM_AGG_ENCODING_MSGPACK = SumAggEncoding(3) + SumAggEncoding_SUM_AGG_ENCODING_MSGPACK_DELTA = SumAggEncoding(4) +) + +// SumAggEncoding identifies how the SumAgg payload bytes are encoded. +type SumAggEncoding int32 + +var SumAggEncoding_name = map[int32]string{ + 0: "SUM_AGG_ENCODING_UNSPECIFIED", + 1: "SUM_AGG_ENCODING_PROTO", + 2: "SUM_AGG_ENCODING_PROTO_DELTA", + 3: "SUM_AGG_ENCODING_MSGPACK", + 4: "SUM_AGG_ENCODING_MSGPACK_DELTA", +} + +var SumAggEncoding_value = map[string]int32{ + "SUM_AGG_ENCODING_UNSPECIFIED": 0, + "SUM_AGG_ENCODING_PROTO": 1, + "SUM_AGG_ENCODING_PROTO_DELTA": 2, + "SUM_AGG_ENCODING_MSGPACK": 3, + "SUM_AGG_ENCODING_MSGPACK_DELTA": 4, +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_metric.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_metric.go index 401561273..f4479f5dd 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_metric.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_metric.go @@ -131,6 +131,17 @@ func (m *Metric) GetHLLSketch() *HLLSketch { return nil } +type Metric_SumAgg struct { + SumAgg *SumAgg +} + +func (m *Metric) GetSumAgg() *SumAgg { + if v, ok := m.GetData().(*Metric_SumAgg); ok { + return v.SumAgg + } + return nil +} + // Metric represents one metric as a collection of datapoints. // See Metric definition in OTLP: https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/metrics/v1/metrics.proto type Metric struct { @@ -207,6 +218,12 @@ var ( return &Metric_HLLSketch{} }, } + + ProtoPoolMetric_SumAgg = sync.Pool{ + New: func() any { + return &Metric_SumAgg{} + }, + } ) func NewMetric() *Metric { @@ -267,6 +284,10 @@ func DeleteMetric(orig *Metric, nullable bool) { DeleteHLLSketch(ov.HLLSketch, true) ov.HLLSketch = nil ProtoPoolMetric_HLLSketch.Put(ov) + case *Metric_SumAgg: + DeleteSumAgg(ov.SumAgg, true) + ov.SumAgg = nil + ProtoPoolMetric_SumAgg.Put(ov) } for i := range orig.Metadata { @@ -409,6 +430,17 @@ func CopyMetric(dest, src *Metric) *Metric { CopyHLLSketch(ov.HLLSketch, t.HLLSketch) dest.Data = ov + case *Metric_SumAgg: + var ov *Metric_SumAgg + if !UseProtoPooling.IsEnabled() { + ov = &Metric_SumAgg{} + } else { + ov = ProtoPoolMetric_SumAgg.Get().(*Metric_SumAgg) + } + ov.SumAgg = NewSumAgg() + CopySumAgg(ov.SumAgg, t.SumAgg) + dest.Data = ov + default: dest.Data = nil } @@ -535,6 +567,11 @@ func (orig *Metric) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("hLLSketch") orig.HLLSketch.MarshalJSON(dest) } + case *Metric_SumAgg: + if orig.SumAgg != nil { + dest.WriteObjectField("sumAgg") + orig.SumAgg.MarshalJSON(dest) + } } if len(orig.Metadata) > 0 { dest.WriteObjectField("metadata") @@ -690,6 +727,19 @@ func (orig *Metric) UnmarshalJSON(iter *json.Iterator) { orig.Data = ov } + case "sumAgg", "sum_agg": + { + var ov *Metric_SumAgg + if !UseProtoPooling.IsEnabled() { + ov = &Metric_SumAgg{} + } else { + ov = ProtoPoolMetric_SumAgg.Get().(*Metric_SumAgg) + } + ov.SumAgg = NewSumAgg() + ov.SumAgg.UnmarshalJSON(iter) + orig.Data = ov + } + case "metadata": for iter.ReadArray() { orig.Metadata = append(orig.Metadata, KeyValue{}) @@ -772,6 +822,11 @@ func (orig *Metric) SizeProto() int { l = orig.HLLSketch.SizeProto() n += 2 + proto.Sov(uint64(l)) + l } + case *Metric_SumAgg: + if orig.SumAgg != nil { + l = orig.SumAgg.SizeProto() + n += 2 + proto.Sov(uint64(l)) + l + } } for i := range orig.Metadata { l = orig.Metadata[i].SizeProto() @@ -893,6 +948,16 @@ func (orig *Metric) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x8a } + case *Metric_SumAgg: + if orig.SumAgg != nil { + l = orig.SumAgg.MarshalProto(buf[:pos]) + pos -= l + pos = proto.EncodeVarint(buf, pos, uint64(l)) + pos-- + buf[pos] = 0x1 + pos-- + buf[pos] = 0x92 + } } for i := len(orig.Metadata) - 1; i >= 0; i-- { l = orig.Metadata[i].MarshalProto(buf[:pos]) @@ -1185,6 +1250,29 @@ func (orig *Metric) UnmarshalProto(buf []byte) error { } orig.Data = ov + case 18: + if wireType != proto.WireTypeLen { + return fmt.Errorf("proto: wrong wireType = %d for field SumAgg", wireType) + } + var length int + length, pos, err = proto.ConsumeLen(buf, pos) + if err != nil { + return err + } + startPos := pos - length + var ov *Metric_SumAgg + if !UseProtoPooling.IsEnabled() { + ov = &Metric_SumAgg{} + } else { + ov = ProtoPoolMetric_SumAgg.Get().(*Metric_SumAgg) + } + ov.SumAgg = NewSumAgg() + err = ov.SumAgg.UnmarshalProto(buf[startPos:pos]) + if err != nil { + return err + } + orig.Data = ov + case 12: if wireType != proto.WireTypeLen { return fmt.Errorf("proto: wrong wireType = %d for field Metadata", wireType) diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_metric_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_metric_test.go index d199ef788..8528d4a93 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_metric_test.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_metric_test.go @@ -225,8 +225,11 @@ func genTestFailingUnmarshalProtoValuesMetric() map[string][]byte { "HLLSketch/wrong_wire_type": {0x8c, 0x1}, "HLLSketch/missing_value": {0x8a, 0x1}, - "Metadata/wrong_wire_type": {0x64}, - "Metadata/missing_value": {0x62}, + + "SumAgg/wrong_wire_type": {0x94, 0x1}, + "SumAgg/missing_value": {0x92, 0x1}, + "Metadata/wrong_wire_type": {0x64}, + "Metadata/missing_value": {0x62}, } } @@ -256,6 +259,8 @@ func genTestEncodingValuesMetric() map[string]*Metric { "CountMinSketch/test": {Data: &Metric_CountMinSketch{CountMinSketch: GenTestCountMinSketch()}}, "HLLSketch/default": {Data: &Metric_HLLSketch{HLLSketch: &HLLSketch{}}}, "HLLSketch/test": {Data: &Metric_HLLSketch{HLLSketch: GenTestHLLSketch()}}, + "SumAgg/default": {Data: &Metric_SumAgg{SumAgg: &SumAgg{}}}, + "SumAgg/test": {Data: &Metric_SumAgg{SumAgg: GenTestSumAgg()}}, "Metadata/test": {Metadata: []KeyValue{{}, *GenTestKeyValue()}}, } } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_sumagg.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_sumagg.go new file mode 100644 index 000000000..87becd055 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_sumagg.go @@ -0,0 +1,276 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "fmt" + "sync" + + "go.opentelemetry.io/collector/pdata/internal/json" + "go.opentelemetry.io/collector/pdata/internal/proto" +) + +// SumAgg represents a first-class scalar Sum aggregate (AggregationKind = Sum), carried as a portable {sum,count} envelope in the Sketch bytes. It is NOT a sketch; it rides the modified-OTLP metric data oneof alongside the sketch families so the backend can decode it via the same envelope path (into the ExactAgg(Sum) accumulator). +type SumAgg struct { + DataPoints []*SumAggDataPoint + AggregationTemporality AggregationTemporality +} + +var ( + protoPoolSumAgg = sync.Pool{ + New: func() any { + return &SumAgg{} + }, + } +) + +func NewSumAgg() *SumAgg { + if !UseProtoPooling.IsEnabled() { + return &SumAgg{} + } + return protoPoolSumAgg.Get().(*SumAgg) +} + +func DeleteSumAgg(orig *SumAgg, nullable bool) { + if orig == nil { + return + } + + if !UseProtoPooling.IsEnabled() { + orig.Reset() + return + } + + for i := range orig.DataPoints { + DeleteSumAggDataPoint(orig.DataPoints[i], true) + } + + orig.Reset() + if nullable { + protoPoolSumAgg.Put(orig) + } +} + +func CopySumAgg(dest, src *SumAgg) *SumAgg { + // If copying to same object, just return. + if src == dest { + return dest + } + + if src == nil { + return nil + } + + if dest == nil { + dest = NewSumAgg() + } + dest.DataPoints = CopySumAggDataPointPtrSlice(dest.DataPoints, src.DataPoints) + + dest.AggregationTemporality = src.AggregationTemporality + + return dest +} + +func CopySumAggSlice(dest, src []SumAgg) []SumAgg { + var newDest []SumAgg + if cap(dest) < len(src) { + newDest = make([]SumAgg, len(src)) + } else { + newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). + for i := len(src); i < len(dest); i++ { + DeleteSumAgg(&dest[i], false) + } + } + for i := range src { + CopySumAgg(&newDest[i], &src[i]) + } + return newDest +} + +func CopySumAggPtrSlice(dest, src []*SumAgg) []*SumAgg { + var newDest []*SumAgg + if cap(dest) < len(src) { + newDest = make([]*SumAgg, len(src)) + // Copy old pointers to re-use. + copy(newDest, dest) + // Add new pointers for missing elements from len(dest) to len(srt). + for i := len(dest); i < len(src); i++ { + newDest[i] = NewSumAgg() + } + } else { + newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). + for i := len(src); i < len(dest); i++ { + DeleteSumAgg(dest[i], true) + dest[i] = nil + } + // Add new pointers for missing elements. + // This can happen when len(dest) < len(src) < cap(dest). + for i := len(dest); i < len(src); i++ { + newDest[i] = NewSumAgg() + } + } + for i := range src { + CopySumAgg(newDest[i], src[i]) + } + return newDest +} + +func (orig *SumAgg) Reset() { + *orig = SumAgg{} +} + +// MarshalJSON marshals all properties from the current struct to the destination stream. +func (orig *SumAgg) MarshalJSON(dest *json.Stream) { + dest.WriteObjectStart() + if len(orig.DataPoints) > 0 { + dest.WriteObjectField("dataPoints") + dest.WriteArrayStart() + orig.DataPoints[0].MarshalJSON(dest) + for i := 1; i < len(orig.DataPoints); i++ { + dest.WriteMore() + orig.DataPoints[i].MarshalJSON(dest) + } + dest.WriteArrayEnd() + } + + if int32(orig.AggregationTemporality) != 0 { + dest.WriteObjectField("aggregationTemporality") + dest.WriteInt32(int32(orig.AggregationTemporality)) + } + dest.WriteObjectEnd() +} + +// UnmarshalJSON unmarshals all properties from the current struct from the source iterator. +func (orig *SumAgg) UnmarshalJSON(iter *json.Iterator) { + for f := iter.ReadObject(); f != ""; f = iter.ReadObject() { + switch f { + case "dataPoints", "data_points": + for iter.ReadArray() { + orig.DataPoints = append(orig.DataPoints, NewSumAggDataPoint()) + orig.DataPoints[len(orig.DataPoints)-1].UnmarshalJSON(iter) + } + + case "aggregationTemporality", "aggregation_temporality": + orig.AggregationTemporality = AggregationTemporality(iter.ReadEnumValue(AggregationTemporality_value)) + default: + iter.Skip() + } + } +} + +func (orig *SumAgg) SizeProto() int { + var n int + var l int + _ = l + for i := range orig.DataPoints { + l = orig.DataPoints[i].SizeProto() + n += 1 + proto.Sov(uint64(l)) + l + } + if orig.AggregationTemporality != 0 { + n += 1 + proto.Sov(uint64(orig.AggregationTemporality)) + } + return n +} + +func (orig *SumAgg) MarshalProto(buf []byte) int { + pos := len(buf) + var l int + _ = l + for i := len(orig.DataPoints) - 1; i >= 0; i-- { + l = orig.DataPoints[i].MarshalProto(buf[:pos]) + pos -= l + pos = proto.EncodeVarint(buf, pos, uint64(l)) + pos-- + buf[pos] = 0xa + } + if orig.AggregationTemporality != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.AggregationTemporality)) + pos-- + buf[pos] = 0x10 + } + return len(buf) - pos +} + +func (orig *SumAgg) UnmarshalProto(buf []byte) error { + var err error + var fieldNum int32 + var wireType proto.WireType + + l := len(buf) + pos := 0 + for pos < l { + // If in a group parsing, move to the next tag. + fieldNum, wireType, pos, err = proto.ConsumeTag(buf, pos) + if err != nil { + return err + } + switch fieldNum { + + case 1: + if wireType != proto.WireTypeLen { + return fmt.Errorf("proto: wrong wireType = %d for field DataPoints", wireType) + } + var length int + length, pos, err = proto.ConsumeLen(buf, pos) + if err != nil { + return err + } + startPos := pos - length + orig.DataPoints = append(orig.DataPoints, NewSumAggDataPoint()) + err = orig.DataPoints[len(orig.DataPoints)-1].UnmarshalProto(buf[startPos:pos]) + if err != nil { + return err + } + + case 2: + if wireType != proto.WireTypeVarint { + return fmt.Errorf("proto: wrong wireType = %d for field AggregationTemporality", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeVarint(buf, pos) + if err != nil { + return err + } + + orig.AggregationTemporality = AggregationTemporality(num) + default: + pos, err = proto.ConsumeUnknown(buf, pos, wireType) + if err != nil { + return err + } + } + } + return nil +} + +func GenTestSumAgg() *SumAgg { + orig := NewSumAgg() + orig.DataPoints = []*SumAggDataPoint{{}, GenTestSumAggDataPoint()} + orig.AggregationTemporality = AggregationTemporality(13) + return orig +} + +func GenTestSumAggPtrSlice() []*SumAgg { + orig := make([]*SumAgg, 5) + orig[0] = NewSumAgg() + orig[1] = GenTestSumAgg() + orig[2] = NewSumAgg() + orig[3] = GenTestSumAgg() + orig[4] = NewSumAgg() + return orig +} + +func GenTestSumAggSlice() []SumAgg { + orig := make([]SumAgg, 5) + orig[1] = *GenTestSumAgg() + orig[3] = *GenTestSumAgg() + return orig +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_sumagg_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_sumagg_test.go new file mode 100644 index 000000000..7be6820c7 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_sumagg_test.go @@ -0,0 +1,205 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gootlpmetrics "go.opentelemetry.io/proto/slim/otlp/metrics/v1" + "google.golang.org/protobuf/proto" + + "go.opentelemetry.io/collector/featuregate" + "go.opentelemetry.io/collector/pdata/internal/json" +) + +func TestCopySumAgg(t *testing.T) { + for name, src := range genTestEncodingValuesSumAgg() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + dest := NewSumAgg() + CopySumAgg(dest, src) + assert.Equal(t, src, dest) + CopySumAgg(dest, dest) + assert.Equal(t, src, dest) + }) + } + } +} + +func TestCopySumAggSlice(t *testing.T) { + src := []SumAgg{} + dest := []SumAgg{} + // Test CopyTo empty + dest = CopySumAggSlice(dest, src) + assert.Equal(t, []SumAgg{}, dest) + + // Test CopyTo larger slice + src = GenTestSumAggSlice() + dest = CopySumAggSlice(dest, src) + assert.Equal(t, GenTestSumAggSlice(), dest) + + // Test CopyTo same size slice + dest = CopySumAggSlice(dest, src) + assert.Equal(t, GenTestSumAggSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopySumAggSlice(dest, []SumAgg{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopySumAggSlice(dest, src) + assert.Equal(t, GenTestSumAggSlice(), dest) +} + +func TestCopySumAggPtrSlice(t *testing.T) { + src := []*SumAgg{} + dest := []*SumAgg{} + // Test CopyTo empty + dest = CopySumAggPtrSlice(dest, src) + assert.Equal(t, []*SumAgg{}, dest) + + // Test CopyTo larger slice + src = GenTestSumAggPtrSlice() + dest = CopySumAggPtrSlice(dest, src) + assert.Equal(t, GenTestSumAggPtrSlice(), dest) + + // Test CopyTo same size slice + dest = CopySumAggPtrSlice(dest, src) + assert.Equal(t, GenTestSumAggPtrSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopySumAggPtrSlice(dest, []*SumAgg{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopySumAggPtrSlice(dest, src) + assert.Equal(t, GenTestSumAggPtrSlice(), dest) +} + +func TestMarshalAndUnmarshalJSONSumAggUnknown(t *testing.T) { + iter := json.BorrowIterator([]byte(`{"unknown": "string"}`)) + defer json.ReturnIterator(iter) + dest := NewSumAgg() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + assert.Equal(t, NewSumAgg(), dest) +} + +func TestMarshalAndUnmarshalJSONSumAgg(t *testing.T) { + for name, src := range genTestEncodingValuesSumAgg() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + stream := json.BorrowStream(nil) + defer json.ReturnStream(stream) + src.MarshalJSON(stream) + require.NoError(t, stream.Error()) + + iter := json.BorrowIterator(stream.Buffer()) + defer json.ReturnIterator(iter) + dest := NewSumAgg() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + + assert.Equal(t, src, dest) + DeleteSumAgg(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoSumAggFailing(t *testing.T) { + for name, buf := range genTestFailingUnmarshalProtoValuesSumAgg() { + t.Run(name, func(t *testing.T) { + dest := NewSumAgg() + require.Error(t, dest.UnmarshalProto(buf)) + }) + } +} + +func TestMarshalAndUnmarshalProtoSumAggUnknown(t *testing.T) { + dest := NewSumAgg() + // message Test { required int64 field = 1313; } encoding { "field": "1234" } + require.NoError(t, dest.UnmarshalProto([]byte{0x88, 0x52, 0xD2, 0x09})) + assert.Equal(t, NewSumAgg(), dest) +} + +func TestMarshalAndUnmarshalProtoSumAgg(t *testing.T) { + for name, src := range genTestEncodingValuesSumAgg() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + dest := NewSumAgg() + require.NoError(t, dest.UnmarshalProto(buf)) + + assert.Equal(t, src, dest) + DeleteSumAgg(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoViaProtobufSumAgg(t *testing.T) { + for name, src := range genTestEncodingValuesSumAgg() { + t.Run(name, func(t *testing.T) { + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + goDest := &gootlpmetrics.SumAgg{} + require.NoError(t, proto.Unmarshal(buf, goDest)) + + goBuf, err := proto.Marshal(goDest) + require.NoError(t, err) + + dest := NewSumAgg() + require.NoError(t, dest.UnmarshalProto(goBuf)) + assert.Equal(t, src, dest) + }) + } +} + +func genTestFailingUnmarshalProtoValuesSumAgg() map[string][]byte { + return map[string][]byte{ + "invalid_field": {0x02}, + "DataPoints/wrong_wire_type": {0xc}, + "DataPoints/missing_value": {0xa}, + "AggregationTemporality/wrong_wire_type": {0x14}, + "AggregationTemporality/missing_value": {0x10}, + } +} + +func genTestEncodingValuesSumAgg() map[string]*SumAgg { + return map[string]*SumAgg{ + "empty": NewSumAgg(), + "DataPoints/test": {DataPoints: []*SumAggDataPoint{{}, GenTestSumAggDataPoint()}}, + "AggregationTemporality/test": {AggregationTemporality: AggregationTemporality(13)}, + } +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_sumaggdatapoint.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_sumaggdatapoint.go new file mode 100644 index 000000000..5e3c7466c --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_sumaggdatapoint.go @@ -0,0 +1,489 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "encoding/binary" + "fmt" + "sync" + + "go.opentelemetry.io/collector/pdata/internal/json" + "go.opentelemetry.io/collector/pdata/internal/proto" +) + +// SumAggDataPoint is a single data point carrying a scalar Sum aggregate as a portable {sum,count} envelope in the Sketch bytes. +type SumAggDataPoint struct { + Attributes []KeyValue + SeriesID uint64 + StartTimeUnixNano uint64 + TimeUnixNano uint64 + Sketch []byte + Encoding SumAggEncoding + Exemplars []Exemplar + Flags uint32 +} + +var ( + protoPoolSumAggDataPoint = sync.Pool{ + New: func() any { + return &SumAggDataPoint{} + }, + } +) + +func NewSumAggDataPoint() *SumAggDataPoint { + if !UseProtoPooling.IsEnabled() { + return &SumAggDataPoint{} + } + return protoPoolSumAggDataPoint.Get().(*SumAggDataPoint) +} + +func DeleteSumAggDataPoint(orig *SumAggDataPoint, nullable bool) { + if orig == nil { + return + } + + if !UseProtoPooling.IsEnabled() { + orig.Reset() + return + } + + for i := range orig.Attributes { + DeleteKeyValue(&orig.Attributes[i], false) + } + for i := range orig.Exemplars { + DeleteExemplar(&orig.Exemplars[i], false) + } + + orig.Reset() + if nullable { + protoPoolSumAggDataPoint.Put(orig) + } +} + +func CopySumAggDataPoint(dest, src *SumAggDataPoint) *SumAggDataPoint { + // If copying to same object, just return. + if src == dest { + return dest + } + + if src == nil { + return nil + } + + if dest == nil { + dest = NewSumAggDataPoint() + } + dest.Attributes = CopyKeyValueSlice(dest.Attributes, src.Attributes) + + dest.SeriesID = src.SeriesID + + dest.StartTimeUnixNano = src.StartTimeUnixNano + + dest.TimeUnixNano = src.TimeUnixNano + + dest.Sketch = src.Sketch + + dest.Encoding = src.Encoding + + dest.Exemplars = CopyExemplarSlice(dest.Exemplars, src.Exemplars) + + dest.Flags = src.Flags + + return dest +} + +func CopySumAggDataPointSlice(dest, src []SumAggDataPoint) []SumAggDataPoint { + var newDest []SumAggDataPoint + if cap(dest) < len(src) { + newDest = make([]SumAggDataPoint, len(src)) + } else { + newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). + for i := len(src); i < len(dest); i++ { + DeleteSumAggDataPoint(&dest[i], false) + } + } + for i := range src { + CopySumAggDataPoint(&newDest[i], &src[i]) + } + return newDest +} + +func CopySumAggDataPointPtrSlice(dest, src []*SumAggDataPoint) []*SumAggDataPoint { + var newDest []*SumAggDataPoint + if cap(dest) < len(src) { + newDest = make([]*SumAggDataPoint, len(src)) + // Copy old pointers to re-use. + copy(newDest, dest) + // Add new pointers for missing elements from len(dest) to len(srt). + for i := len(dest); i < len(src); i++ { + newDest[i] = NewSumAggDataPoint() + } + } else { + newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). + for i := len(src); i < len(dest); i++ { + DeleteSumAggDataPoint(dest[i], true) + dest[i] = nil + } + // Add new pointers for missing elements. + // This can happen when len(dest) < len(src) < cap(dest). + for i := len(dest); i < len(src); i++ { + newDest[i] = NewSumAggDataPoint() + } + } + for i := range src { + CopySumAggDataPoint(newDest[i], src[i]) + } + return newDest +} + +func (orig *SumAggDataPoint) Reset() { + *orig = SumAggDataPoint{} +} + +// MarshalJSON marshals all properties from the current struct to the destination stream. +func (orig *SumAggDataPoint) MarshalJSON(dest *json.Stream) { + dest.WriteObjectStart() + if len(orig.Attributes) > 0 { + dest.WriteObjectField("attributes") + dest.WriteArrayStart() + orig.Attributes[0].MarshalJSON(dest) + for i := 1; i < len(orig.Attributes); i++ { + dest.WriteMore() + orig.Attributes[i].MarshalJSON(dest) + } + dest.WriteArrayEnd() + } + if orig.SeriesID != uint64(0) { + dest.WriteObjectField("seriesID") + dest.WriteUint64(orig.SeriesID) + } + if orig.StartTimeUnixNano != uint64(0) { + dest.WriteObjectField("startTimeUnixNano") + dest.WriteUint64(orig.StartTimeUnixNano) + } + if orig.TimeUnixNano != uint64(0) { + dest.WriteObjectField("timeUnixNano") + dest.WriteUint64(orig.TimeUnixNano) + } + + if len(orig.Sketch) > 0 { + dest.WriteObjectField("sketch") + dest.WriteBytes(orig.Sketch) + } + + if int32(orig.Encoding) != 0 { + dest.WriteObjectField("encoding") + dest.WriteInt32(int32(orig.Encoding)) + } + if len(orig.Exemplars) > 0 { + dest.WriteObjectField("exemplars") + dest.WriteArrayStart() + orig.Exemplars[0].MarshalJSON(dest) + for i := 1; i < len(orig.Exemplars); i++ { + dest.WriteMore() + orig.Exemplars[i].MarshalJSON(dest) + } + dest.WriteArrayEnd() + } + if orig.Flags != uint32(0) { + dest.WriteObjectField("flags") + dest.WriteUint32(orig.Flags) + } + dest.WriteObjectEnd() +} + +// UnmarshalJSON unmarshals all properties from the current struct from the source iterator. +func (orig *SumAggDataPoint) UnmarshalJSON(iter *json.Iterator) { + for f := iter.ReadObject(); f != ""; f = iter.ReadObject() { + switch f { + case "attributes": + for iter.ReadArray() { + orig.Attributes = append(orig.Attributes, KeyValue{}) + orig.Attributes[len(orig.Attributes)-1].UnmarshalJSON(iter) + } + + case "seriesID", "series_id": + orig.SeriesID = iter.ReadUint64() + case "startTimeUnixNano", "start_time_unix_nano": + orig.StartTimeUnixNano = iter.ReadUint64() + case "timeUnixNano", "time_unix_nano": + orig.TimeUnixNano = iter.ReadUint64() + case "sketch": + orig.Sketch = iter.ReadBytes() + case "encoding": + orig.Encoding = SumAggEncoding(iter.ReadEnumValue(SumAggEncoding_value)) + case "exemplars": + for iter.ReadArray() { + orig.Exemplars = append(orig.Exemplars, Exemplar{}) + orig.Exemplars[len(orig.Exemplars)-1].UnmarshalJSON(iter) + } + + case "flags": + orig.Flags = iter.ReadUint32() + default: + iter.Skip() + } + } +} + +func (orig *SumAggDataPoint) SizeProto() int { + var n int + var l int + _ = l + for i := range orig.Attributes { + l = orig.Attributes[i].SizeProto() + n += 1 + proto.Sov(uint64(l)) + l + } + if orig.SeriesID != 0 { + n += 2 + proto.Sov(uint64(orig.SeriesID)) + } + if orig.StartTimeUnixNano != 0 { + n += 9 + } + if orig.TimeUnixNano != 0 { + n += 9 + } + l = len(orig.Sketch) + if l > 0 { + n += 1 + proto.Sov(uint64(l)) + l + } + if orig.Encoding != 0 { + n += 1 + proto.Sov(uint64(orig.Encoding)) + } + for i := range orig.Exemplars { + l = orig.Exemplars[i].SizeProto() + n += 1 + proto.Sov(uint64(l)) + l + } + if orig.Flags != 0 { + n += 1 + proto.Sov(uint64(orig.Flags)) + } + return n +} + +func (orig *SumAggDataPoint) MarshalProto(buf []byte) int { + pos := len(buf) + var l int + _ = l + for i := len(orig.Attributes) - 1; i >= 0; i-- { + l = orig.Attributes[i].MarshalProto(buf[:pos]) + pos -= l + pos = proto.EncodeVarint(buf, pos, uint64(l)) + pos-- + buf[pos] = 0x4a + } + if orig.SeriesID != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.SeriesID)) + pos-- + buf[pos] = 0x1 + pos-- + buf[pos] = 0x80 + } + if orig.StartTimeUnixNano != 0 { + pos -= 8 + binary.LittleEndian.PutUint64(buf[pos:], uint64(orig.StartTimeUnixNano)) + pos-- + buf[pos] = 0x11 + } + if orig.TimeUnixNano != 0 { + pos -= 8 + binary.LittleEndian.PutUint64(buf[pos:], uint64(orig.TimeUnixNano)) + pos-- + buf[pos] = 0x19 + } + l = len(orig.Sketch) + if l > 0 { + pos -= l + copy(buf[pos:], orig.Sketch) + pos = proto.EncodeVarint(buf, pos, uint64(l)) + pos-- + buf[pos] = 0x42 + } + if orig.Encoding != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.Encoding)) + pos-- + buf[pos] = 0x50 + } + for i := len(orig.Exemplars) - 1; i >= 0; i-- { + l = orig.Exemplars[i].MarshalProto(buf[:pos]) + pos -= l + pos = proto.EncodeVarint(buf, pos, uint64(l)) + pos-- + buf[pos] = 0x5a + } + if orig.Flags != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.Flags)) + pos-- + buf[pos] = 0x78 + } + return len(buf) - pos +} + +func (orig *SumAggDataPoint) UnmarshalProto(buf []byte) error { + var err error + var fieldNum int32 + var wireType proto.WireType + + l := len(buf) + pos := 0 + for pos < l { + // If in a group parsing, move to the next tag. + fieldNum, wireType, pos, err = proto.ConsumeTag(buf, pos) + if err != nil { + return err + } + switch fieldNum { + + case 9: + if wireType != proto.WireTypeLen { + return fmt.Errorf("proto: wrong wireType = %d for field Attributes", wireType) + } + var length int + length, pos, err = proto.ConsumeLen(buf, pos) + if err != nil { + return err + } + startPos := pos - length + orig.Attributes = append(orig.Attributes, KeyValue{}) + err = orig.Attributes[len(orig.Attributes)-1].UnmarshalProto(buf[startPos:pos]) + if err != nil { + return err + } + + case 16: + if wireType != proto.WireTypeVarint { + return fmt.Errorf("proto: wrong wireType = %d for field SeriesID", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeVarint(buf, pos) + if err != nil { + return err + } + + orig.SeriesID = uint64(num) + + case 2: + if wireType != proto.WireTypeI64 { + return fmt.Errorf("proto: wrong wireType = %d for field StartTimeUnixNano", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeI64(buf, pos) + if err != nil { + return err + } + + orig.StartTimeUnixNano = uint64(num) + + case 3: + if wireType != proto.WireTypeI64 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeUnixNano", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeI64(buf, pos) + if err != nil { + return err + } + + orig.TimeUnixNano = uint64(num) + + case 8: + if wireType != proto.WireTypeLen { + return fmt.Errorf("proto: wrong wireType = %d for field Sketch", wireType) + } + var length int + length, pos, err = proto.ConsumeLen(buf, pos) + if err != nil { + return err + } + startPos := pos - length + if length != 0 { + orig.Sketch = make([]byte, length) + copy(orig.Sketch, buf[startPos:pos]) + } + + case 10: + if wireType != proto.WireTypeVarint { + return fmt.Errorf("proto: wrong wireType = %d for field Encoding", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeVarint(buf, pos) + if err != nil { + return err + } + + orig.Encoding = SumAggEncoding(num) + + case 11: + if wireType != proto.WireTypeLen { + return fmt.Errorf("proto: wrong wireType = %d for field Exemplars", wireType) + } + var length int + length, pos, err = proto.ConsumeLen(buf, pos) + if err != nil { + return err + } + startPos := pos - length + orig.Exemplars = append(orig.Exemplars, Exemplar{}) + err = orig.Exemplars[len(orig.Exemplars)-1].UnmarshalProto(buf[startPos:pos]) + if err != nil { + return err + } + + case 15: + if wireType != proto.WireTypeVarint { + return fmt.Errorf("proto: wrong wireType = %d for field Flags", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeVarint(buf, pos) + if err != nil { + return err + } + + orig.Flags = uint32(num) + default: + pos, err = proto.ConsumeUnknown(buf, pos, wireType) + if err != nil { + return err + } + } + } + return nil +} + +func GenTestSumAggDataPoint() *SumAggDataPoint { + orig := NewSumAggDataPoint() + orig.Attributes = []KeyValue{{}, *GenTestKeyValue()} + orig.SeriesID = uint64(13) + orig.StartTimeUnixNano = uint64(13) + orig.TimeUnixNano = uint64(13) + orig.Sketch = []byte{1, 2, 3} + orig.Encoding = SumAggEncoding(13) + orig.Exemplars = []Exemplar{{}, *GenTestExemplar()} + orig.Flags = uint32(13) + return orig +} + +func GenTestSumAggDataPointPtrSlice() []*SumAggDataPoint { + orig := make([]*SumAggDataPoint, 5) + orig[0] = NewSumAggDataPoint() + orig[1] = GenTestSumAggDataPoint() + orig[2] = NewSumAggDataPoint() + orig[3] = GenTestSumAggDataPoint() + orig[4] = NewSumAggDataPoint() + return orig +} + +func GenTestSumAggDataPointSlice() []SumAggDataPoint { + orig := make([]SumAggDataPoint, 5) + orig[1] = *GenTestSumAggDataPoint() + orig[3] = *GenTestSumAggDataPoint() + return orig +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_sumaggdatapoint_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_sumaggdatapoint_test.go new file mode 100644 index 000000000..82d5c4e3f --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_sumaggdatapoint_test.go @@ -0,0 +1,223 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gootlpmetrics "go.opentelemetry.io/proto/slim/otlp/metrics/v1" + "google.golang.org/protobuf/proto" + + "go.opentelemetry.io/collector/featuregate" + "go.opentelemetry.io/collector/pdata/internal/json" +) + +func TestCopySumAggDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesSumAggDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + dest := NewSumAggDataPoint() + CopySumAggDataPoint(dest, src) + assert.Equal(t, src, dest) + CopySumAggDataPoint(dest, dest) + assert.Equal(t, src, dest) + }) + } + } +} + +func TestCopySumAggDataPointSlice(t *testing.T) { + src := []SumAggDataPoint{} + dest := []SumAggDataPoint{} + // Test CopyTo empty + dest = CopySumAggDataPointSlice(dest, src) + assert.Equal(t, []SumAggDataPoint{}, dest) + + // Test CopyTo larger slice + src = GenTestSumAggDataPointSlice() + dest = CopySumAggDataPointSlice(dest, src) + assert.Equal(t, GenTestSumAggDataPointSlice(), dest) + + // Test CopyTo same size slice + dest = CopySumAggDataPointSlice(dest, src) + assert.Equal(t, GenTestSumAggDataPointSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopySumAggDataPointSlice(dest, []SumAggDataPoint{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopySumAggDataPointSlice(dest, src) + assert.Equal(t, GenTestSumAggDataPointSlice(), dest) +} + +func TestCopySumAggDataPointPtrSlice(t *testing.T) { + src := []*SumAggDataPoint{} + dest := []*SumAggDataPoint{} + // Test CopyTo empty + dest = CopySumAggDataPointPtrSlice(dest, src) + assert.Equal(t, []*SumAggDataPoint{}, dest) + + // Test CopyTo larger slice + src = GenTestSumAggDataPointPtrSlice() + dest = CopySumAggDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestSumAggDataPointPtrSlice(), dest) + + // Test CopyTo same size slice + dest = CopySumAggDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestSumAggDataPointPtrSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopySumAggDataPointPtrSlice(dest, []*SumAggDataPoint{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopySumAggDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestSumAggDataPointPtrSlice(), dest) +} + +func TestMarshalAndUnmarshalJSONSumAggDataPointUnknown(t *testing.T) { + iter := json.BorrowIterator([]byte(`{"unknown": "string"}`)) + defer json.ReturnIterator(iter) + dest := NewSumAggDataPoint() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + assert.Equal(t, NewSumAggDataPoint(), dest) +} + +func TestMarshalAndUnmarshalJSONSumAggDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesSumAggDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + stream := json.BorrowStream(nil) + defer json.ReturnStream(stream) + src.MarshalJSON(stream) + require.NoError(t, stream.Error()) + + iter := json.BorrowIterator(stream.Buffer()) + defer json.ReturnIterator(iter) + dest := NewSumAggDataPoint() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + + assert.Equal(t, src, dest) + DeleteSumAggDataPoint(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoSumAggDataPointFailing(t *testing.T) { + for name, buf := range genTestFailingUnmarshalProtoValuesSumAggDataPoint() { + t.Run(name, func(t *testing.T) { + dest := NewSumAggDataPoint() + require.Error(t, dest.UnmarshalProto(buf)) + }) + } +} + +func TestMarshalAndUnmarshalProtoSumAggDataPointUnknown(t *testing.T) { + dest := NewSumAggDataPoint() + // message Test { required int64 field = 1313; } encoding { "field": "1234" } + require.NoError(t, dest.UnmarshalProto([]byte{0x88, 0x52, 0xD2, 0x09})) + assert.Equal(t, NewSumAggDataPoint(), dest) +} + +func TestMarshalAndUnmarshalProtoSumAggDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesSumAggDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + dest := NewSumAggDataPoint() + require.NoError(t, dest.UnmarshalProto(buf)) + + assert.Equal(t, src, dest) + DeleteSumAggDataPoint(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoViaProtobufSumAggDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesSumAggDataPoint() { + t.Run(name, func(t *testing.T) { + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + goDest := &gootlpmetrics.SumAggDataPoint{} + require.NoError(t, proto.Unmarshal(buf, goDest)) + + goBuf, err := proto.Marshal(goDest) + require.NoError(t, err) + + dest := NewSumAggDataPoint() + require.NoError(t, dest.UnmarshalProto(goBuf)) + assert.Equal(t, src, dest) + }) + } +} + +func genTestFailingUnmarshalProtoValuesSumAggDataPoint() map[string][]byte { + return map[string][]byte{ + "invalid_field": {0x02}, + "Attributes/wrong_wire_type": {0x4c}, + "Attributes/missing_value": {0x4a}, + "SeriesID/wrong_wire_type": {0x84, 0x1}, + "SeriesID/missing_value": {0x80, 0x1}, + "StartTimeUnixNano/wrong_wire_type": {0x14}, + "StartTimeUnixNano/missing_value": {0x11}, + "TimeUnixNano/wrong_wire_type": {0x1c}, + "TimeUnixNano/missing_value": {0x19}, + "Sketch/wrong_wire_type": {0x44}, + "Sketch/missing_value": {0x42}, + "Encoding/wrong_wire_type": {0x54}, + "Encoding/missing_value": {0x50}, + "Exemplars/wrong_wire_type": {0x5c}, + "Exemplars/missing_value": {0x5a}, + "Flags/wrong_wire_type": {0x7c}, + "Flags/missing_value": {0x78}, + } +} + +func genTestEncodingValuesSumAggDataPoint() map[string]*SumAggDataPoint { + return map[string]*SumAggDataPoint{ + "empty": NewSumAggDataPoint(), + "Attributes/test": {Attributes: []KeyValue{{}, *GenTestKeyValue()}}, + "SeriesID/test": {SeriesID: uint64(13)}, + "StartTimeUnixNano/test": {StartTimeUnixNano: uint64(13)}, + "TimeUnixNano/test": {TimeUnixNano: uint64(13)}, + "Sketch/test": {Sketch: []byte{1, 2, 3}}, + "Encoding/test": {Encoding: SumAggEncoding(13)}, + "Exemplars/test": {Exemplars: []Exemplar{{}, *GenTestExemplar()}}, + "Flags/test": {Flags: uint32(13)}, + } +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_metric.go b/opentelemetry-collector-patch/pdata/pmetric/generated_metric.go index 9a7a47b1c..f767873d6 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_metric.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_metric.go @@ -106,6 +106,8 @@ func (ms Metric) Type() MetricType { return MetricTypeCountMinSketch case *internal.Metric_HLLSketch: return MetricTypeHLLSketch + case *internal.Metric_SumAgg: + return MetricTypeSumAgg } return MetricTypeEmpty } @@ -430,6 +432,38 @@ func (ms Metric) SetEmptyHLLSketch() HLLSketch { return newHLLSketch(ov.HLLSketch, ms.state) } +// SumAgg returns the sumagg associated with this Metric. +// +// Calling this function when Type() != MetricTypeSumAgg returns an invalid +// zero-initialized instance of SumAgg. Note that using such SumAgg instance can cause panic. +// +// Calling this function on zero-initialized Metric will cause a panic. +func (ms Metric) SumAgg() SumAgg { + v, ok := ms.orig.GetData().(*internal.Metric_SumAgg) + if !ok { + return SumAgg{} + } + return newSumAgg(v.SumAgg, ms.state) +} + +// SetEmptySumAgg sets an empty sumagg to this Metric. +// +// After this, Type() function will return MetricTypeSumAgg". +// +// Calling this function on zero-initialized Metric will cause a panic. +func (ms Metric) SetEmptySumAgg() SumAgg { + ms.state.AssertMutable() + var ov *internal.Metric_SumAgg + if !internal.UseProtoPooling.IsEnabled() { + ov = &internal.Metric_SumAgg{} + } else { + ov = internal.ProtoPoolMetric_SumAgg.Get().(*internal.Metric_SumAgg) + } + ov.SumAgg = internal.NewSumAgg() + ms.orig.Data = ov + return newSumAgg(ov.SumAgg, ms.state) +} + // Metadata returns the Metadata associated with this Metric. func (ms Metric) Metadata() pcommon.Map { return pcommon.Map(internal.NewMapWrapper(&ms.orig.Metadata, ms.state)) diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_metric_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_metric_test.go index b79e3107a..7919dd262 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_metric_test.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_metric_test.go @@ -197,6 +197,18 @@ func TestMetric_HLLSketch(t *testing.T) { assert.Panics(t, func() { newMetric(internal.NewMetric(), sharedState).SetEmptyHLLSketch() }) } +func TestMetric_SumAgg(t *testing.T) { + ms := NewMetric() + ms.SetEmptySumAgg() + assert.Equal(t, NewSumAgg(), ms.SumAgg()) + ms.orig.GetData().(*internal.Metric_SumAgg).SumAgg = internal.GenTestSumAgg() + assert.Equal(t, MetricTypeSumAgg, ms.Type()) + assert.Equal(t, generateTestSumAgg(), ms.SumAgg()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newMetric(internal.NewMetric(), sharedState).SetEmptySumAgg() }) +} + func TestMetric_Metadata(t *testing.T) { ms := NewMetric() assert.Equal(t, pcommon.NewMap(), ms.Metadata()) diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_sumagg.go b/opentelemetry-collector-patch/pdata/pmetric/generated_sumagg.go new file mode 100644 index 000000000..896506004 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_sumagg.go @@ -0,0 +1,70 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "go.opentelemetry.io/collector/pdata/internal" +) + +// SumAgg represents a first-class scalar Sum aggregate (AggregationKind = Sum), carried as a portable {sum,count} envelope in the Sketch bytes. It is NOT a sketch; it rides the modified-OTLP metric data oneof alongside the sketch families so the backend can decode it via the same envelope path (into the ExactAgg(Sum) accumulator). +// +// This is a reference type, if passed by value and callee modifies it the +// caller will see the modification. +// +// Must use NewSumAgg function to create new instances. +// Important: zero-initialized instance is not valid for use. +type SumAgg struct { + orig *internal.SumAgg + state *internal.State +} + +func newSumAgg(orig *internal.SumAgg, state *internal.State) SumAgg { + return SumAgg{orig: orig, state: state} +} + +// NewSumAgg creates a new empty SumAgg. +// +// This must be used only in testing code. Users should use "AppendEmpty" when part of a Slice, +// OR directly access the member if this is embedded in another struct. +func NewSumAgg() SumAgg { + return newSumAgg(internal.NewSumAgg(), internal.NewState()) +} + +// MoveTo moves all properties from the current struct overriding the destination and +// resetting the current instance to its zero value +func (ms SumAgg) MoveTo(dest SumAgg) { + ms.state.AssertMutable() + dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. + if ms.orig == dest.orig { + return + } + internal.DeleteSumAgg(dest.orig, false) + *dest.orig, *ms.orig = *ms.orig, *dest.orig +} + +// DataPoints returns the DataPoints associated with this SumAgg. +func (ms SumAgg) DataPoints() SumAggDataPointSlice { + return newSumAggDataPointSlice(&ms.orig.DataPoints, ms.state) +} + +// AggregationTemporality returns the aggregationtemporality associated with this SumAgg. +func (ms SumAgg) AggregationTemporality() AggregationTemporality { + return AggregationTemporality(ms.orig.AggregationTemporality) +} + +// SetAggregationTemporality replaces the aggregationtemporality associated with this SumAgg. +func (ms SumAgg) SetAggregationTemporality(v AggregationTemporality) { + ms.state.AssertMutable() + ms.orig.AggregationTemporality = internal.AggregationTemporality(v) +} + +// CopyTo copies all properties from the current struct overriding the destination. +func (ms SumAgg) CopyTo(dest SumAgg) { + dest.state.AssertMutable() + internal.CopySumAgg(dest.orig, ms.orig) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_sumagg_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_sumagg_test.go new file mode 100644 index 000000000..d9c2e565b --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_sumagg_test.go @@ -0,0 +1,61 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" +) + +func TestSumAgg_MoveTo(t *testing.T) { + ms := generateTestSumAgg() + dest := NewSumAgg() + ms.MoveTo(dest) + assert.Equal(t, NewSumAgg(), ms) + assert.Equal(t, generateTestSumAgg(), dest) + dest.MoveTo(dest) + assert.Equal(t, generateTestSumAgg(), dest) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.MoveTo(newSumAgg(internal.NewSumAgg(), sharedState)) }) + assert.Panics(t, func() { newSumAgg(internal.NewSumAgg(), sharedState).MoveTo(dest) }) +} + +func TestSumAgg_CopyTo(t *testing.T) { + ms := NewSumAgg() + orig := NewSumAgg() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + orig = generateTestSumAgg() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.CopyTo(newSumAgg(internal.NewSumAgg(), sharedState)) }) +} + +func TestSumAgg_DataPoints(t *testing.T) { + ms := NewSumAgg() + assert.Equal(t, NewSumAggDataPointSlice(), ms.DataPoints()) + ms.orig.DataPoints = internal.GenTestSumAggDataPointPtrSlice() + assert.Equal(t, generateTestSumAggDataPointSlice(), ms.DataPoints()) +} + +func TestSumAgg_AggregationTemporality(t *testing.T) { + ms := NewSumAgg() + assert.Equal(t, AggregationTemporality(internal.AggregationTemporality(0)), ms.AggregationTemporality()) + testValAggregationTemporality := AggregationTemporality(internal.AggregationTemporality(1)) + ms.SetAggregationTemporality(testValAggregationTemporality) + assert.Equal(t, testValAggregationTemporality, ms.AggregationTemporality()) +} + +func generateTestSumAgg() SumAgg { + return newSumAgg(internal.GenTestSumAgg(), internal.NewState()) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapoint.go b/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapoint.go new file mode 100644 index 000000000..e6ce2c429 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapoint.go @@ -0,0 +1,131 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "go.opentelemetry.io/collector/pdata/internal" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +// SumAggDataPoint is a single data point carrying a scalar Sum aggregate as a portable {sum,count} envelope in the Sketch bytes. +// +// This is a reference type, if passed by value and callee modifies it the +// caller will see the modification. +// +// Must use NewSumAggDataPoint function to create new instances. +// Important: zero-initialized instance is not valid for use. +type SumAggDataPoint struct { + orig *internal.SumAggDataPoint + state *internal.State +} + +func newSumAggDataPoint(orig *internal.SumAggDataPoint, state *internal.State) SumAggDataPoint { + return SumAggDataPoint{orig: orig, state: state} +} + +// NewSumAggDataPoint creates a new empty SumAggDataPoint. +// +// This must be used only in testing code. Users should use "AppendEmpty" when part of a Slice, +// OR directly access the member if this is embedded in another struct. +func NewSumAggDataPoint() SumAggDataPoint { + return newSumAggDataPoint(internal.NewSumAggDataPoint(), internal.NewState()) +} + +// MoveTo moves all properties from the current struct overriding the destination and +// resetting the current instance to its zero value +func (ms SumAggDataPoint) MoveTo(dest SumAggDataPoint) { + ms.state.AssertMutable() + dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. + if ms.orig == dest.orig { + return + } + internal.DeleteSumAggDataPoint(dest.orig, false) + *dest.orig, *ms.orig = *ms.orig, *dest.orig +} + +// Attributes returns the Attributes associated with this SumAggDataPoint. +func (ms SumAggDataPoint) Attributes() pcommon.Map { + return pcommon.Map(internal.NewMapWrapper(&ms.orig.Attributes, ms.state)) +} + +// SeriesID returns the seriesid associated with this SumAggDataPoint. +func (ms SumAggDataPoint) SeriesID() uint64 { + return ms.orig.SeriesID +} + +// SetSeriesID replaces the seriesid associated with this SumAggDataPoint. +func (ms SumAggDataPoint) SetSeriesID(v uint64) { + ms.state.AssertMutable() + ms.orig.SeriesID = v +} + +// StartTimestamp returns the starttimestamp associated with this SumAggDataPoint. +func (ms SumAggDataPoint) StartTimestamp() pcommon.Timestamp { + return pcommon.Timestamp(ms.orig.StartTimeUnixNano) +} + +// SetStartTimestamp replaces the starttimestamp associated with this SumAggDataPoint. +func (ms SumAggDataPoint) SetStartTimestamp(v pcommon.Timestamp) { + ms.state.AssertMutable() + ms.orig.StartTimeUnixNano = uint64(v) +} + +// Timestamp returns the timestamp associated with this SumAggDataPoint. +func (ms SumAggDataPoint) Timestamp() pcommon.Timestamp { + return pcommon.Timestamp(ms.orig.TimeUnixNano) +} + +// SetTimestamp replaces the timestamp associated with this SumAggDataPoint. +func (ms SumAggDataPoint) SetTimestamp(v pcommon.Timestamp) { + ms.state.AssertMutable() + ms.orig.TimeUnixNano = uint64(v) +} + +// Sketch returns the sketch associated with this SumAggDataPoint. +func (ms SumAggDataPoint) Sketch() []byte { + return ms.orig.Sketch +} + +// SetSketch replaces the sketch associated with this SumAggDataPoint. +func (ms SumAggDataPoint) SetSketch(v []byte) { + ms.state.AssertMutable() + ms.orig.Sketch = v +} + +// Encoding returns the encoding associated with this SumAggDataPoint. +func (ms SumAggDataPoint) Encoding() SumAggEncoding { + return SumAggEncoding(ms.orig.Encoding) +} + +// SetEncoding replaces the encoding associated with this SumAggDataPoint. +func (ms SumAggDataPoint) SetEncoding(v SumAggEncoding) { + ms.state.AssertMutable() + ms.orig.Encoding = internal.SumAggEncoding(v) +} + +// Exemplars returns the Exemplars associated with this SumAggDataPoint. +func (ms SumAggDataPoint) Exemplars() ExemplarSlice { + return newExemplarSlice(&ms.orig.Exemplars, ms.state) +} + +// Flags returns the flags associated with this SumAggDataPoint. +func (ms SumAggDataPoint) Flags() DataPointFlags { + return DataPointFlags(ms.orig.Flags) +} + +// SetFlags replaces the flags associated with this SumAggDataPoint. +func (ms SumAggDataPoint) SetFlags(v DataPointFlags) { + ms.state.AssertMutable() + ms.orig.Flags = uint32(v) +} + +// CopyTo copies all properties from the current struct overriding the destination. +func (ms SumAggDataPoint) CopyTo(dest SumAggDataPoint) { + dest.state.AssertMutable() + internal.CopySumAggDataPoint(dest.orig, ms.orig) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapoint_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapoint_test.go new file mode 100644 index 000000000..e8d5c09d1 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapoint_test.go @@ -0,0 +1,113 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +func TestSumAggDataPoint_MoveTo(t *testing.T) { + ms := generateTestSumAggDataPoint() + dest := NewSumAggDataPoint() + ms.MoveTo(dest) + assert.Equal(t, NewSumAggDataPoint(), ms) + assert.Equal(t, generateTestSumAggDataPoint(), dest) + dest.MoveTo(dest) + assert.Equal(t, generateTestSumAggDataPoint(), dest) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.MoveTo(newSumAggDataPoint(internal.NewSumAggDataPoint(), sharedState)) }) + assert.Panics(t, func() { newSumAggDataPoint(internal.NewSumAggDataPoint(), sharedState).MoveTo(dest) }) +} + +func TestSumAggDataPoint_CopyTo(t *testing.T) { + ms := NewSumAggDataPoint() + orig := NewSumAggDataPoint() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + orig = generateTestSumAggDataPoint() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.CopyTo(newSumAggDataPoint(internal.NewSumAggDataPoint(), sharedState)) }) +} + +func TestSumAggDataPoint_Attributes(t *testing.T) { + ms := NewSumAggDataPoint() + assert.Equal(t, pcommon.NewMap(), ms.Attributes()) + ms.orig.Attributes = internal.GenTestKeyValueSlice() + assert.Equal(t, pcommon.Map(internal.GenTestMapWrapper()), ms.Attributes()) +} + +func TestSumAggDataPoint_SeriesID(t *testing.T) { + ms := NewSumAggDataPoint() + assert.Equal(t, uint64(0), ms.SeriesID()) + ms.SetSeriesID(uint64(13)) + assert.Equal(t, uint64(13), ms.SeriesID()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newSumAggDataPoint(internal.NewSumAggDataPoint(), sharedState).SetSeriesID(uint64(13)) }) +} + +func TestSumAggDataPoint_StartTimestamp(t *testing.T) { + ms := NewSumAggDataPoint() + assert.Equal(t, pcommon.Timestamp(0), ms.StartTimestamp()) + testValStartTimestamp := pcommon.Timestamp(1234567890) + ms.SetStartTimestamp(testValStartTimestamp) + assert.Equal(t, testValStartTimestamp, ms.StartTimestamp()) +} + +func TestSumAggDataPoint_Timestamp(t *testing.T) { + ms := NewSumAggDataPoint() + assert.Equal(t, pcommon.Timestamp(0), ms.Timestamp()) + testValTimestamp := pcommon.Timestamp(1234567890) + ms.SetTimestamp(testValTimestamp) + assert.Equal(t, testValTimestamp, ms.Timestamp()) +} + +func TestSumAggDataPoint_Sketch(t *testing.T) { + ms := NewSumAggDataPoint() + assert.Equal(t, nil, ms.Sketch()) + ms.SetSketch([]byte{1, 2, 3}) + assert.Equal(t, []byte{1, 2, 3}, ms.Sketch()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newSumAggDataPoint(internal.NewSumAggDataPoint(), sharedState).SetSketch([]byte{1, 2, 3}) }) +} + +func TestSumAggDataPoint_Encoding(t *testing.T) { + ms := NewSumAggDataPoint() + assert.Equal(t, SumAggEncoding(internal.SumAggEncoding(0)), ms.Encoding()) + testValEncoding := SumAggEncoding(internal.SumAggEncoding(1)) + ms.SetEncoding(testValEncoding) + assert.Equal(t, testValEncoding, ms.Encoding()) +} + +func TestSumAggDataPoint_Exemplars(t *testing.T) { + ms := NewSumAggDataPoint() + assert.Equal(t, NewExemplarSlice(), ms.Exemplars()) + ms.orig.Exemplars = internal.GenTestExemplarSlice() + assert.Equal(t, generateTestExemplarSlice(), ms.Exemplars()) +} + +func TestSumAggDataPoint_Flags(t *testing.T) { + ms := NewSumAggDataPoint() + assert.Equal(t, DataPointFlags(0), ms.Flags()) + testValFlags := DataPointFlags(1) + ms.SetFlags(testValFlags) + assert.Equal(t, testValFlags, ms.Flags()) +} + +func generateTestSumAggDataPoint() SumAggDataPoint { + return newSumAggDataPoint(internal.GenTestSumAggDataPoint(), internal.NewState()) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapointslice.go b/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapointslice.go new file mode 100644 index 000000000..75e632251 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapointslice.go @@ -0,0 +1,163 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "iter" + "sort" + + "go.opentelemetry.io/collector/pdata/internal" +) + +// SumAggDataPointSlice logically represents a slice of SumAggDataPoint. +// +// This is a reference type. If passed by value and callee modifies it, the +// caller will see the modification. +// +// Must use NewSumAggDataPointSlice function to create new instances. +// Important: zero-initialized instance is not valid for use. +type SumAggDataPointSlice struct { + orig *[]*internal.SumAggDataPoint + state *internal.State +} + +func newSumAggDataPointSlice(orig *[]*internal.SumAggDataPoint, state *internal.State) SumAggDataPointSlice { + return SumAggDataPointSlice{orig: orig, state: state} +} + +// NewSumAggDataPointSlice creates a SumAggDataPointSliceWrapper with 0 elements. +// Can use "EnsureCapacity" to initialize with a given capacity. +func NewSumAggDataPointSlice() SumAggDataPointSlice { + orig := []*internal.SumAggDataPoint(nil) + return newSumAggDataPointSlice(&orig, internal.NewState()) +} + +// Len returns the number of elements in the slice. +// +// Returns "0" for a newly instance created with "NewSumAggDataPointSlice()". +func (es SumAggDataPointSlice) Len() int { + return len(*es.orig) +} + +// At returns the element at the given index. +// +// This function is used mostly for iterating over all the values in the slice: +// +// for i := 0; i < es.Len(); i++ { +// e := es.At(i) +// ... // Do something with the element +// } +func (es SumAggDataPointSlice) At(i int) SumAggDataPoint { + return newSumAggDataPoint((*es.orig)[i], es.state) +} + +// All returns an iterator over index-value pairs in the slice. +// +// for i, v := range es.All() { +// ... // Do something with index-value pair +// } +func (es SumAggDataPointSlice) All() iter.Seq2[int, SumAggDataPoint] { + return func(yield func(int, SumAggDataPoint) bool) { + for i := 0; i < es.Len(); i++ { + if !yield(i, es.At(i)) { + return + } + } + } +} + +// EnsureCapacity is an operation that ensures the slice has at least the specified capacity. +// 1. If the newCap <= cap then no change in capacity. +// 2. If the newCap > cap then the slice capacity will be expanded to equal newCap. +// +// Here is how a new SumAggDataPointSlice can be initialized: +// +// es := NewSumAggDataPointSlice() +// es.EnsureCapacity(4) +// for i := 0; i < 4; i++ { +// e := es.AppendEmpty() +// // Here should set all the values for e. +// } +func (es SumAggDataPointSlice) EnsureCapacity(newCap int) { + es.state.AssertMutable() + oldCap := cap(*es.orig) + if newCap <= oldCap { + return + } + + newOrig := make([]*internal.SumAggDataPoint, len(*es.orig), newCap) + copy(newOrig, *es.orig) + *es.orig = newOrig +} + +// AppendEmpty will append to the end of the slice an empty SumAggDataPoint. +// It returns the newly added SumAggDataPoint. +func (es SumAggDataPointSlice) AppendEmpty() SumAggDataPoint { + es.state.AssertMutable() + *es.orig = append(*es.orig, internal.NewSumAggDataPoint()) + return es.At(es.Len() - 1) +} + +// MoveAndAppendTo moves all elements from the current slice and appends them to the dest. +// The current slice will be cleared. +func (es SumAggDataPointSlice) MoveAndAppendTo(dest SumAggDataPointSlice) { + es.state.AssertMutable() + dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. + if es.orig == dest.orig { + return + } + if *dest.orig == nil { + // We can simply move the entire vector and avoid any allocations. + *dest.orig = *es.orig + } else { + *dest.orig = append(*dest.orig, *es.orig...) + } + *es.orig = nil +} + +// RemoveIf calls f sequentially for each element present in the slice. +// If f returns true, the element is removed from the slice. +func (es SumAggDataPointSlice) RemoveIf(f func(SumAggDataPoint) bool) { + es.state.AssertMutable() + newLen := 0 + for i := 0; i < len(*es.orig); i++ { + if f(es.At(i)) { + internal.DeleteSumAggDataPoint((*es.orig)[i], true) + (*es.orig)[i] = nil + + continue + } + if newLen == i { + // Nothing to move, element is at the right place. + newLen++ + continue + } + (*es.orig)[newLen] = (*es.orig)[i] + // Cannot delete here since we just move the data(or pointer to data) to a different position in the slice. + (*es.orig)[i] = nil + newLen++ + } + *es.orig = (*es.orig)[:newLen] +} + +// CopyTo copies all elements from the current slice overriding the destination. +func (es SumAggDataPointSlice) CopyTo(dest SumAggDataPointSlice) { + dest.state.AssertMutable() + if es.orig == dest.orig { + return + } + *dest.orig = internal.CopySumAggDataPointPtrSlice(*dest.orig, *es.orig) +} + +// Sort sorts the SumAggDataPoint elements within SumAggDataPointSlice given the +// provided less function so that two instances of SumAggDataPointSlice +// can be compared. +func (es SumAggDataPointSlice) Sort(less func(a, b SumAggDataPoint) bool) { + es.state.AssertMutable() + sort.SliceStable(*es.orig, func(i, j int) bool { return less(es.At(i), es.At(j)) }) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapointslice_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapointslice_test.go new file mode 100644 index 000000000..971928757 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_sumaggdatapointslice_test.go @@ -0,0 +1,166 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" +) + +func TestSumAggDataPointSlice(t *testing.T) { + es := NewSumAggDataPointSlice() + assert.Equal(t, 0, es.Len()) + es = newSumAggDataPointSlice(&[]*internal.SumAggDataPoint{}, internal.NewState()) + assert.Equal(t, 0, es.Len()) + + emptyVal := NewSumAggDataPoint() + testVal := generateTestSumAggDataPoint() + for i := 0; i < 7; i++ { + es.AppendEmpty() + assert.Equal(t, emptyVal, es.At(i)) + (*es.orig)[i] = internal.GenTestSumAggDataPoint() + assert.Equal(t, testVal, es.At(i)) + } + assert.Equal(t, 7, es.Len()) +} + +func TestSumAggDataPointSliceReadOnly(t *testing.T) { + sharedState := internal.NewState() + sharedState.MarkReadOnly() + es := newSumAggDataPointSlice(&[]*internal.SumAggDataPoint{}, sharedState) + assert.Equal(t, 0, es.Len()) + assert.Panics(t, func() { es.AppendEmpty() }) + assert.Panics(t, func() { es.EnsureCapacity(2) }) + es2 := NewSumAggDataPointSlice() + es.CopyTo(es2) + assert.Panics(t, func() { es2.CopyTo(es) }) + assert.Panics(t, func() { es.MoveAndAppendTo(es2) }) + assert.Panics(t, func() { es2.MoveAndAppendTo(es) }) +} + +func TestSumAggDataPointSlice_CopyTo(t *testing.T) { + dest := NewSumAggDataPointSlice() + src := generateTestSumAggDataPointSlice() + src.CopyTo(dest) + assert.Equal(t, generateTestSumAggDataPointSlice(), dest) + dest.CopyTo(dest) + assert.Equal(t, generateTestSumAggDataPointSlice(), dest) +} + +func TestSumAggDataPointSlice_EnsureCapacity(t *testing.T) { + es := generateTestSumAggDataPointSlice() + + // Test ensure smaller capacity. + const ensureSmallLen = 4 + es.EnsureCapacity(ensureSmallLen) + assert.Less(t, ensureSmallLen, es.Len()) + assert.Equal(t, es.Len(), cap(*es.orig)) + assert.Equal(t, generateTestSumAggDataPointSlice(), es) + + // Test ensure larger capacity + const ensureLargeLen = 9 + es.EnsureCapacity(ensureLargeLen) + assert.Less(t, generateTestSumAggDataPointSlice().Len(), ensureLargeLen) + assert.Equal(t, ensureLargeLen, cap(*es.orig)) + assert.Equal(t, generateTestSumAggDataPointSlice(), es) +} + +func TestSumAggDataPointSlice_MoveAndAppendTo(t *testing.T) { + // Test MoveAndAppendTo to empty + expectedSlice := generateTestSumAggDataPointSlice() + dest := NewSumAggDataPointSlice() + src := generateTestSumAggDataPointSlice() + src.MoveAndAppendTo(dest) + assert.Equal(t, generateTestSumAggDataPointSlice(), dest) + assert.Equal(t, 0, src.Len()) + assert.Equal(t, expectedSlice.Len(), dest.Len()) + + // Test MoveAndAppendTo empty slice + src.MoveAndAppendTo(dest) + assert.Equal(t, generateTestSumAggDataPointSlice(), dest) + assert.Equal(t, 0, src.Len()) + assert.Equal(t, expectedSlice.Len(), dest.Len()) + + // Test MoveAndAppendTo not empty slice + generateTestSumAggDataPointSlice().MoveAndAppendTo(dest) + assert.Equal(t, 2*expectedSlice.Len(), dest.Len()) + for i := 0; i < expectedSlice.Len(); i++ { + assert.Equal(t, expectedSlice.At(i), dest.At(i)) + assert.Equal(t, expectedSlice.At(i), dest.At(i+expectedSlice.Len())) + } + + dest.MoveAndAppendTo(dest) + assert.Equal(t, 2*expectedSlice.Len(), dest.Len()) + for i := 0; i < expectedSlice.Len(); i++ { + assert.Equal(t, expectedSlice.At(i), dest.At(i)) + assert.Equal(t, expectedSlice.At(i), dest.At(i+expectedSlice.Len())) + } +} + +func TestSumAggDataPointSlice_RemoveIf(t *testing.T) { + // Test RemoveIf on empty slice + emptySlice := NewSumAggDataPointSlice() + emptySlice.RemoveIf(func(el SumAggDataPoint) bool { + t.Fail() + return false + }) + + // Test RemoveIf + filtered := generateTestSumAggDataPointSlice() + pos := 0 + filtered.RemoveIf(func(el SumAggDataPoint) bool { + pos++ + return pos%2 == 1 + }) + assert.Equal(t, 2, filtered.Len()) +} + +func TestSumAggDataPointSlice_RemoveIfAll(t *testing.T) { + got := generateTestSumAggDataPointSlice() + got.RemoveIf(func(el SumAggDataPoint) bool { + return true + }) + assert.Equal(t, 0, got.Len()) +} + +func TestSumAggDataPointSliceAll(t *testing.T) { + ms := generateTestSumAggDataPointSlice() + assert.NotEmpty(t, ms.Len()) + + var c int + for i, v := range ms.All() { + assert.Equal(t, ms.At(i), v, "element should match") + c++ + } + assert.Equal(t, ms.Len(), c, "All elements should have been visited") +} + +func TestSumAggDataPointSlice_Sort(t *testing.T) { + es := generateTestSumAggDataPointSlice() + es.Sort(func(a, b SumAggDataPoint) bool { + return uintptr(unsafe.Pointer(a.orig)) < uintptr(unsafe.Pointer(b.orig)) + }) + for i := 1; i < es.Len(); i++ { + assert.Less(t, uintptr(unsafe.Pointer(es.At(i-1).orig)), uintptr(unsafe.Pointer(es.At(i).orig))) + } + es.Sort(func(a, b SumAggDataPoint) bool { + return uintptr(unsafe.Pointer(a.orig)) > uintptr(unsafe.Pointer(b.orig)) + }) + for i := 1; i < es.Len(); i++ { + assert.Greater(t, uintptr(unsafe.Pointer(es.At(i-1).orig)), uintptr(unsafe.Pointer(es.At(i).orig))) + } +} + +func generateTestSumAggDataPointSlice() SumAggDataPointSlice { + ms := NewSumAggDataPointSlice() + *ms.orig = internal.GenTestSumAggDataPointPtrSlice() + return ms +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/metric_type.go b/opentelemetry-collector-patch/pdata/pmetric/metric_type.go index d3ae2fb1d..6e08bd6a8 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/metric_type.go +++ b/opentelemetry-collector-patch/pdata/pmetric/metric_type.go @@ -19,6 +19,7 @@ const ( MetricTypeCountSketch MetricTypeCountMinSketch MetricTypeHLLSketch + MetricTypeSumAgg ) // String returns the string representation of the MetricType. @@ -46,6 +47,8 @@ func (mdt MetricType) String() string { return "CountMinSketch" case MetricTypeHLLSketch: return "HLLSketch" + case MetricTypeSumAgg: + return "SumAgg" } return "" } diff --git a/opentelemetry-collector-patch/pdata/pmetric/metrics.go b/opentelemetry-collector-patch/pdata/pmetric/metrics.go index 8d7dc55d8..3da31ad70 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/metrics.go +++ b/opentelemetry-collector-patch/pdata/pmetric/metrics.go @@ -60,6 +60,8 @@ func (ms Metrics) DataPointCount() (dataPointCount int) { dataPointCount += m.CountMinSketch().DataPoints().Len() case MetricTypeHLLSketch: dataPointCount += m.HLLSketch().DataPoints().Len() + case MetricTypeSumAgg: + dataPointCount += m.SumAgg().DataPoints().Len() } } } diff --git a/opentelemetry-collector-patch/pdata/pmetric/sum_agg_encoding.go b/opentelemetry-collector-patch/pdata/pmetric/sum_agg_encoding.go new file mode 100644 index 000000000..dd857229c --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/sum_agg_encoding.go @@ -0,0 +1,41 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package pmetric // import "go.opentelemetry.io/collector/pdata/pmetric" + +import "go.opentelemetry.io/collector/pdata/internal" + +// SumAggEncoding identifies how the SumAgg payload bytes are encoded. +type SumAggEncoding int32 + +const ( + // SumAggEncodingUnspecified indicates the encoding is not specified. + SumAggEncodingUnspecified = SumAggEncoding(internal.SumAggEncoding_SUM_AGG_ENCODING_UNSPECIFIED) + // SumAggEncodingProto indicates the payload is the sketchlib SketchEnvelope + // proto carrying a SumState{sum,count} (the asap-precompute-go SumWrapper + // full-state form; decoded by the backend's SumAccumulator). + SumAggEncodingProto = SumAggEncoding(internal.SumAggEncoding_SUM_AGG_ENCODING_PROTO) + // SumAggEncodingProtoDelta reserves the per-window proto delta form. + SumAggEncodingProtoDelta = SumAggEncoding(internal.SumAggEncoding_SUM_AGG_ENCODING_PROTO_DELTA) + // SumAggEncodingMsgpack reserves a msgpack full-state form. + SumAggEncodingMsgpack = SumAggEncoding(internal.SumAggEncoding_SUM_AGG_ENCODING_MSGPACK) + // SumAggEncodingMsgpackDelta reserves a msgpack delta form. + SumAggEncodingMsgpackDelta = SumAggEncoding(internal.SumAggEncoding_SUM_AGG_ENCODING_MSGPACK_DELTA) +) + +// String returns the string representation of the SumAggEncoding. +func (e SumAggEncoding) String() string { + switch e { + case SumAggEncodingUnspecified: + return "Unspecified" + case SumAggEncodingProto: + return "Proto" + case SumAggEncodingProtoDelta: + return "ProtoDelta" + case SumAggEncodingMsgpack: + return "Msgpack" + case SumAggEncodingMsgpackDelta: + return "MsgpackDelta" + } + return "" +}