From bf7384f12c46ba32db752834502f9ff021274196 Mon Sep 17 00:00:00 2001 From: Leo Wang Date: Sun, 7 Dec 2025 00:21:18 -0500 Subject: [PATCH 1/5] Correct restore/backup telegraf scripts --- backup_telegraf_patches.sh | 2 +- restore_telegraf_patches.sh | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/backup_telegraf_patches.sh b/backup_telegraf_patches.sh index 5f619d05..6831e7d8 100755 --- a/backup_telegraf_patches.sh +++ b/backup_telegraf_patches.sh @@ -3,7 +3,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SRC_TELEGRAF="${ROOT_DIR}/telegraf" -DEST_DIR="${ROOT_DIR}/telegraf-plugins" +DEST_DIR="${ROOT_DIR}/telegraf-patch" copy_path() { local src="$1" diff --git a/restore_telegraf_patches.sh b/restore_telegraf_patches.sh index 68d5ff77..b4f4b20f 100755 --- a/restore_telegraf_patches.sh +++ b/restore_telegraf_patches.sh @@ -2,7 +2,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SRC_ROOT="${ROOT_DIR}/telegraf-plugins/plugins" +SRC_ROOT="${ROOT_DIR}/telegraf-patch/plugins" DEST_ROOT="${ROOT_DIR}/telegraf/plugins" if [[ ! -d "${SRC_ROOT}" ]]; then @@ -16,10 +16,9 @@ copied=0 for path in "${SRC_ROOT}"/*; do rel="$(basename "${path}")" dest="${DEST_ROOT}/${rel}" - rm -rf "${dest}" - cp -R "${path}" "${dest}" + cp -R -T "${path}" "${dest}" echo "Restored ${rel} into ${dest}" copied=$((copied + 1)) done -echo "Restored ${copied} items from telegraf-plugins/plugins into telegraf/plugins/" +echo "Restored ${copied} items from telegraf-patch/plugins into telegraf/plugins/" From 708a63121677078f6164ad97f71f2ec123946dc8 Mon Sep 17 00:00:00 2001 From: Leo Wang Date: Sun, 7 Dec 2025 21:01:51 -0500 Subject: [PATCH 2/5] wrap kll implementation from precompute engine --- .gitignore | 4 + telegraf-patch/go.mod | 4 +- telegraf-patch/go.sum | 6 +- telegraf-patch/plugins/aggregators/all/kll.go | 5 + .../plugins/aggregators/kll/heights.go | 142 ++++++++ telegraf-patch/plugins/aggregators/kll/kll.go | 119 +++++++ .../plugins/aggregators/kll/sample.conf | 5 + .../plugins/aggregators/kll/wrapper.go | 312 ++++++++++++++++++ 8 files changed, 593 insertions(+), 4 deletions(-) create mode 100644 telegraf-patch/plugins/aggregators/all/kll.go create mode 100644 telegraf-patch/plugins/aggregators/kll/heights.go create mode 100755 telegraf-patch/plugins/aggregators/kll/kll.go create mode 100755 telegraf-patch/plugins/aggregators/kll/sample.conf create mode 100644 telegraf-patch/plugins/aggregators/kll/wrapper.go diff --git a/.gitignore b/.gitignore index a414f134..a36d8944 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,7 @@ benchmarks/__pycache__/* # Local checkouts opentelemetry-go + +# Telegraf artifacts +config.toml +out.json \ No newline at end of file diff --git a/telegraf-patch/go.mod b/telegraf-patch/go.mod index 9754c06f..e5cd1a51 100644 --- a/telegraf-patch/go.mod +++ b/telegraf-patch/go.mod @@ -25,7 +25,6 @@ require ( github.com/BurntSushi/toml v1.5.0 github.com/ClickHouse/clickhouse-go/v2 v2.40.3 github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/DataDog/sketches-go v1.4.1 github.com/IBM/nzgo/v12 v12.0.10 github.com/IBM/sarama v1.46.3 github.com/Masterminds/semver/v3 v3.4.0 @@ -139,7 +138,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 github.com/karrick/godirwalk v1.16.2 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 - github.com/klauspost/compress v1.18.1 + github.com/klauspost/compress v1.18.2 github.com/klauspost/pgzip v1.2.6 github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b github.com/leodido/go-syslog/v4 v4.3.0 @@ -288,6 +287,7 @@ require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect github.com/ClickHouse/ch-go v0.68.0 // indirect + github.com/DataDog/sketches-go v1.4.7 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect diff --git a/telegraf-patch/go.sum b/telegraf-patch/go.sum index c78cbdb5..a6c49436 100644 --- a/telegraf-patch/go.sum +++ b/telegraf-patch/go.sum @@ -738,8 +738,8 @@ github.com/ClickHouse/clickhouse-go/v2 v2.40.3/go.mod h1:qO0HwvjCnTB4BPL/k6EE3l4 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/sketches-go v1.4.1 h1:j5G6as+9FASM2qC36lvpvQAj9qsv/jUs3FtO8CwZNAY= -github.com/DataDog/sketches-go v1.4.1/go.mod h1:xJIXldczJyyjnbDop7ZZcLxJdV3+7Kra7H1KMgpgkLk= +github.com/DataDog/sketches-go v1.4.7 h1:eHs5/0i2Sdf20Zkj0udVFWuCrXGRFig2Dcfm5rtcTxc= +github.com/DataDog/sketches-go v1.4.7/go.mod h1:eAmQ/EBmtSO+nQp7IZMZVRPT4BQTmIc5RZQ+deGlTPM= github.com/Files-com/files-sdk-go/v3 v3.2.97 h1:c+mQoiES/21JrHDAxJLCYICJO+bu8Clv0ZDNZe7Ndyk= github.com/Files-com/files-sdk-go/v3 v3.2.97/go.mod h1:Y/bCHoPJNPKz2hw1ADXjQXJP378HODwK+g/5SR2gqfU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= @@ -1790,6 +1790,8 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= diff --git a/telegraf-patch/plugins/aggregators/all/kll.go b/telegraf-patch/plugins/aggregators/all/kll.go new file mode 100644 index 00000000..3cd4c0c3 --- /dev/null +++ b/telegraf-patch/plugins/aggregators/all/kll.go @@ -0,0 +1,5 @@ +//go:build !custom || aggregators || aggregators.kll + +package all + +import _ "github.com/influxdata/telegraf/plugins/aggregators/kll" // register plugin diff --git a/telegraf-patch/plugins/aggregators/kll/heights.go b/telegraf-patch/plugins/aggregators/kll/heights.go new file mode 100644 index 00000000..02addece --- /dev/null +++ b/telegraf-patch/plugins/aggregators/kll/heights.go @@ -0,0 +1,142 @@ +package kll + +import "math" + + +func computeHeight(h int) float64 { + if h < len(heightsCache) { + return heightsCache[h] + } + return math.Pow((2.0 / 3.0), float64(h)) +} + +var heightsCache = [...]float64{ + 1, + 0.6666666666666666, + 0.4444444444444444, + 0.2962962962962963, + 0.19753086419753085, + 0.1316872427983539, + 0.0877914951989026, + 0.05852766346593506, + 0.039018442310623375, + 0.026012294873748915, + 0.01734152991583261, + 0.011561019943888407, + 0.007707346629258938, + 0.005138231086172625, + 0.00342548739078175, + 0.0022836582605211663, + 0.0015224388403474443, + 0.0010149592268982961, + 0.0006766394845988641, + 0.00045109298973257606, + 0.0003007286598217174, + 0.00020048577321447823, + 0.0001336571821429855, + 8.910478809532365e-05, + 5.9403192063549106e-05, + 3.960212804236607e-05, + 2.640141869491071e-05, + 1.760094579660714e-05, + 1.1733963864404761e-05, + 7.82264257626984e-06, + 5.21509505084656e-06, + 3.4767300338977064e-06, + 2.3178200225984708e-06, + 1.5452133483989804e-06, + 1.030142232265987e-06, + 6.867614881773246e-07, + 4.5784099211821645e-07, + 3.0522732807881095e-07, + 2.0348488538587396e-07, + 1.356565902572493e-07, + 9.04377268381662e-08, + 6.02918178921108e-08, + 4.019454526140719e-08, + 2.67963635076048e-08, + 1.78642423384032e-08, + 1.1909494892268798e-08, + 7.939663261512532e-09, + 5.293108841008354e-09, + 3.528739227338903e-09, + 2.352492818225935e-09, + 1.5683285454839568e-09, + 1.0455523636559712e-09, + 6.970349091039809e-10, + 4.646899394026538e-10, + 3.097932929351026e-10, + 2.0652886195673503e-10, + 1.3768590797115669e-10, + 9.179060531410445e-11, + 6.119373687606963e-11, + 4.0795824584046424e-11, + 2.7197216389364282e-11, + 1.813147759290952e-11, + 1.2087651728606347e-11, + 8.058434485737563e-12, + 5.372289657158376e-12, + 3.581526438105584e-12, + 2.3876842920703892e-12, + 1.5917895280469262e-12, + 1.0611930186979508e-12, + 7.074620124653005e-13, + 4.716413416435336e-13, + 3.1442756109568906e-13, + 2.0961837406379272e-13, + 1.3974558270919513e-13, + 9.316372180613009e-14, + 6.21091478707534e-14, + 4.140609858050226e-14, + 2.760406572033484e-14, + 1.8402710480223226e-14, + 1.226847365348215e-14, + 8.178982435654766e-15, + 5.452654957103177e-15, + 3.635103304735452e-15, + 2.423402203156968e-15, + 1.615601468771312e-15, + 1.0770676458475411e-15, + 7.180450972316942e-16, + 4.786967314877961e-16, + 3.191311543251974e-16, + 2.1275410288346492e-16, + 1.418360685889766e-16, + 9.455737905931773e-17, + 6.303825270621183e-17, + 4.2025501804141215e-17, + 2.801700120276081e-17, + 1.8678000801840538e-17, + 1.2452000534560357e-17, + 8.301333689706904e-18, + 5.534222459804603e-18, + 3.6894816398697355e-18, + 2.459654426579824e-18, + 1.6397696177198825e-18, + 1.0931797451465883e-18, + 7.287864967643922e-19, + 4.858576645095947e-19, + 3.239051096730632e-19, + 2.1593673978204208e-19, + 1.439578265213614e-19, + 9.597188434757427e-20, + 6.398125623171617e-20, + 4.2654170821144116e-20, + 2.843611388076274e-20, + 1.8957409253841826e-20, + 1.263827283589455e-20, + 8.4255152239297e-21, + 5.6170101492864665e-21, + 3.744673432857645e-21, + 2.4964489552384296e-21, + 1.6642993034922866e-21, + 1.1095328689948576e-21, + 7.39688579329905e-22, + 4.931257195532699e-22, + 3.2875047970218e-22, + 2.1916698646812e-22, + 1.4611132431208001e-22, + 9.740754954138666e-23, + 6.493836636092445e-23, + 4.3292244240616286e-23, +} \ No newline at end of file diff --git a/telegraf-patch/plugins/aggregators/kll/kll.go b/telegraf-patch/plugins/aggregators/kll/kll.go new file mode 100755 index 00000000..98bf0a3b --- /dev/null +++ b/telegraf-patch/plugins/aggregators/kll/kll.go @@ -0,0 +1,119 @@ +//go:generate ../../../tools/readme_config_includer/generator +package kll + +import ( + _ "embed" + "fmt" + "slices" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/aggregators" +) + +type quantile struct { + seen []float64; // for debugging, save the seen values + sketch *Sketch; +} + +type metric struct { + name string; + fields map[string]*quantile; +} + +type KLL struct { + K int `toml:"k"`; + Quantiles []float64 `toml:"quantiles"`; + WriteSeen bool `toml:"write_seen"`; + + cache map[uint64]*metric; // state for each metric/field, key is metric.HashID() + suffixes map[float64]string; // suffix to attach to output quantiles, e.g.. _p50, _p99, ... +} + +//go:embed sample.conf +var sampleConfig string +func (*KLL) SampleConfig() string { return sampleConfig; } + +func (kll *KLL) Init() error { + // TODO: sanitize inputs! + + kll.cache = make(map[uint64]*metric); + kll.suffixes = make(map[float64]string); + for _, q := range kll.Quantiles { kll.suffixes[q] = fmt.Sprintf("_p%d", int(q * 100)); } + + return nil; +} + +// for each numeric field in each metric, update the backing KLL sketch +func (kll *KLL) Add(in telegraf.Metric) { + var id uint64 = in.HashID(); + + // get saved metric + m, ok := kll.cache[id]; + if !ok { + kll.cache[id] = &metric{name: in.Name(), fields: make(map[string]*quantile)}; + m = kll.cache[id]; + } + + // for each field, get associated sketch + var fields []*telegraf.Field = in.FieldList(); + for _, field := range fields { + var val float64; + + // conversion referenced from minmax aggregator + switch field.Value.(type) { + case float64: + val = field.Value.(float64); + case int64: + val = float64(field.Value.(int64)); + case uint64: + val = float64(field.Value.(uint64)); + default: + continue; + } + + // get sketch + sketch, ok := m.fields[field.Key]; + if !ok { + m.fields[field.Key] = &quantile{seen: nil, sketch: New(kll.K)}; + if kll.WriteSeen { m.fields[field.Key].seen = make([]float64, 0); } + + sketch = m.fields[field.Key]; + } + + if kll.WriteSeen { sketch.seen = append(sketch.seen, val); } + + sketch.sketch.Update(val); + } +} + +func (kll *KLL) Push(acc telegraf.Accumulator) { + for _, m := range kll.cache { + out := make(map[string]any); + + fields := m.fields; + for name, sketch := range fields { + // get the desired quantile + cdf := sketch.sketch.CDF(); + for q, str := range kll.suffixes { out[name + str] = cdf.Query(q); } + + if kll.WriteSeen { + slices.Sort(sketch.seen) + out[name + "_seen"] = fmt.Sprintf("%v", sketch.seen); + } + } + + acc.AddSummary(m.name + "_KLL", out, nil); + } +} + +func (kll *KLL) Reset() { + for _, m := range kll.cache { + for k := range m.fields { + if kll.WriteSeen { clear(m.fields[k].seen); } + m.fields[k].sketch = New(kll.K); + } + } +} + +func init() { + aggregators.Add("kll", func() telegraf.Aggregator { return &KLL{}; }); +} diff --git a/telegraf-patch/plugins/aggregators/kll/sample.conf b/telegraf-patch/plugins/aggregators/kll/sample.conf new file mode 100755 index 00000000..bd03644d --- /dev/null +++ b/telegraf-patch/plugins/aggregators/kll/sample.conf @@ -0,0 +1,5 @@ +[[aggregators.kll]] + k = 256 + # 0 - 1 (float) + quantiles = [0.5, 0.99] + write_seen = true \ No newline at end of file diff --git a/telegraf-patch/plugins/aggregators/kll/wrapper.go b/telegraf-patch/plugins/aggregators/kll/wrapper.go new file mode 100644 index 00000000..726365fd --- /dev/null +++ b/telegraf-patch/plugins/aggregators/kll/wrapper.go @@ -0,0 +1,312 @@ +// implementation ported from PrecomputeEngine https://github.com/approx-telemetry/PrecomputeEngine/blob/main/aggregators/KLL/kll.go +package kll + +import ( + "math/rand" + "math" + "unsafe" + "sort" +) + +// 64-bit xorshift multiply rng from http://vigna.di.unimi.it/ftp/papers/xorshift.pdf +func xorshiftMult64(x uint64) uint64 { + x ^= x >> 12 // a + x ^= x << 25 // b + x ^= x >> 27 // c + return x * 2685821657736338717 +} + +// coin is a simple struct to let us get random bools and make minimum calls +// to the random number generator. +type coin struct { + st uint64 + mask uint64 +} + +// v is either 0 or 1 +func (c *coin) toss() (v int) { + if c.mask == 0 { + if c.st == 0 { + c.st = uint64(rand.Int63()) + } + c.st = xorshiftMult64(c.st) + c.mask = 1 + } + if c.st&c.mask > 0 { + v = 1 + } + c.mask <<= 1 + return v +} + + +// Sketch is a streaming quantiles sketch +type Sketch struct { + Compactors []Compactor + k int + H int + size int + maxSize int + + co coin +} + +// New returns a new Sketch. k controls the maximum memory used by the stream, which is 3*k + lg(n). +func New(k int) *Sketch { + s := Sketch{ + k: k, + } + s.grow() + return &s +} + +func (s *Sketch) GetSize() int { + return s.size +} + +func (s *Sketch) grow() { + s.Compactors = append(s.Compactors, Compactor{}) + s.H = len(s.Compactors) + + s.maxSize = 0 + for h := 0; h < s.H; h++ { + s.maxSize += s.capacity(h) + } +} + +func (s *Sketch) capacity(h int) int { + return int(math.Ceil(float64(s.k)*computeHeight(s.H-h-1))) + 1 +} + +// Update adds x to the stream. +func (s *Sketch) Update(x float64) { + s.Compactors[0] = append(s.Compactors[0], x) + s.size++ + s.compact() +} + +func (s *Sketch) compact() { + for s.size >= s.maxSize { + for h := 0; h < len(s.Compactors); h++ { + if len(s.Compactors[h]) >= s.capacity(h) { + if h+1 >= s.H { + s.grow() + } + + prev_h := len(s.Compactors[h]) + prev_h1 := len(s.Compactors[h+1]) + + s.Compactors[h+1] = s.Compactors[h].compact( + &s.co, s.Compactors[h+1]) + + s.size += len(s.Compactors[h]) - prev_h + s.size += len(s.Compactors[h+1]) - prev_h1 + + if s.size < s.maxSize { + break + } + } + } + } +} + +func (s *Sketch) updateSize() { + s.size = 0 + for _, c := range s.Compactors { + s.size += len(c) + } +} + +// Merge merges a second sketch into this one +func (s *Sketch) Merge(t *Sketch) { + for s.H < t.H { + s.grow() + } + + for h, c := range t.Compactors { + s.Compactors[h] = append(s.Compactors[h], c...) + } + + s.updateSize() + s.compact() +} + +// Rank estimates the rank of the value x in the stream. +func (s *Sketch) Rank(x float64) int { + var r int + for h, c := range s.Compactors { + for _, v := range c { + if v <= x { + r += 1 << uint(h) + } + } + } + return r +} + +func (s *Sketch) Count() int { + var n int + for h, c := range s.Compactors { + n += len(c) * (1 << uint(h)) + } + return n +} + +// Quantile estimates the quantile of the value x in the stream. +func (s *Sketch) Quantile(x float64) float64 { + var r, n int + for h, c := range s.Compactors { + for _, v := range c { + w := 1 << uint(h) + if v <= x { + r += w + } + n += w + } + } + return float64(r) / float64(n) +} + +type CDF []Quantile + +func (q CDF) Len() int { return len(q) } + +func (q CDF) Less(i int, j int) bool { return q[i].V < q[j].V } + +func (q CDF) Swap(i int, j int) { q[i], q[j] = q[j], q[i] } + +type Quantile struct { + Q float64 + V float64 +} + +func (s *Sketch) GetMemoryBytes() float64 { + var total_mem float64 = 0 + total_mem += float64(unsafe.Sizeof(*s)) + for i := range s.Compactors { + total_mem += float64(len(s.Compactors[i])) * 8 + } + return total_mem // Bytes +} + +func (s *Sketch) CDF() CDF { + q := make(CDF, 0, s.size) + + var totalW float64 + for h, c := range s.Compactors { + weight := float64(int(1 << uint(h))) + for _, v := range c { + q = append(q, Quantile{Q: weight, V: v}) + } + totalW += float64(len(c)) * weight + } + + sort.Sort(q) + + var curW float64 + for i := range q { + curW += q[i].Q + q[i].Q = curW / totalW + } + + return q +} + +// Quantile estimates the quantile of the value x in the stream. +func (q CDF) Quantile(x float64) float64 { + idx := sort.Search(len(q), func(i int) bool { return q[i].V >= x }) + if idx == 0 { + return 0 + } + return q[idx-1].Q +} + +// Query estimates the value given quantile p. +func (q CDF) Query(p float64) float64 { + idx := sort.Search(len(q), func(i int) bool { return q[i].Q >= p }) + if idx == len(q) { + return q[len(q)-1].V + } + return q[idx].V +} + +// QuantileLI estimates the quantile of the value x in the stream using linear interpolation. +func (q CDF) QuantileLI(x float64) float64 { + idx := sort.Search(len(q), func(i int) bool { return q[i].V >= x }) + if idx == len(q) { + return 1 + } + if idx == 0 { + return 0 + } + // a < x <= b + a, aq := q[idx-1].V, q[idx-1].Q + b, bq := q[idx].V, q[idx].Q + return ((a-x)*bq + (x-b)*aq) / (a - b) +} + +// QueryLI estimates the value given quantile p using linear interpolation. +func (q CDF) QueryLI(p float64) float64 { + idx := sort.Search(len(q), func(i int) bool { return q[i].Q >= p }) + if idx == len(q) { + return q[len(q)-1].V + } + if idx == 0 { + return q[0].V + } + // aq < p <= b + a, aq := q[idx-1].V, q[idx-1].Q + b, bq := q[idx].V, q[idx].Q + return ((aq-p)*b + (p-bq)*a) / (aq - bq) +} + +type Compactor []float64 + +func (c *Compactor) compact(co *coin, dst []float64) []float64 { + l := len(*c) + + if l == 0 || l == 1 { + } else if l == 2 { + c := *c + if c[0] > c[1] { + c[0], c[1] = c[1], c[0] + } + } else if l > 100 { + sort.Float64s([]float64(*c)) + } else { + c.insertionSort() + } + + free := cap(dst) - len(dst) + if free < len(*c)/2 { + extra := len(*c)/2 - free + newdst := make([]float64, len(dst), cap(dst)+extra) + copy(newdst, dst) + dst = newdst + } + + // choose either the evens or the odds + offs := co.toss() + for len(*c) >= 2 { + l := len(*c) - 2 + dst = append(dst, (*c)[l+offs]) + *c = (*c)[:l] + } + + return dst +} + +func (c Compactor) insertionSort() { + l := len(c) + for i := 1; i < l; i++ { + v := c[i] + j := i + for ; j > 0 && c[j-1] > v; j-- { + } + if j == i { + continue + } + copy(c[j+1:], c[j:i]) + c[j] = v + } +} From b6af613484966b2f269c3c8767b439b4475383cd Mon Sep 17 00:00:00 2001 From: Leo Wang Date: Sun, 7 Dec 2025 23:45:18 -0500 Subject: [PATCH 3/5] verification and benchmarks --- .../max-throughput-prometheus-input.conf | 50 +++++ benchmarks/tests/kll.conf | 4 + .../plugins/aggregators/kll/kll_test.go | 211 ++++++++++++++++++ .../aggregators/kll/{wrapper.go => src.go} | 0 4 files changed, 265 insertions(+) create mode 100644 benchmarks/max-throughput-prometheus-input.conf create mode 100644 benchmarks/tests/kll.conf create mode 100644 telegraf-patch/plugins/aggregators/kll/kll_test.go rename telegraf-patch/plugins/aggregators/kll/{wrapper.go => src.go} (100%) diff --git a/benchmarks/max-throughput-prometheus-input.conf b/benchmarks/max-throughput-prometheus-input.conf new file mode 100644 index 00000000..0a65844e --- /dev/null +++ b/benchmarks/max-throughput-prometheus-input.conf @@ -0,0 +1,50 @@ +# Read data directly from FakePrometheusExporter +# based off max-throughput-prometheus-client.conf + +[global_tags] + scenario = "prometheus_inp" + +[agent] + interval = "1ms" + flush_interval = "1ms" + flush_jitter = "0s" + collection_jitter = "0s" + round_interval = false + metric_batch_size = 10000 + metric_buffer_limit = 500000 + precision = "1ms" + debug = true + logfile = "/tmp/telegraf-prometheus-inp.log" + +[[inputs.prometheus]] + urls = [ + "http://localhost:8000/metrics", + "http://localhost:8001/metrics", + ] + metric_version = 2 + url_tag = "scrape_url" + name_prefix = "gorilla_local_" + timeout = "5s" + interval = "250ms" + metric_batch_size = 10000 + metric_buffer_limit = 250000 + +[[inputs.internal]] + per_instance = true + collect_gostats = true + +[[outputs.file]] + files = ["../benchmarks/results/prometheus-inp.lp"] + data_format = "influx" + use_batch_format = false + rotation_interval = "1m" + rotation_max_size = "50MB" + rotation_max_archives = 15 + namepass = ["internal_*"] + +[[outputs.prometheus_client]] + listen = ":9203" + path = "/metrics" + metric_version = 2 + export_timestamp = true + expiration_interval = "30s" diff --git a/benchmarks/tests/kll.conf b/benchmarks/tests/kll.conf new file mode 100644 index 00000000..66418f4d --- /dev/null +++ b/benchmarks/tests/kll.conf @@ -0,0 +1,4 @@ +[[aggregators.kll]] + k = 256 + quantiles = [0.5, 0.99] + write_seen = false \ No newline at end of file diff --git a/telegraf-patch/plugins/aggregators/kll/kll_test.go b/telegraf-patch/plugins/aggregators/kll/kll_test.go new file mode 100644 index 00000000..bc927091 --- /dev/null +++ b/telegraf-patch/plugins/aggregators/kll/kll_test.go @@ -0,0 +1,211 @@ +// Tests taken from Precompute Engine https://github.com/approx-telemetry/PrecomputeEngine/blob/main/aggregators/KLL/kll_test.go +package kll + +import ( + "math" + "sort" + "testing" +) + +// Helper: build a sorted float slice [start..end] inclusive. +func seq(start, end int) []float64 { + n := end - start + 1 + out := make([]float64, 0, n) + for i := start; i <= end; i++ { + out = append(out, float64(i)) + } + return out +} + +// Insert values into a sketch. +func insertAll(s *Sketch, vals []float64) { + for _, v := range vals { + s.Update(v) + } +} + +// With sufficiently large k and small n, the sketch will not compact, +// so results should be exact for Rank/Quantile/Count and the CDF mapping. +func TestKLL_NoCompaction_Exactness(t *testing.T) { + k := 256 + vals := seq(1, 20) // n=20 << capacity(0) for most reasonable computeHeight + s := New(k) + insertAll(s, vals) + + // Count is exact when no compaction occurs. + if got := s.Count(); got != len(vals) { + t.Fatalf("Count() mismatch: got=%d want=%d", got, len(vals)) + } + + // Rank and Quantile should align exactly on midpoints. + if got := s.Rank(10.5); got != 10 { + t.Fatalf("Rank(10.5)=%d want=10", got) + } + if got := s.Quantile(10.5); math.Abs(got-0.5) > 1e-9 { + t.Fatalf("Quantile(10.5)=%.6f want=0.5", got) + } + + // CDF should map 0.5 back to ~10 exactly (discrete). + cdf := s.CDF() + if got := cdf.Query(0.5); got != 10 { + t.Fatalf("CDF.Query(0.5)=%.6f want=10", got) + } + + // Endpoints + if q0 := cdf.Query(0.0); q0 != 1 { + t.Fatalf("CDF.Query(0.0)=%.6f want=1 (min value)", q0) + } + if q1 := cdf.Query(1.0); q1 != 20 { + t.Fatalf("CDF.Query(1.0)=%.6f want=20 (max value)", q1) + } +} + +func TestKLL_CDF_Monotonicity_And_Inverses(t *testing.T) { + k := 512 + vals := append(seq(1, 50), seq(100, 110)...) // gaps, still no compaction + s := New(k) + insertAll(s, vals) + cdf := s.CDF() + + // Monotonicity by value and cumulative probability. + for i := 1; i < len(cdf); i++ { + if !(cdf[i].V >= cdf[i-1].V) { + t.Fatalf("CDF not non-decreasing by value at i=%d: %v !>= %v", i, cdf[i].V, cdf[i-1].V) + } + if !(cdf[i].Q >= cdf[i-1].Q) { + t.Fatalf("CDF not non-decreasing by Q at i=%d: %v !>= %v", i, cdf[i].Q, cdf[i-1].Q) + } + } + + // Inverse checks (stepwise-safe): for x in the support, + // let q = CDF.Quantile(x) be the left-closed CDF; then + // Query(q+ε) must be >= x for any tiny ε>0. + points := []float64{ + vals[len(vals)/4], + vals[len(vals)/2], + vals[3*len(vals)/4], + } + const eps = 1e-12 + for _, x := range points { + q := cdf.Quantile(x) + vRight := cdf.Query(math.Min(q+eps, 1.0)) + if vRight < x { + t.Fatalf("Inverse sandwich fails at x=%v: Query(Quantile(x)+eps)=%v < x", x, vRight) + } + // Optional additional check: x should not be greater than the next step’s inverse either. + // (i.e., x is bracketed by the step to the right) + if q > 0 { + vLeft := cdf.Query(math.Max(q-eps, 0.0)) + // vLeft can be <= x (often strictly < x). This should always hold: + if vLeft > vRight { + t.Fatalf("Query left/right ordering broken at x=%v: %v > %v", x, vLeft, vRight) + } + } + } +} + +// Merge two sketches built on disjoint ranges (no compaction) and verify exactness. +func TestKLL_Merge_Disjoint_NoCompaction(t *testing.T) { + k := 256 + s1 := New(k) + s2 := New(k) + insertAll(s1, seq(1, 50)) + insertAll(s2, seq(51, 100)) + + // Before merge sanity + if s1.Count() != 50 || s2.Count() != 50 { + t.Fatalf("pre-merge counts wrong: s1=%d s2=%d", s1.Count(), s2.Count()) + } + + // Merge s2 into s1 + s1.Merge(s2) + + // Total count should be 100 + if s1.Count() != 100 { + t.Fatalf("post-merge Count()=%d want=100", s1.Count()) + } + + // Rank/Quantile exact on uniform 1..100 + if s1.Rank(50) != 50 { + t.Fatalf("Rank(50)=%d want=50", s1.Rank(50)) + } + if math.Abs(s1.Quantile(50)-0.5) > 1e-9 { + t.Fatalf("Quantile(50)=%.6f want=0.5", s1.Quantile(50)) + } + + // CDF Query: with uniform integers, Q(V)=V/100 + cdf := s1.CDF() + if q95 := cdf.Query(0.95); q95 != 95 { + t.Fatalf("CDF.Query(0.95)=%.6f want=95", q95) + } + if q05 := cdf.Query(0.05); q05 != 5 { + t.Fatalf("CDF.Query(0.05)=%.6f want=5", q05) + } +} + +// Linear interpolation behavior at midpoints. +// We create a small sketch with evenly spaced values so LI is easy to verify. +func TestKLL_LinearInterpolation(t *testing.T) { + k := 128 + vals := []float64{0, 10, 20, 30} + s := New(k) + insertAll(s, vals) + cdf := s.CDF() + + // With four items, cumulative Q are 0.25, 0.5, 0.75, 1.0 + // QuantileLI at x=5 should be halfway between 0.25 and 0.5 => 0.375 + if got := cdf.QuantileLI(5); math.Abs(got-0.375) > 1e-9 { + t.Fatalf("QuantileLI(5)=%.6f want=0.375", got) + } + + // QueryLI at p=0.6 should be between 10 and 20, closer to 10: + // Expected 14 via linear interpolation. + if got := cdf.QueryLI(0.6); math.Abs(got-14.0) > 1e-9 { + t.Fatalf("QueryLI(0.6)=%.6f want=14", got) + } + + // Endpoints behavior + if got := cdf.QuantileLI(-1); got != 0 { + t.Fatalf("QuantileLI(-1)=%.6f want=0", got) + } + if got := cdf.QueryLI(1.0); got != 30 { + t.Fatalf("QueryLI(1.0)=%.6f want=30", got) + } +} + +// Memory sanity: just ensure GetMemoryBytes reports a positive footprint. +func TestKLL_MemoryBytes_Positive(t *testing.T) { + s := New(64) + insertAll(s, seq(1, 10)) + if mb := s.GetMemoryBytes(); !(mb > 0) { + t.Fatalf("GetMemoryBytes()=%.2f want>0", mb) + } +} + +// Quantile vs Rank/Count consistency on a random-ish permutation (no compaction). +func TestKLL_Quantile_Rank_Consistency(t *testing.T) { + k := 256 + vals := []float64{7, 2, 9, 1, 6, 3, 8, 4, 10, 5} // permutation 1..10 + s := New(k) + insertAll(s, vals) + + // For x in [1..10], Quantile(x) should equal Rank(x)/10 exactly (no compaction). + for x := 1.0; x <= 10.0; x++ { + r := s.Rank(x) + q := s.Quantile(x) + expect := float64(r) / float64(s.Count()) + if math.Abs(q-expect) > 1e-12 { + t.Fatalf("Quantile(%v)=%.6f, Rank/Count=%.6f", x, q, expect) + } + } + + // CDF order matches sorted input. + cdf := s.CDF() + sorted := append([]float64(nil), vals...) + sort.Float64s(sorted) + for i := range sorted { + if cdf[i].V != sorted[i] { + t.Fatalf("CDF sorted mismatch at %d: got=%v want=%v", i, cdf[i].V, sorted[i]) + } + } +} \ No newline at end of file diff --git a/telegraf-patch/plugins/aggregators/kll/wrapper.go b/telegraf-patch/plugins/aggregators/kll/src.go similarity index 100% rename from telegraf-patch/plugins/aggregators/kll/wrapper.go rename to telegraf-patch/plugins/aggregators/kll/src.go From 1bae7a5f73a19abf4d896ebebc4afe17b8afc32a Mon Sep 17 00:00:00 2001 From: Leo Wang Date: Mon, 8 Dec 2025 00:03:32 -0500 Subject: [PATCH 4/5] cleanup --- .gitignore | 2 +- telegraf-patch/plugins/aggregators/kll/kll.go | 7 +++++-- telegraf-patch/plugins/aggregators/kll/sample.conf | 12 ++++++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index a36d8944..3f7b212b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,7 @@ go.work.sum # Editor/IDE # .idea/ -# .vscode/ +.vscode/ benchmarks/results/* benchmarks/__pycache__/* diff --git a/telegraf-patch/plugins/aggregators/kll/kll.go b/telegraf-patch/plugins/aggregators/kll/kll.go index 98bf0a3b..36364b0e 100755 --- a/telegraf-patch/plugins/aggregators/kll/kll.go +++ b/telegraf-patch/plugins/aggregators/kll/kll.go @@ -33,11 +33,14 @@ var sampleConfig string func (*KLL) SampleConfig() string { return sampleConfig; } func (kll *KLL) Init() error { - // TODO: sanitize inputs! + if kll.K < 2 { return fmt.Errorf("Invalid Argument. k must be >= 2 (k=%d)", kll.K); } kll.cache = make(map[uint64]*metric); kll.suffixes = make(map[float64]string); - for _, q := range kll.Quantiles { kll.suffixes[q] = fmt.Sprintf("_p%d", int(q * 100)); } + for _, q := range kll.Quantiles { + if q < 0 || q > 1 { return fmt.Errorf("Invalid Argument. Quantiles must be in [0, 1] (q=%f)", q); } + kll.suffixes[q] = fmt.Sprintf("_p%d", int(q * 100)); + } return nil; } diff --git a/telegraf-patch/plugins/aggregators/kll/sample.conf b/telegraf-patch/plugins/aggregators/kll/sample.conf index bd03644d..524117ea 100755 --- a/telegraf-patch/plugins/aggregators/kll/sample.conf +++ b/telegraf-patch/plugins/aggregators/kll/sample.conf @@ -1,5 +1,13 @@ +# Outputs quantiles for each numeric field for each metric +# Aggregated metric has name _KLL [[aggregators.kll]] + ## Size of topmost compactor + ## Total memory used by sketch is 3*k + lg(n) k = 256 - # 0 - 1 (float) + + ## Quantiles to retrieve. Must be in range [0, 1] and float (e.g. 0 must be 0.0) + ## Values will be output as field_p, e.g. cpu_p50 quantiles = [0.5, 0.99] - write_seen = true \ No newline at end of file + + ## Debugging option to also output the sorted ingested values (with suffix _seen) + write_seen = false From 88bb84519ad5975d2b603e4a6ddbae373c0320d6 Mon Sep 17 00:00:00 2001 From: Leo Wang Date: Mon, 8 Dec 2025 13:02:04 -0500 Subject: [PATCH 5/5] move sketch implementation to library --- telegraf-patch/go.mod | 5 +- telegraf-patch/go.sum | 7 +- .../plugins/aggregators/kll/heights.go | 142 -------- telegraf-patch/plugins/aggregators/kll/kll.go | 47 +-- .../plugins/aggregators/kll/kll_test.go | 211 ------------ telegraf-patch/plugins/aggregators/kll/src.go | 312 ------------------ 6 files changed, 28 insertions(+), 696 deletions(-) delete mode 100644 telegraf-patch/plugins/aggregators/kll/heights.go delete mode 100644 telegraf-patch/plugins/aggregators/kll/kll_test.go delete mode 100644 telegraf-patch/plugins/aggregators/kll/src.go diff --git a/telegraf-patch/go.mod b/telegraf-patch/go.mod index e5cd1a51..87c0dcdc 100644 --- a/telegraf-patch/go.mod +++ b/telegraf-patch/go.mod @@ -50,7 +50,6 @@ require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/awnumar/memguard v0.23.0 github.com/aws/aws-msk-iam-sasl-signer-go v1.0.4 - github.com/aws/aws-sdk-go v1.44.263 github.com/aws/aws-sdk-go-v2 v1.39.5 github.com/aws/aws-sdk-go-v2/config v1.31.16 github.com/aws/aws-sdk-go-v2/credentials v1.18.20 @@ -138,7 +137,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 github.com/karrick/godirwalk v1.16.2 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 - github.com/klauspost/compress v1.18.2 + github.com/klauspost/compress v1.18.1 github.com/klauspost/pgzip v1.2.6 github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b github.com/leodido/go-syslog/v4 v4.3.0 @@ -287,7 +286,6 @@ require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect github.com/ClickHouse/ch-go v0.68.0 // indirect - github.com/DataDog/sketches-go v1.4.7 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect @@ -545,6 +543,7 @@ require ( github.com/zitadel/logging v0.6.2 // indirect github.com/zitadel/oidc/v3 v3.45.0 // indirect github.com/zitadel/schema v1.3.1 // indirect + github.com/zzylol/go-kll v0.0.1 go.bug.st/serial v1.6.4 // indirect go.etcd.io/etcd/api/v3 v3.5.4 // indirect go.opencensus.io v0.24.0 // indirect diff --git a/telegraf-patch/go.sum b/telegraf-patch/go.sum index a6c49436..be6ba31d 100644 --- a/telegraf-patch/go.sum +++ b/telegraf-patch/go.sum @@ -738,8 +738,6 @@ github.com/ClickHouse/clickhouse-go/v2 v2.40.3/go.mod h1:qO0HwvjCnTB4BPL/k6EE3l4 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/sketches-go v1.4.7 h1:eHs5/0i2Sdf20Zkj0udVFWuCrXGRFig2Dcfm5rtcTxc= -github.com/DataDog/sketches-go v1.4.7/go.mod h1:eAmQ/EBmtSO+nQp7IZMZVRPT4BQTmIc5RZQ+deGlTPM= github.com/Files-com/files-sdk-go/v3 v3.2.97 h1:c+mQoiES/21JrHDAxJLCYICJO+bu8Clv0ZDNZe7Ndyk= github.com/Files-com/files-sdk-go/v3 v3.2.97/go.mod h1:Y/bCHoPJNPKz2hw1ADXjQXJP378HODwK+g/5SR2gqfU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= @@ -892,7 +890,6 @@ github.com/aws/aws-msk-iam-sasl-signer-go v1.0.4 h1:2jAwFwA0Xgcx94dUId+K24yFabsK github.com/aws/aws-msk-iam-sasl-signer-go v1.0.4/go.mod h1:MVYeeOhILFFemC/XlYTClvBjYZrg/EPd3ts885KrNTI= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.29.11/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= -github.com/aws/aws-sdk-go v1.44.263 h1:Dkt5fcdtL8QtK3cz0bOTQ84m9dGx+YDeTsDl+wY2yW4= github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.39.5 h1:e/SXuia3rkFtapghJROrydtQpfQaaUgd1cUvyO1mp2w= @@ -1790,8 +1787,6 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= -github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= -github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= @@ -2521,6 +2516,8 @@ github.com/zitadel/oidc/v3 v3.45.0 h1:SaVJ2kdcJi/zdEWWlAns+81VxmfdYX4E+2mWFVIH7E github.com/zitadel/oidc/v3 v3.45.0/go.mod h1:UeK0iVOoqfMuDVgSfv56BqTz8YQC2M+tGRIXZ7Ii3VY= github.com/zitadel/schema v1.3.1 h1:QT3kwiRIRXXLVAs6gCK/u044WmUVh6IlbLXUsn6yRQU= github.com/zitadel/schema v1.3.1/go.mod h1:071u7D2LQacy1HAN+YnMd/mx1qVE2isb0Mjeqg46xnU= +github.com/zzylol/go-kll v0.0.1 h1:54O2mqW41PG7OgkgZL88x+YiqBE4g5FtW4iVEdENgXM= +github.com/zzylol/go-kll v0.0.1/go.mod h1:yBYzN7orOcqHVW7NcjYS/K90f4bOuXr4OkhsadygrnM= go.bug.st/serial v1.6.4 h1:7FmqNPgVp3pu2Jz5PoPtbZ9jJO5gnEnZIvnI1lzve8A= go.bug.st/serial v1.6.4/go.mod h1:nofMJxTeNVny/m6+KaafC6vJGj3miwQZ6vW4BZUGJPI= go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps= diff --git a/telegraf-patch/plugins/aggregators/kll/heights.go b/telegraf-patch/plugins/aggregators/kll/heights.go deleted file mode 100644 index 02addece..00000000 --- a/telegraf-patch/plugins/aggregators/kll/heights.go +++ /dev/null @@ -1,142 +0,0 @@ -package kll - -import "math" - - -func computeHeight(h int) float64 { - if h < len(heightsCache) { - return heightsCache[h] - } - return math.Pow((2.0 / 3.0), float64(h)) -} - -var heightsCache = [...]float64{ - 1, - 0.6666666666666666, - 0.4444444444444444, - 0.2962962962962963, - 0.19753086419753085, - 0.1316872427983539, - 0.0877914951989026, - 0.05852766346593506, - 0.039018442310623375, - 0.026012294873748915, - 0.01734152991583261, - 0.011561019943888407, - 0.007707346629258938, - 0.005138231086172625, - 0.00342548739078175, - 0.0022836582605211663, - 0.0015224388403474443, - 0.0010149592268982961, - 0.0006766394845988641, - 0.00045109298973257606, - 0.0003007286598217174, - 0.00020048577321447823, - 0.0001336571821429855, - 8.910478809532365e-05, - 5.9403192063549106e-05, - 3.960212804236607e-05, - 2.640141869491071e-05, - 1.760094579660714e-05, - 1.1733963864404761e-05, - 7.82264257626984e-06, - 5.21509505084656e-06, - 3.4767300338977064e-06, - 2.3178200225984708e-06, - 1.5452133483989804e-06, - 1.030142232265987e-06, - 6.867614881773246e-07, - 4.5784099211821645e-07, - 3.0522732807881095e-07, - 2.0348488538587396e-07, - 1.356565902572493e-07, - 9.04377268381662e-08, - 6.02918178921108e-08, - 4.019454526140719e-08, - 2.67963635076048e-08, - 1.78642423384032e-08, - 1.1909494892268798e-08, - 7.939663261512532e-09, - 5.293108841008354e-09, - 3.528739227338903e-09, - 2.352492818225935e-09, - 1.5683285454839568e-09, - 1.0455523636559712e-09, - 6.970349091039809e-10, - 4.646899394026538e-10, - 3.097932929351026e-10, - 2.0652886195673503e-10, - 1.3768590797115669e-10, - 9.179060531410445e-11, - 6.119373687606963e-11, - 4.0795824584046424e-11, - 2.7197216389364282e-11, - 1.813147759290952e-11, - 1.2087651728606347e-11, - 8.058434485737563e-12, - 5.372289657158376e-12, - 3.581526438105584e-12, - 2.3876842920703892e-12, - 1.5917895280469262e-12, - 1.0611930186979508e-12, - 7.074620124653005e-13, - 4.716413416435336e-13, - 3.1442756109568906e-13, - 2.0961837406379272e-13, - 1.3974558270919513e-13, - 9.316372180613009e-14, - 6.21091478707534e-14, - 4.140609858050226e-14, - 2.760406572033484e-14, - 1.8402710480223226e-14, - 1.226847365348215e-14, - 8.178982435654766e-15, - 5.452654957103177e-15, - 3.635103304735452e-15, - 2.423402203156968e-15, - 1.615601468771312e-15, - 1.0770676458475411e-15, - 7.180450972316942e-16, - 4.786967314877961e-16, - 3.191311543251974e-16, - 2.1275410288346492e-16, - 1.418360685889766e-16, - 9.455737905931773e-17, - 6.303825270621183e-17, - 4.2025501804141215e-17, - 2.801700120276081e-17, - 1.8678000801840538e-17, - 1.2452000534560357e-17, - 8.301333689706904e-18, - 5.534222459804603e-18, - 3.6894816398697355e-18, - 2.459654426579824e-18, - 1.6397696177198825e-18, - 1.0931797451465883e-18, - 7.287864967643922e-19, - 4.858576645095947e-19, - 3.239051096730632e-19, - 2.1593673978204208e-19, - 1.439578265213614e-19, - 9.597188434757427e-20, - 6.398125623171617e-20, - 4.2654170821144116e-20, - 2.843611388076274e-20, - 1.8957409253841826e-20, - 1.263827283589455e-20, - 8.4255152239297e-21, - 5.6170101492864665e-21, - 3.744673432857645e-21, - 2.4964489552384296e-21, - 1.6642993034922866e-21, - 1.1095328689948576e-21, - 7.39688579329905e-22, - 4.931257195532699e-22, - 3.2875047970218e-22, - 2.1916698646812e-22, - 1.4611132431208001e-22, - 9.740754954138666e-23, - 6.493836636092445e-23, - 4.3292244240616286e-23, -} \ No newline at end of file diff --git a/telegraf-patch/plugins/aggregators/kll/kll.go b/telegraf-patch/plugins/aggregators/kll/kll.go index 36364b0e..43bf403f 100755 --- a/telegraf-patch/plugins/aggregators/kll/kll.go +++ b/telegraf-patch/plugins/aggregators/kll/kll.go @@ -7,11 +7,12 @@ import ( "slices" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/aggregators" + "github.com/zzylol/go-kll" ) type quantile struct { seen []float64; // for debugging, save the seen values - sketch *Sketch; + sketch *kll.Sketch; } type metric struct { @@ -32,28 +33,28 @@ type KLL struct { var sampleConfig string func (*KLL) SampleConfig() string { return sampleConfig; } -func (kll *KLL) Init() error { - if kll.K < 2 { return fmt.Errorf("Invalid Argument. k must be >= 2 (k=%d)", kll.K); } +func (k *KLL) Init() error { + if k.K < 2 { return fmt.Errorf("Invalid Argument. k must be >= 2 (k=%d)", k.K); } - kll.cache = make(map[uint64]*metric); - kll.suffixes = make(map[float64]string); - for _, q := range kll.Quantiles { + k.cache = make(map[uint64]*metric); + k.suffixes = make(map[float64]string); + for _, q := range k.Quantiles { if q < 0 || q > 1 { return fmt.Errorf("Invalid Argument. Quantiles must be in [0, 1] (q=%f)", q); } - kll.suffixes[q] = fmt.Sprintf("_p%d", int(q * 100)); + k.suffixes[q] = fmt.Sprintf("_p%d", int(q * 100)); } return nil; } // for each numeric field in each metric, update the backing KLL sketch -func (kll *KLL) Add(in telegraf.Metric) { +func (k *KLL) Add(in telegraf.Metric) { var id uint64 = in.HashID(); // get saved metric - m, ok := kll.cache[id]; + m, ok := k.cache[id]; if !ok { - kll.cache[id] = &metric{name: in.Name(), fields: make(map[string]*quantile)}; - m = kll.cache[id]; + k.cache[id] = &metric{name: in.Name(), fields: make(map[string]*quantile)}; + m = k.cache[id]; } // for each field, get associated sketch @@ -76,29 +77,29 @@ func (kll *KLL) Add(in telegraf.Metric) { // get sketch sketch, ok := m.fields[field.Key]; if !ok { - m.fields[field.Key] = &quantile{seen: nil, sketch: New(kll.K)}; - if kll.WriteSeen { m.fields[field.Key].seen = make([]float64, 0); } + m.fields[field.Key] = &quantile{seen: nil, sketch: kll.New(k.K)}; + if k.WriteSeen { m.fields[field.Key].seen = make([]float64, 0); } sketch = m.fields[field.Key]; } - if kll.WriteSeen { sketch.seen = append(sketch.seen, val); } + if k.WriteSeen { sketch.seen = append(sketch.seen, val); } sketch.sketch.Update(val); } } -func (kll *KLL) Push(acc telegraf.Accumulator) { - for _, m := range kll.cache { +func (k *KLL) Push(acc telegraf.Accumulator) { + for _, m := range k.cache { out := make(map[string]any); fields := m.fields; for name, sketch := range fields { // get the desired quantile cdf := sketch.sketch.CDF(); - for q, str := range kll.suffixes { out[name + str] = cdf.Query(q); } + for q, str := range k.suffixes { out[name + str] = cdf.Query(q); } - if kll.WriteSeen { + if k.WriteSeen { slices.Sort(sketch.seen) out[name + "_seen"] = fmt.Sprintf("%v", sketch.seen); } @@ -108,11 +109,11 @@ func (kll *KLL) Push(acc telegraf.Accumulator) { } } -func (kll *KLL) Reset() { - for _, m := range kll.cache { - for k := range m.fields { - if kll.WriteSeen { clear(m.fields[k].seen); } - m.fields[k].sketch = New(kll.K); +func (k *KLL) Reset() { + for _, m := range k.cache { + for key := range m.fields { + if k.WriteSeen { clear(m.fields[key].seen); } + m.fields[key].sketch = kll.New(k.K); } } } diff --git a/telegraf-patch/plugins/aggregators/kll/kll_test.go b/telegraf-patch/plugins/aggregators/kll/kll_test.go deleted file mode 100644 index bc927091..00000000 --- a/telegraf-patch/plugins/aggregators/kll/kll_test.go +++ /dev/null @@ -1,211 +0,0 @@ -// Tests taken from Precompute Engine https://github.com/approx-telemetry/PrecomputeEngine/blob/main/aggregators/KLL/kll_test.go -package kll - -import ( - "math" - "sort" - "testing" -) - -// Helper: build a sorted float slice [start..end] inclusive. -func seq(start, end int) []float64 { - n := end - start + 1 - out := make([]float64, 0, n) - for i := start; i <= end; i++ { - out = append(out, float64(i)) - } - return out -} - -// Insert values into a sketch. -func insertAll(s *Sketch, vals []float64) { - for _, v := range vals { - s.Update(v) - } -} - -// With sufficiently large k and small n, the sketch will not compact, -// so results should be exact for Rank/Quantile/Count and the CDF mapping. -func TestKLL_NoCompaction_Exactness(t *testing.T) { - k := 256 - vals := seq(1, 20) // n=20 << capacity(0) for most reasonable computeHeight - s := New(k) - insertAll(s, vals) - - // Count is exact when no compaction occurs. - if got := s.Count(); got != len(vals) { - t.Fatalf("Count() mismatch: got=%d want=%d", got, len(vals)) - } - - // Rank and Quantile should align exactly on midpoints. - if got := s.Rank(10.5); got != 10 { - t.Fatalf("Rank(10.5)=%d want=10", got) - } - if got := s.Quantile(10.5); math.Abs(got-0.5) > 1e-9 { - t.Fatalf("Quantile(10.5)=%.6f want=0.5", got) - } - - // CDF should map 0.5 back to ~10 exactly (discrete). - cdf := s.CDF() - if got := cdf.Query(0.5); got != 10 { - t.Fatalf("CDF.Query(0.5)=%.6f want=10", got) - } - - // Endpoints - if q0 := cdf.Query(0.0); q0 != 1 { - t.Fatalf("CDF.Query(0.0)=%.6f want=1 (min value)", q0) - } - if q1 := cdf.Query(1.0); q1 != 20 { - t.Fatalf("CDF.Query(1.0)=%.6f want=20 (max value)", q1) - } -} - -func TestKLL_CDF_Monotonicity_And_Inverses(t *testing.T) { - k := 512 - vals := append(seq(1, 50), seq(100, 110)...) // gaps, still no compaction - s := New(k) - insertAll(s, vals) - cdf := s.CDF() - - // Monotonicity by value and cumulative probability. - for i := 1; i < len(cdf); i++ { - if !(cdf[i].V >= cdf[i-1].V) { - t.Fatalf("CDF not non-decreasing by value at i=%d: %v !>= %v", i, cdf[i].V, cdf[i-1].V) - } - if !(cdf[i].Q >= cdf[i-1].Q) { - t.Fatalf("CDF not non-decreasing by Q at i=%d: %v !>= %v", i, cdf[i].Q, cdf[i-1].Q) - } - } - - // Inverse checks (stepwise-safe): for x in the support, - // let q = CDF.Quantile(x) be the left-closed CDF; then - // Query(q+ε) must be >= x for any tiny ε>0. - points := []float64{ - vals[len(vals)/4], - vals[len(vals)/2], - vals[3*len(vals)/4], - } - const eps = 1e-12 - for _, x := range points { - q := cdf.Quantile(x) - vRight := cdf.Query(math.Min(q+eps, 1.0)) - if vRight < x { - t.Fatalf("Inverse sandwich fails at x=%v: Query(Quantile(x)+eps)=%v < x", x, vRight) - } - // Optional additional check: x should not be greater than the next step’s inverse either. - // (i.e., x is bracketed by the step to the right) - if q > 0 { - vLeft := cdf.Query(math.Max(q-eps, 0.0)) - // vLeft can be <= x (often strictly < x). This should always hold: - if vLeft > vRight { - t.Fatalf("Query left/right ordering broken at x=%v: %v > %v", x, vLeft, vRight) - } - } - } -} - -// Merge two sketches built on disjoint ranges (no compaction) and verify exactness. -func TestKLL_Merge_Disjoint_NoCompaction(t *testing.T) { - k := 256 - s1 := New(k) - s2 := New(k) - insertAll(s1, seq(1, 50)) - insertAll(s2, seq(51, 100)) - - // Before merge sanity - if s1.Count() != 50 || s2.Count() != 50 { - t.Fatalf("pre-merge counts wrong: s1=%d s2=%d", s1.Count(), s2.Count()) - } - - // Merge s2 into s1 - s1.Merge(s2) - - // Total count should be 100 - if s1.Count() != 100 { - t.Fatalf("post-merge Count()=%d want=100", s1.Count()) - } - - // Rank/Quantile exact on uniform 1..100 - if s1.Rank(50) != 50 { - t.Fatalf("Rank(50)=%d want=50", s1.Rank(50)) - } - if math.Abs(s1.Quantile(50)-0.5) > 1e-9 { - t.Fatalf("Quantile(50)=%.6f want=0.5", s1.Quantile(50)) - } - - // CDF Query: with uniform integers, Q(V)=V/100 - cdf := s1.CDF() - if q95 := cdf.Query(0.95); q95 != 95 { - t.Fatalf("CDF.Query(0.95)=%.6f want=95", q95) - } - if q05 := cdf.Query(0.05); q05 != 5 { - t.Fatalf("CDF.Query(0.05)=%.6f want=5", q05) - } -} - -// Linear interpolation behavior at midpoints. -// We create a small sketch with evenly spaced values so LI is easy to verify. -func TestKLL_LinearInterpolation(t *testing.T) { - k := 128 - vals := []float64{0, 10, 20, 30} - s := New(k) - insertAll(s, vals) - cdf := s.CDF() - - // With four items, cumulative Q are 0.25, 0.5, 0.75, 1.0 - // QuantileLI at x=5 should be halfway between 0.25 and 0.5 => 0.375 - if got := cdf.QuantileLI(5); math.Abs(got-0.375) > 1e-9 { - t.Fatalf("QuantileLI(5)=%.6f want=0.375", got) - } - - // QueryLI at p=0.6 should be between 10 and 20, closer to 10: - // Expected 14 via linear interpolation. - if got := cdf.QueryLI(0.6); math.Abs(got-14.0) > 1e-9 { - t.Fatalf("QueryLI(0.6)=%.6f want=14", got) - } - - // Endpoints behavior - if got := cdf.QuantileLI(-1); got != 0 { - t.Fatalf("QuantileLI(-1)=%.6f want=0", got) - } - if got := cdf.QueryLI(1.0); got != 30 { - t.Fatalf("QueryLI(1.0)=%.6f want=30", got) - } -} - -// Memory sanity: just ensure GetMemoryBytes reports a positive footprint. -func TestKLL_MemoryBytes_Positive(t *testing.T) { - s := New(64) - insertAll(s, seq(1, 10)) - if mb := s.GetMemoryBytes(); !(mb > 0) { - t.Fatalf("GetMemoryBytes()=%.2f want>0", mb) - } -} - -// Quantile vs Rank/Count consistency on a random-ish permutation (no compaction). -func TestKLL_Quantile_Rank_Consistency(t *testing.T) { - k := 256 - vals := []float64{7, 2, 9, 1, 6, 3, 8, 4, 10, 5} // permutation 1..10 - s := New(k) - insertAll(s, vals) - - // For x in [1..10], Quantile(x) should equal Rank(x)/10 exactly (no compaction). - for x := 1.0; x <= 10.0; x++ { - r := s.Rank(x) - q := s.Quantile(x) - expect := float64(r) / float64(s.Count()) - if math.Abs(q-expect) > 1e-12 { - t.Fatalf("Quantile(%v)=%.6f, Rank/Count=%.6f", x, q, expect) - } - } - - // CDF order matches sorted input. - cdf := s.CDF() - sorted := append([]float64(nil), vals...) - sort.Float64s(sorted) - for i := range sorted { - if cdf[i].V != sorted[i] { - t.Fatalf("CDF sorted mismatch at %d: got=%v want=%v", i, cdf[i].V, sorted[i]) - } - } -} \ No newline at end of file diff --git a/telegraf-patch/plugins/aggregators/kll/src.go b/telegraf-patch/plugins/aggregators/kll/src.go deleted file mode 100644 index 726365fd..00000000 --- a/telegraf-patch/plugins/aggregators/kll/src.go +++ /dev/null @@ -1,312 +0,0 @@ -// implementation ported from PrecomputeEngine https://github.com/approx-telemetry/PrecomputeEngine/blob/main/aggregators/KLL/kll.go -package kll - -import ( - "math/rand" - "math" - "unsafe" - "sort" -) - -// 64-bit xorshift multiply rng from http://vigna.di.unimi.it/ftp/papers/xorshift.pdf -func xorshiftMult64(x uint64) uint64 { - x ^= x >> 12 // a - x ^= x << 25 // b - x ^= x >> 27 // c - return x * 2685821657736338717 -} - -// coin is a simple struct to let us get random bools and make minimum calls -// to the random number generator. -type coin struct { - st uint64 - mask uint64 -} - -// v is either 0 or 1 -func (c *coin) toss() (v int) { - if c.mask == 0 { - if c.st == 0 { - c.st = uint64(rand.Int63()) - } - c.st = xorshiftMult64(c.st) - c.mask = 1 - } - if c.st&c.mask > 0 { - v = 1 - } - c.mask <<= 1 - return v -} - - -// Sketch is a streaming quantiles sketch -type Sketch struct { - Compactors []Compactor - k int - H int - size int - maxSize int - - co coin -} - -// New returns a new Sketch. k controls the maximum memory used by the stream, which is 3*k + lg(n). -func New(k int) *Sketch { - s := Sketch{ - k: k, - } - s.grow() - return &s -} - -func (s *Sketch) GetSize() int { - return s.size -} - -func (s *Sketch) grow() { - s.Compactors = append(s.Compactors, Compactor{}) - s.H = len(s.Compactors) - - s.maxSize = 0 - for h := 0; h < s.H; h++ { - s.maxSize += s.capacity(h) - } -} - -func (s *Sketch) capacity(h int) int { - return int(math.Ceil(float64(s.k)*computeHeight(s.H-h-1))) + 1 -} - -// Update adds x to the stream. -func (s *Sketch) Update(x float64) { - s.Compactors[0] = append(s.Compactors[0], x) - s.size++ - s.compact() -} - -func (s *Sketch) compact() { - for s.size >= s.maxSize { - for h := 0; h < len(s.Compactors); h++ { - if len(s.Compactors[h]) >= s.capacity(h) { - if h+1 >= s.H { - s.grow() - } - - prev_h := len(s.Compactors[h]) - prev_h1 := len(s.Compactors[h+1]) - - s.Compactors[h+1] = s.Compactors[h].compact( - &s.co, s.Compactors[h+1]) - - s.size += len(s.Compactors[h]) - prev_h - s.size += len(s.Compactors[h+1]) - prev_h1 - - if s.size < s.maxSize { - break - } - } - } - } -} - -func (s *Sketch) updateSize() { - s.size = 0 - for _, c := range s.Compactors { - s.size += len(c) - } -} - -// Merge merges a second sketch into this one -func (s *Sketch) Merge(t *Sketch) { - for s.H < t.H { - s.grow() - } - - for h, c := range t.Compactors { - s.Compactors[h] = append(s.Compactors[h], c...) - } - - s.updateSize() - s.compact() -} - -// Rank estimates the rank of the value x in the stream. -func (s *Sketch) Rank(x float64) int { - var r int - for h, c := range s.Compactors { - for _, v := range c { - if v <= x { - r += 1 << uint(h) - } - } - } - return r -} - -func (s *Sketch) Count() int { - var n int - for h, c := range s.Compactors { - n += len(c) * (1 << uint(h)) - } - return n -} - -// Quantile estimates the quantile of the value x in the stream. -func (s *Sketch) Quantile(x float64) float64 { - var r, n int - for h, c := range s.Compactors { - for _, v := range c { - w := 1 << uint(h) - if v <= x { - r += w - } - n += w - } - } - return float64(r) / float64(n) -} - -type CDF []Quantile - -func (q CDF) Len() int { return len(q) } - -func (q CDF) Less(i int, j int) bool { return q[i].V < q[j].V } - -func (q CDF) Swap(i int, j int) { q[i], q[j] = q[j], q[i] } - -type Quantile struct { - Q float64 - V float64 -} - -func (s *Sketch) GetMemoryBytes() float64 { - var total_mem float64 = 0 - total_mem += float64(unsafe.Sizeof(*s)) - for i := range s.Compactors { - total_mem += float64(len(s.Compactors[i])) * 8 - } - return total_mem // Bytes -} - -func (s *Sketch) CDF() CDF { - q := make(CDF, 0, s.size) - - var totalW float64 - for h, c := range s.Compactors { - weight := float64(int(1 << uint(h))) - for _, v := range c { - q = append(q, Quantile{Q: weight, V: v}) - } - totalW += float64(len(c)) * weight - } - - sort.Sort(q) - - var curW float64 - for i := range q { - curW += q[i].Q - q[i].Q = curW / totalW - } - - return q -} - -// Quantile estimates the quantile of the value x in the stream. -func (q CDF) Quantile(x float64) float64 { - idx := sort.Search(len(q), func(i int) bool { return q[i].V >= x }) - if idx == 0 { - return 0 - } - return q[idx-1].Q -} - -// Query estimates the value given quantile p. -func (q CDF) Query(p float64) float64 { - idx := sort.Search(len(q), func(i int) bool { return q[i].Q >= p }) - if idx == len(q) { - return q[len(q)-1].V - } - return q[idx].V -} - -// QuantileLI estimates the quantile of the value x in the stream using linear interpolation. -func (q CDF) QuantileLI(x float64) float64 { - idx := sort.Search(len(q), func(i int) bool { return q[i].V >= x }) - if idx == len(q) { - return 1 - } - if idx == 0 { - return 0 - } - // a < x <= b - a, aq := q[idx-1].V, q[idx-1].Q - b, bq := q[idx].V, q[idx].Q - return ((a-x)*bq + (x-b)*aq) / (a - b) -} - -// QueryLI estimates the value given quantile p using linear interpolation. -func (q CDF) QueryLI(p float64) float64 { - idx := sort.Search(len(q), func(i int) bool { return q[i].Q >= p }) - if idx == len(q) { - return q[len(q)-1].V - } - if idx == 0 { - return q[0].V - } - // aq < p <= b - a, aq := q[idx-1].V, q[idx-1].Q - b, bq := q[idx].V, q[idx].Q - return ((aq-p)*b + (p-bq)*a) / (aq - bq) -} - -type Compactor []float64 - -func (c *Compactor) compact(co *coin, dst []float64) []float64 { - l := len(*c) - - if l == 0 || l == 1 { - } else if l == 2 { - c := *c - if c[0] > c[1] { - c[0], c[1] = c[1], c[0] - } - } else if l > 100 { - sort.Float64s([]float64(*c)) - } else { - c.insertionSort() - } - - free := cap(dst) - len(dst) - if free < len(*c)/2 { - extra := len(*c)/2 - free - newdst := make([]float64, len(dst), cap(dst)+extra) - copy(newdst, dst) - dst = newdst - } - - // choose either the evens or the odds - offs := co.toss() - for len(*c) >= 2 { - l := len(*c) - 2 - dst = append(dst, (*c)[l+offs]) - *c = (*c)[:l] - } - - return dst -} - -func (c Compactor) insertionSort() { - l := len(c) - for i := 1; i < l; i++ { - v := c[i] - j := i - for ; j > 0 && c[j-1] > v; j-- { - } - if j == i { - continue - } - copy(c[j+1:], c[j:i]) - c[j] = v - } -}