diff --git a/.gitignore b/.gitignore
index a414f134..3f7b212b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,10 +29,14 @@ go.work.sum
# Editor/IDE
# .idea/
-# .vscode/
+.vscode/
benchmarks/results/*
benchmarks/__pycache__/*
# Local checkouts
opentelemetry-go
+
+# Telegraf artifacts
+config.toml
+out.json
\ No newline at end of file
diff --git a/backup_telegraf_patches.sh b/backup_telegraf_patches.sh
index 5f619d05..6831e7d8 100755
--- a/backup_telegraf_patches.sh
+++ b/backup_telegraf_patches.sh
@@ -3,7 +3,7 @@ set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SRC_TELEGRAF="${ROOT_DIR}/telegraf"
-DEST_DIR="${ROOT_DIR}/telegraf-plugins"
+DEST_DIR="${ROOT_DIR}/telegraf-patch"
copy_path() {
local src="$1"
diff --git a/benchmarks/max-throughput-prometheus-input.conf b/benchmarks/max-throughput-prometheus-input.conf
new file mode 100644
index 00000000..0a65844e
--- /dev/null
+++ b/benchmarks/max-throughput-prometheus-input.conf
@@ -0,0 +1,50 @@
+# Read data directly from FakePrometheusExporter
+# based off max-throughput-prometheus-client.conf
+
+[global_tags]
+ scenario = "prometheus_inp"
+
+[agent]
+ interval = "1ms"
+ flush_interval = "1ms"
+ flush_jitter = "0s"
+ collection_jitter = "0s"
+ round_interval = false
+ metric_batch_size = 10000
+ metric_buffer_limit = 500000
+ precision = "1ms"
+ debug = true
+ logfile = "/tmp/telegraf-prometheus-inp.log"
+
+[[inputs.prometheus]]
+ urls = [
+ "http://localhost:8000/metrics",
+ "http://localhost:8001/metrics",
+ ]
+ metric_version = 2
+ url_tag = "scrape_url"
+ name_prefix = "gorilla_local_"
+ timeout = "5s"
+ interval = "250ms"
+ metric_batch_size = 10000
+ metric_buffer_limit = 250000
+
+[[inputs.internal]]
+ per_instance = true
+ collect_gostats = true
+
+[[outputs.file]]
+ files = ["../benchmarks/results/prometheus-inp.lp"]
+ data_format = "influx"
+ use_batch_format = false
+ rotation_interval = "1m"
+ rotation_max_size = "50MB"
+ rotation_max_archives = 15
+ namepass = ["internal_*"]
+
+[[outputs.prometheus_client]]
+ listen = ":9203"
+ path = "/metrics"
+ metric_version = 2
+ export_timestamp = true
+ expiration_interval = "30s"
diff --git a/benchmarks/tests/kll.conf b/benchmarks/tests/kll.conf
new file mode 100644
index 00000000..66418f4d
--- /dev/null
+++ b/benchmarks/tests/kll.conf
@@ -0,0 +1,4 @@
+[[aggregators.kll]]
+ k = 256
+ quantiles = [0.5, 0.99]
+ write_seen = false
\ No newline at end of file
diff --git a/restore_telegraf_patches.sh b/restore_telegraf_patches.sh
index 68d5ff77..b4f4b20f 100755
--- a/restore_telegraf_patches.sh
+++ b/restore_telegraf_patches.sh
@@ -2,7 +2,7 @@
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SRC_ROOT="${ROOT_DIR}/telegraf-plugins/plugins"
+SRC_ROOT="${ROOT_DIR}/telegraf-patch/plugins"
DEST_ROOT="${ROOT_DIR}/telegraf/plugins"
if [[ ! -d "${SRC_ROOT}" ]]; then
@@ -16,10 +16,9 @@ copied=0
for path in "${SRC_ROOT}"/*; do
rel="$(basename "${path}")"
dest="${DEST_ROOT}/${rel}"
- rm -rf "${dest}"
- cp -R "${path}" "${dest}"
+ cp -R -T "${path}" "${dest}"
echo "Restored ${rel} into ${dest}"
copied=$((copied + 1))
done
-echo "Restored ${copied} items from telegraf-plugins/plugins into telegraf/plugins/"
+echo "Restored ${copied} items from telegraf-patch/plugins into telegraf/plugins/"
diff --git a/telegraf-patch/go.mod b/telegraf-patch/go.mod
index 9754c06f..87c0dcdc 100644
--- a/telegraf-patch/go.mod
+++ b/telegraf-patch/go.mod
@@ -25,7 +25,6 @@ require (
github.com/BurntSushi/toml v1.5.0
github.com/ClickHouse/clickhouse-go/v2 v2.40.3
github.com/DATA-DOG/go-sqlmock v1.5.2
- github.com/DataDog/sketches-go v1.4.1
github.com/IBM/nzgo/v12 v12.0.10
github.com/IBM/sarama v1.46.3
github.com/Masterminds/semver/v3 v3.4.0
@@ -51,7 +50,6 @@ require (
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
github.com/awnumar/memguard v0.23.0
github.com/aws/aws-msk-iam-sasl-signer-go v1.0.4
- github.com/aws/aws-sdk-go v1.44.263
github.com/aws/aws-sdk-go-v2 v1.39.5
github.com/aws/aws-sdk-go-v2/config v1.31.16
github.com/aws/aws-sdk-go-v2/credentials v1.18.20
@@ -545,6 +543,7 @@ require (
github.com/zitadel/logging v0.6.2 // indirect
github.com/zitadel/oidc/v3 v3.45.0 // indirect
github.com/zitadel/schema v1.3.1 // indirect
+ github.com/zzylol/go-kll v0.0.1
go.bug.st/serial v1.6.4 // indirect
go.etcd.io/etcd/api/v3 v3.5.4 // indirect
go.opencensus.io v0.24.0 // indirect
diff --git a/telegraf-patch/go.sum b/telegraf-patch/go.sum
index c78cbdb5..be6ba31d 100644
--- a/telegraf-patch/go.sum
+++ b/telegraf-patch/go.sum
@@ -738,8 +738,6 @@ github.com/ClickHouse/clickhouse-go/v2 v2.40.3/go.mod h1:qO0HwvjCnTB4BPL/k6EE3l4
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
-github.com/DataDog/sketches-go v1.4.1 h1:j5G6as+9FASM2qC36lvpvQAj9qsv/jUs3FtO8CwZNAY=
-github.com/DataDog/sketches-go v1.4.1/go.mod h1:xJIXldczJyyjnbDop7ZZcLxJdV3+7Kra7H1KMgpgkLk=
github.com/Files-com/files-sdk-go/v3 v3.2.97 h1:c+mQoiES/21JrHDAxJLCYICJO+bu8Clv0ZDNZe7Ndyk=
github.com/Files-com/files-sdk-go/v3 v3.2.97/go.mod h1:Y/bCHoPJNPKz2hw1ADXjQXJP378HODwK+g/5SR2gqfU=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4=
@@ -892,7 +890,6 @@ github.com/aws/aws-msk-iam-sasl-signer-go v1.0.4 h1:2jAwFwA0Xgcx94dUId+K24yFabsK
github.com/aws/aws-msk-iam-sasl-signer-go v1.0.4/go.mod h1:MVYeeOhILFFemC/XlYTClvBjYZrg/EPd3ts885KrNTI=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.29.11/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
-github.com/aws/aws-sdk-go v1.44.263 h1:Dkt5fcdtL8QtK3cz0bOTQ84m9dGx+YDeTsDl+wY2yW4=
github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2 v1.39.5 h1:e/SXuia3rkFtapghJROrydtQpfQaaUgd1cUvyO1mp2w=
@@ -2519,6 +2516,8 @@ github.com/zitadel/oidc/v3 v3.45.0 h1:SaVJ2kdcJi/zdEWWlAns+81VxmfdYX4E+2mWFVIH7E
github.com/zitadel/oidc/v3 v3.45.0/go.mod h1:UeK0iVOoqfMuDVgSfv56BqTz8YQC2M+tGRIXZ7Ii3VY=
github.com/zitadel/schema v1.3.1 h1:QT3kwiRIRXXLVAs6gCK/u044WmUVh6IlbLXUsn6yRQU=
github.com/zitadel/schema v1.3.1/go.mod h1:071u7D2LQacy1HAN+YnMd/mx1qVE2isb0Mjeqg46xnU=
+github.com/zzylol/go-kll v0.0.1 h1:54O2mqW41PG7OgkgZL88x+YiqBE4g5FtW4iVEdENgXM=
+github.com/zzylol/go-kll v0.0.1/go.mod h1:yBYzN7orOcqHVW7NcjYS/K90f4bOuXr4OkhsadygrnM=
go.bug.st/serial v1.6.4 h1:7FmqNPgVp3pu2Jz5PoPtbZ9jJO5gnEnZIvnI1lzve8A=
go.bug.st/serial v1.6.4/go.mod h1:nofMJxTeNVny/m6+KaafC6vJGj3miwQZ6vW4BZUGJPI=
go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
diff --git a/telegraf-patch/plugins/aggregators/all/kll.go b/telegraf-patch/plugins/aggregators/all/kll.go
new file mode 100644
index 00000000..3cd4c0c3
--- /dev/null
+++ b/telegraf-patch/plugins/aggregators/all/kll.go
@@ -0,0 +1,5 @@
+//go:build !custom || aggregators || aggregators.kll
+
+package all
+
+import _ "github.com/influxdata/telegraf/plugins/aggregators/kll" // register plugin
diff --git a/telegraf-patch/plugins/aggregators/kll/kll.go b/telegraf-patch/plugins/aggregators/kll/kll.go
new file mode 100755
index 00000000..43bf403f
--- /dev/null
+++ b/telegraf-patch/plugins/aggregators/kll/kll.go
@@ -0,0 +1,123 @@
+//go:generate ../../../tools/readme_config_includer/generator
+package kll
+
+import (
+ _ "embed"
+ "fmt"
+ "slices"
+ "github.com/influxdata/telegraf"
+ "github.com/influxdata/telegraf/plugins/aggregators"
+ "github.com/zzylol/go-kll"
+)
+
+type quantile struct {
+ seen []float64; // for debugging, save the seen values
+ sketch *kll.Sketch;
+}
+
+type metric struct {
+ name string;
+ fields map[string]*quantile;
+}
+
+type KLL struct {
+ K int `toml:"k"`;
+ Quantiles []float64 `toml:"quantiles"`;
+ WriteSeen bool `toml:"write_seen"`;
+
+ cache map[uint64]*metric; // state for each metric/field, key is metric.HashID()
+ suffixes map[float64]string; // suffix to attach to output quantiles, e.g.. _p50, _p99, ...
+}
+
+//go:embed sample.conf
+var sampleConfig string
+func (*KLL) SampleConfig() string { return sampleConfig; }
+
+func (k *KLL) Init() error {
+ if k.K < 2 { return fmt.Errorf("Invalid Argument. k must be >= 2 (k=%d)", k.K); }
+
+ k.cache = make(map[uint64]*metric);
+ k.suffixes = make(map[float64]string);
+ for _, q := range k.Quantiles {
+ if q < 0 || q > 1 { return fmt.Errorf("Invalid Argument. Quantiles must be in [0, 1] (q=%f)", q); }
+ k.suffixes[q] = fmt.Sprintf("_p%d", int(q * 100));
+ }
+
+ return nil;
+}
+
+// for each numeric field in each metric, update the backing KLL sketch
+func (k *KLL) Add(in telegraf.Metric) {
+ var id uint64 = in.HashID();
+
+ // get saved metric
+ m, ok := k.cache[id];
+ if !ok {
+ k.cache[id] = &metric{name: in.Name(), fields: make(map[string]*quantile)};
+ m = k.cache[id];
+ }
+
+ // for each field, get associated sketch
+ var fields []*telegraf.Field = in.FieldList();
+ for _, field := range fields {
+ var val float64;
+
+ // conversion referenced from minmax aggregator
+ switch field.Value.(type) {
+ case float64:
+ val = field.Value.(float64);
+ case int64:
+ val = float64(field.Value.(int64));
+ case uint64:
+ val = float64(field.Value.(uint64));
+ default:
+ continue;
+ }
+
+ // get sketch
+ sketch, ok := m.fields[field.Key];
+ if !ok {
+ m.fields[field.Key] = &quantile{seen: nil, sketch: kll.New(k.K)};
+ if k.WriteSeen { m.fields[field.Key].seen = make([]float64, 0); }
+
+ sketch = m.fields[field.Key];
+ }
+
+ if k.WriteSeen { sketch.seen = append(sketch.seen, val); }
+
+ sketch.sketch.Update(val);
+ }
+}
+
+func (k *KLL) Push(acc telegraf.Accumulator) {
+ for _, m := range k.cache {
+ out := make(map[string]any);
+
+ fields := m.fields;
+ for name, sketch := range fields {
+ // get the desired quantile
+ cdf := sketch.sketch.CDF();
+ for q, str := range k.suffixes { out[name + str] = cdf.Query(q); }
+
+ if k.WriteSeen {
+ slices.Sort(sketch.seen)
+ out[name + "_seen"] = fmt.Sprintf("%v", sketch.seen);
+ }
+ }
+
+ acc.AddSummary(m.name + "_KLL", out, nil);
+ }
+}
+
+func (k *KLL) Reset() {
+ for _, m := range k.cache {
+ for key := range m.fields {
+ if k.WriteSeen { clear(m.fields[key].seen); }
+ m.fields[key].sketch = kll.New(k.K);
+ }
+ }
+}
+
+func init() {
+ aggregators.Add("kll", func() telegraf.Aggregator { return &KLL{}; });
+}
diff --git a/telegraf-patch/plugins/aggregators/kll/sample.conf b/telegraf-patch/plugins/aggregators/kll/sample.conf
new file mode 100755
index 00000000..524117ea
--- /dev/null
+++ b/telegraf-patch/plugins/aggregators/kll/sample.conf
@@ -0,0 +1,13 @@
+# Outputs quantiles for each numeric field for each metric
+# Aggregated metric has name _KLL
+[[aggregators.kll]]
+ ## Size of topmost compactor
+ ## Total memory used by sketch is 3*k + lg(n)
+ k = 256
+
+ ## Quantiles to retrieve. Must be in range [0, 1] and float (e.g. 0 must be 0.0)
+ ## Values will be output as field_p, e.g. cpu_p50
+ quantiles = [0.5, 0.99]
+
+ ## Debugging option to also output the sorted ingested values (with suffix _seen)
+ write_seen = false