diff --git a/.gitignore b/.gitignore index a414f134..3f7b212b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,10 +29,14 @@ go.work.sum # Editor/IDE # .idea/ -# .vscode/ +.vscode/ benchmarks/results/* benchmarks/__pycache__/* # Local checkouts opentelemetry-go + +# Telegraf artifacts +config.toml +out.json \ No newline at end of file diff --git a/backup_telegraf_patches.sh b/backup_telegraf_patches.sh index 5f619d05..6831e7d8 100755 --- a/backup_telegraf_patches.sh +++ b/backup_telegraf_patches.sh @@ -3,7 +3,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SRC_TELEGRAF="${ROOT_DIR}/telegraf" -DEST_DIR="${ROOT_DIR}/telegraf-plugins" +DEST_DIR="${ROOT_DIR}/telegraf-patch" copy_path() { local src="$1" diff --git a/benchmarks/max-throughput-prometheus-input.conf b/benchmarks/max-throughput-prometheus-input.conf new file mode 100644 index 00000000..0a65844e --- /dev/null +++ b/benchmarks/max-throughput-prometheus-input.conf @@ -0,0 +1,50 @@ +# Read data directly from FakePrometheusExporter +# based off max-throughput-prometheus-client.conf + +[global_tags] + scenario = "prometheus_inp" + +[agent] + interval = "1ms" + flush_interval = "1ms" + flush_jitter = "0s" + collection_jitter = "0s" + round_interval = false + metric_batch_size = 10000 + metric_buffer_limit = 500000 + precision = "1ms" + debug = true + logfile = "/tmp/telegraf-prometheus-inp.log" + +[[inputs.prometheus]] + urls = [ + "http://localhost:8000/metrics", + "http://localhost:8001/metrics", + ] + metric_version = 2 + url_tag = "scrape_url" + name_prefix = "gorilla_local_" + timeout = "5s" + interval = "250ms" + metric_batch_size = 10000 + metric_buffer_limit = 250000 + +[[inputs.internal]] + per_instance = true + collect_gostats = true + +[[outputs.file]] + files = ["../benchmarks/results/prometheus-inp.lp"] + data_format = "influx" + use_batch_format = false + rotation_interval = "1m" + rotation_max_size = "50MB" + rotation_max_archives = 15 + namepass = ["internal_*"] + +[[outputs.prometheus_client]] + listen = ":9203" + path = "/metrics" + metric_version = 2 + export_timestamp = true + expiration_interval = "30s" diff --git a/benchmarks/tests/kll.conf b/benchmarks/tests/kll.conf new file mode 100644 index 00000000..66418f4d --- /dev/null +++ b/benchmarks/tests/kll.conf @@ -0,0 +1,4 @@ +[[aggregators.kll]] + k = 256 + quantiles = [0.5, 0.99] + write_seen = false \ No newline at end of file diff --git a/restore_telegraf_patches.sh b/restore_telegraf_patches.sh index 68d5ff77..b4f4b20f 100755 --- a/restore_telegraf_patches.sh +++ b/restore_telegraf_patches.sh @@ -2,7 +2,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SRC_ROOT="${ROOT_DIR}/telegraf-plugins/plugins" +SRC_ROOT="${ROOT_DIR}/telegraf-patch/plugins" DEST_ROOT="${ROOT_DIR}/telegraf/plugins" if [[ ! -d "${SRC_ROOT}" ]]; then @@ -16,10 +16,9 @@ copied=0 for path in "${SRC_ROOT}"/*; do rel="$(basename "${path}")" dest="${DEST_ROOT}/${rel}" - rm -rf "${dest}" - cp -R "${path}" "${dest}" + cp -R -T "${path}" "${dest}" echo "Restored ${rel} into ${dest}" copied=$((copied + 1)) done -echo "Restored ${copied} items from telegraf-plugins/plugins into telegraf/plugins/" +echo "Restored ${copied} items from telegraf-patch/plugins into telegraf/plugins/" diff --git a/telegraf-patch/go.mod b/telegraf-patch/go.mod index 9754c06f..87c0dcdc 100644 --- a/telegraf-patch/go.mod +++ b/telegraf-patch/go.mod @@ -25,7 +25,6 @@ require ( github.com/BurntSushi/toml v1.5.0 github.com/ClickHouse/clickhouse-go/v2 v2.40.3 github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/DataDog/sketches-go v1.4.1 github.com/IBM/nzgo/v12 v12.0.10 github.com/IBM/sarama v1.46.3 github.com/Masterminds/semver/v3 v3.4.0 @@ -51,7 +50,6 @@ require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/awnumar/memguard v0.23.0 github.com/aws/aws-msk-iam-sasl-signer-go v1.0.4 - github.com/aws/aws-sdk-go v1.44.263 github.com/aws/aws-sdk-go-v2 v1.39.5 github.com/aws/aws-sdk-go-v2/config v1.31.16 github.com/aws/aws-sdk-go-v2/credentials v1.18.20 @@ -545,6 +543,7 @@ require ( github.com/zitadel/logging v0.6.2 // indirect github.com/zitadel/oidc/v3 v3.45.0 // indirect github.com/zitadel/schema v1.3.1 // indirect + github.com/zzylol/go-kll v0.0.1 go.bug.st/serial v1.6.4 // indirect go.etcd.io/etcd/api/v3 v3.5.4 // indirect go.opencensus.io v0.24.0 // indirect diff --git a/telegraf-patch/go.sum b/telegraf-patch/go.sum index c78cbdb5..be6ba31d 100644 --- a/telegraf-patch/go.sum +++ b/telegraf-patch/go.sum @@ -738,8 +738,6 @@ github.com/ClickHouse/clickhouse-go/v2 v2.40.3/go.mod h1:qO0HwvjCnTB4BPL/k6EE3l4 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/sketches-go v1.4.1 h1:j5G6as+9FASM2qC36lvpvQAj9qsv/jUs3FtO8CwZNAY= -github.com/DataDog/sketches-go v1.4.1/go.mod h1:xJIXldczJyyjnbDop7ZZcLxJdV3+7Kra7H1KMgpgkLk= github.com/Files-com/files-sdk-go/v3 v3.2.97 h1:c+mQoiES/21JrHDAxJLCYICJO+bu8Clv0ZDNZe7Ndyk= github.com/Files-com/files-sdk-go/v3 v3.2.97/go.mod h1:Y/bCHoPJNPKz2hw1ADXjQXJP378HODwK+g/5SR2gqfU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= @@ -892,7 +890,6 @@ github.com/aws/aws-msk-iam-sasl-signer-go v1.0.4 h1:2jAwFwA0Xgcx94dUId+K24yFabsK github.com/aws/aws-msk-iam-sasl-signer-go v1.0.4/go.mod h1:MVYeeOhILFFemC/XlYTClvBjYZrg/EPd3ts885KrNTI= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.29.11/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= -github.com/aws/aws-sdk-go v1.44.263 h1:Dkt5fcdtL8QtK3cz0bOTQ84m9dGx+YDeTsDl+wY2yW4= github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.39.5 h1:e/SXuia3rkFtapghJROrydtQpfQaaUgd1cUvyO1mp2w= @@ -2519,6 +2516,8 @@ github.com/zitadel/oidc/v3 v3.45.0 h1:SaVJ2kdcJi/zdEWWlAns+81VxmfdYX4E+2mWFVIH7E github.com/zitadel/oidc/v3 v3.45.0/go.mod h1:UeK0iVOoqfMuDVgSfv56BqTz8YQC2M+tGRIXZ7Ii3VY= github.com/zitadel/schema v1.3.1 h1:QT3kwiRIRXXLVAs6gCK/u044WmUVh6IlbLXUsn6yRQU= github.com/zitadel/schema v1.3.1/go.mod h1:071u7D2LQacy1HAN+YnMd/mx1qVE2isb0Mjeqg46xnU= +github.com/zzylol/go-kll v0.0.1 h1:54O2mqW41PG7OgkgZL88x+YiqBE4g5FtW4iVEdENgXM= +github.com/zzylol/go-kll v0.0.1/go.mod h1:yBYzN7orOcqHVW7NcjYS/K90f4bOuXr4OkhsadygrnM= go.bug.st/serial v1.6.4 h1:7FmqNPgVp3pu2Jz5PoPtbZ9jJO5gnEnZIvnI1lzve8A= go.bug.st/serial v1.6.4/go.mod h1:nofMJxTeNVny/m6+KaafC6vJGj3miwQZ6vW4BZUGJPI= go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps= diff --git a/telegraf-patch/plugins/aggregators/all/kll.go b/telegraf-patch/plugins/aggregators/all/kll.go new file mode 100644 index 00000000..3cd4c0c3 --- /dev/null +++ b/telegraf-patch/plugins/aggregators/all/kll.go @@ -0,0 +1,5 @@ +//go:build !custom || aggregators || aggregators.kll + +package all + +import _ "github.com/influxdata/telegraf/plugins/aggregators/kll" // register plugin diff --git a/telegraf-patch/plugins/aggregators/kll/kll.go b/telegraf-patch/plugins/aggregators/kll/kll.go new file mode 100755 index 00000000..43bf403f --- /dev/null +++ b/telegraf-patch/plugins/aggregators/kll/kll.go @@ -0,0 +1,123 @@ +//go:generate ../../../tools/readme_config_includer/generator +package kll + +import ( + _ "embed" + "fmt" + "slices" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/aggregators" + "github.com/zzylol/go-kll" +) + +type quantile struct { + seen []float64; // for debugging, save the seen values + sketch *kll.Sketch; +} + +type metric struct { + name string; + fields map[string]*quantile; +} + +type KLL struct { + K int `toml:"k"`; + Quantiles []float64 `toml:"quantiles"`; + WriteSeen bool `toml:"write_seen"`; + + cache map[uint64]*metric; // state for each metric/field, key is metric.HashID() + suffixes map[float64]string; // suffix to attach to output quantiles, e.g.. _p50, _p99, ... +} + +//go:embed sample.conf +var sampleConfig string +func (*KLL) SampleConfig() string { return sampleConfig; } + +func (k *KLL) Init() error { + if k.K < 2 { return fmt.Errorf("Invalid Argument. k must be >= 2 (k=%d)", k.K); } + + k.cache = make(map[uint64]*metric); + k.suffixes = make(map[float64]string); + for _, q := range k.Quantiles { + if q < 0 || q > 1 { return fmt.Errorf("Invalid Argument. Quantiles must be in [0, 1] (q=%f)", q); } + k.suffixes[q] = fmt.Sprintf("_p%d", int(q * 100)); + } + + return nil; +} + +// for each numeric field in each metric, update the backing KLL sketch +func (k *KLL) Add(in telegraf.Metric) { + var id uint64 = in.HashID(); + + // get saved metric + m, ok := k.cache[id]; + if !ok { + k.cache[id] = &metric{name: in.Name(), fields: make(map[string]*quantile)}; + m = k.cache[id]; + } + + // for each field, get associated sketch + var fields []*telegraf.Field = in.FieldList(); + for _, field := range fields { + var val float64; + + // conversion referenced from minmax aggregator + switch field.Value.(type) { + case float64: + val = field.Value.(float64); + case int64: + val = float64(field.Value.(int64)); + case uint64: + val = float64(field.Value.(uint64)); + default: + continue; + } + + // get sketch + sketch, ok := m.fields[field.Key]; + if !ok { + m.fields[field.Key] = &quantile{seen: nil, sketch: kll.New(k.K)}; + if k.WriteSeen { m.fields[field.Key].seen = make([]float64, 0); } + + sketch = m.fields[field.Key]; + } + + if k.WriteSeen { sketch.seen = append(sketch.seen, val); } + + sketch.sketch.Update(val); + } +} + +func (k *KLL) Push(acc telegraf.Accumulator) { + for _, m := range k.cache { + out := make(map[string]any); + + fields := m.fields; + for name, sketch := range fields { + // get the desired quantile + cdf := sketch.sketch.CDF(); + for q, str := range k.suffixes { out[name + str] = cdf.Query(q); } + + if k.WriteSeen { + slices.Sort(sketch.seen) + out[name + "_seen"] = fmt.Sprintf("%v", sketch.seen); + } + } + + acc.AddSummary(m.name + "_KLL", out, nil); + } +} + +func (k *KLL) Reset() { + for _, m := range k.cache { + for key := range m.fields { + if k.WriteSeen { clear(m.fields[key].seen); } + m.fields[key].sketch = kll.New(k.K); + } + } +} + +func init() { + aggregators.Add("kll", func() telegraf.Aggregator { return &KLL{}; }); +} diff --git a/telegraf-patch/plugins/aggregators/kll/sample.conf b/telegraf-patch/plugins/aggregators/kll/sample.conf new file mode 100755 index 00000000..524117ea --- /dev/null +++ b/telegraf-patch/plugins/aggregators/kll/sample.conf @@ -0,0 +1,13 @@ +# Outputs quantiles for each numeric field for each metric +# Aggregated metric has name _KLL +[[aggregators.kll]] + ## Size of topmost compactor + ## Total memory used by sketch is 3*k + lg(n) + k = 256 + + ## Quantiles to retrieve. Must be in range [0, 1] and float (e.g. 0 must be 0.0) + ## Values will be output as field_p, e.g. cpu_p50 + quantiles = [0.5, 0.99] + + ## Debugging option to also output the sorted ingested values (with suffix _seen) + write_seen = false