Fix Thanos crash on XOR2-encoded chunks (SRE-791)

pkg/compact/downsample/downsample.go

@@ -941,7 +941,7 @@ func downsampleAggrLoop(
 
 func expandChunkIterator(it chunkenc.Iterator, encoding chunkenc.Encoding, samples *[]sample) error {
 	switch encoding {
-	case chunkenc.EncXOR:
+	case chunkenc.EncXOR, chunkenc.EncXOR2:
 		return expandXorChunkIterator(it, samples)
 	case chunkenc.EncFloatHistogram:
 		return expandFloatHistogramChunkIterator(it, samples)

pkg/compact/downsample/downsample_test.go

@@ -1211,6 +1211,22 @@ func TestExpandChunkIterator(t *testing.T) {
 	testutil.Equals(t, []sample{{t: 100, v: 1}, {t: 200, v: 2}, {t: 200, v: 3}, {t: 201, v: 4}, {t: 300, v: 6}, {t: 500, v: 5}}, res)
 }
 
+func TestExpandChunkIteratorXOR2(t *testing.T) {
+	// XOR2 (chunkenc.EncXOR2) is the Prometheus 3.11+ float chunk encoding used to
+	// persist real start timestamps; it must expand identically to plain XOR.
+	var res []sample
+	testutil.Ok(t,
+		expandChunkIterator(
+			newSampleIterator([]sample{
+				{t: 100, v: 1}, {t: 200, v: 2}, {t: 200, v: 3}, {t: 201, v: 4}, {t: 200, v: 5},
+				{t: 300, v: 6}, {t: 400, v: math.Float64frombits(value.StaleNaN)}, {t: 500, v: 5},
+			}), chunkenc.EncXOR2, &res,
+		),
+	)
+
+	testutil.Equals(t, []sample{{t: 100, v: 1}, {t: 200, v: 2}, {t: 200, v: 3}, {t: 201, v: 4}, {t: 300, v: 6}, {t: 500, v: 5}}, res)
+}
+
 var (
 	// Decoded excerpt of pkg/query/testdata/issue2401-seriesresponses.json without overlaps (downsampling works directly on blocks).
 	realisticChkDataWithStaleMarker = [][]sample{
@@ -2881,4 +2897,7 @@ func TestDownsampleNHCutNewChunk(t *testing.T) {
 	require.False(t, cutNewChunk(chunkenc.EncFloatHistogram, chunkenc.EncFloatHistogram))
 	require.True(t, cutNewChunk(chunkenc.EncXOR, chunkenc.EncFloatHistogram))
 	require.True(t, cutNewChunk(chunkenc.EncXOR, chunkenc.EncHistogram))
+	require.True(t, cutNewChunk(chunkenc.EncXOR2, chunkenc.EncHistogram))
+	require.False(t, cutNewChunk(chunkenc.EncXOR2, chunkenc.EncXOR))
+	require.False(t, cutNewChunk(chunkenc.EncXOR, chunkenc.EncXOR2))
 }

pkg/query/iter.go

@@ -188,6 +188,8 @@ func chunkEncoding(e storepb.Chunk_Encoding) chunkenc.Encoding {
 	switch e {
 	case storepb.Chunk_XOR:
 		return chunkenc.EncXOR
+	case storepb.Chunk_XOR2:
+		return chunkenc.EncXOR2
 	case storepb.Chunk_HISTOGRAM:
 		return chunkenc.EncHistogram
 	case storepb.Chunk_FLOAT_HISTOGRAM:

pkg/query/iter_test.go

@@ -0,0 +1,20 @@
+// Copyright (c) The Thanos Authors.
+// Licensed under the Apache License 2.0.
+
+package query
+
+import (
+	"testing"
+
+	"github.com/efficientgo/core/testutil"
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+
+	"github.com/thanos-io/thanos/pkg/store/storepb"
+)
+
+func TestChunkEncoding_XOR2(t *testing.T) {
+	// XOR2 (chunkenc.EncXOR2) is the Prometheus 3.11+ float chunk encoding used to
+	// persist real start timestamps. The querier must be able to decode it back from
+	// the storepb wire encoding after a store-gateway serves it.
+	testutil.Equals(t, chunkenc.EncXOR2, chunkEncoding(storepb.Chunk_XOR2))
+}

pkg/store/bucket.go

@@ -1491,6 +1491,8 @@ func chunkToStoreEncoding(in chunkenc.Encoding) storepb.Chunk_Encoding {
 	switch in {
 	case chunkenc.EncXOR:
 		return storepb.Chunk_XOR
+	case chunkenc.EncXOR2:
+		return storepb.Chunk_XOR2
 	case chunkenc.EncHistogram:
 		return storepb.Chunk_HISTOGRAM
 	case chunkenc.EncFloatHistogram:

pkg/store/bucket_test.go

@@ -77,6 +77,12 @@ func TestRawChunkReset(t *testing.T) {
 	testutil.Equals(t, []byte(r), []byte{3, 4})
 }
 
+func TestChunkToStoreEncoding_XOR2(t *testing.T) {
+	// XOR2 (chunkenc.EncXOR2) is the Prometheus 3.11+ float chunk encoding used to
+	// persist real start timestamps. Serving it via the store-gateway must not panic.
+	testutil.Equals(t, storepb.Chunk_XOR2, chunkToStoreEncoding(chunkenc.EncXOR2))
+}
+
 func TestBucketBlock_Property(t *testing.T) {
 	t.Parallel()
 

pkg/store/storepb/custom.go

@@ -501,9 +501,9 @@ func LabelsToPromLabelsUnsafe(lset []Label) labels.Labels {
 	return labelpb.ZLabelsToPromLabels(lset)
 }
 
-// XORNumSamples return number of samples. Returns 0 if it's not XOR chunk.
+// XORNumSamples return number of samples. Returns 0 if it's not an XOR or XOR2 chunk.
 func (m *Chunk) XORNumSamples() int {
-	if m.Type == Chunk_XOR {
+	if m.Type == Chunk_XOR || m.Type == Chunk_XOR2 {
 		return int(binary.BigEndian.Uint16(m.Data))
 	}
 	return 0

pkg/store/storepb/custom_test.go

@@ -4,6 +4,7 @@
 package storepb
 
 import (
+	"encoding/binary"
 	"fmt"
 	"path/filepath"
 	"sort"
@@ -25,6 +26,17 @@ type sample struct {
 	v float64
 }
 
+func TestChunk_XORNumSamples_XOR2(t *testing.T) {
+	// XOR2 (Chunk_XOR2) is the Prometheus 3.11+ float chunk encoding used to persist
+	// real start timestamps; it shares XOR's wire header (sample count as first 2 bytes)
+	// and must be counted the same way for series stats.
+	data := make([]byte, 2)
+	binary.BigEndian.PutUint16(data, 7)
+	c := &Chunk{Type: Chunk_XOR2, Data: data}
+
+	testutil.Equals(t, 7, c.XORNumSamples())
+}
+
 type listSeriesSet struct {
 	series []Series
 	idx    int

pkg/store/storepb/types.proto

@@ -25,6 +25,7 @@ message Chunk {
     XOR = 0;
     HISTOGRAM = 1;
     FLOAT_HISTOGRAM = 2;
+    XOR2 = 3;
   }
   Encoding type  = 1;
   bytes data     = 2;