From c90feb4798a58de22c7ef832aafe8c875be53074 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 2 Sep 2026 08:48:19 -0700 Subject: [PATCH 01/11] feat(blob): support reading map blob values --- .../format/blob/blob_file_batch_reader.cpp | 324 +++++++++++++++++- .../format/blob/blob_file_batch_reader.h | 1 + .../blob/blob_file_batch_reader_test.cpp | 115 +++++++ 3 files changed, 435 insertions(+), 5 deletions(-) diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index f5e3ef9c..a12c9e04 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -19,14 +19,20 @@ #include "paimon/format/blob/blob_file_batch_reader.h" #include +#include #include #include +#include +#include #include "arrow/api.h" #include "arrow/array/builder_dict.h" #include "arrow/array/builder_nested.h" #include "arrow/c/bridge.h" #include "arrow/util/bit_util.h" +#include "arrow/util/decimal.h" +#include "arrow/util/endian.h" +#include "arrow/util/ubsan.h" #include "fmt/format.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/io/offset_input_stream.h" @@ -39,6 +45,110 @@ #include "paimon/data/blob.h" namespace paimon::blob { +namespace { + +constexpr int32_t kMapBlobMagicNumber = 0x4D424342; +constexpr int8_t kMapBlobVersion = 1; +constexpr int32_t kMapBlobHeaderLength = 9; +constexpr int32_t kMapBlobIndexLengthsSize = 8; +constexpr int32_t kMapBlobMinPayloadLength = kMapBlobHeaderLength + kMapBlobIndexLengthsSize; + +template +T ReadLittleEndian(const uint8_t* data) { + return arrow::bit_util::FromLittleEndian(arrow::util::SafeLoadAs(data)); +} + +bool IsMapBlobField(const std::shared_ptr& field) { + if (field->type()->id() != arrow::Type::MAP) { + return false; + } + const auto& map_type = static_cast(*field->type()); + // Nested field metadata is not retained by Arrow's C schema bridge for MapType. Paimon's + // regular binary types use BINARY, so LARGE_BINARY here uniquely identifies a BLOB value. + return map_type.item_type()->id() == arrow::Type::LARGE_BINARY; +} + +Result GetMapBlobFixedKeyLength(const std::shared_ptr& key_type) { + switch (key_type->id()) { + case arrow::Type::BOOL: + case arrow::Type::INT8: + return 1; + case arrow::Type::INT16: + return 2; + case arrow::Type::INT32: + case arrow::Type::DATE32: + case arrow::Type::TIME32: + return 4; + case arrow::Type::INT64: + return 8; + case arrow::Type::DECIMAL128: { + const auto& decimal_type = static_cast(*key_type); + return decimal_type.precision() <= 18 ? 8 : -1; + } + case arrow::Type::STRING: + case arrow::Type::BINARY: + return -1; + default: + return Status::Invalid( + fmt::format("unsupported MAP<..., BLOB> key type: {}", key_type->ToString())); + } +} + +Status AppendMapBlobKey(const std::shared_ptr& key_type, const uint8_t* data, + int32_t length, arrow::ArrayBuilder* builder) { + switch (key_type->id()) { + case arrow::Type::BOOL: { + if (data[0] != 0 && data[0] != 1) { + return Status::Invalid("invalid MAP<..., BLOB> boolean key"); + } + return ToPaimonStatus( + checked_cast(builder)->Append(data[0] == 1)); + } + case arrow::Type::INT8: + return ToPaimonStatus( + checked_cast(builder)->Append(static_cast(data[0]))); + case arrow::Type::INT16: + return ToPaimonStatus(checked_cast(builder)->Append( + ReadLittleEndian(data))); + case arrow::Type::INT32: + return ToPaimonStatus(checked_cast(builder)->Append( + ReadLittleEndian(data))); + case arrow::Type::INT64: + return ToPaimonStatus(checked_cast(builder)->Append( + ReadLittleEndian(data))); + case arrow::Type::DATE32: + return ToPaimonStatus(checked_cast(builder)->Append( + ReadLittleEndian(data))); + case arrow::Type::TIME32: + return ToPaimonStatus(checked_cast(builder)->Append( + ReadLittleEndian(data))); + case arrow::Type::STRING: + return ToPaimonStatus( + checked_cast(builder)->Append(data, length)); + case arrow::Type::BINARY: + return ToPaimonStatus( + checked_cast(builder)->Append(data, length)); + case arrow::Type::DECIMAL128: { + const auto& decimal_type = static_cast(*key_type); + arrow::Decimal128 value; + if (decimal_type.precision() <= 18) { + value = arrow::Decimal128(ReadLittleEndian(data)); + } else { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(value, + arrow::Decimal128::FromBigEndian(data, length)); + } + if (!value.FitsInPrecision(decimal_type.precision())) { + return Status::Invalid("MAP<..., BLOB> decimal key exceeds declared precision"); + } + return ToPaimonStatus(checked_cast(builder)->Append(value)); + } + default: + return Status::Invalid( + fmt::format("unsupported MAP<..., BLOB> key type: {}", key_type->ToString())); + } +} + +} // namespace Result> BlobFileBatchReader::Create( const std::shared_ptr& input_stream, int32_t batch_size, bool blob_as_descriptor, @@ -133,9 +243,14 @@ Status BlobFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, return Status::Invalid( fmt::format("read schema field number {} is not 1", arrow_schema->num_fields())); } - if (!BlobUtils::IsBlobField(arrow_schema->field(0))) { - return Status::Invalid( - fmt::format("field {} is not BLOB", arrow_schema->field(0)->ToString())); + std::shared_ptr read_field = arrow_schema->field(0); + if (!BlobUtils::IsBlobField(read_field) && !IsMapBlobField(read_field)) { + return Status::Invalid(fmt::format("field {} is not BLOB", read_field->ToString())); + } + if (IsMapBlobField(read_field)) { + const auto& map_type = static_cast(*read_field->type()); + PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] int32_t fixed_key_length, + GetMapBlobFixedKeyLength(map_type.key_type())); } if (selection_bitmap != std::nullopt) { int32_t cardinality = selection_bitmap->Cardinality(); @@ -160,7 +275,7 @@ Status BlobFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, target_blob_offsets_ = new_offsets; target_blob_row_indexes_ = new_row_indexes; } - target_type_ = arrow::struct_(arrow_schema->fields()); + target_type_ = arrow::struct_({read_field}); current_pos_ = 0; previous_batch_start_pos_ = std::numeric_limits::max(); previous_batch_row_count_ = 0; @@ -252,9 +367,208 @@ Result> BlobFileBatchReader::BuildContentArray( return std::make_shared(struct_array_data); } +Result> BlobFileBatchReader::BuildMapBlobArray( + int32_t rows_to_read) const { + const auto& struct_type = static_cast(*target_type_); + const std::shared_ptr& map_field = struct_type.field(0); + auto map_type = checked_pointer_cast(map_field->type()); + const std::shared_ptr& key_type = map_type->key_type(); + PAIMON_ASSIGN_OR_RAISE(int32_t fixed_key_length, GetMapBlobFixedKeyLength(key_type)); + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr key_builder_unique, + arrow::MakeBuilder(key_type, arrow_pool_.get())); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr item_builder_unique, + arrow::MakeBuilder(map_type->item_type(), arrow_pool_.get())); + std::shared_ptr key_builder(std::move(key_builder_unique)); + std::shared_ptr item_builder(std::move(item_builder_unique)); + if (!item_builder || !item_builder->type() || + item_builder->type()->id() != arrow::Type::LARGE_BINARY) { + return Status::Invalid("cast MAP<..., BLOB> item builder to large binary builder failed"); + } + auto* blob_builder = checked_cast(item_builder.get()); + arrow::MapBuilder map_builder(arrow_pool_.get(), key_builder, item_builder, map_type); + + for (int32_t k = 0; k < rows_to_read; ++k) { + const size_t row_index = current_pos_ + k; + if (IsTargetNull(row_index)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.AppendNull()); + continue; + } + if (target_blob_lengths_[row_index] < 0) { + return Status::Invalid(fmt::format("unsupported MAP<..., BLOB> record length: {}", + target_blob_lengths_[row_index])); + } + + const int64_t payload_offset = GetTargetContentOffset(row_index); + const int64_t payload_length = GetTargetContentLength(row_index); + if (payload_length < kMapBlobMinPayloadLength) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> payload length: {}", payload_length)); + } + + std::array header; + PAIMON_RETURN_NOT_OK(ReadBlobContentAt(payload_offset, header.size(), header.data())); + const auto magic_number = ReadLittleEndian(header.data()); + if (magic_number != kMapBlobMagicNumber) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> payload magic number: {}", magic_number)); + } + const auto version = static_cast(header[4]); + if (version != kMapBlobVersion) { + return Status::NotImplemented( + fmt::format("unsupported MAP<..., BLOB> payload version: {}", version)); + } + const auto entry_count = ReadLittleEndian(header.data() + 5); + if (entry_count < 0) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> entry count: {}", entry_count)); + } + + const int64_t index_lengths_offset = + payload_offset + payload_length - kMapBlobIndexLengthsSize; + std::array index_lengths; + PAIMON_RETURN_NOT_OK( + ReadBlobContentAt(index_lengths_offset, index_lengths.size(), index_lengths.data())); + const auto key_index_length = ReadLittleEndian(index_lengths.data()); + const auto value_index_length = + ReadLittleEndian(index_lengths.data() + sizeof(int32_t)); + const int64_t maximum_indexes_length = payload_length - kMapBlobMinPayloadLength; + if (key_index_length < 0 || key_index_length > maximum_indexes_length) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> key index length: {}", key_index_length)); + } + if (value_index_length < 0 || value_index_length > maximum_indexes_length) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> value index length: {}", value_index_length)); + } + if (static_cast(key_index_length) + value_index_length > maximum_indexes_length) { + return Status::Invalid("MAP<..., BLOB> indexes exceed the payload length"); + } + if (entry_count > key_index_length || entry_count > value_index_length) { + return Status::Invalid("MAP<..., BLOB> entry count exceeds index length"); + } + + const int64_t value_index_offset = index_lengths_offset - value_index_length; + const int64_t key_index_offset = value_index_offset - key_index_length; + std::vector key_index_bytes(key_index_length); + std::vector value_index_bytes(value_index_length); + PAIMON_RETURN_NOT_OK(ReadBlobContentAt(key_index_offset, key_index_length, + reinterpret_cast(key_index_bytes.data()))); + PAIMON_RETURN_NOT_OK( + ReadBlobContentAt(value_index_offset, value_index_length, + reinterpret_cast(value_index_bytes.data()))); + PAIMON_ASSIGN_OR_RAISE(std::vector key_lengths, + DeltaVarintCompressor::Decompress(key_index_bytes)); + PAIMON_ASSIGN_OR_RAISE(std::vector value_lengths, + DeltaVarintCompressor::Decompress(value_index_bytes)); + if (key_lengths.size() != static_cast(entry_count)) { + return Status::Invalid("MAP<..., BLOB> entry count does not match key index length"); + } + if (value_lengths.size() != static_cast(entry_count)) { + return Status::Invalid("MAP<..., BLOB> entry count does not match value index length"); + } + + const int64_t data_offset = payload_offset + kMapBlobHeaderLength; + const int64_t data_length = key_index_offset - data_offset; + int64_t key_data_length = 0; + for (int64_t key_length : key_lengths) { + if (key_length < 0) { + return Status::Invalid("MAP<..., BLOB> keys cannot be null"); + } + if (key_length > std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("MAP<..., BLOB> key is too large: {}", key_length)); + } + if (fixed_key_length >= 0 && key_length != fixed_key_length) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> fixed-width key length: {}", key_length)); + } + if (key_length > data_length - key_data_length) { + return Status::Invalid("MAP<..., BLOB> key lengths exceed the payload data length"); + } + key_data_length += key_length; + } + + const int64_t maximum_value_data_length = data_length - key_data_length; + int64_t value_data_length = 0; + for (int64_t value_length : value_lengths) { + if (value_length == BlobDefs::kNullBinLength) { + continue; + } + if (value_length < 0) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> value length: {}", value_length)); + } + if (!blob_as_descriptor_ && value_length > std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("MAP<..., BLOB> inline value is too large: {}", value_length)); + } + if (value_length > maximum_value_data_length - value_data_length) { + return Status::Invalid( + "MAP<..., BLOB> value lengths exceed the payload data length"); + } + value_data_length += value_length; + } + if (value_data_length != maximum_value_data_length) { + return Status::Invalid( + "MAP<..., BLOB> key/value lengths do not match the payload data length"); + } + + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.Append()); + int64_t key_offset = data_offset; + std::set serialized_keys; + for (int32_t entry = 0; entry < entry_count; ++entry) { + const auto key_length = static_cast(key_lengths[entry]); + std::vector key_bytes(key_length); + PAIMON_RETURN_NOT_OK(ReadBlobContentAt(key_offset, key_length, key_bytes.data())); + if (!serialized_keys.emplace(key_bytes.begin(), key_bytes.end()).second) { + return Status::Invalid("invalid MAP<..., BLOB> payload: duplicate key"); + } + PAIMON_RETURN_NOT_OK( + AppendMapBlobKey(key_type, key_bytes.data(), key_length, key_builder.get())); + key_offset += key_length; + } + + int64_t value_offset = data_offset + key_data_length; + for (int64_t value_length : value_lengths) { + if (value_length == BlobDefs::kNullBinLength) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->AppendNull()); + continue; + } + if (blob_as_descriptor_) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr blob, + Blob::FromPath(file_path_, value_offset, value_length)); + PAIMON_UNIQUE_PTR descriptor = blob->ToDescriptor(pool_); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + blob_builder->Append(descriptor->data(), descriptor->size())); + } else { + std::vector value_bytes(static_cast(value_length)); + PAIMON_RETURN_NOT_OK( + ReadBlobContentAt(value_offset, value_length, value_bytes.data())); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + blob_builder->Append(value_bytes.data(), value_length)); + } + value_offset += value_length; + } + } + + std::shared_ptr built_map_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.Finish(&built_map_array)); + auto map_array = std::make_shared( + map_type, built_map_array->length(), built_map_array->value_offsets(), + built_map_array->keys(), built_map_array->items(), built_map_array->null_bitmap(), + built_map_array->null_count(), built_map_array->offset()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr struct_array, + arrow::StructArray::Make({map_array}, {map_field})); + return struct_array; +} + Result> BlobFileBatchReader::BuildTargetArray( int32_t rows_to_read) const { - std::shared_ptr blob_array; + const auto& struct_type = static_cast(*target_type_); + if (struct_type.field(0)->type()->id() == arrow::Type::MAP) { + return BuildMapBlobArray(rows_to_read); + } if (!blob_as_descriptor_) { return BuildContentArray(rows_to_read); } diff --git a/src/paimon/format/blob/blob_file_batch_reader.h b/src/paimon/format/blob/blob_file_batch_reader.h index 9a05c15b..23cb24bb 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.h +++ b/src/paimon/format/blob/blob_file_batch_reader.h @@ -168,6 +168,7 @@ class BlobFileBatchReader : public FileBatchReader { /// Builds a null bitmap buffer for the given rows. Returns nullptr if no nulls. Result> BuildNullBitmap(int32_t rows_to_read) const; Result> BuildContentArray(int32_t rows_to_read) const; + Result> BuildMapBlobArray(int32_t rows_to_read) const; Result> BuildTargetArray(int32_t rows_to_read) const; /// Returns true if the blob at the given index is null (bin_length == kNullBinLength). diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 17482af7..1377b8ad 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -18,6 +18,8 @@ #include "paimon/format/blob/blob_file_batch_reader.h" +#include + #include "arrow/api.h" #include "arrow/c/helpers.h" #include "gtest/gtest.h" @@ -32,6 +34,21 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::blob::test { +namespace { + +std::string HexToBytes(std::string_view hex) { + auto hex_value = [](char c) -> uint8_t { + return c <= '9' ? static_cast(c - '0') : static_cast(c - 'a' + 10); + }; + std::string bytes; + bytes.reserve(hex.size() / 2); + for (size_t i = 0; i < hex.size(); i += 2) { + bytes.push_back(static_cast((hex_value(hex[i]) << 4) | hex_value(hex[i + 1]))); + } + return bytes; +} + +} // namespace TEST(BlobReaderBuilderTest, RejectsNullMemoryPool) { BlobReaderBuilder builder(/*batch_size=*/10, /*options=*/{}); @@ -108,6 +125,28 @@ class BlobFileBatchReaderTest : public testing::Test, public ::testing::WithPara } } + Result ReadMapBlobValue(const std::shared_ptr& blob_array, + int64_t index, bool blob_as_descriptor, + const std::shared_ptr& file_system) { + std::string stored_value = blob_array->GetString(index); + if (!blob_as_descriptor) { + return stored_value; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr blob, + Blob::FromDescriptor(stored_value.data(), stored_value.size())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr input_stream, + blob->NewInputStream(file_system)); + PAIMON_ASSIGN_OR_RAISE(int64_t length, input_stream->Length()); + std::string value(length, '\0'); + if (length > 0) { + PAIMON_ASSIGN_OR_RAISE(int64_t actual_length, input_stream->Read(value.data(), length)); + if (actual_length != length) { + return Status::IOError("failed to read MAP<..., BLOB> descriptor content"); + } + } + return value; + } + private: std::string blob_field_name_; std::shared_ptr pool_; @@ -132,6 +171,82 @@ TEST_P(BlobFileBatchReaderTest, TestSimple) { {"blob_9_f54d253c.bin"}, blob_as_descriptor); } +TEST_P(BlobFileBatchReaderTest, TestMapBlob) { + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string file_path = dir->Str() + "/map-blob.blob"; + std::shared_ptr file_system = std::make_shared(); + + // Java-compatible MAP golden file. Rows are: + // {alpha: "hello", empty: "", missing: null}, null, {}, {omega: "world"}. + const std::string file_bytes = HexToBytes( + "cf114e584243424d0103000000616c706861656d7074796d697373696e676865" + "6c6c6f0a00040a090103000000030000003d000000000000002a64aaabcf114e" + "584243424d0100000000000000000000000021000000000000008360591ecf11" + "4e584243424d01010000006f6d656761776f726c640a0a01000000010000002d" + "00000000000000248fe4237a7b44180400000001"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr output_stream, + file_system->Create(file_path, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(int64_t written, + output_stream->Write(file_bytes.data(), file_bytes.size())); + ASSERT_EQ(file_bytes.size(), written); + ASSERT_OK(output_stream->Close()); + + std::shared_ptr blob_item = BlobUtils::ToArrowField("value", true); + auto key_field = arrow::field("key", arrow::utf8(), false); + auto map_type = std::make_shared(key_field, blob_item); + ASSERT_TRUE(BlobUtils::IsBlobField(map_type->item_field())); + auto map_field = arrow::field("blob_map", map_type, true); + auto schema = arrow::schema({map_field}); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system->Open(file_path)); + const bool blob_as_descriptor = GetParam(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create( + input_stream, /*batch_size=*/2, blob_as_descriptor, + /*emit_placeholder_sentinel=*/false, pool_, GetArrowPool(pool_))); + ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr chunked_array, + paimon::test::ReadResultCollector::CollectResult(std::move(reader))); + std::shared_ptr combined_array = + arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + + auto struct_array = std::dynamic_pointer_cast(combined_array); + ASSERT_TRUE(struct_array); + auto map_array = std::dynamic_pointer_cast(struct_array->field(0)); + ASSERT_TRUE(map_array); + ASSERT_EQ(arrow::Type::LARGE_BINARY, map_array->map_type()->item_type()->id()); + ASSERT_EQ(4, map_array->length()); + ASSERT_EQ(3, map_array->value_length(0)); + ASSERT_TRUE(map_array->IsNull(1)); + ASSERT_EQ(0, map_array->value_length(2)); + ASSERT_EQ(1, map_array->value_length(3)); + + auto keys = std::dynamic_pointer_cast(map_array->keys()); + auto values = std::dynamic_pointer_cast(map_array->items()); + ASSERT_TRUE(keys); + ASSERT_TRUE(values); + ASSERT_EQ("alpha", keys->GetString(0)); + ASSERT_EQ("empty", keys->GetString(1)); + ASSERT_EQ("missing", keys->GetString(2)); + ASSERT_EQ("omega", keys->GetString(3)); + ASSERT_FALSE(values->IsNull(0)); + ASSERT_FALSE(values->IsNull(1)); + ASSERT_TRUE(values->IsNull(2)); + ASSERT_FALSE(values->IsNull(3)); + ASSERT_OK_AND_ASSIGN(std::string first_value, + ReadMapBlobValue(values, 0, blob_as_descriptor, file_system)); + ASSERT_OK_AND_ASSIGN(std::string empty_value, + ReadMapBlobValue(values, 1, blob_as_descriptor, file_system)); + ASSERT_OK_AND_ASSIGN(std::string last_value, + ReadMapBlobValue(values, 3, blob_as_descriptor, file_system)); + ASSERT_EQ("hello", first_value); + ASSERT_EQ("", empty_value); + ASSERT_EQ("world", last_value); +} + TEST_P(BlobFileBatchReaderTest, TestPushdownBitmap) { std::string test_data_path = paimon::test::GetDataDir() + "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/db_with_blob.db/table_with_blob/"; auto dir = paimon::test::UniqueTestDirectory::Create(); From ce4b0b6ab1b2d218d1932394051c202efc262de2 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 2 Sep 2026 09:38:22 -0700 Subject: [PATCH 02/11] fix(blob): support map blob table and fallback reads --- src/paimon/common/data/blob_utils.cpp | 10 ++ src/paimon/common/data/blob_utils.h | 2 + .../reader/blob_fallback_batch_reader.cpp | 30 +++- .../reader/blob_fallback_batch_reader.h | 5 +- src/paimon/common/types/data_type.cpp | 4 + .../common/types/data_type_json_parser.cpp | 18 +++ .../types/data_type_json_parser_test.cpp | 11 ++ src/paimon/common/types/data_type_test.cpp | 2 + .../core/schema/arrow_schema_validator.cpp | 10 +- .../schema/arrow_schema_validator_test.cpp | 19 +++ .../core/schema/schema_validation_test.cpp | 26 +-- src/paimon/core/schema/table_schema_test.cpp | 30 ++++ .../format/blob/blob_file_batch_reader.cpp | 29 ++-- .../format/blob/blob_file_batch_reader.h | 5 +- .../blob/blob_file_batch_reader_test.cpp | 152 +++++++++++++++++- 15 files changed, 311 insertions(+), 42 deletions(-) diff --git a/src/paimon/common/data/blob_utils.cpp b/src/paimon/common/data/blob_utils.cpp index ad2ad315..3dd98a4d 100644 --- a/src/paimon/common/data/blob_utils.cpp +++ b/src/paimon/common/data/blob_utils.cpp @@ -108,6 +108,16 @@ bool BlobUtils::IsBlobField(const std::shared_ptr& field) { return IsBlobMetadata(field->metadata()); } +bool BlobUtils::IsMapBlobField(const std::shared_ptr& field) { + if (field == nullptr || field->type()->id() != arrow::Type::MAP) { + return false; + } + const auto& map_type = checked_cast(*field->type()); + // Arrow's C schema bridge does not retain nested field metadata for MapType. Paimon's + // ordinary binary type is BINARY, so LARGE_BINARY uniquely identifies a BLOB value here. + return map_type.item_type()->id() == arrow::Type::LARGE_BINARY; +} + bool BlobUtils::IsBlobMetadata(const std::shared_ptr& metadata) { if (!metadata) { return false; diff --git a/src/paimon/common/data/blob_utils.h b/src/paimon/common/data/blob_utils.h index c13e24f9..17196545 100644 --- a/src/paimon/common/data/blob_utils.h +++ b/src/paimon/common/data/blob_utils.h @@ -73,6 +73,8 @@ class PAIMON_EXPORT BlobUtils { const std::set& inline_fields); static bool IsBlobField(const std::shared_ptr& field); + /// Returns whether the field is a top-level MAP whose values are BLOBs. + static bool IsMapBlobField(const std::shared_ptr& field); static bool IsBlobMetadata(const std::shared_ptr& metadata); static bool IsBlobFile(const std::string& file_name); diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.cpp b/src/paimon/common/reader/blob_fallback_batch_reader.cpp index f3bcdd61..1a23ec2d 100644 --- a/src/paimon/common/reader/blob_fallback_batch_reader.cpp +++ b/src/paimon/common/reader/blob_fallback_batch_reader.cpp @@ -56,7 +56,8 @@ Result> BlobFallbackBatchReader::Create } int32_t blob_field_idx = -1; for (int32_t i = 0; i < read_schema->num_fields(); i++) { - if (BlobUtils::IsBlobField(read_schema->field(i))) { + if (BlobUtils::IsBlobField(read_schema->field(i)) || + BlobUtils::IsMapBlobField(read_schema->field(i))) { if (blob_field_idx != -1) { return Status::Invalid( "Blob fallback read schema should contain exactly one blob field."); @@ -193,10 +194,29 @@ Result> BlobFallbackBatchReader::ComputePlaceholderFlags( std::fill(flags.begin() + pos, flags.begin() + pos + chunk.length, true); } else { std::shared_ptr blob_col = chunk.array->field(blob_field_idx_); - if (!blob_col || blob_col->type_id() != arrow::Type::LARGE_BINARY) { - return Status::Invalid(fmt::format( - "Blob fallback expects the blob column to be large binary, but got {}", - blob_col ? blob_col->type()->ToString() : "null")); + if (!blob_col) { + return Status::Invalid("Blob fallback got a null blob column."); + } + if (blob_col->type_id() == arrow::Type::MAP) { + auto map_col = checked_pointer_cast(blob_col); + const std::shared_ptr& keys = map_col->keys(); + const std::shared_ptr& items = map_col->items(); + for (int64_t k = 0; k < chunk.length; k++) { + int64_t idx = chunk.offset + k; + if (!map_col->IsNull(idx) && map_col->value_length(idx) == 2) { + int64_t entry_idx = map_col->value_offset(idx); + flags[pos + k] = + items->IsNull(entry_idx) && items->IsNull(entry_idx + 1) && + keys->RangeEquals(entry_idx, entry_idx + 1, entry_idx + 1, *keys); + } + } + pos += chunk.length; + continue; + } + if (blob_col->type_id() != arrow::Type::LARGE_BINARY) { + return Status::Invalid( + fmt::format("Blob fallback expects a BLOB or MAP<..., BLOB> column, but got {}", + blob_col->type()->ToString())); } auto binary_col = checked_pointer_cast(blob_col); for (int64_t k = 0; k < chunk.length; k++) { diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.h b/src/paimon/common/reader/blob_fallback_batch_reader.h index 011b3ff1..3664b0c1 100644 --- a/src/paimon/common/reader/blob_fallback_batch_reader.h +++ b/src/paimon/common/reader/blob_fallback_batch_reader.h @@ -54,9 +54,8 @@ namespace paimon { /// vector has to reach every group the same way, through the file segments' readers and /// through the row ids the caller leaves in a gap segment's `gap_selected_ranges`. /// 3. Each output row takes the first group, in max-sequence order, whose row is not a -/// placeholder. Placeholder rows are identified by exact equality with the -/// BlobDefs::kPlaceholderSentinel bytes, emitted by the blob format reader when -/// BlobDefs::kEmitPlaceholderSentinelKey is set. +/// placeholder. The blob format reader emits placeholders as BlobDefs::kPlaceholderSentinel +/// bytes for scalar BLOBs, or as a two-entry map with duplicate keys for MAP<..., BLOB>. /// 4. A row that is a placeholder in every group degrades to a null blob: it keeps its /// _ROW_ID, reports -1 as its _SEQUENCE_NUMBER, and returns null for every other field. class BlobFallbackBatchReader : public BatchReader { diff --git a/src/paimon/common/types/data_type.cpp b/src/paimon/common/types/data_type.cpp index 2bf5d73c..f2798666 100644 --- a/src/paimon/common/types/data_type.cpp +++ b/src/paimon/common/types/data_type.cpp @@ -112,6 +112,10 @@ std::string DataType::DataTypeToString(const std::shared_ptr& t return "BYTES"; case arrow::Type::type::DATE32: return "DATE"; + case arrow::Type::type::TIME32: { + const auto& time_type = checked_cast(*type); + return time_type.unit() == arrow::TimeUnit::SECOND ? "TIME(0)" : "TIME(3)"; + } case arrow::Type::type::DECIMAL128: { auto status = DecimalUtils::CheckDecimalType(*type); if (!status.ok()) { diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp index d04e5ade..a910bd78 100644 --- a/src/paimon/common/types/data_type_json_parser.cpp +++ b/src/paimon/common/types/data_type_json_parser.cpp @@ -250,6 +250,7 @@ class TokenParser { Result> ParseStringType(); Result> ParseDecimalType(); Result> ParseDoubleType(); + Result> ParseTimeType(); Result> ParseTimestampType(); Result> ParseTimestampLtzType(); Result> ParseVectorType(); @@ -523,6 +524,8 @@ Result> TokenParser::ParseTypeByKeyword( return ParseDoubleType(); case Keyword::DATE: return arrow::date32(); + case Keyword::TIME: + return ParseTimeType(); case Keyword::TIMESTAMP: return ParseTimestampType(); case Keyword::TIMESTAMP_LTZ: @@ -582,6 +585,21 @@ Result> TokenParser::ParseDoubleType() { return arrow::float64(); } +Result> TokenParser::ParseTimeType() { + PAIMON_ASSIGN_OR_RAISE(int32_t precision, ParseOptionalPrecision(/*default_precision=*/0)); + if (precision < 0 || precision > 9) { + return Status::Invalid("TIME precision must be between 0 and 9"); + } + if (HasNextToken({Keyword::WITHOUT})) { + PAIMON_RETURN_NOT_OK(NextToken(Keyword::WITHOUT)); + PAIMON_RETURN_NOT_OK(NextToken(Keyword::TIME)); + PAIMON_RETURN_NOT_OK(NextToken(Keyword::ZONE)); + } + // Paimon stores TIME as the number of milliseconds since midnight for every supported + // precision. Arrow's corresponding physical type is therefore time32[ms]. + return arrow::time32(arrow::TimeUnit::MILLI); +} + Result> TokenParser::ParseTimestampType() { PAIMON_ASSIGN_OR_RAISE(int32_t precision, ParseOptionalPrecision(Timestamp::DEFAULT_PRECISION)); bool with_timezone = false; diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp index e5dfbc21..f249420a 100644 --- a/src/paimon/common/types/data_type_json_parser_test.cpp +++ b/src/paimon/common/types/data_type_json_parser_test.cpp @@ -158,6 +158,11 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { {"NUMERIC", arrow::decimal128(10, 0)}, {"NUMERIC(10)", arrow::decimal128(10, 0)}, {"NUMERIC(10, 3)", arrow::decimal128(10, 3)}, + {"TIME", arrow::time32(arrow::TimeUnit::MILLI)}, + {"TIME(0)", arrow::time32(arrow::TimeUnit::MILLI)}, + {"TIME(3)", arrow::time32(arrow::TimeUnit::MILLI)}, + {"TIME(9)", arrow::time32(arrow::TimeUnit::MILLI)}, + {"TIME(3) WITHOUT TIME ZONE", arrow::time32(arrow::TimeUnit::MILLI)}, {"TIMESTAMP(0)", arrow::timestamp(arrow::TimeUnit::SECOND)}, {"TIMESTAMP(3)", arrow::timestamp(arrow::TimeUnit::MILLI)}, {"TIMESTAMP(6)", arrow::timestamp(arrow::TimeUnit::MICRO)}, @@ -215,6 +220,12 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("field_name", value), "length must be between 1 and 2147483647"); } + { + rapidjson::Document invalid_doc; + rapidjson::Value value("TIME(10)", invalid_doc.GetAllocator()); + ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("field_name", value), + "TIME precision must be between 0 and 9"); + } { rapidjson::Document invalid_doc; rapidjson::Value value("TIMESTAMP(4)", invalid_doc.GetAllocator()); diff --git a/src/paimon/common/types/data_type_test.cpp b/src/paimon/common/types/data_type_test.cpp index d6eacdc1..bd94bdc9 100644 --- a/src/paimon/common/types/data_type_test.cpp +++ b/src/paimon/common/types/data_type_test.cpp @@ -82,6 +82,8 @@ TEST(DataTypeTest, DataTypeToString) { ASSERT_EQ(std::string(json_value.GetString()), "VARIANT"); } ASSERT_EQ(dummy_data_type.DataTypeToString(arrow::date32()), "DATE"); + ASSERT_EQ(dummy_data_type.DataTypeToString(arrow::time32(arrow::TimeUnit::SECOND)), "TIME(0)"); + ASSERT_EQ(dummy_data_type.DataTypeToString(arrow::time32(arrow::TimeUnit::MILLI)), "TIME(3)"); auto decimal_type1 = arrow::decimal128(10, 2); ASSERT_EQ(dummy_data_type.DataTypeToString(decimal_type1), "DECIMAL(10, 2)"); diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index 0db1145f..29d8fb47 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -121,6 +121,7 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId( case arrow::Type::type::STRING: case arrow::Type::type::BINARY: case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: case arrow::Type::type::DECIMAL128: case arrow::Type::type::TIMESTAMP: return Status::OK(); @@ -164,8 +165,10 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId( const auto& item_field = checked_cast(type.get())->item_field(); PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId( key_field->type(), key_field->metadata(), /*allow_blob=*/false, field_id_set)); + bool allow_direct_blob_item = + allow_blob && item_field->type()->id() == arrow::Type::LARGE_BINARY; PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId( - item_field->type(), item_field->metadata(), /*allow_blob=*/false, field_id_set)); + item_field->type(), item_field->metadata(), allow_direct_blob_item, field_id_set)); break; } case arrow::Type::type::LARGE_BINARY: { @@ -202,6 +205,7 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& case arrow::Type::type::STRING: case arrow::Type::type::BINARY: case arrow::Type::type::DATE32: + case arrow::Type::type::TIME32: case arrow::Type::type::TIMESTAMP: break; case arrow::Type::type::DECIMAL128: @@ -248,7 +252,9 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& const auto& item_field = checked_cast(*field->type()).item_field(); PAIMON_RETURN_NOT_OK(ValidateField(key_field, /*allow_blob=*/false)); - PAIMON_RETURN_NOT_OK(ValidateField(item_field, /*allow_blob=*/false)); + bool allow_direct_blob_item = + allow_blob && item_field->type()->id() == arrow::Type::LARGE_BINARY; + PAIMON_RETURN_NOT_OK(ValidateField(item_field, allow_direct_blob_item)); break; } case arrow::Type::type::LARGE_BINARY: { diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp index ed56b637..d644e563 100644 --- a/src/paimon/core/schema/arrow_schema_validator_test.cpp +++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp @@ -195,6 +195,16 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema), "Blob field must be a top-level field."); } + { + auto map_blob_field = arrow::field( + "map_blob", arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", true))); + auto arrow_schema = arrow::schema(arrow::FieldVector({map_blob_field})); + ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow_schema)); + + std::vector fields = {DataField(0, map_blob_field)}; + arrow_schema = DataField::ConvertDataFieldsToArrowSchema(fields); + ASSERT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema)); + } { auto map_blob_field = arrow::field( "map_blob", @@ -203,6 +213,15 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) { ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema), "Blob field must be a top-level field."); } + { + auto nested_map_blob_field = arrow::field( + "nested_map_blob", + arrow::map(arrow::utf8(), + arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", true)))); + auto arrow_schema = arrow::schema(arrow::FieldVector({nested_map_blob_field})); + ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema), + "Blob field must be a top-level field."); + } { std::vector nested_fields = { DataField(1, BlobUtils::ToArrowField("blob", true))}; diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 054f76da..1d9731b9 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -1169,15 +1169,23 @@ TEST(SchemaValidationTest, TestMapSharedShreddingRejectsBlobValue) { {"fields.f1.map.storage-layout", "shared-shredding"}, }; - for (const auto& map_type : {direct_blob_map, nested_blob_map}) { - auto schema = arrow::schema({ - arrow::field("f0", arrow::utf8()), - arrow::field("f1", map_type), - }); - ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options), - "Blob field must be a top-level field."); - } + auto direct_schema = arrow::schema({ + arrow::field("f0", arrow::utf8()), + arrow::field("f1", direct_blob_map), + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, direct_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "MAP shared-shredding currently cannot contain BLOB fields."); + + auto nested_schema = arrow::schema({ + arrow::field("f0", arrow::utf8()), + arrow::field("f1", nested_blob_map), + }); + ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, nested_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options), + "Blob field must be a top-level field."); } TEST(SchemaValidationTest, TestMapSharedShreddingCompression) { diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index 0acbee0b..0245b7b4 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -23,6 +23,7 @@ #include "arrow/api.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" @@ -1331,6 +1332,35 @@ TEST_F(TableSchemaTest, NullableMapKeySchemaIsSupported) { ASSERT_TRUE(direct_map_type->key_field()->nullable()); } +TEST_F(TableSchemaTest, MapBlobSchemaLoadsFromJson) { + std::string table_schema_str = R"json({ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "string_blob_map", + "type" : {"type":"MAP", "key":"STRING", "value":"BLOB"} + }, { + "id" : 1, + "name" : "time_blob_map", + "type" : {"type":"MAP", "key":"TIME(0)", "value":"BLOB"} + } ], + "highestFieldId" : 1, + "partitionKeys" : [], + "primaryKeys" : [], + "options" : {}, + "timeMillis" : 1721614341162 + })json"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_schema, + TableSchema::CreateFromJson(table_schema_str)); + ASSERT_TRUE(BlobUtils::IsMapBlobField( + DataField::ConvertDataFieldToArrowField(table_schema->Fields()[0]))); + ASSERT_TRUE(BlobUtils::IsMapBlobField( + DataField::ConvertDataFieldToArrowField(table_schema->Fields()[1]))); + auto time_map = checked_pointer_cast(table_schema->Fields()[1].Type()); + ASSERT_EQ(time_map->key_type()->id(), arrow::Type::TIME32); +} + TEST_F(TableSchemaTest, MapKeysSortedIsNormalized) { auto sorted_map = std::make_shared(arrow::field("key", arrow::utf8(), /*nullable=*/false), diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index a12c9e04..afc4fb17 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -58,16 +58,6 @@ T ReadLittleEndian(const uint8_t* data) { return arrow::bit_util::FromLittleEndian(arrow::util::SafeLoadAs(data)); } -bool IsMapBlobField(const std::shared_ptr& field) { - if (field->type()->id() != arrow::Type::MAP) { - return false; - } - const auto& map_type = static_cast(*field->type()); - // Nested field metadata is not retained by Arrow's C schema bridge for MapType. Paimon's - // regular binary types use BINARY, so LARGE_BINARY here uniquely identifies a BLOB value. - return map_type.item_type()->id() == arrow::Type::LARGE_BINARY; -} - Result GetMapBlobFixedKeyLength(const std::shared_ptr& key_type) { switch (key_type->id()) { case arrow::Type::BOOL: @@ -134,6 +124,13 @@ Status AppendMapBlobKey(const std::shared_ptr& key_type, const if (decimal_type.precision() <= 18) { value = arrow::Decimal128(ReadLittleEndian(data)); } else { + // Java BigInteger.toByteArray() uses the shortest big-endian two's-complement + // representation. This makes byte-wise duplicate detection equivalent to + // decoded decimal equality. + if (length <= 0 || (length > 1 && ((data[0] == 0x00 && (data[1] & 0x80) == 0) || + (data[0] == 0xFF && (data[1] & 0x80) != 0)))) { + return Status::Invalid("invalid MAP<..., BLOB> non-canonical decimal key"); + } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(value, arrow::Decimal128::FromBigEndian(data, length)); } @@ -244,10 +241,10 @@ Status BlobFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, fmt::format("read schema field number {} is not 1", arrow_schema->num_fields())); } std::shared_ptr read_field = arrow_schema->field(0); - if (!BlobUtils::IsBlobField(read_field) && !IsMapBlobField(read_field)) { + if (!BlobUtils::IsBlobField(read_field) && !BlobUtils::IsMapBlobField(read_field)) { return Status::Invalid(fmt::format("field {} is not BLOB", read_field->ToString())); } - if (IsMapBlobField(read_field)) { + if (BlobUtils::IsMapBlobField(read_field)) { const auto& map_type = static_cast(*read_field->type()); PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] int32_t fixed_key_length, GetMapBlobFixedKeyLength(map_type.key_type())); @@ -394,6 +391,14 @@ Result> BlobFileBatchReader::BuildMapBlobArray( PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.AppendNull()); continue; } + if (IsTargetPlaceholder(row_index)) { + // Duplicate map keys cannot occur in a valid Paimon map, so two empty/default keys + // with null values form an unambiguous, Arrow-valid internal sentinel. + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.Append()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->AppendEmptyValues(2)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->AppendNulls(2)); + continue; + } if (target_blob_lengths_[row_index] < 0) { return Status::Invalid(fmt::format("unsupported MAP<..., BLOB> record length: {}", target_blob_lengths_[row_index])); diff --git a/src/paimon/format/blob/blob_file_batch_reader.h b/src/paimon/format/blob/blob_file_batch_reader.h index 23cb24bb..bf6a3f59 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.h +++ b/src/paimon/format/blob/blob_file_batch_reader.h @@ -99,9 +99,8 @@ class BlobFileBatchReader : public FileBatchReader { /// `emit_placeholder_sentinel` controls how placeholder entries (bin_length == /// BlobDefs::kPlaceholderBinLength) are read: when false they fail the read, as resolving /// them requires the data-evolution blob fallback path; when true they are returned as the - /// non-null BlobDefs::kPlaceholderSentinel bytes for that path to merge away. Stored values - /// are returned verbatim; see BlobDefs::kPlaceholderSentinel for the accepted collision - /// with a user value exactly equal to the sentinel. + /// non-null BlobDefs::kPlaceholderSentinel bytes for scalar BLOB, or a two-entry map with + /// duplicate keys for MAP<..., BLOB>. The fallback path removes these internal values. static Result> Create( const std::shared_ptr& input_stream, int32_t batch_size, bool blob_as_descriptor, bool emit_placeholder_sentinel, diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 1377b8ad..9b73e337 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -21,9 +21,12 @@ #include #include "arrow/api.h" +#include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/reader/blob_fallback_batch_reader.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/data/blob.h" #include "paimon/format/blob/blob_format_writer.h" @@ -48,6 +51,17 @@ std::string HexToBytes(std::string_view hex) { return bytes; } +std::string MapBlobGoldenBytes() { + // Java-compatible MAP golden file. Rows are: + // {alpha: "hello", empty: "", missing: null}, null, {}, {omega: "world"}. + return HexToBytes( + "cf114e584243424d0103000000616c706861656d7074796d697373696e676865" + "6c6c6f0a00040a090103000000030000003d000000000000002a64aaabcf114e" + "584243424d0100000000000000000000000021000000000000008360591ecf11" + "4e584243424d01010000006f6d656761776f726c640a0a01000000010000002d" + "00000000000000248fe4237a7b44180400000001"); +} + } // namespace TEST(BlobReaderBuilderTest, RejectsNullMemoryPool) { @@ -177,14 +191,7 @@ TEST_P(BlobFileBatchReaderTest, TestMapBlob) { const std::string file_path = dir->Str() + "/map-blob.blob"; std::shared_ptr file_system = std::make_shared(); - // Java-compatible MAP golden file. Rows are: - // {alpha: "hello", empty: "", missing: null}, null, {}, {omega: "world"}. - const std::string file_bytes = HexToBytes( - "cf114e584243424d0103000000616c706861656d7074796d697373696e676865" - "6c6c6f0a00040a090103000000030000003d000000000000002a64aaabcf114e" - "584243424d0100000000000000000000000021000000000000008360591ecf11" - "4e584243424d01010000006f6d656761776f726c640a0a01000000010000002d" - "00000000000000248fe4237a7b44180400000001"); + const std::string file_bytes = MapBlobGoldenBytes(); ASSERT_OK_AND_ASSIGN(std::shared_ptr output_stream, file_system->Create(file_path, /*overwrite=*/true)); ASSERT_OK_AND_ASSIGN(int64_t written, @@ -247,6 +254,135 @@ TEST_P(BlobFileBatchReaderTest, TestMapBlob) { ASSERT_EQ("world", last_value); } +TEST_P(BlobFileBatchReaderTest, MapBlobFallbackAcrossSequenceLayers) { + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::shared_ptr file_system = std::make_shared(); + const std::string old_file_path = dir->Str() + "/old-map.blob"; + const std::string new_file_path = dir->Str() + "/new-placeholder.blob"; + + const std::string old_bytes = MapBlobGoldenBytes(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr old_output, + file_system->Create(old_file_path, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(int64_t old_written, + old_output->Write(old_bytes.data(), old_bytes.size())); + ASSERT_EQ(old_bytes.size(), old_written); + ASSERT_OK(old_output->Close()); + + // Generate four genuine -2 outer-file entries through the scalar writer. The outer blob + // index is type-independent; the map reader turns them into its map placeholder sentinel. + std::shared_ptr scalar_blob_field = BlobUtils::ToArrowField("blob_map", true); + auto scalar_struct_type = arrow::struct_({scalar_blob_field}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr new_output, + file_system->Create(new_file_path, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + BlobFormatWriter::Create(new_output, scalar_struct_type, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/true, file_system, pool_)); + arrow::LargeBinaryBuilder scalar_builder; + const std::string sentinel(BlobDefs::PlaceholderSentinelView()); + for (int32_t i = 0; i < 4; i++) { + ASSERT_TRUE(scalar_builder.Append(sentinel).ok()); + } + std::shared_ptr scalar_values; + ASSERT_TRUE(scalar_builder.Finish(&scalar_values).ok()); + std::shared_ptr scalar_rows = + arrow::StructArray::Make({scalar_values}, {scalar_blob_field}).ValueOrDie(); + for (int32_t i = 0; i < 4; i++) { + ::ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*scalar_rows->Slice(i, 1), &c_array).ok()); + ASSERT_OK(writer->AddBatch(&c_array)); + } + ASSERT_OK(writer->Finish()); + ASSERT_OK(new_output->Close()); + + auto map_type = std::make_shared(arrow::field("key", arrow::utf8(), false), + BlobUtils::ToArrowField("value", true)); + std::shared_ptr map_field = arrow::field("blob_map", map_type, true); + auto map_schema = arrow::schema({map_field}); + const bool blob_as_descriptor = GetParam(); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr new_input, file_system->Open(new_file_path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr new_reader, + BlobFileBatchReader::Create( + new_input, /*batch_size=*/2, blob_as_descriptor, + /*emit_placeholder_sentinel=*/true, pool_, GetArrowPool(pool_))); + ::ArrowSchema new_schema; + ASSERT_TRUE(arrow::ExportSchema(*map_schema, &new_schema).ok()); + ASSERT_OK(new_reader->SetReadSchema(&new_schema, nullptr, std::nullopt)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr old_input, file_system->Open(old_file_path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr old_reader, + BlobFileBatchReader::Create( + old_input, /*batch_size=*/2, blob_as_descriptor, + /*emit_placeholder_sentinel=*/true, pool_, GetArrowPool(pool_))); + ::ArrowSchema old_schema; + ASSERT_TRUE(arrow::ExportSchema(*map_schema, &old_schema).ok()); + ASSERT_OK(old_reader->SetReadSchema(&old_schema, nullptr, std::nullopt)); + + std::vector> groups(2); + groups[0].push_back({std::move(new_reader), {}}); + groups[1].push_back({std::move(old_reader), {}}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback, + BlobFallbackBatchReader::Create(std::move(groups), map_schema, /*read_batch_size=*/2, + GetArrowPool(pool_))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + paimon::test::ReadResultCollector::CollectResult(std::move(fallback))); + std::shared_ptr combined = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(combined); + ASSERT_TRUE(struct_array); + auto map_array = std::dynamic_pointer_cast(struct_array->field(0)); + ASSERT_TRUE(map_array); + ASSERT_EQ(4, map_array->length()); + ASSERT_EQ(3, map_array->value_length(0)); + ASSERT_TRUE(map_array->IsNull(1)); + ASSERT_EQ(0, map_array->value_length(2)); + ASSERT_EQ(1, map_array->value_length(3)); + auto keys = std::dynamic_pointer_cast(map_array->keys()); + auto values = std::dynamic_pointer_cast(map_array->items()); + ASSERT_EQ("alpha", keys->GetString(0)); + ASSERT_EQ("omega", keys->GetString(3)); + ASSERT_OK_AND_ASSIGN(std::string first_value, + ReadMapBlobValue(values, 0, blob_as_descriptor, file_system)); + ASSERT_OK_AND_ASSIGN(std::string last_value, + ReadMapBlobValue(values, 3, blob_as_descriptor, file_system)); + ASSERT_EQ("hello", first_value); + ASSERT_EQ("world", last_value); +} + +TEST_F(BlobFileBatchReaderTest, RejectsNonCanonicalDecimalMapKey) { + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string file_path = dir->Str() + "/bad-decimal-map.blob"; + std::shared_ptr file_system = std::make_shared(); + // The two raw keys 00 and 0000 both decode to decimal zero. The second is not the shortest + // Java BigInteger two's-complement representation and must not bypass duplicate detection. + const std::string file_bytes = HexToBytes( + "cf114e584243424d010200000000000002020000020000000200000028000000" + "0000000000000000d0000200000001"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr output, + file_system->Create(file_path, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(int64_t written, output->Write(file_bytes.data(), file_bytes.size())); + ASSERT_EQ(file_bytes.size(), written); + ASSERT_OK(output->Close()); + + auto map_type = + std::make_shared(arrow::field("key", arrow::decimal128(20, 0), false), + BlobUtils::ToArrowField("value", true)); + auto schema = arrow::schema({arrow::field("blob_map", map_type, true)}); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, file_system->Open(file_path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create( + input, /*batch_size=*/1, /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_, GetArrowPool(pool_))); + ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "non-canonical decimal key"); +} + TEST_P(BlobFileBatchReaderTest, TestPushdownBitmap) { std::string test_data_path = paimon::test::GetDataDir() + "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/db_with_blob.db/table_with_blob/"; auto dir = paimon::test::UniqueTestDirectory::Create(); From eddb8ab568b2c601e2cbfb4311847fb828b39679 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 2 Sep 2026 16:11:20 -0700 Subject: [PATCH 03/11] fix(blob): harden map blob schema validation --- src/paimon/common/types/data_type.cpp | 9 ++++++ .../common/types/data_type_json_parser.cpp | 15 ++++++++-- .../types/data_type_json_parser_test.cpp | 17 +++++++++++ .../core/schema/schema_validation_test.cpp | 30 ++++++++++++++----- src/paimon/core/schema/table_schema.cpp | 6 ++++ src/paimon/core/schema/table_schema_test.cpp | 24 ++++++++++++++- .../format/blob/blob_file_batch_reader.cpp | 7 +++++ .../blob/blob_file_batch_reader_test.cpp | 28 +++++++++++++++++ 8 files changed, 124 insertions(+), 12 deletions(-) diff --git a/src/paimon/common/types/data_type.cpp b/src/paimon/common/types/data_type.cpp index f2798666..9b7e653c 100644 --- a/src/paimon/common/types/data_type.cpp +++ b/src/paimon/common/types/data_type.cpp @@ -40,6 +40,9 @@ #include "rapidjson/rapidjson.h" namespace paimon { +namespace { +constexpr char kTimePrecisionMetadata[] = "paimon.time.precision"; +} // namespace DataType::DataType(const std::shared_ptr& type, bool nullable, const std::shared_ptr& metadata) @@ -113,6 +116,12 @@ std::string DataType::DataTypeToString(const std::shared_ptr& t case arrow::Type::type::DATE32: return "DATE"; case arrow::Type::type::TIME32: { + if (metadata_) { + auto precision = metadata_->Get(kTimePrecisionMetadata); + if (precision.ok()) { + return fmt::format("TIME({})", precision.ValueUnsafe()); + } + } const auto& time_type = checked_cast(*type); return time_type.unit() == arrow::TimeUnit::SECOND ? "TIME(0)" : "TIME(3)"; } diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp index a910bd78..0a30273b 100644 --- a/src/paimon/common/types/data_type_json_parser.cpp +++ b/src/paimon/common/types/data_type_json_parser.cpp @@ -53,6 +53,7 @@ static constexpr char CHAR_LIST_SEPARATOR = ','; static constexpr char CHAR_STRING = '\''; static constexpr char CHAR_IDENTIFIER = '`'; static constexpr char CHAR_DOT = '.'; +static constexpr char TIME_PRECISION_METADATA[] = "paimon.time.precision"; enum class TokenType : int32_t { // e.g. "ROW<" @@ -90,6 +91,7 @@ struct Token { struct AtomicTypeAttributes { bool is_blob = false; bool is_variant = false; + std::optional time_precision; }; // nullptr is returned in the case of parsing failed @@ -250,7 +252,7 @@ class TokenParser { Result> ParseStringType(); Result> ParseDecimalType(); Result> ParseDoubleType(); - Result> ParseTimeType(); + Result> ParseTimeType(AtomicTypeAttributes* attributes); Result> ParseTimestampType(); Result> ParseTimestampLtzType(); Result> ParseVectorType(); @@ -525,7 +527,7 @@ Result> TokenParser::ParseTypeByKeyword( case Keyword::DATE: return arrow::date32(); case Keyword::TIME: - return ParseTimeType(); + return ParseTimeType(attributes); case Keyword::TIMESTAMP: return ParseTimestampType(); case Keyword::TIMESTAMP_LTZ: @@ -585,7 +587,8 @@ Result> TokenParser::ParseDoubleType() { return arrow::float64(); } -Result> TokenParser::ParseTimeType() { +Result> TokenParser::ParseTimeType( + AtomicTypeAttributes* attributes) { PAIMON_ASSIGN_OR_RAISE(int32_t precision, ParseOptionalPrecision(/*default_precision=*/0)); if (precision < 0 || precision > 9) { return Status::Invalid("TIME precision must be between 0 and 9"); @@ -595,6 +598,7 @@ Result> TokenParser::ParseTimeType() { PAIMON_RETURN_NOT_OK(NextToken(Keyword::TIME)); PAIMON_RETURN_NOT_OK(NextToken(Keyword::ZONE)); } + attributes->time_precision = precision; // Paimon stores TIME as the number of milliseconds since midnight for every supported // precision. Arrow's corresponding physical type is therefore time32[ms]. return arrow::time32(arrow::TimeUnit::MILLI); @@ -686,6 +690,11 @@ Result> DataTypeJsonParser::ParseAtomicTypeField( return BlobUtils::ToArrowField(name, nullable); } else if (attributes.is_variant) { return VariantTypeUtils::ToArrowField(name, nullable); + } else if (attributes.time_precision) { + return arrow::field( + name, type, nullable, + arrow::KeyValueMetadata::Make({TIME_PRECISION_METADATA}, + {std::to_string(attributes.time_precision.value())})); } else { return arrow::field(name, type, nullable); } diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp index f249420a..f690b1ce 100644 --- a/src/paimon/common/types/data_type_json_parser_test.cpp +++ b/src/paimon/common/types/data_type_json_parser_test.cpp @@ -22,8 +22,10 @@ #include #include +#include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/status.h" @@ -240,4 +242,19 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { } } +TEST(DataTypeJsonParserTest, TimePrecisionRoundTrip) { + for (int32_t precision = 0; precision <= 9; ++precision) { + const std::string type_string = fmt::format("TIME({})", precision); + rapidjson::Document doc; + rapidjson::Value value(type_string.c_str(), doc.GetAllocator()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr field, + DataTypeJsonParser::ParseType("time_field", value)); + std::unique_ptr data_type = + DataType::Create(field->type(), field->nullable(), field->metadata()); + rapidjson::Value serialized = data_type->ToJson(&doc.GetAllocator()); + ASSERT_TRUE(serialized.IsString()); + ASSERT_EQ(serialized.GetString(), type_string); + } +} + } // namespace paimon::test diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 1d9731b9..3e708f87 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -25,6 +25,7 @@ #include "gtest/gtest.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/core/schema/table_schema.h" #include "paimon/defs.h" #include "paimon/testing/utils/testharness.h" @@ -1159,8 +1160,6 @@ TEST(SchemaValidationTest, TestMapSharedShreddingRequiresNonNullableKey) { } TEST(SchemaValidationTest, TestMapSharedShreddingRejectsBlobValue) { - auto direct_blob_map = - arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", /*nullable=*/true)); auto nested_blob_map = arrow::map( arrow::utf8(), arrow::field("value", arrow::struct_({BlobUtils::ToArrowField("blob")}))); std::map options = { @@ -1169,13 +1168,28 @@ TEST(SchemaValidationTest, TestMapSharedShreddingRejectsBlobValue) { {"fields.f1.map.storage-layout", "shared-shredding"}, }; - auto direct_schema = arrow::schema({ - arrow::field("f0", arrow::utf8()), - arrow::field("f1", direct_blob_map), - }); + const std::string loaded_schema = R"json({ + "version": 3, + "id": 0, + "fields": [ + {"id": 0, "name": "f0", "type": "STRING"}, + {"id": 1, "name": "f1", + "type": {"type": "MAP", "key": "STRING", "value": "BLOB"}} + ], + "highestFieldId": 1, + "partitionKeys": [], + "primaryKeys": [], + "options": { + "bucket": "1", + "bucket-key": "f0", + "fields.f1.map.storage-layout": "shared-shredding" + }, + "timeMillis": 0 + })json"; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, - TableSchema::Create(/*schema_id=*/0, direct_schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options)); + TableSchema::CreateFromJson(loaded_schema)); + auto loaded_map = checked_pointer_cast(table_schema->Fields()[1].Type()); + ASSERT_TRUE(BlobUtils::IsBlobField(loaded_map->item_field())); ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "MAP shared-shredding currently cannot contain BLOB fields."); diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 6e2d8747..cfcf5306 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -27,6 +27,7 @@ #include "arrow/api.h" #include "arrow/c/bridge.h" #include "fmt/format.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" @@ -57,6 +58,11 @@ Result> TableSchema::Create( primary_key_set.insert(primary_key); } for (const auto& field : schema->fields()) { + if (BlobUtils::IsMapBlobField(field)) { + return Status::NotImplemented( + "Creating a table with MAP<..., BLOB> is not supported by the C++ writer; " + "loading an existing table is supported."); + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr field_with_id, AssignFieldIdsRecursively(field, /*set_field_id=*/true, &field_id)); if (primary_key_set.count(field_with_id->name())) { diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index 0245b7b4..3126614d 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -1344,8 +1344,12 @@ TEST_F(TableSchemaTest, MapBlobSchemaLoadsFromJson) { "id" : 1, "name" : "time_blob_map", "type" : {"type":"MAP", "key":"TIME(0)", "value":"BLOB"} + }, { + "id" : 2, + "name" : "time9", + "type" : "TIME(9)" } ], - "highestFieldId" : 1, + "highestFieldId" : 2, "partitionKeys" : [], "primaryKeys" : [], "options" : {}, @@ -1359,6 +1363,24 @@ TEST_F(TableSchemaTest, MapBlobSchemaLoadsFromJson) { DataField::ConvertDataFieldToArrowField(table_schema->Fields()[1]))); auto time_map = checked_pointer_cast(table_schema->Fields()[1].Type()); ASSERT_EQ(time_map->key_type()->id(), arrow::Type::TIME32); + ASSERT_TRUE(time_map->key_field()->HasMetadata()); + ASSERT_TRUE(time_map->key_field()->metadata()->Contains("paimon.time.precision")); + ASSERT_TRUE(BlobUtils::IsBlobField(time_map->item_field())); + ASSERT_OK_AND_ASSIGN(std::string serialized, table_schema->ToJsonString()); + ASSERT_NE(serialized.find("\"TIME(0) NOT NULL\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"TIME(9)\""), std::string::npos) << serialized; + ASSERT_OK_AND_ASSIGN(std::unique_ptr restored, + TableSchema::CreateFromJson(serialized)); + ASSERT_OK_AND_ASSIGN(std::string restored_json, restored->ToJsonString()); + ASSERT_EQ(restored_json, serialized); +} + +TEST_F(TableSchemaTest, CreatingMapBlobSchemaIsRejected) { + auto map_type = arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", /*nullable=*/true)); + ASSERT_NOK_WITH_MSG( + TableSchema::Create(/*schema_id=*/0, arrow::schema({arrow::field("blob_map", map_type)}), + /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{}), + "not supported by the C++ writer"); } TEST_F(TableSchemaTest, MapKeysSortedIsNormalized) { diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index afc4fb17..f5e374c7 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -33,6 +33,7 @@ #include "arrow/util/decimal.h" #include "arrow/util/endian.h" #include "arrow/util/ubsan.h" +#include "arrow/util/utf8.h" #include "fmt/format.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/io/offset_input_stream.h" @@ -113,6 +114,9 @@ Status AppendMapBlobKey(const std::shared_ptr& key_type, const return ToPaimonStatus(checked_cast(builder)->Append( ReadLittleEndian(data))); case arrow::Type::STRING: + if (!arrow::util::ValidateUTF8(data, length)) { + return Status::Invalid("invalid UTF-8 in MAP key"); + } return ToPaimonStatus( checked_cast(builder)->Append(data, length)); case arrow::Type::BINARY: @@ -370,6 +374,9 @@ Result> BlobFileBatchReader::BuildMapBlobArray( const std::shared_ptr& map_field = struct_type.field(0); auto map_type = checked_pointer_cast(map_field->type()); const std::shared_ptr& key_type = map_type->key_type(); + if (key_type->id() == arrow::Type::STRING) { + arrow::util::InitializeUTF8(); + } PAIMON_ASSIGN_OR_RAISE(int32_t fixed_key_length, GetMapBlobFixedKeyLength(key_type)); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr key_builder_unique, diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 9b73e337..64fce911 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -383,6 +383,34 @@ TEST_F(BlobFileBatchReaderTest, RejectsNonCanonicalDecimalMapKey) { ASSERT_NOK_WITH_MSG(reader->NextBatch(), "non-canonical decimal key"); } +TEST_F(BlobFileBatchReaderTest, RejectsInvalidUtf8StringMapKey) { + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string file_path = dir->Str() + "/bad-string-map.blob"; + std::shared_ptr file_system = std::make_shared(); + std::string file_bytes = MapBlobGoldenBytes(); + ASSERT_GT(file_bytes.size(), 13); + // The first key starts after the outer magic and the map header. + file_bytes[13] = static_cast(0xFF); + ASSERT_OK_AND_ASSIGN(std::shared_ptr output, + file_system->Create(file_path, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(int64_t written, output->Write(file_bytes.data(), file_bytes.size())); + ASSERT_EQ(file_bytes.size(), written); + ASSERT_OK(output->Close()); + + auto map_type = arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", /*nullable=*/true)); + auto schema = arrow::schema({arrow::field("blob_map", map_type)}); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, file_system->Open(file_path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create( + input, /*batch_size=*/1, /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_, GetArrowPool(pool_))); + ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "invalid UTF-8"); +} + TEST_P(BlobFileBatchReaderTest, TestPushdownBitmap) { std::string test_data_path = paimon::test::GetDataDir() + "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/db_with_blob.db/table_with_blob/"; auto dir = paimon::test::UniqueTestDirectory::Create(); From ba315537076d128c063da4daf8f01598595b5f6b Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 2 Sep 2026 23:08:30 -0700 Subject: [PATCH 04/11] fix(blob): reject writing map blob tables --- .../core/operation/file_store_write.cpp | 9 +++++- .../core/operation/file_store_write_test.cpp | 28 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index fa1294d1..1129f9d0 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -23,6 +23,7 @@ #include #include "fmt/format.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/core/core_options.h" @@ -109,6 +110,13 @@ Result> FileStoreWrite::Create(std::unique_ptrFields()); + for (const auto& field : arrow_schema->fields()) { + if (BlobUtils::IsMapBlobField(field)) { + return Status::NotImplemented( + "Writing a table with MAP<..., BLOB> is not supported by the C++ writer."); + } + } auto opts = schema->Options(); for (const auto& [key, value] : ctx->GetOptions()) { opts[key] = value; @@ -116,7 +124,6 @@ Result> FileStoreWrite::Create(std::unique_ptrGetSpecificFileSystem(), ctx->GetFileSystemSchemeToIdentifierMap())); - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 3e1b3ea4..23633488 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -75,6 +75,34 @@ TEST(FileStoreWriteTest, TestCreateAppendTable) { FileStoreWrite::Create(std::move(write_context))); } +TEST(FileStoreWriteTest, TestCreateWriterForLoadedMapBlobTable) { + auto dir = UniqueTestDirectory::Create(); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + auto fs = std::make_shared(); + SchemaManager schema_manager(fs, table_path); + std::string schema_json = R"json({ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "blob_map", + "type" : {"type":"MAP", "key":"STRING", "value":"BLOB"} + } ], + "highestFieldId" : 0, + "partitionKeys" : [], + "primaryKeys" : [], + "options" : {}, + "timeMillis" : 1721614341162 + })json"; + ASSERT_OK(fs->AtomicStore(PathUtil::JoinPath(schema_manager.SchemaDirectory(), "schema-0"), + schema_json)); + + WriteContextBuilder context_builder(table_path, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + "Writing a table with MAP<..., BLOB> is not supported by the C++ writer"); +} + TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidBucket) { auto dir = UniqueTestDirectory::Create(); arrow::FieldVector fields = { From 128d47c9455de5c07895c3f7e93578a7e4d56186 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 3 Sep 2026 00:00:49 -0700 Subject: [PATCH 05/11] fix(blob): guard append compaction for map blob --- src/paimon/common/data/blob_utils.cpp | 10 ++++++++ src/paimon/common/data/blob_utils.h | 2 ++ .../append/append_compact_coordinator.cpp | 7 ++++-- .../append_compact_coordinator_test.cpp | 25 +++++++++++++++++++ .../core/operation/file_store_write.cpp | 7 +----- src/paimon/core/schema/table_schema.cpp | 6 +---- 6 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/paimon/common/data/blob_utils.cpp b/src/paimon/common/data/blob_utils.cpp index 3dd98a4d..d7d8f336 100644 --- a/src/paimon/common/data/blob_utils.cpp +++ b/src/paimon/common/data/blob_utils.cpp @@ -118,6 +118,16 @@ bool BlobUtils::IsMapBlobField(const std::shared_ptr& field) { return map_type.item_type()->id() == arrow::Type::LARGE_BINARY; } +Status BlobUtils::ValidateMapBlobWriteSchema(const std::shared_ptr& schema) { + for (const auto& field : schema->fields()) { + if (IsMapBlobField(field)) { + return Status::NotImplemented( + "Writing a table with MAP<..., BLOB> is not supported by the C++ writer."); + } + } + return Status::OK(); +} + bool BlobUtils::IsBlobMetadata(const std::shared_ptr& metadata) { if (!metadata) { return false; diff --git a/src/paimon/common/data/blob_utils.h b/src/paimon/common/data/blob_utils.h index 17196545..86e0ef37 100644 --- a/src/paimon/common/data/blob_utils.h +++ b/src/paimon/common/data/blob_utils.h @@ -75,6 +75,8 @@ class PAIMON_EXPORT BlobUtils { static bool IsBlobField(const std::shared_ptr& field); /// Returns whether the field is a top-level MAP whose values are BLOBs. static bool IsMapBlobField(const std::shared_ptr& field); + /// Rejects schemas that the C++ writer cannot safely mutate. + static Status ValidateMapBlobWriteSchema(const std::shared_ptr& schema); static bool IsBlobMetadata(const std::shared_ptr& metadata); static bool IsBlobFile(const std::string& file_name); diff --git a/src/paimon/core/append/append_compact_coordinator.cpp b/src/paimon/core/append/append_compact_coordinator.cpp index b6afc15c..8ff580f3 100644 --- a/src/paimon/core/append/append_compact_coordinator.cpp +++ b/src/paimon/core/append/append_compact_coordinator.cpp @@ -25,6 +25,7 @@ #include #include "paimon/common/data/binary_row.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/linked_hash_map.h" #include "paimon/core/append/append_compact_task.h" @@ -198,7 +199,9 @@ Result, CoreOptions>> LoadSchemaAndOption /// Validate that the table is an append-only unaware-bucket table without DV. Status ValidateTable(const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options) { + PAIMON_RETURN_NOT_OK(BlobUtils::ValidateMapBlobWriteSchema(arrow_schema)); if (!table_schema->PrimaryKeys().empty() || core_options.GetBucket() != -1) { return Status::Invalid( "AppendCompactCoordinator only supports append-only tables " @@ -320,12 +323,12 @@ Result>> AppendCompactCoordinator::Ru PAIMON_ASSIGN_OR_RAISE(schema_and_options, LoadSchemaAndOptions(table_path, options, file_system)); const auto& [table_schema, core_options] = schema_and_options; + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); // Validate table type - PAIMON_RETURN_NOT_OK(ValidateTable(table_schema, core_options)); + PAIMON_RETURN_NOT_OK(ValidateTable(table_schema, arrow_schema, core_options)); // Build shared objects - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); diff --git a/src/paimon/core/append/append_compact_coordinator_test.cpp b/src/paimon/core/append/append_compact_coordinator_test.cpp index 1c8c7aff..bb166fe5 100644 --- a/src/paimon/core/append/append_compact_coordinator_test.cpp +++ b/src/paimon/core/append/append_compact_coordinator_test.cpp @@ -511,6 +511,31 @@ TEST_F(AppendCompactCoordinatorTest, TestValidateFailsOnDvTable) { "not support for dv in UNAWARE_BUCKET mode"); } +TEST_F(AppendCompactCoordinatorTest, TestValidateFailsOnLoadedMapBlobTable) { + auto fs = dir_->GetFileSystem(); + SchemaManager schema_manager(fs, TablePath()); + std::string schema_json = R"json({ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "blob_map", + "type" : {"type":"MAP", "key":"STRING", "value":"BLOB"} + } ], + "highestFieldId" : 0, + "partitionKeys" : [], + "primaryKeys" : [], + "options" : {"bucket":"-1"}, + "timeMillis" : 1721614341162 + })json"; + ASSERT_OK(fs->AtomicStore(PathUtil::JoinPath(schema_manager.SchemaDirectory(), "schema-0"), + schema_json)); + + ASSERT_NOK_WITH_MSG( + AppendCompactCoordinator::Run(TablePath(), /*options=*/{}, /*partitions=*/{}, fs, pool_), + "Writing a table with MAP<..., BLOB> is not supported by the C++ writer"); +} + /// Test that compact output files are written to external path when configured. TEST_F(AppendCompactCoordinatorTest, TestCompactWithExternalPath) { auto external_dir = UniqueTestDirectory::Create("local"); diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index 1129f9d0..74f56dc9 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -111,12 +111,7 @@ Result> FileStoreWrite::Create(std::unique_ptrFields()); - for (const auto& field : arrow_schema->fields()) { - if (BlobUtils::IsMapBlobField(field)) { - return Status::NotImplemented( - "Writing a table with MAP<..., BLOB> is not supported by the C++ writer."); - } - } + PAIMON_RETURN_NOT_OK(BlobUtils::ValidateMapBlobWriteSchema(arrow_schema)); auto opts = schema->Options(); for (const auto& [key, value] : ctx->GetOptions()) { opts[key] = value; diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index cfcf5306..9d240560 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -57,12 +57,8 @@ Result> TableSchema::Create( for (const auto& primary_key : primary_keys) { primary_key_set.insert(primary_key); } + PAIMON_RETURN_NOT_OK(BlobUtils::ValidateMapBlobWriteSchema(schema)); for (const auto& field : schema->fields()) { - if (BlobUtils::IsMapBlobField(field)) { - return Status::NotImplemented( - "Creating a table with MAP<..., BLOB> is not supported by the C++ writer; " - "loading an existing table is supported."); - } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr field_with_id, AssignFieldIdsRecursively(field, /*set_field_id=*/true, &field_id)); if (primary_key_set.count(field_with_id->name())) { From 29f54bea03af2891e3be23a80b81c639304ff0c0 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 3 Sep 2026 00:39:40 -0700 Subject: [PATCH 06/11] fix(schema): require millisecond time32 values --- src/paimon/common/types/data_type.cpp | 19 ++++++++++-- src/paimon/common/types/data_type_test.cpp | 3 +- .../core/schema/arrow_schema_validator.cpp | 31 ++++++++++++++++++- .../schema/arrow_schema_validator_test.cpp | 21 +++++++++++++ .../blob/blob_file_batch_reader_test.cpp | 30 ++++++++++++++++++ 5 files changed, 99 insertions(+), 5 deletions(-) diff --git a/src/paimon/common/types/data_type.cpp b/src/paimon/common/types/data_type.cpp index 9b7e653c..668b8bb0 100644 --- a/src/paimon/common/types/data_type.cpp +++ b/src/paimon/common/types/data_type.cpp @@ -20,6 +20,7 @@ #include "paimon/common/types/data_type.h" #include +#include #include #include "arrow/api.h" @@ -34,6 +35,7 @@ #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/decimal_utils.h" #include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/status.h" #include "rapidjson/allocators.h" #include "rapidjson/document.h" @@ -116,14 +118,25 @@ std::string DataType::DataTypeToString(const std::shared_ptr& t case arrow::Type::type::DATE32: return "DATE"; case arrow::Type::type::TIME32: { + const auto& time_type = checked_cast(*type); + if (time_type.unit() != arrow::TimeUnit::MILLI) { + throw std::invalid_argument( + "Paimon TIME fields must use Arrow time32[ms], but got " + type->ToString()); + } if (metadata_) { auto precision = metadata_->Get(kTimePrecisionMetadata); if (precision.ok()) { - return fmt::format("TIME({})", precision.ValueUnsafe()); + std::optional parsed_precision = + StringUtils::StringToValue(precision.ValueUnsafe()); + if (!parsed_precision || parsed_precision.value() < 0 || + parsed_precision.value() > 9) { + throw std::invalid_argument( + "paimon.time.precision must be an integer between 0 and 9"); + } + return fmt::format("TIME({})", parsed_precision.value()); } } - const auto& time_type = checked_cast(*type); - return time_type.unit() == arrow::TimeUnit::SECOND ? "TIME(0)" : "TIME(3)"; + return "TIME(3)"; } case arrow::Type::type::DECIMAL128: { auto status = DecimalUtils::CheckDecimalType(*type); diff --git a/src/paimon/common/types/data_type_test.cpp b/src/paimon/common/types/data_type_test.cpp index bd94bdc9..dfee0c89 100644 --- a/src/paimon/common/types/data_type_test.cpp +++ b/src/paimon/common/types/data_type_test.cpp @@ -82,7 +82,8 @@ TEST(DataTypeTest, DataTypeToString) { ASSERT_EQ(std::string(json_value.GetString()), "VARIANT"); } ASSERT_EQ(dummy_data_type.DataTypeToString(arrow::date32()), "DATE"); - ASSERT_EQ(dummy_data_type.DataTypeToString(arrow::time32(arrow::TimeUnit::SECOND)), "TIME(0)"); + ASSERT_THROW(dummy_data_type.DataTypeToString(arrow::time32(arrow::TimeUnit::SECOND)), + std::invalid_argument); ASSERT_EQ(dummy_data_type.DataTypeToString(arrow::time32(arrow::TimeUnit::MILLI)), "TIME(3)"); auto decimal_type1 = arrow::decimal128(10, 2); diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index 29d8fb47..1d9a3960 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -19,6 +19,7 @@ #include "paimon/core/schema/arrow_schema_validator.h" +#include #include #include @@ -39,6 +40,32 @@ class KeyValueMetadata; } // namespace arrow namespace paimon { +namespace { + +constexpr char kTimePrecisionMetadata[] = "paimon.time.precision"; + +Status ValidateTime32Field(const std::shared_ptr& field) { + const auto& time_type = checked_cast(*field->type()); + if (time_type.unit() != arrow::TimeUnit::MILLI) { + return Status::Invalid("Paimon TIME fields must use Arrow time32[ms], but got ", + field->type()->ToString()); + } + if (!field->HasMetadata() || !field->metadata()->Contains(kTimePrecisionMetadata)) { + return Status::OK(); + } + auto precision_result = field->metadata()->Get(kTimePrecisionMetadata); + if (!precision_result.ok()) { + return Status::Invalid("Cannot read paimon.time.precision metadata"); + } + std::optional precision = + StringUtils::StringToValue(precision_result.ValueUnsafe()); + if (!precision || precision.value() < 0 || precision.value() > 9) { + return Status::Invalid("paimon.time.precision must be an integer between 0 and 9"); + } + return Status::OK(); +} + +} // namespace bool ArrowSchemaValidator::IsNestedType(const std::shared_ptr& data_type) { return (data_type->id() == arrow::Type::MAP || data_type->id() == arrow::Type::LIST || @@ -205,9 +232,11 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& case arrow::Type::type::STRING: case arrow::Type::type::BINARY: case arrow::Type::type::DATE32: - case arrow::Type::type::TIME32: case arrow::Type::type::TIMESTAMP: break; + case arrow::Type::type::TIME32: + PAIMON_RETURN_NOT_OK(ValidateTime32Field(field)); + break; case arrow::Type::type::DECIMAL128: PAIMON_RETURN_NOT_OK(DecimalUtils::CheckDecimalType(*field->type())); break; diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp index d644e563..691c581a 100644 --- a/src/paimon/core/schema/arrow_schema_validator_test.cpp +++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp @@ -62,6 +62,27 @@ TEST(ArrowSchemaValidatorTest, TestSimple) { ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow_schema)); } +TEST(ArrowSchemaValidatorTest, TestTime32) { + ASSERT_OK(ArrowSchemaValidator::ValidateSchema( + *arrow::schema({arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI))}))); + ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow::schema( + {arrow::field("time", arrow::time32(arrow::TimeUnit::SECOND))})), + "Paimon TIME fields must use Arrow time32[ms]"); + + for (const std::string& precision : {"0", "9"}) { + auto metadata = arrow::KeyValueMetadata::Make({"paimon.time.precision"}, {precision}); + ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow::schema({arrow::field( + "time", arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true, metadata)}))); + } + for (const std::string& precision : {"-1", "10", "invalid"}) { + auto metadata = arrow::KeyValueMetadata::Make({"paimon.time.precision"}, {precision}); + ASSERT_NOK_WITH_MSG( + ArrowSchemaValidator::ValidateSchema(*arrow::schema({arrow::field( + "time", arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true, metadata)})), + "paimon.time.precision must be an integer between 0 and 9"); + } +} + TEST(ArrowSchemaValidatorTest, TestVectorElementType) { for (const auto& element_type : {arrow::boolean(), arrow::int8(), arrow::int16(), arrow::int32(), arrow::int64(), diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 64fce911..f514e9b6 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -411,6 +411,36 @@ TEST_F(BlobFileBatchReaderTest, RejectsInvalidUtf8StringMapKey) { ASSERT_NOK_WITH_MSG(reader->NextBatch(), "invalid UTF-8"); } +TEST_F(BlobFileBatchReaderTest, RejectsNullMapKey) { + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string file_path = dir->Str() + "/null-key-map.blob"; + std::shared_ptr file_system = std::make_shared(); + std::string file_bytes = MapBlobGoldenBytes(); + const std::string key_index = HexToBytes("0a0004"); + const size_t key_index_offset = file_bytes.find(key_index); + ASSERT_NE(std::string::npos, key_index_offset); + // The first delta-varint changes from key length 5 to Java's null marker -1. + file_bytes[key_index_offset] = static_cast(0x01); + ASSERT_OK_AND_ASSIGN(std::shared_ptr output, + file_system->Create(file_path, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(int64_t written, output->Write(file_bytes.data(), file_bytes.size())); + ASSERT_EQ(file_bytes.size(), written); + ASSERT_OK(output->Close()); + + auto map_type = arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", /*nullable=*/true)); + auto schema = arrow::schema({arrow::field("blob_map", map_type)}); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, file_system->Open(file_path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create( + input, /*batch_size=*/1, /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_, GetArrowPool(pool_))); + ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "MAP<..., BLOB> keys cannot be null"); +} + TEST_P(BlobFileBatchReaderTest, TestPushdownBitmap) { std::string test_data_path = paimon::test::GetDataDir() + "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/db_with_blob.db/table_with_blob/"; auto dir = paimon::test::UniqueTestDirectory::Create(); From aa5b2045d66f84971cdbc83ea345698bec87855d Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 3 Sep 2026 00:52:21 -0700 Subject: [PATCH 07/11] fix(test): satisfy gcc range loop warnings --- src/paimon/core/schema/arrow_schema_validator_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp index 691c581a..13a9faa1 100644 --- a/src/paimon/core/schema/arrow_schema_validator_test.cpp +++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp @@ -69,12 +69,12 @@ TEST(ArrowSchemaValidatorTest, TestTime32) { {arrow::field("time", arrow::time32(arrow::TimeUnit::SECOND))})), "Paimon TIME fields must use Arrow time32[ms]"); - for (const std::string& precision : {"0", "9"}) { + for (const char* precision : {"0", "9"}) { auto metadata = arrow::KeyValueMetadata::Make({"paimon.time.precision"}, {precision}); ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow::schema({arrow::field( "time", arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true, metadata)}))); } - for (const std::string& precision : {"-1", "10", "invalid"}) { + for (const char* precision : {"-1", "10", "invalid"}) { auto metadata = arrow::KeyValueMetadata::Make({"paimon.time.precision"}, {precision}); ASSERT_NOK_WITH_MSG( ArrowSchemaValidator::ValidateSchema(*arrow::schema({arrow::field( From 00da44a5535cfb279b5e6b05fb965365cb6448bd Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Fri, 4 Sep 2026 02:55:58 -0700 Subject: [PATCH 08/11] fix(blob): address map blob review feedback --- src/paimon/common/types/data_type.cpp | 26 -- .../common/types/data_type_json_parser.cpp | 27 -- .../types/data_type_json_parser_test.cpp | 28 -- src/paimon/common/types/data_type_test.cpp | 3 - .../core/schema/arrow_schema_validator.cpp | 31 -- .../schema/arrow_schema_validator_test.cpp | 21 - src/paimon/core/schema/table_schema_test.cpp | 19 +- .../core/utils/nested_projection_utils.cpp | 5 + .../utils/nested_projection_utils_test.cpp | 11 + .../format/blob/blob_file_batch_reader.cpp | 393 ++++++++++-------- .../format/blob/blob_file_batch_reader.h | 18 + .../blob/blob_file_batch_reader_test.cpp | 98 ++--- 12 files changed, 287 insertions(+), 393 deletions(-) diff --git a/src/paimon/common/types/data_type.cpp b/src/paimon/common/types/data_type.cpp index 668b8bb0..2bf5d73c 100644 --- a/src/paimon/common/types/data_type.cpp +++ b/src/paimon/common/types/data_type.cpp @@ -20,7 +20,6 @@ #include "paimon/common/types/data_type.h" #include -#include #include #include "arrow/api.h" @@ -35,16 +34,12 @@ #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/decimal_utils.h" #include "paimon/common/utils/rapidjson_util.h" -#include "paimon/common/utils/string_utils.h" #include "paimon/status.h" #include "rapidjson/allocators.h" #include "rapidjson/document.h" #include "rapidjson/rapidjson.h" namespace paimon { -namespace { -constexpr char kTimePrecisionMetadata[] = "paimon.time.precision"; -} // namespace DataType::DataType(const std::shared_ptr& type, bool nullable, const std::shared_ptr& metadata) @@ -117,27 +112,6 @@ std::string DataType::DataTypeToString(const std::shared_ptr& t return "BYTES"; case arrow::Type::type::DATE32: return "DATE"; - case arrow::Type::type::TIME32: { - const auto& time_type = checked_cast(*type); - if (time_type.unit() != arrow::TimeUnit::MILLI) { - throw std::invalid_argument( - "Paimon TIME fields must use Arrow time32[ms], but got " + type->ToString()); - } - if (metadata_) { - auto precision = metadata_->Get(kTimePrecisionMetadata); - if (precision.ok()) { - std::optional parsed_precision = - StringUtils::StringToValue(precision.ValueUnsafe()); - if (!parsed_precision || parsed_precision.value() < 0 || - parsed_precision.value() > 9) { - throw std::invalid_argument( - "paimon.time.precision must be an integer between 0 and 9"); - } - return fmt::format("TIME({})", parsed_precision.value()); - } - } - return "TIME(3)"; - } case arrow::Type::type::DECIMAL128: { auto status = DecimalUtils::CheckDecimalType(*type); if (!status.ok()) { diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp index 0a30273b..d04e5ade 100644 --- a/src/paimon/common/types/data_type_json_parser.cpp +++ b/src/paimon/common/types/data_type_json_parser.cpp @@ -53,7 +53,6 @@ static constexpr char CHAR_LIST_SEPARATOR = ','; static constexpr char CHAR_STRING = '\''; static constexpr char CHAR_IDENTIFIER = '`'; static constexpr char CHAR_DOT = '.'; -static constexpr char TIME_PRECISION_METADATA[] = "paimon.time.precision"; enum class TokenType : int32_t { // e.g. "ROW<" @@ -91,7 +90,6 @@ struct Token { struct AtomicTypeAttributes { bool is_blob = false; bool is_variant = false; - std::optional time_precision; }; // nullptr is returned in the case of parsing failed @@ -252,7 +250,6 @@ class TokenParser { Result> ParseStringType(); Result> ParseDecimalType(); Result> ParseDoubleType(); - Result> ParseTimeType(AtomicTypeAttributes* attributes); Result> ParseTimestampType(); Result> ParseTimestampLtzType(); Result> ParseVectorType(); @@ -526,8 +523,6 @@ Result> TokenParser::ParseTypeByKeyword( return ParseDoubleType(); case Keyword::DATE: return arrow::date32(); - case Keyword::TIME: - return ParseTimeType(attributes); case Keyword::TIMESTAMP: return ParseTimestampType(); case Keyword::TIMESTAMP_LTZ: @@ -587,23 +582,6 @@ Result> TokenParser::ParseDoubleType() { return arrow::float64(); } -Result> TokenParser::ParseTimeType( - AtomicTypeAttributes* attributes) { - PAIMON_ASSIGN_OR_RAISE(int32_t precision, ParseOptionalPrecision(/*default_precision=*/0)); - if (precision < 0 || precision > 9) { - return Status::Invalid("TIME precision must be between 0 and 9"); - } - if (HasNextToken({Keyword::WITHOUT})) { - PAIMON_RETURN_NOT_OK(NextToken(Keyword::WITHOUT)); - PAIMON_RETURN_NOT_OK(NextToken(Keyword::TIME)); - PAIMON_RETURN_NOT_OK(NextToken(Keyword::ZONE)); - } - attributes->time_precision = precision; - // Paimon stores TIME as the number of milliseconds since midnight for every supported - // precision. Arrow's corresponding physical type is therefore time32[ms]. - return arrow::time32(arrow::TimeUnit::MILLI); -} - Result> TokenParser::ParseTimestampType() { PAIMON_ASSIGN_OR_RAISE(int32_t precision, ParseOptionalPrecision(Timestamp::DEFAULT_PRECISION)); bool with_timezone = false; @@ -690,11 +668,6 @@ Result> DataTypeJsonParser::ParseAtomicTypeField( return BlobUtils::ToArrowField(name, nullable); } else if (attributes.is_variant) { return VariantTypeUtils::ToArrowField(name, nullable); - } else if (attributes.time_precision) { - return arrow::field( - name, type, nullable, - arrow::KeyValueMetadata::Make({TIME_PRECISION_METADATA}, - {std::to_string(attributes.time_precision.value())})); } else { return arrow::field(name, type, nullable); } diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp index f690b1ce..e5dfbc21 100644 --- a/src/paimon/common/types/data_type_json_parser_test.cpp +++ b/src/paimon/common/types/data_type_json_parser_test.cpp @@ -22,10 +22,8 @@ #include #include -#include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/common/data/variant/variant_type_utils.h" -#include "paimon/common/types/data_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/status.h" @@ -160,11 +158,6 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { {"NUMERIC", arrow::decimal128(10, 0)}, {"NUMERIC(10)", arrow::decimal128(10, 0)}, {"NUMERIC(10, 3)", arrow::decimal128(10, 3)}, - {"TIME", arrow::time32(arrow::TimeUnit::MILLI)}, - {"TIME(0)", arrow::time32(arrow::TimeUnit::MILLI)}, - {"TIME(3)", arrow::time32(arrow::TimeUnit::MILLI)}, - {"TIME(9)", arrow::time32(arrow::TimeUnit::MILLI)}, - {"TIME(3) WITHOUT TIME ZONE", arrow::time32(arrow::TimeUnit::MILLI)}, {"TIMESTAMP(0)", arrow::timestamp(arrow::TimeUnit::SECOND)}, {"TIMESTAMP(3)", arrow::timestamp(arrow::TimeUnit::MILLI)}, {"TIMESTAMP(6)", arrow::timestamp(arrow::TimeUnit::MICRO)}, @@ -222,12 +215,6 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("field_name", value), "length must be between 1 and 2147483647"); } - { - rapidjson::Document invalid_doc; - rapidjson::Value value("TIME(10)", invalid_doc.GetAllocator()); - ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("field_name", value), - "TIME precision must be between 0 and 9"); - } { rapidjson::Document invalid_doc; rapidjson::Value value("TIMESTAMP(4)", invalid_doc.GetAllocator()); @@ -242,19 +229,4 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { } } -TEST(DataTypeJsonParserTest, TimePrecisionRoundTrip) { - for (int32_t precision = 0; precision <= 9; ++precision) { - const std::string type_string = fmt::format("TIME({})", precision); - rapidjson::Document doc; - rapidjson::Value value(type_string.c_str(), doc.GetAllocator()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr field, - DataTypeJsonParser::ParseType("time_field", value)); - std::unique_ptr data_type = - DataType::Create(field->type(), field->nullable(), field->metadata()); - rapidjson::Value serialized = data_type->ToJson(&doc.GetAllocator()); - ASSERT_TRUE(serialized.IsString()); - ASSERT_EQ(serialized.GetString(), type_string); - } -} - } // namespace paimon::test diff --git a/src/paimon/common/types/data_type_test.cpp b/src/paimon/common/types/data_type_test.cpp index dfee0c89..d6eacdc1 100644 --- a/src/paimon/common/types/data_type_test.cpp +++ b/src/paimon/common/types/data_type_test.cpp @@ -82,9 +82,6 @@ TEST(DataTypeTest, DataTypeToString) { ASSERT_EQ(std::string(json_value.GetString()), "VARIANT"); } ASSERT_EQ(dummy_data_type.DataTypeToString(arrow::date32()), "DATE"); - ASSERT_THROW(dummy_data_type.DataTypeToString(arrow::time32(arrow::TimeUnit::SECOND)), - std::invalid_argument); - ASSERT_EQ(dummy_data_type.DataTypeToString(arrow::time32(arrow::TimeUnit::MILLI)), "TIME(3)"); auto decimal_type1 = arrow::decimal128(10, 2); ASSERT_EQ(dummy_data_type.DataTypeToString(decimal_type1), "DECIMAL(10, 2)"); diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index 1d9a3960..00997936 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -19,7 +19,6 @@ #include "paimon/core/schema/arrow_schema_validator.h" -#include #include #include @@ -40,32 +39,6 @@ class KeyValueMetadata; } // namespace arrow namespace paimon { -namespace { - -constexpr char kTimePrecisionMetadata[] = "paimon.time.precision"; - -Status ValidateTime32Field(const std::shared_ptr& field) { - const auto& time_type = checked_cast(*field->type()); - if (time_type.unit() != arrow::TimeUnit::MILLI) { - return Status::Invalid("Paimon TIME fields must use Arrow time32[ms], but got ", - field->type()->ToString()); - } - if (!field->HasMetadata() || !field->metadata()->Contains(kTimePrecisionMetadata)) { - return Status::OK(); - } - auto precision_result = field->metadata()->Get(kTimePrecisionMetadata); - if (!precision_result.ok()) { - return Status::Invalid("Cannot read paimon.time.precision metadata"); - } - std::optional precision = - StringUtils::StringToValue(precision_result.ValueUnsafe()); - if (!precision || precision.value() < 0 || precision.value() > 9) { - return Status::Invalid("paimon.time.precision must be an integer between 0 and 9"); - } - return Status::OK(); -} - -} // namespace bool ArrowSchemaValidator::IsNestedType(const std::shared_ptr& data_type) { return (data_type->id() == arrow::Type::MAP || data_type->id() == arrow::Type::LIST || @@ -148,7 +121,6 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId( case arrow::Type::type::STRING: case arrow::Type::type::BINARY: case arrow::Type::type::DATE32: - case arrow::Type::type::TIME32: case arrow::Type::type::DECIMAL128: case arrow::Type::type::TIMESTAMP: return Status::OK(); @@ -234,9 +206,6 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& case arrow::Type::type::DATE32: case arrow::Type::type::TIMESTAMP: break; - case arrow::Type::type::TIME32: - PAIMON_RETURN_NOT_OK(ValidateTime32Field(field)); - break; case arrow::Type::type::DECIMAL128: PAIMON_RETURN_NOT_OK(DecimalUtils::CheckDecimalType(*field->type())); break; diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp index 13a9faa1..d644e563 100644 --- a/src/paimon/core/schema/arrow_schema_validator_test.cpp +++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp @@ -62,27 +62,6 @@ TEST(ArrowSchemaValidatorTest, TestSimple) { ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow_schema)); } -TEST(ArrowSchemaValidatorTest, TestTime32) { - ASSERT_OK(ArrowSchemaValidator::ValidateSchema( - *arrow::schema({arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI))}))); - ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow::schema( - {arrow::field("time", arrow::time32(arrow::TimeUnit::SECOND))})), - "Paimon TIME fields must use Arrow time32[ms]"); - - for (const char* precision : {"0", "9"}) { - auto metadata = arrow::KeyValueMetadata::Make({"paimon.time.precision"}, {precision}); - ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow::schema({arrow::field( - "time", arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true, metadata)}))); - } - for (const char* precision : {"-1", "10", "invalid"}) { - auto metadata = arrow::KeyValueMetadata::Make({"paimon.time.precision"}, {precision}); - ASSERT_NOK_WITH_MSG( - ArrowSchemaValidator::ValidateSchema(*arrow::schema({arrow::field( - "time", arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true, metadata)})), - "paimon.time.precision must be an integer between 0 and 9"); - } -} - TEST(ArrowSchemaValidatorTest, TestVectorElementType) { for (const auto& element_type : {arrow::boolean(), arrow::int8(), arrow::int16(), arrow::int32(), arrow::int64(), diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index 3126614d..e195802c 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -1340,16 +1340,8 @@ TEST_F(TableSchemaTest, MapBlobSchemaLoadsFromJson) { "id" : 0, "name" : "string_blob_map", "type" : {"type":"MAP", "key":"STRING", "value":"BLOB"} - }, { - "id" : 1, - "name" : "time_blob_map", - "type" : {"type":"MAP", "key":"TIME(0)", "value":"BLOB"} - }, { - "id" : 2, - "name" : "time9", - "type" : "TIME(9)" } ], - "highestFieldId" : 2, + "highestFieldId" : 0, "partitionKeys" : [], "primaryKeys" : [], "options" : {}, @@ -1359,16 +1351,7 @@ TEST_F(TableSchemaTest, MapBlobSchemaLoadsFromJson) { TableSchema::CreateFromJson(table_schema_str)); ASSERT_TRUE(BlobUtils::IsMapBlobField( DataField::ConvertDataFieldToArrowField(table_schema->Fields()[0]))); - ASSERT_TRUE(BlobUtils::IsMapBlobField( - DataField::ConvertDataFieldToArrowField(table_schema->Fields()[1]))); - auto time_map = checked_pointer_cast(table_schema->Fields()[1].Type()); - ASSERT_EQ(time_map->key_type()->id(), arrow::Type::TIME32); - ASSERT_TRUE(time_map->key_field()->HasMetadata()); - ASSERT_TRUE(time_map->key_field()->metadata()->Contains("paimon.time.precision")); - ASSERT_TRUE(BlobUtils::IsBlobField(time_map->item_field())); ASSERT_OK_AND_ASSIGN(std::string serialized, table_schema->ToJsonString()); - ASSERT_NE(serialized.find("\"TIME(0) NOT NULL\""), std::string::npos) << serialized; - ASSERT_NE(serialized.find("\"TIME(9)\""), std::string::npos) << serialized; ASSERT_OK_AND_ASSIGN(std::unique_ptr restored, TableSchema::CreateFromJson(serialized)); ASSERT_OK_AND_ASSIGN(std::string restored_json, restored->ToJsonString()); diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index f956ca8f..03712e93 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -33,6 +33,7 @@ #include "arrow/compute/cast.h" #include "arrow/type.h" #include "fmt/format.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/data/variant/variant_access_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/checked_cast.h" @@ -434,6 +435,10 @@ Result> NestedProjectionUtils::GetMapSelectedKeys( if (!get_result.ok()) { return result; } + if (BlobUtils::IsMapBlobField(field)) { + return Status::NotImplemented( + "paimon.map.selected-keys is not supported for MAP<..., BLOB>"); + } auto tokens = StringUtils::Split(get_result.ValueUnsafe(), ",", /*ignore_empty=*/false); std::unordered_set deduplicated; deduplicated.reserve(tokens.size()); diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index 153d90e1..8adf7b24 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -29,6 +29,7 @@ #include "arrow/memory_pool.h" #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/data/variant/variant_access_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" @@ -548,6 +549,16 @@ TEST(NestedProjectionUtilsTest, GetMapSelectedKeysDuplicateKey) { "Duplicate selected key 'a'"); } +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysRejectsMapBlob) { + auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a"}); + auto map_type = arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value")); + auto field = arrow::field("m", map_type, /*nullable=*/true, metadata); + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::GetMapSelectedKeys(field), + "paimon.map.selected-keys is not supported for MAP<..., BLOB>"); + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::HasMapSelectedKeysRecursively(field), + "paimon.map.selected-keys is not supported for MAP<..., BLOB>"); +} + // ============== MapSharedShreddingAccessField ============== TEST(NestedProjectionUtilsTest, IsMapSharedShreddingAccessField) { diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index f5e374c7..d2700c93 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -31,7 +31,6 @@ #include "arrow/c/bridge.h" #include "arrow/util/bit_util.h" #include "arrow/util/decimal.h" -#include "arrow/util/endian.h" #include "arrow/util/ubsan.h" #include "arrow/util/utf8.h" #include "fmt/format.h" @@ -42,6 +41,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/delta_varint_compressor.h" +#include "paimon/common/utils/math.h" #include "paimon/common/utils/stream_utils.h" #include "paimon/data/blob.h" @@ -56,7 +56,7 @@ constexpr int32_t kMapBlobMinPayloadLength = kMapBlobHeaderLength + kMapBlobInde template T ReadLittleEndian(const uint8_t* data) { - return arrow::bit_util::FromLittleEndian(arrow::util::SafeLoadAs(data)); + return FromLittleEndian(arrow::util::SafeLoadAs(data)); } Result GetMapBlobFixedKeyLength(const std::shared_ptr& key_type) { @@ -68,7 +68,6 @@ Result GetMapBlobFixedKeyLength(const std::shared_ptr& return 2; case arrow::Type::INT32: case arrow::Type::DATE32: - case arrow::Type::TIME32: return 4; case arrow::Type::INT64: return 8; @@ -92,36 +91,48 @@ Status AppendMapBlobKey(const std::shared_ptr& key_type, const if (data[0] != 0 && data[0] != 1) { return Status::Invalid("invalid MAP<..., BLOB> boolean key"); } - return ToPaimonStatus( + PAIMON_RETURN_NOT_OK_FROM_ARROW( checked_cast(builder)->Append(data[0] == 1)); + return Status::OK(); } - case arrow::Type::INT8: - return ToPaimonStatus( + case arrow::Type::INT8: { + PAIMON_RETURN_NOT_OK_FROM_ARROW( checked_cast(builder)->Append(static_cast(data[0]))); - case arrow::Type::INT16: - return ToPaimonStatus(checked_cast(builder)->Append( + return Status::OK(); + } + case arrow::Type::INT16: { + PAIMON_RETURN_NOT_OK_FROM_ARROW(checked_cast(builder)->Append( ReadLittleEndian(data))); - case arrow::Type::INT32: - return ToPaimonStatus(checked_cast(builder)->Append( + return Status::OK(); + } + case arrow::Type::INT32: { + PAIMON_RETURN_NOT_OK_FROM_ARROW(checked_cast(builder)->Append( ReadLittleEndian(data))); - case arrow::Type::INT64: - return ToPaimonStatus(checked_cast(builder)->Append( + return Status::OK(); + } + case arrow::Type::INT64: { + PAIMON_RETURN_NOT_OK_FROM_ARROW(checked_cast(builder)->Append( ReadLittleEndian(data))); - case arrow::Type::DATE32: - return ToPaimonStatus(checked_cast(builder)->Append( - ReadLittleEndian(data))); - case arrow::Type::TIME32: - return ToPaimonStatus(checked_cast(builder)->Append( + return Status::OK(); + } + case arrow::Type::DATE32: { + PAIMON_RETURN_NOT_OK_FROM_ARROW(checked_cast(builder)->Append( ReadLittleEndian(data))); - case arrow::Type::STRING: + return Status::OK(); + } + case arrow::Type::STRING: { if (!arrow::util::ValidateUTF8(data, length)) { return Status::Invalid("invalid UTF-8 in MAP key"); } - return ToPaimonStatus( + PAIMON_RETURN_NOT_OK_FROM_ARROW( checked_cast(builder)->Append(data, length)); - case arrow::Type::BINARY: - return ToPaimonStatus( + return Status::OK(); + } + case arrow::Type::BINARY: { + PAIMON_RETURN_NOT_OK_FROM_ARROW( checked_cast(builder)->Append(data, length)); + return Status::OK(); + } case arrow::Type::DECIMAL128: { const auto& decimal_type = static_cast(*key_type); arrow::Decimal128 value; @@ -141,7 +152,9 @@ Status AppendMapBlobKey(const std::shared_ptr& key_type, const if (!value.FitsInPrecision(decimal_type.precision())) { return Status::Invalid("MAP<..., BLOB> decimal key exceeds declared precision"); } - return ToPaimonStatus(checked_cast(builder)->Append(value)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + checked_cast(builder)->Append(value)); + return Status::OK(); } default: return Status::Invalid( @@ -368,6 +381,184 @@ Result> BlobFileBatchReader::BuildContentArray( return std::make_shared(struct_array_data); } +Result BlobFileBatchReader::ReadMapBlobPayload( + size_t row_index, int32_t fixed_key_length) const { + if (target_blob_lengths_[row_index] < 0) { + return Status::Invalid(fmt::format("unsupported MAP<..., BLOB> record length: {}", + target_blob_lengths_[row_index])); + } + + const int64_t payload_offset = GetTargetContentOffset(row_index); + const int64_t payload_length = GetTargetContentLength(row_index); + if (payload_length < kMapBlobMinPayloadLength) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> payload length: {}", payload_length)); + } + + std::array header; + PAIMON_RETURN_NOT_OK(ReadBlobContentAt(payload_offset, header.size(), header.data())); + const int32_t magic_number = ReadLittleEndian(header.data()); + if (magic_number != kMapBlobMagicNumber) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> payload magic number: {}", magic_number)); + } + const int8_t version = static_cast(header[4]); + if (version != kMapBlobVersion) { + return Status::NotImplemented( + fmt::format("unsupported MAP<..., BLOB> payload version: {}", version)); + } + const int32_t entry_count = ReadLittleEndian(header.data() + 5); + if (entry_count < 0) { + return Status::Invalid(fmt::format("invalid MAP<..., BLOB> entry count: {}", entry_count)); + } + + const int64_t index_lengths_offset = payload_offset + payload_length - kMapBlobIndexLengthsSize; + std::array index_lengths; + PAIMON_RETURN_NOT_OK( + ReadBlobContentAt(index_lengths_offset, index_lengths.size(), index_lengths.data())); + const int32_t key_index_length = ReadLittleEndian(index_lengths.data()); + const int32_t value_index_length = + ReadLittleEndian(index_lengths.data() + sizeof(int32_t)); + const int64_t maximum_indexes_length = payload_length - kMapBlobMinPayloadLength; + if (key_index_length < 0 || key_index_length > maximum_indexes_length) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> key index length: {}", key_index_length)); + } + if (value_index_length < 0 || value_index_length > maximum_indexes_length) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> value index length: {}", value_index_length)); + } + if (static_cast(key_index_length) + value_index_length > maximum_indexes_length) { + return Status::Invalid("MAP<..., BLOB> indexes exceed the payload length"); + } + if (entry_count > key_index_length || entry_count > value_index_length) { + return Status::Invalid("MAP<..., BLOB> entry count exceeds index length"); + } + + const int64_t value_index_offset = index_lengths_offset - value_index_length; + const int64_t key_index_offset = value_index_offset - key_index_length; + std::vector key_index_bytes(key_index_length); + std::vector value_index_bytes(value_index_length); + PAIMON_RETURN_NOT_OK(ReadBlobContentAt(key_index_offset, key_index_length, + reinterpret_cast(key_index_bytes.data()))); + PAIMON_RETURN_NOT_OK(ReadBlobContentAt(value_index_offset, value_index_length, + reinterpret_cast(value_index_bytes.data()))); + PAIMON_ASSIGN_OR_RAISE(std::vector key_lengths, + DeltaVarintCompressor::Decompress(key_index_bytes)); + PAIMON_ASSIGN_OR_RAISE(std::vector value_lengths, + DeltaVarintCompressor::Decompress(value_index_bytes)); + if (key_lengths.size() != static_cast(entry_count)) { + return Status::Invalid("MAP<..., BLOB> entry count does not match key index length"); + } + if (value_lengths.size() != static_cast(entry_count)) { + return Status::Invalid("MAP<..., BLOB> entry count does not match value index length"); + } + + const int64_t data_offset = payload_offset + kMapBlobHeaderLength; + const int64_t data_length = key_index_offset - data_offset; + int64_t key_data_length = 0; + for (int64_t key_length : key_lengths) { + if (key_length < 0) { + return Status::Invalid("MAP<..., BLOB> keys cannot be null"); + } + if (key_length > std::numeric_limits::max()) { + return Status::Invalid(fmt::format("MAP<..., BLOB> key is too large: {}", key_length)); + } + if (fixed_key_length >= 0 && key_length != fixed_key_length) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> fixed-width key length: {}", key_length)); + } + if (key_length > data_length - key_data_length) { + return Status::Invalid("MAP<..., BLOB> key lengths exceed the payload data length"); + } + key_data_length += key_length; + } + + const int64_t maximum_value_data_length = data_length - key_data_length; + int64_t value_data_length = 0; + for (int64_t value_length : value_lengths) { + if (value_length == BlobDefs::kNullBinLength) { + continue; + } + if (value_length < 0) { + return Status::Invalid( + fmt::format("invalid MAP<..., BLOB> value length: {}", value_length)); + } + if (!blob_as_descriptor_ && value_length > std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("MAP<..., BLOB> inline value is too large: {}", value_length)); + } + if (value_length > maximum_value_data_length - value_data_length) { + return Status::Invalid("MAP<..., BLOB> value lengths exceed the payload data length"); + } + value_data_length += value_length; + } + if (value_data_length != maximum_value_data_length) { + return Status::Invalid( + "MAP<..., BLOB> key/value lengths do not match the payload data length"); + } + return MapBlobPayload{std::move(key_lengths), std::move(value_lengths), data_offset, + key_data_length}; +} + +Status BlobFileBatchReader::AppendMapBlobKeys(const MapBlobPayload& payload, + const std::shared_ptr& key_type, + arrow::ArrayBuilder* key_builder) const { + int64_t key_offset = payload.data_offset; + std::set serialized_keys; + for (int64_t key_length_64 : payload.key_lengths) { + const int32_t key_length = static_cast(key_length_64); + PAIMON_UNIQUE_PTR key_bytes = + Bytes::AllocateBytes(static_cast(key_length), pool_.get()); + if (key_length > 0) { + PAIMON_RETURN_NOT_OK(ReadBlobContentAt(key_offset, key_length, + reinterpret_cast(key_bytes->data()))); + } + std::string serialized_key; + if (key_length > 0) { + serialized_key.assign(key_bytes->data(), key_length); + } + if (!serialized_keys.emplace(std::move(serialized_key)).second) { + return Status::Invalid("invalid MAP<..., BLOB> payload: duplicate key"); + } + const uint8_t empty_key = 0; + const uint8_t* key_data = + key_length == 0 ? &empty_key : reinterpret_cast(key_bytes->data()); + PAIMON_RETURN_NOT_OK(AppendMapBlobKey(key_type, key_data, key_length, key_builder)); + key_offset += key_length; + } + return Status::OK(); +} + +Status BlobFileBatchReader::AppendMapBlobValues(const MapBlobPayload& payload, + arrow::LargeBinaryBuilder* blob_builder) const { + int64_t value_offset = payload.data_offset + payload.key_data_length; + for (int64_t value_length : payload.value_lengths) { + if (value_length == BlobDefs::kNullBinLength) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->AppendNull()); + continue; + } + if (blob_as_descriptor_) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr blob, + Blob::FromPath(file_path_, value_offset, value_length)); + PAIMON_UNIQUE_PTR descriptor = blob->ToDescriptor(pool_); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + blob_builder->Append(descriptor->data(), descriptor->size())); + } else { + PAIMON_UNIQUE_PTR value_bytes = + Bytes::AllocateBytes(static_cast(value_length), pool_.get()); + if (value_length > 0) { + PAIMON_RETURN_NOT_OK(ReadBlobContentAt( + value_offset, value_length, reinterpret_cast(value_bytes->data()))); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW( + blob_builder->Append(value_bytes->data(), value_length)); + } + value_offset += value_length; + } + return Status::OK(); +} + Result> BlobFileBatchReader::BuildMapBlobArray( int32_t rows_to_read) const { const auto& struct_type = static_cast(*target_type_); @@ -406,162 +597,12 @@ Result> BlobFileBatchReader::BuildMapBlobArray( PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->AppendNulls(2)); continue; } - if (target_blob_lengths_[row_index] < 0) { - return Status::Invalid(fmt::format("unsupported MAP<..., BLOB> record length: {}", - target_blob_lengths_[row_index])); - } - - const int64_t payload_offset = GetTargetContentOffset(row_index); - const int64_t payload_length = GetTargetContentLength(row_index); - if (payload_length < kMapBlobMinPayloadLength) { - return Status::Invalid( - fmt::format("invalid MAP<..., BLOB> payload length: {}", payload_length)); - } - - std::array header; - PAIMON_RETURN_NOT_OK(ReadBlobContentAt(payload_offset, header.size(), header.data())); - const auto magic_number = ReadLittleEndian(header.data()); - if (magic_number != kMapBlobMagicNumber) { - return Status::Invalid( - fmt::format("invalid MAP<..., BLOB> payload magic number: {}", magic_number)); - } - const auto version = static_cast(header[4]); - if (version != kMapBlobVersion) { - return Status::NotImplemented( - fmt::format("unsupported MAP<..., BLOB> payload version: {}", version)); - } - const auto entry_count = ReadLittleEndian(header.data() + 5); - if (entry_count < 0) { - return Status::Invalid( - fmt::format("invalid MAP<..., BLOB> entry count: {}", entry_count)); - } - - const int64_t index_lengths_offset = - payload_offset + payload_length - kMapBlobIndexLengthsSize; - std::array index_lengths; - PAIMON_RETURN_NOT_OK( - ReadBlobContentAt(index_lengths_offset, index_lengths.size(), index_lengths.data())); - const auto key_index_length = ReadLittleEndian(index_lengths.data()); - const auto value_index_length = - ReadLittleEndian(index_lengths.data() + sizeof(int32_t)); - const int64_t maximum_indexes_length = payload_length - kMapBlobMinPayloadLength; - if (key_index_length < 0 || key_index_length > maximum_indexes_length) { - return Status::Invalid( - fmt::format("invalid MAP<..., BLOB> key index length: {}", key_index_length)); - } - if (value_index_length < 0 || value_index_length > maximum_indexes_length) { - return Status::Invalid( - fmt::format("invalid MAP<..., BLOB> value index length: {}", value_index_length)); - } - if (static_cast(key_index_length) + value_index_length > maximum_indexes_length) { - return Status::Invalid("MAP<..., BLOB> indexes exceed the payload length"); - } - if (entry_count > key_index_length || entry_count > value_index_length) { - return Status::Invalid("MAP<..., BLOB> entry count exceeds index length"); - } - - const int64_t value_index_offset = index_lengths_offset - value_index_length; - const int64_t key_index_offset = value_index_offset - key_index_length; - std::vector key_index_bytes(key_index_length); - std::vector value_index_bytes(value_index_length); - PAIMON_RETURN_NOT_OK(ReadBlobContentAt(key_index_offset, key_index_length, - reinterpret_cast(key_index_bytes.data()))); - PAIMON_RETURN_NOT_OK( - ReadBlobContentAt(value_index_offset, value_index_length, - reinterpret_cast(value_index_bytes.data()))); - PAIMON_ASSIGN_OR_RAISE(std::vector key_lengths, - DeltaVarintCompressor::Decompress(key_index_bytes)); - PAIMON_ASSIGN_OR_RAISE(std::vector value_lengths, - DeltaVarintCompressor::Decompress(value_index_bytes)); - if (key_lengths.size() != static_cast(entry_count)) { - return Status::Invalid("MAP<..., BLOB> entry count does not match key index length"); - } - if (value_lengths.size() != static_cast(entry_count)) { - return Status::Invalid("MAP<..., BLOB> entry count does not match value index length"); - } - - const int64_t data_offset = payload_offset + kMapBlobHeaderLength; - const int64_t data_length = key_index_offset - data_offset; - int64_t key_data_length = 0; - for (int64_t key_length : key_lengths) { - if (key_length < 0) { - return Status::Invalid("MAP<..., BLOB> keys cannot be null"); - } - if (key_length > std::numeric_limits::max()) { - return Status::Invalid( - fmt::format("MAP<..., BLOB> key is too large: {}", key_length)); - } - if (fixed_key_length >= 0 && key_length != fixed_key_length) { - return Status::Invalid( - fmt::format("invalid MAP<..., BLOB> fixed-width key length: {}", key_length)); - } - if (key_length > data_length - key_data_length) { - return Status::Invalid("MAP<..., BLOB> key lengths exceed the payload data length"); - } - key_data_length += key_length; - } - - const int64_t maximum_value_data_length = data_length - key_data_length; - int64_t value_data_length = 0; - for (int64_t value_length : value_lengths) { - if (value_length == BlobDefs::kNullBinLength) { - continue; - } - if (value_length < 0) { - return Status::Invalid( - fmt::format("invalid MAP<..., BLOB> value length: {}", value_length)); - } - if (!blob_as_descriptor_ && value_length > std::numeric_limits::max()) { - return Status::Invalid( - fmt::format("MAP<..., BLOB> inline value is too large: {}", value_length)); - } - if (value_length > maximum_value_data_length - value_data_length) { - return Status::Invalid( - "MAP<..., BLOB> value lengths exceed the payload data length"); - } - value_data_length += value_length; - } - if (value_data_length != maximum_value_data_length) { - return Status::Invalid( - "MAP<..., BLOB> key/value lengths do not match the payload data length"); - } + PAIMON_ASSIGN_OR_RAISE(MapBlobPayload payload, + ReadMapBlobPayload(row_index, fixed_key_length)); PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.Append()); - int64_t key_offset = data_offset; - std::set serialized_keys; - for (int32_t entry = 0; entry < entry_count; ++entry) { - const auto key_length = static_cast(key_lengths[entry]); - std::vector key_bytes(key_length); - PAIMON_RETURN_NOT_OK(ReadBlobContentAt(key_offset, key_length, key_bytes.data())); - if (!serialized_keys.emplace(key_bytes.begin(), key_bytes.end()).second) { - return Status::Invalid("invalid MAP<..., BLOB> payload: duplicate key"); - } - PAIMON_RETURN_NOT_OK( - AppendMapBlobKey(key_type, key_bytes.data(), key_length, key_builder.get())); - key_offset += key_length; - } - - int64_t value_offset = data_offset + key_data_length; - for (int64_t value_length : value_lengths) { - if (value_length == BlobDefs::kNullBinLength) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->AppendNull()); - continue; - } - if (blob_as_descriptor_) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr blob, - Blob::FromPath(file_path_, value_offset, value_length)); - PAIMON_UNIQUE_PTR descriptor = blob->ToDescriptor(pool_); - PAIMON_RETURN_NOT_OK_FROM_ARROW( - blob_builder->Append(descriptor->data(), descriptor->size())); - } else { - std::vector value_bytes(static_cast(value_length)); - PAIMON_RETURN_NOT_OK( - ReadBlobContentAt(value_offset, value_length, value_bytes.data())); - PAIMON_RETURN_NOT_OK_FROM_ARROW( - blob_builder->Append(value_bytes.data(), value_length)); - } - value_offset += value_length; - } + PAIMON_RETURN_NOT_OK(AppendMapBlobKeys(payload, key_type, key_builder.get())); + PAIMON_RETURN_NOT_OK(AppendMapBlobValues(payload, blob_builder)); } std::shared_ptr built_map_array; diff --git a/src/paimon/format/blob/blob_file_batch_reader.h b/src/paimon/format/blob/blob_file_batch_reader.h index bf6a3f59..b1512f79 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.h +++ b/src/paimon/format/blob/blob_file_batch_reader.h @@ -36,6 +36,11 @@ #include "paimon/result.h" #include "paimon/utils/roaring_bitmap32.h" +namespace arrow { +class ArrayBuilder; +class LargeBinaryBuilder; +} // namespace arrow + namespace paimon::blob { /// Binary Blob File Layout Specification @@ -149,6 +154,13 @@ class BlobFileBatchReader : public FileBatchReader { } private: + struct MapBlobPayload { + std::vector key_lengths; + std::vector value_lengths; + int64_t data_offset; + int64_t key_data_length; + }; + static constexpr uint64_t kDefaultReadChunkSize = 1024 * 1024; static int32_t GetIndexLength(const int8_t* bytes, int32_t offset); @@ -167,6 +179,12 @@ class BlobFileBatchReader : public FileBatchReader { /// Builds a null bitmap buffer for the given rows. Returns nullptr if no nulls. Result> BuildNullBitmap(int32_t rows_to_read) const; Result> BuildContentArray(int32_t rows_to_read) const; + Result ReadMapBlobPayload(size_t row_index, int32_t fixed_key_length) const; + Status AppendMapBlobKeys(const MapBlobPayload& payload, + const std::shared_ptr& key_type, + arrow::ArrayBuilder* key_builder) const; + Status AppendMapBlobValues(const MapBlobPayload& payload, + arrow::LargeBinaryBuilder* blob_builder) const; Result> BuildMapBlobArray(int32_t rows_to_read) const; Result> BuildTargetArray(int32_t rows_to_read) const; diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index f514e9b6..0525e3ac 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -23,6 +23,7 @@ #include "arrow/api.h" #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_utils.h" @@ -148,17 +149,11 @@ class BlobFileBatchReaderTest : public testing::Test, public ::testing::WithPara } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr blob, Blob::FromDescriptor(stored_value.data(), stored_value.size())); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr input_stream, - blob->NewInputStream(file_system)); - PAIMON_ASSIGN_OR_RAISE(int64_t length, input_stream->Length()); - std::string value(length, '\0'); - if (length > 0) { - PAIMON_ASSIGN_OR_RAISE(int64_t actual_length, input_stream->Read(value.data(), length)); - if (actual_length != length) { - return Status::IOError("failed to read MAP<..., BLOB> descriptor content"); - } + PAIMON_ASSIGN_OR_RAISE(PAIMON_UNIQUE_PTR value, blob->ToData(file_system, pool_)); + if (value->size() == 0) { + return std::string(); } - return value; + return std::string(value->data(), value->size()); } private: @@ -192,12 +187,7 @@ TEST_P(BlobFileBatchReaderTest, TestMapBlob) { std::shared_ptr file_system = std::make_shared(); const std::string file_bytes = MapBlobGoldenBytes(); - ASSERT_OK_AND_ASSIGN(std::shared_ptr output_stream, - file_system->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int64_t written, - output_stream->Write(file_bytes.data(), file_bytes.size())); - ASSERT_EQ(file_bytes.size(), written); - ASSERT_OK(output_stream->Close()); + ASSERT_OK(file_system->WriteFile(file_path, file_bytes, /*overwrite=*/true)); std::shared_ptr blob_item = BlobUtils::ToArrowField("value", true); auto key_field = arrow::field("key", arrow::utf8(), false); @@ -224,34 +214,33 @@ TEST_P(BlobFileBatchReaderTest, TestMapBlob) { ASSERT_TRUE(struct_array); auto map_array = std::dynamic_pointer_cast(struct_array->field(0)); ASSERT_TRUE(map_array); - ASSERT_EQ(arrow::Type::LARGE_BINARY, map_array->map_type()->item_type()->id()); - ASSERT_EQ(4, map_array->length()); - ASSERT_EQ(3, map_array->value_length(0)); - ASSERT_TRUE(map_array->IsNull(1)); - ASSERT_EQ(0, map_array->value_length(2)); - ASSERT_EQ(1, map_array->value_length(3)); - - auto keys = std::dynamic_pointer_cast(map_array->keys()); auto values = std::dynamic_pointer_cast(map_array->items()); - ASSERT_TRUE(keys); ASSERT_TRUE(values); - ASSERT_EQ("alpha", keys->GetString(0)); - ASSERT_EQ("empty", keys->GetString(1)); - ASSERT_EQ("missing", keys->GetString(2)); - ASSERT_EQ("omega", keys->GetString(3)); - ASSERT_FALSE(values->IsNull(0)); - ASSERT_FALSE(values->IsNull(1)); - ASSERT_TRUE(values->IsNull(2)); - ASSERT_FALSE(values->IsNull(3)); - ASSERT_OK_AND_ASSIGN(std::string first_value, - ReadMapBlobValue(values, 0, blob_as_descriptor, file_system)); - ASSERT_OK_AND_ASSIGN(std::string empty_value, - ReadMapBlobValue(values, 1, blob_as_descriptor, file_system)); - ASSERT_OK_AND_ASSIGN(std::string last_value, - ReadMapBlobValue(values, 3, blob_as_descriptor, file_system)); - ASSERT_EQ("hello", first_value); - ASSERT_EQ("", empty_value); - ASSERT_EQ("world", last_value); + arrow::LargeBinaryBuilder normalized_values_builder; + for (int64_t i = 0; i < values->length(); ++i) { + if (values->IsNull(i)) { + ASSERT_TRUE(normalized_values_builder.AppendNull().ok()); + continue; + } + ASSERT_OK_AND_ASSIGN(std::string value, + ReadMapBlobValue(values, i, blob_as_descriptor, file_system)); + ASSERT_TRUE(normalized_values_builder.Append(value).ok()); + } + std::shared_ptr normalized_values; + ASSERT_TRUE(normalized_values_builder.Finish(&normalized_values).ok()); + auto normalized_map = std::make_shared( + map_type, map_array->length(), map_array->value_offsets(), map_array->keys(), + normalized_values, map_array->null_bitmap(), map_array->null_count(), map_array->offset()); + std::shared_ptr expected = arrow::ipc::internal::json::ArrayFromJSON(map_type, + R"json([ + [["alpha", "hello"], ["empty", ""], ["missing", null]], + null, + [], + [["omega", "world"]] + ])json") + .ValueOrDie(); + ASSERT_TRUE(expected->Equals(normalized_map)) + << "expected: " << expected->ToString() << "\nactual: " << normalized_map->ToString(); } TEST_P(BlobFileBatchReaderTest, MapBlobFallbackAcrossSequenceLayers) { @@ -262,12 +251,7 @@ TEST_P(BlobFileBatchReaderTest, MapBlobFallbackAcrossSequenceLayers) { const std::string new_file_path = dir->Str() + "/new-placeholder.blob"; const std::string old_bytes = MapBlobGoldenBytes(); - ASSERT_OK_AND_ASSIGN(std::shared_ptr old_output, - file_system->Create(old_file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int64_t old_written, - old_output->Write(old_bytes.data(), old_bytes.size())); - ASSERT_EQ(old_bytes.size(), old_written); - ASSERT_OK(old_output->Close()); + ASSERT_OK(file_system->WriteFile(old_file_path, old_bytes, /*overwrite=*/true)); // Generate four genuine -2 outer-file entries through the scalar writer. The outer blob // index is type-independent; the map reader turns them into its map placeholder sentinel. @@ -362,11 +346,7 @@ TEST_F(BlobFileBatchReaderTest, RejectsNonCanonicalDecimalMapKey) { const std::string file_bytes = HexToBytes( "cf114e584243424d010200000000000002020000020000000200000028000000" "0000000000000000d0000200000001"); - ASSERT_OK_AND_ASSIGN(std::shared_ptr output, - file_system->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int64_t written, output->Write(file_bytes.data(), file_bytes.size())); - ASSERT_EQ(file_bytes.size(), written); - ASSERT_OK(output->Close()); + ASSERT_OK(file_system->WriteFile(file_path, file_bytes, /*overwrite=*/true)); auto map_type = std::make_shared(arrow::field("key", arrow::decimal128(20, 0), false), @@ -392,11 +372,7 @@ TEST_F(BlobFileBatchReaderTest, RejectsInvalidUtf8StringMapKey) { ASSERT_GT(file_bytes.size(), 13); // The first key starts after the outer magic and the map header. file_bytes[13] = static_cast(0xFF); - ASSERT_OK_AND_ASSIGN(std::shared_ptr output, - file_system->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int64_t written, output->Write(file_bytes.data(), file_bytes.size())); - ASSERT_EQ(file_bytes.size(), written); - ASSERT_OK(output->Close()); + ASSERT_OK(file_system->WriteFile(file_path, file_bytes, /*overwrite=*/true)); auto map_type = arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", /*nullable=*/true)); auto schema = arrow::schema({arrow::field("blob_map", map_type)}); @@ -422,11 +398,7 @@ TEST_F(BlobFileBatchReaderTest, RejectsNullMapKey) { ASSERT_NE(std::string::npos, key_index_offset); // The first delta-varint changes from key length 5 to Java's null marker -1. file_bytes[key_index_offset] = static_cast(0x01); - ASSERT_OK_AND_ASSIGN(std::shared_ptr output, - file_system->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int64_t written, output->Write(file_bytes.data(), file_bytes.size())); - ASSERT_EQ(file_bytes.size(), written); - ASSERT_OK(output->Close()); + ASSERT_OK(file_system->WriteFile(file_path, file_bytes, /*overwrite=*/true)); auto map_type = arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", /*nullable=*/true)); auto schema = arrow::schema({arrow::field("blob_map", map_type)}); From 635b1d132ef5d9ee79f53eed7eaa28d8b27a8eac Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Fri, 4 Sep 2026 06:06:27 -0700 Subject: [PATCH 09/11] fix(blob): satisfy clang-tidy --- src/paimon/format/blob/blob_file_batch_reader.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index d2700c93..84c73ee9 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -397,17 +397,17 @@ Result BlobFileBatchReader::ReadMapBlobPayl std::array header; PAIMON_RETURN_NOT_OK(ReadBlobContentAt(payload_offset, header.size(), header.data())); - const int32_t magic_number = ReadLittleEndian(header.data()); + const auto magic_number = ReadLittleEndian(header.data()); if (magic_number != kMapBlobMagicNumber) { return Status::Invalid( fmt::format("invalid MAP<..., BLOB> payload magic number: {}", magic_number)); } - const int8_t version = static_cast(header[4]); + const auto version = static_cast(header[4]); if (version != kMapBlobVersion) { return Status::NotImplemented( fmt::format("unsupported MAP<..., BLOB> payload version: {}", version)); } - const int32_t entry_count = ReadLittleEndian(header.data() + 5); + const auto entry_count = ReadLittleEndian(header.data() + 5); if (entry_count < 0) { return Status::Invalid(fmt::format("invalid MAP<..., BLOB> entry count: {}", entry_count)); } @@ -416,8 +416,8 @@ Result BlobFileBatchReader::ReadMapBlobPayl std::array index_lengths; PAIMON_RETURN_NOT_OK( ReadBlobContentAt(index_lengths_offset, index_lengths.size(), index_lengths.data())); - const int32_t key_index_length = ReadLittleEndian(index_lengths.data()); - const int32_t value_index_length = + const auto key_index_length = ReadLittleEndian(index_lengths.data()); + const auto value_index_length = ReadLittleEndian(index_lengths.data() + sizeof(int32_t)); const int64_t maximum_indexes_length = payload_length - kMapBlobMinPayloadLength; if (key_index_length < 0 || key_index_length > maximum_indexes_length) { @@ -507,7 +507,7 @@ Status BlobFileBatchReader::AppendMapBlobKeys(const MapBlobPayload& payload, int64_t key_offset = payload.data_offset; std::set serialized_keys; for (int64_t key_length_64 : payload.key_lengths) { - const int32_t key_length = static_cast(key_length_64); + const auto key_length = static_cast(key_length_64); PAIMON_UNIQUE_PTR key_bytes = Bytes::AllocateBytes(static_cast(key_length), pool_.get()); if (key_length > 0) { From d287f03f460ced67f7ed57ed714093c2bc6c09cc Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Fri, 4 Sep 2026 07:07:22 -0700 Subject: [PATCH 10/11] test: expand map blob read coverage --- .../blob/blob_file_batch_reader_test.cpp | 68 ++++++++ test/inte/blob_table_inte_test.cpp | 151 ++++++++++++++++++ .../map_blob_java.db/map_blob_java/README.md | 26 +++ ...a677b74-2f94-4d34-9166-82aaedb2b84b-0.blob | Bin 0 -> 9 bytes ...8d56-71bd-453f-ab1f-4312f3bf480e-0.parquet | Bin 0 -> 309 bytes ...be78d56-71bd-453f-ab1f-4312f3bf480e-1.blob | Bin 0 -> 161 bytes ...e78d56-71bd-453f-ab1f-4312f3bf480e-10.blob | Bin 0 -> 94 bytes ...be78d56-71bd-453f-ab1f-4312f3bf480e-2.blob | Bin 0 -> 67 bytes ...be78d56-71bd-453f-ab1f-4312f3bf480e-3.blob | Bin 0 -> 82 bytes ...be78d56-71bd-453f-ab1f-4312f3bf480e-4.blob | Bin 0 -> 88 bytes ...be78d56-71bd-453f-ab1f-4312f3bf480e-5.blob | Bin 0 -> 88 bytes ...be78d56-71bd-453f-ab1f-4312f3bf480e-6.blob | Bin 0 -> 100 bytes ...be78d56-71bd-453f-ab1f-4312f3bf480e-7.blob | Bin 0 -> 79 bytes ...be78d56-71bd-453f-ab1f-4312f3bf480e-8.blob | Bin 0 -> 76 bytes ...be78d56-71bd-453f-ab1f-4312f3bf480e-9.blob | Bin 0 -> 96 bytes ...5380318-987b-4af0-9073-9c8e8162b568-0.blob | Bin 0 -> 9 bytes ...est-0e14efd4-f0a2-46cf-99ba-993506575ea4-0 | Bin 0 -> 2373 bytes ...est-1856e00c-bc22-481b-a8ac-c6304d6307fb-0 | Bin 0 -> 2188 bytes ...est-cbd224f1-d718-466b-89c4-f6ee7b673e2f-0 | Bin 0 -> 2194 bytes ...ist-437bd3e4-24fc-40e8-9695-532ce23df015-0 | Bin 0 -> 1006 bytes ...ist-437bd3e4-24fc-40e8-9695-532ce23df015-1 | Bin 0 -> 1109 bytes ...ist-7475337a-832d-4e57-969d-c0cedd1b28be-0 | Bin 0 -> 1109 bytes ...ist-7475337a-832d-4e57-969d-c0cedd1b28be-1 | Bin 0 -> 1111 bytes ...ist-da204778-ac4b-4d76-af5e-bd55d0287b14-0 | Bin 0 -> 1140 bytes ...ist-da204778-ac4b-4d76-af5e-bd55d0287b14-1 | Bin 0 -> 1111 bytes .../map_blob_java/schema/schema-0 | 99 ++++++++++++ .../map_blob_java/snapshot/EARLIEST | 1 + .../map_blob_java/snapshot/LATEST | 1 + .../map_blob_java/snapshot/snapshot-1 | 18 +++ .../map_blob_java/snapshot/snapshot-2 | 18 +++ .../map_blob_java/snapshot/snapshot-3 | 18 +++ 31 files changed, 400 insertions(+) create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/README.md create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1a677b74-2f94-4d34-9166-82aaedb2b84b-0.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-0.parquet create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-1.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-10.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-2.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-3.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-4.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-5.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-6.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-7.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-8.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-9.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-d5380318-987b-4af0-9073-9c8e8162b568-0.blob create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-0e14efd4-f0a2-46cf-99ba-993506575ea4-0 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-1856e00c-bc22-481b-a8ac-c6304d6307fb-0 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-cbd224f1-d718-466b-89c4-f6ee7b673e2f-0 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-437bd3e4-24fc-40e8-9695-532ce23df015-0 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-437bd3e4-24fc-40e8-9695-532ce23df015-1 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-7475337a-832d-4e57-969d-c0cedd1b28be-0 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-7475337a-832d-4e57-969d-c0cedd1b28be-1 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-da204778-ac4b-4d76-af5e-bd55d0287b14-0 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-da204778-ac4b-4d76-af5e-bd55d0287b14-1 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/schema/schema-0 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/snapshot/EARLIEST create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/snapshot/LATEST create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/snapshot/snapshot-2 create mode 100644 test/test_data/parquet/map_blob_java.db/map_blob_java/snapshot/snapshot-3 diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 0525e3ac..012c051e 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -18,6 +18,7 @@ #include "paimon/format/blob/blob_file_batch_reader.h" +#include #include #include "arrow/api.h" @@ -156,6 +157,28 @@ class BlobFileBatchReaderTest : public testing::Test, public ::testing::WithPara return std::string(value->data(), value->size()); } + void CheckMapBlobReadFails(const std::string& file_bytes, + const std::shared_ptr& key_type, + const std::string& expected_message) { + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string file_path = dir->Str() + "/corrupt-map.blob"; + std::shared_ptr file_system = std::make_shared(); + ASSERT_OK(file_system->WriteFile(file_path, file_bytes, /*overwrite=*/true)); + + auto map_type = arrow::map(key_type, BlobUtils::ToArrowField("value", /*nullable=*/true)); + auto schema = arrow::schema({arrow::field("blob_map", map_type)}); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, file_system->Open(file_path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create( + input, /*batch_size=*/1, /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_, GetArrowPool(pool_))); + ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), expected_message); + } + private: std::string blob_field_name_; std::shared_ptr pool_; @@ -413,6 +436,51 @@ TEST_F(BlobFileBatchReaderTest, RejectsNullMapKey) { ASSERT_NOK_WITH_MSG(reader->NextBatch(), "MAP<..., BLOB> keys cannot be null"); } +TEST_F(BlobFileBatchReaderTest, RejectsCorruptMapPayloadMetadata) { + // The Java golden's first bin starts with the outer BLOB magic (4 bytes), followed by a + // 45-byte MAP payload: header [4, 13), key/value data [13, 35), indexes [35, 41), and the + // two index lengths [41, 49). Mutating the payload does not require updating the outer CRC, + // which BlobFileBatchReader intentionally does not validate. + const std::string golden = MapBlobGoldenBytes(); + + std::string corrupted = golden; + corrupted[4] = 0; + CheckMapBlobReadFails(corrupted, arrow::utf8(), "invalid MAP<..., BLOB> payload magic number"); + + corrupted = golden; + corrupted[8] = 2; + CheckMapBlobReadFails(corrupted, arrow::utf8(), "unsupported MAP<..., BLOB> payload version"); + + corrupted = golden; + std::fill(corrupted.begin() + 9, corrupted.begin() + 13, static_cast(0xFF)); + CheckMapBlobReadFails(corrupted, arrow::utf8(), "invalid MAP<..., BLOB> entry count"); + + corrupted = golden; + corrupted[41] = static_cast(0xFF); + CheckMapBlobReadFails(corrupted, arrow::utf8(), "invalid MAP<..., BLOB> key index length"); + + corrupted = golden; + corrupted[9] = 2; + CheckMapBlobReadFails(corrupted, arrow::utf8(), "entry count does not match key index length"); + + CheckMapBlobReadFails(golden, arrow::int32(), "invalid MAP<..., BLOB> fixed-width key length"); + + corrupted = golden; + corrupted[38] = 0x0C; + corrupted[39] = 0x0B; + CheckMapBlobReadFails(corrupted, arrow::utf8(), "value lengths exceed the payload data length"); + + corrupted = golden; + corrupted[38] = 0x08; + corrupted[39] = 0x07; + CheckMapBlobReadFails(corrupted, arrow::utf8(), + "key/value lengths do not match the payload data length"); + + corrupted = golden; + std::copy_n(corrupted.begin() + 13, 5, corrupted.begin() + 18); + CheckMapBlobReadFails(corrupted, arrow::utf8(), "payload: duplicate key"); +} + TEST_P(BlobFileBatchReaderTest, TestPushdownBitmap) { std::string test_data_path = paimon::test::GetDataDir() + "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/db_with_blob.db/table_with_blob/"; auto dir = paimon::test::UniqueTestDirectory::Create(); diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 4b76cef1..4419b356 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -535,6 +535,50 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter }); } + Result> NormalizeMapBlobValues( + const std::shared_ptr& map_array, bool blob_as_descriptor) const { + const auto& values = checked_cast(*map_array->items()); + auto fs = std::make_shared(); + arrow::LargeBinaryBuilder builder; + for (int64_t i = 0; i < values.length(); ++i) { + if (values.IsNull(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + continue; + } + std::string_view stored = values.GetView(i); + if (!blob_as_descriptor) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(stored)); + continue; + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr blob, + Blob::FromDescriptor(stored.data(), static_cast(stored.size()))); + PAIMON_ASSIGN_OR_RAISE(PAIMON_UNIQUE_PTR data, blob->ToData(fs, pool_)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(data->data(), data->size())); + } + std::shared_ptr normalized_values; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&normalized_values)); + return std::make_shared(map_array->type(), map_array->length(), + map_array->value_offsets(), map_array->keys(), + normalized_values, map_array->null_bitmap(), + map_array->null_count(), map_array->offset()); + } + + void CheckMapBlobColumn(const std::shared_ptr& rows, + const std::string& field_name, const std::string& expected_json, + bool blob_as_descriptor) const { + auto map_array = + std::dynamic_pointer_cast(rows->GetFieldByName(field_name)); + ASSERT_TRUE(map_array) << field_name; + ASSERT_OK_AND_ASSIGN(auto normalized, + NormalizeMapBlobValues(map_array, blob_as_descriptor)); + auto expected = arrow::ipc::internal::json::ArrayFromJSON(map_array->type(), expected_json) + .ValueOrDie(); + ASSERT_TRUE(expected->Equals(normalized)) + << field_name << " expected: " << expected->ToString() + << " actual: " << normalized->ToString(); + } + /// Verify DataFileMeta properties from a scan plan. /// Each vector element corresponds to one expected DataFileMeta (ordered by file index). static void VerifyDataFileMetas( @@ -4334,4 +4378,111 @@ TEST_P(BlobTableInteTest, TestReadBlobDescriptorFieldFromJava) { ASSERT_TRUE(resolved->Equals(expected_with_rk)); } +TEST_P(BlobTableInteTest, TestReadMapBlobTableFromJava) { + if (GetParam() != "parquet") { + GTEST_SKIP() << "the Java fixture uses Parquet"; + } + const std::string table_path = GetDataDir() + "/parquet/map_blob_java.db/map_blob_java"; + const std::vector read_fields = {"id", + "string_payloads", + "boolean_payloads", + "tinyint_payloads", + "smallint_payloads", + "int_payloads", + "bigint_payloads", + "date_payloads", + "binary_payloads", + "compact_decimal_payloads", + "large_decimal_payloads"}; + + for (int64_t snapshot_id : {1, 3}) { + ScanContextBuilder scan_builder(table_path); + scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); + + size_t string_layer_count = 0; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + for (const auto& file : data_split->DataFiles()) { + if (file->write_cols == + std::optional>({"string_payloads"})) { + ++string_layer_count; + } + } + } + ASSERT_EQ(snapshot_id == 1 ? 1 : 3, string_layer_count); + + for (bool blob_as_descriptor : {false, true}) { + std::map read_options = { + {Options::BLOB_AS_DESCRIPTOR, blob_as_descriptor ? "true" : "false"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, read_fields, plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result); + auto combined = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto rows = std::dynamic_pointer_cast(combined); + ASSERT_TRUE(rows); + ASSERT_EQ(4, rows->length()); + const auto& ids = checked_cast(*rows->GetFieldByName("id")); + for (int64_t i = 0; i < ids.length(); ++i) { + ASSERT_EQ(i + 1, ids.Value(i)); + } + + CheckMapBlobColumn( + rows, "string_payloads", + R"json([[["", "string-empty"], ["alpha", "string-alpha"]], [], null, [["omega", "string-omega"]]])json", + blob_as_descriptor); + CheckMapBlobColumn( + rows, "boolean_payloads", + R"json([[[false, "bool-false"], [true, "bool-true"]], null, null, null])json", + blob_as_descriptor); + CheckMapBlobColumn( + rows, "tinyint_payloads", + R"json([[[-128, "tiny-min"], [-1, "tiny-negative"], [127, "tiny-max"]], null, null, null])json", + blob_as_descriptor); + CheckMapBlobColumn( + rows, "smallint_payloads", + R"json([[[-32768, "small-min"], [-1, "small-negative"], [32767, "small-max"]], null, null, null])json", + blob_as_descriptor); + CheckMapBlobColumn( + rows, "int_payloads", + R"json([[[-2147483648, "int-min"], [-1, "int-negative"], [2147483647, "int-max"]], null, null, null])json", + blob_as_descriptor); + CheckMapBlobColumn( + rows, "bigint_payloads", + R"json([[[-9223372036854775808, "big-min"], [-1, "big-negative"], [9223372036854775807, "big-max"]], null, null, null])json", + blob_as_descriptor); + CheckMapBlobColumn( + rows, "date_payloads", + R"json([[[-1, "date-negative"], [0, "date-epoch"]], null, null, null])json", + blob_as_descriptor); + CheckMapBlobColumn( + rows, "compact_decimal_payloads", + R"json([[["-99999999.99", "compact-negative"], ["99999999.99", "compact-positive"]], null, null, null])json", + blob_as_descriptor); + CheckMapBlobColumn( + rows, "large_decimal_payloads", + R"json([[["-999999999999999999.99", "large-negative"], ["999999999999999999.99", "large-positive"]], null, null, null])json", + blob_as_descriptor); + + auto binary_map = + std::dynamic_pointer_cast(rows->GetFieldByName("binary_payloads")); + ASSERT_TRUE(binary_map); + ASSERT_OK_AND_ASSIGN(auto normalized_binary, + NormalizeMapBlobValues(binary_map, blob_as_descriptor)); + const auto& binary_keys = + checked_cast(*normalized_binary->keys()); + const auto& binary_values = + checked_cast(*normalized_binary->items()); + ASSERT_EQ(2, normalized_binary->value_length(0)); + ASSERT_EQ("", binary_keys.GetString(0)); + ASSERT_EQ(std::string("\0\xff\1\2", 4), binary_keys.GetString(1)); + ASSERT_EQ("binary-empty", binary_values.GetString(0)); + ASSERT_EQ("binary-bytes", binary_values.GetString(1)); + } + } +} + } // namespace paimon::test diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/README.md b/test/test_data/parquet/map_blob_java.db/map_blob_java/README.md new file mode 100644 index 00000000..c1f752fd --- /dev/null +++ b/test/test_data/parquet/map_blob_java.db/map_blob_java/README.md @@ -0,0 +1,26 @@ + + +# Java MAP<K, BLOB> fixture + +Generated by Apache Paimon Java at commit `a176eba1c6f9b0402eceea641bf435b05976470b`. + +Snapshot 1 contains a full write. Snapshots 2 and 3 each add a partial write containing +`BlobMapPlaceholder` values for `string_payloads`, so snapshot 3 exercises fallback across three +sequence layers. The remaining map columns cover BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, DATE, +BINARY, compact DECIMAL, and large DECIMAL keys, including negative and boundary values. diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1a677b74-2f94-4d34-9166-82aaedb2b84b-0.blob b/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1a677b74-2f94-4d34-9166-82aaedb2b84b-0.blob new file mode 100644 index 0000000000000000000000000000000000000000..4dc6a6a31274075eef6eecde238fc4d3d7c2cc9a GIT binary patch literal 9 OcmZQ(U|?VYVnzS}IRFU& literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-0.parquet b/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..830f7d8587f0f396e2d38f72b283bee7e7bfb299 GIT binary patch literal 309 zcmX9)O-sW-5S^~2EL0Jpvvgq(xfmkUm`yaP(2L-qcq)PyFOqDw7%*+!S`YpkZyvq$ zKX~&;=-&{WG>7?kZ+G9D+1b^tM!*qX@pgzTo0sIG;~i+15iqurp~9xzM1b1rw6_=-@!SVF>h&`&_sI&407X-Sfu2oYu~J zwQgN47oHAv8b)4FY^tZyJC9nzi`vBlgPlj*|FQ+*v=ojJS?BvVH#K6Fim{X9ESX@$+nU}7cnp;p(31x%D8Mq`EK#D-v35Y=;lylpG z^Dq-2LJ(3B%4trBl!Gf^1PbQorl!NJ0JFIyK*Atw0@affxK6Nz(a|}J6(j)w(ljA* literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-10.blob b/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-10.blob new file mode 100644 index 0000000000000000000000000000000000000000..e936ae19228ec2ddbdfbb87d3bc07b80c8f58195 GIT binary patch literal 94 zcmX>v=ojJS?BvVH#K6GtYl_9fOLIOjFtTPJkJC-&|NlQHu_!%NH!n3ku_UuB6~ZgX iFU|zGe&rhKW@GREbM3=C{Q9wPwKlNt^H literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-2.blob b/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-2.blob new file mode 100644 index 0000000000000000000000000000000000000000..71e7312c1d836562c1f6462e39441015fa4e9dec GIT binary patch literal 67 zcmX>v=ojJS?BvVH!~g`0N%{FXx@n0y#i?LcNl|Gk6N3m?5~RQiqzDM&jQ97Jlrk`| H0NIQH4a5yR literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-3.blob b/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-3.blob new file mode 100644 index 0000000000000000000000000000000000000000..3a4a5bb91b3d68a01b92f1f5044789d415d44a92 GIT binary patch literal 82 zcmX>v=ojJS?BvVH%)r3V@V~w!Gp|xNH!}}R=cT46mSmQtLIe{lm>3uYxHv&7LD(IL ULEwbd{PsS^enticHXx4?0P8Lk4FCWD literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-4.blob b/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-4.blob new file mode 100644 index 0000000000000000000000000000000000000000..d4a9c49694c7ad4e2534b0acefd91017c65c910c GIT binary patch literal 88 zcmX>v=ojJS?BvVH%m4%p|NsB5FV0QO$v=ojJS?BvVH%m4-r{~@40Gp|H9H!}}J<)x-4mSmQtf`t+*I2aiCxHv)TLD(0f UkHPlU-ygFWXEQP|umMFF0Yw`bvH$=8 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-6.blob b/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-6.blob new file mode 100644 index 0000000000000000000000000000000000000000..5cee202c9f42c68f24a68b4f4ac48df87dcd5f80 GIT binary patch literal 100 zcmX>v=ojJS?BvVH%m4=s|4~7GQf9htZe|{c%1cd8EXgcO1q&rs2rw}4adCq5f^a0% U2$hZUKQ=LLW@KPs1Bx&L0H*{Xa{vGU literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-7.blob b/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-7.blob new file mode 100644 index 0000000000000000000000000000000000000000..d0c44d4f5d82aed442dae91684de7b883b793adb GIT binary patch literal 79 zcmX>v=ojJS?BvVH#K6Gt9|(XHLrP*vs%~CtdSXdtSt^*5T9BWd!NDNK3Q`8bE+BP4 TaJZ^}LI-0fBLf2)kjDrBqr($- literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-8.blob b/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-1be78d56-71bd-453f-ab1f-4312f3bf480e-8.blob new file mode 100644 index 0000000000000000000000000000000000000000..d101f8d0ce02bb3e180961fa185cf67bced1f387 GIT binary patch literal 76 zcmX>v=ojJS?BvVH!~g{U8JUtY^Ad|HbyIT-N-Cl3q{@=iVg?Qg29O#Mb_A&fg4Bbz Pg_;v=ojJS?BvVH#K6G7DD!3Y-~T}Hm^%c-PtMORNK7u#%}Y&BEXgcOh4BmWi!(ud e0R{yIkU9_!1!54`;rgy|72|3~1_m}Dj}ZVcBOBNN literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-d5380318-987b-4af0-9073-9c8e8162b568-0.blob b/test/test_data/parquet/map_blob_java.db/map_blob_java/bucket-0/data-d5380318-987b-4af0-9073-9c8e8162b568-0.blob new file mode 100644 index 0000000000000000000000000000000000000000..4dc6a6a31274075eef6eecde238fc4d3d7c2cc9a GIT binary patch literal 9 OcmZQ(U|?VYVnzS}IRFU& literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-0e14efd4-f0a2-46cf-99ba-993506575ea4-0 b/test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-0e14efd4-f0a2-46cf-99ba-993506575ea4-0 new file mode 100644 index 0000000000000000000000000000000000000000..a51f68820c3a0e12e73dbe1ff784610de6610974 GIT binary patch literal 2373 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ>MRLzoNCAS47GBwJv38(3<^){21Z-{kz1)MTM5U~j? z=Y>}X4Hl&IoXpeW4Ozp(EVsuw z37u#?xyEnS-S_*2c1!xETfcwMKI!J)fA{MJCA4Jp6u$j`w}1DU>H7aU_vHft!x^_; zE>21BImeH-cdcpDIo_<4zBeu1G3ZIc{*$&U9=;prKR$IK>fSjPU4?|jA~tMimdtx^ z=^N9LndacNsi2c%>(6%^u5u}LWKFu!WRWx_$cm-x;EEaRXWx(voS~)|%(|9|%%IsZynkRTR@$L8Mf$Op{wOWHO2KXrpbsOefQH$~?kkT4+lU zRHWdhROn7zxlqKFkA)yrtUD1TuB3>oZuAdWt=^eQW~P}Yso=(4l6$^$zwg|0zNE&d zcD1zgGrATj3X`J3ZyuY+jM;*S^<%myXnGC{V=Axsz84F;VE(A-`3R3bPl}NOFDaTD zvB>hGD(bu;n&lIcDCde;|LA;`2@V6AAyO1NtlTB32388YfCNFs>Yt`@nkCpR1mm-Y zSlmJ}$VC%?txv+HSe%45im4A9B1qtzHqi#c3VNH7%@auBNtX;O8oH$Bw=`{`7~-l2 zC2KVpIyC~LDUR~Y=EEcj3MRl6r*X`$PZG~GgCGXTusG|=vm*hZ$?FtuqlMYsFivv7 zC9cz8a}>pqq%DU_Hj1g!vWy`=RqVyWI(Q3k($EI|sw@-=f1ZYp`l>Yxxm@mhG%N(Gh$jw}G5YVp2{4PWm+c}zv#z7K? zgeUkuMiLAI(!H^EZGa3*<5mb=GdM3*V*)3B-lvYz05=y4WC>4FtA25)tiHm?K6y#t zW!B!8hNOr^gI5YYiX_Vt+D1NKc{W@OkbniyALi<16?4{F(?bD}{IM8_Sp!3~F+s+W z#$mC8->4IN=cG6~>TVsU+*G(xu_qfSg`_KcmBG=hp8Sp2(R%`#LTfsWvvL1yl?B>} zVThm^)>7Hyt7>n}B_5#(^hp1HFqhW}$X`fj$HF%@+FIFC$ol^kw45lk%^^FgIbP2N zc-+QUPrSiqS4`#}tx$Yl*;m!l`?jt{6mA+iMIsS0$3{8#m5sD){v-vOyoaE!+q|-R!*C&A{8$log&#)-#%CjQ@c$3#D+(rwtyD^mD zfJ)na8GdsJBmv?F8PQ(kcD_r7&NP0Vg;>MxdLy#0a2WkNI5)Ew*zs6N!NY z%Hy8k`xpr@1_<}Y>a_uqER9+rbj{$rRFw&w_<5f?LIc!XEIf;Ql3MYLLuK_9g7(Qt z89`?4eQ8LFSTF=7->*orEWvH$^Oa}AVSoTE;Qe8)Tvj$`wKY8y0LqVH0J8?>;l>0> zM;eF44t~8(?43h#bi~~{PPwUYy<$%`UQ+x z*}%i_k>&;JTCijHkEm^`|m5COXt^4m!5Af z|9reR@G8~bzOLMTz3?&h=}xTm*v8C_zCA+&J&Q50@8Xx2T50h9Xz2F3+PxWFdfV0Z z4A(E;Y(4%cF!W#zi+3*HI=3=!Tz|?g^oy%+mcJeBS_%eR0s}2ylV{onuXJH@@BIGW Irs~fA0CM8c8vpM;$A^dR=@#~sWBD}JOm;q?Y6%g2~>ZQrV$vJbLpk^tHFqJGLboylrq!BRD;#XgvK z6qG?5L~KZ>i2xnNEQ7K>EVp}(Oh^p@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhJL|SCtsy99^id$>Cjsj>dv%+QDdwAUj~m@CWa;#0r3f|dH)~qzUZ@E)YzEQTSUkH z+ykwQmqCqp^S0l9yM6g`Zs(Tir(XqdPhD6wdqteK_WFo7r@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG z27&Dps;?{kF)W|^OUdlriyo#Gj2c_@|1x;QGBGr{2#8Nu&HMj=_eG!WqQ=IY-Xc2o z=N@QfybNl*o45V;+wIGjb33<8Km96zd+Nff*(>6-wbw_yIla`9^H{T7-m#l(j7&@n M2ObzRh@e{o0Oa~|-2eap literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-7475337a-832d-4e57-969d-c0cedd1b28be-1 b/test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-7475337a-832d-4e57-969d-c0cedd1b28be-1 new file mode 100644 index 0000000000000000000000000000000000000000..2eb6a32c9c3d0b0347ccc3eb49a5588c0415f2b3 GIT binary patch literal 1111 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhQ4bX0xvA`_MXD^@>mUw5hi literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-da204778-ac4b-4d76-af5e-bd55d0287b14-0 b/test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-da204778-ac4b-4d76-af5e-bd55d0287b14-0 new file mode 100644 index 0000000000000000000000000000000000000000..5ce489a3bff635f77d4ed6c28fe483bfad74e442 GIT binary patch literal 1140 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhNyzxq`1EGr`eYGhWPJ$Qp56$QDdwAUj~m*W`-g+G3g1*bNTO2=Ux|=lKc72-0lgU z`ldJf%gvsrzfE!7aA0$r!X*_!&H3SPX9oT(o-tQ2da_YNQTV$ZkC~WR?Y0X~5)qDi sefnBe^4a-EzxhNh&#Ym(xV9jI`5*@?1Lua8gB^F&4m>bs5J7hZ0NJ&HvH$=8 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-da204778-ac4b-4d76-af5e-bd55d0287b14-1 b/test/test_data/parquet/map_blob_java.db/map_blob_java/manifest/manifest-list-da204778-ac4b-4d76-af5e-bd55d0287b14-1 new file mode 100644 index 0000000000000000000000000000000000000000..f6479b2b8df2c91590117cc14f423cf16ee06dd0 GIT binary patch literal 1111 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhSHA#8j8CV?7kacPTG51$%knTqsCVKzYHF^Obks&1SA(&vj4Y#HsessiFeDi;_^TK z(>uBM)?Z-{#)Gx@zt{e@v)gxpXNPC!)ZCTN78S7`p0;wr!mk2ry Date: Sun, 6 Sep 2026 18:57:56 -0700 Subject: [PATCH 11/11] fix: clarify map blob reader validation --- .../format/blob/blob_file_batch_reader.cpp | 3 +- .../blob/blob_file_batch_reader_test.cpp | 2 +- .../map_blob_java.db/map_blob_java/README.md | 64 +++++++++++++------ 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index 84c73ee9..acbf152e 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -259,7 +259,8 @@ Status BlobFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, } std::shared_ptr read_field = arrow_schema->field(0); if (!BlobUtils::IsBlobField(read_field) && !BlobUtils::IsMapBlobField(read_field)) { - return Status::Invalid(fmt::format("field {} is not BLOB", read_field->ToString())); + return Status::Invalid( + fmt::format("field {} must be BLOB or MAP<..., BLOB>", read_field->ToString())); } if (BlobUtils::IsMapBlobField(read_field)) { const auto& map_type = static_cast(*read_field->type()); diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 012c051e..beefdd8f 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -740,7 +740,7 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { GetArrowPool(pool_))); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), - "field my_blob_field: large_binary is not BLOB"); + "field my_blob_field: large_binary must be BLOB or MAP<..., BLOB>"); } { auto schema = arrow::schema({BlobUtils::ToArrowField("my_blob_field", false)}); diff --git a/test/test_data/parquet/map_blob_java.db/map_blob_java/README.md b/test/test_data/parquet/map_blob_java.db/map_blob_java/README.md index c1f752fd..402a48bb 100644 --- a/test/test_data/parquet/map_blob_java.db/map_blob_java/README.md +++ b/test/test_data/parquet/map_blob_java.db/map_blob_java/README.md @@ -1,26 +1,48 @@ - +Options: +bucket = -1 +data-evolution.enabled = true +file.format = parquet +row-tracking.enabled = true -# Java MAP<K, BLOB> fixture +Msgs: +snapshot-1 +Commit four rows with BatchTableWrite over the full row type. BLOB values below are UTF-8 byte +payloads shown as text: +id 1: + string_payloads: {"": "string-empty", "alpha": "string-alpha"} + boolean_payloads: {false: "bool-false", true: "bool-true"} + tinyint_payloads: {-128: "tiny-min", -1: "tiny-negative", 127: "tiny-max"} + smallint_payloads: {-32768: "small-min", -1: "small-negative", 32767: "small-max"} + int_payloads: {-2147483648: "int-min", -1: "int-negative", 2147483647: "int-max"} + bigint_payloads: {-9223372036854775808: "big-min", -1: "big-negative", 9223372036854775807: "big-max"} + date_payloads: {-1: "date-negative", 0: "date-epoch"} + binary_payloads: {empty bytes: "binary-empty", 00 ff 01 02: "binary-bytes"} + compact_decimal_payloads: {-99999999.99: "compact-negative", 99999999.99: "compact-positive"} + large_decimal_payloads: {-999999999999999999.99: "large-negative", 999999999999999999.99: "large-positive"} +id 2: string_payloads is empty; all other map columns are null +id 3: all map columns are null +id 4: string_payloads is {"omega": "string-omega"}; all other map columns are null -Generated by Apache Paimon Java at commit `a176eba1c6f9b0402eceea641bf435b05976470b`. +snapshot-2 +Create BatchTableWrite with the write type projected to string_payloads and write +BlobMapPlaceholder.INSTANCE for the four row positions. Set the first row id to 0 before +committing. This adds a second sequence layer without changing the logical values. -Snapshot 1 contains a full write. Snapshots 2 and 3 each add a partial write containing -`BlobMapPlaceholder` values for `string_payloads`, so snapshot 3 exercises fallback across three -sequence layers. The remaining map columns cover BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, DATE, -BINARY, compact DECIMAL, and large DECIMAL keys, including negative and boundary values. +snapshot-3 +Repeat the snapshot-2 write to add a third sequence layer. Reading this snapshot exercises +fallback through both newer placeholder layers to the values from snapshot-1.