From c47942795b8fb076893439c3c1d480f59d551238 Mon Sep 17 00:00:00 2001 From: Ran Trifon Date: Tue, 22 Sep 2026 20:32:02 +0300 Subject: [PATCH] Fix stale unlink helpers clearing newer spill summaries --- src/cpp/src/key_directory.cpp | 130 +++++++++++++------------ src/cpp/src/key_directory.hpp | 1 + tests/cpp/directory_schedule_tests.cpp | 99 +++++++++++++++++++ 3 files changed, 166 insertions(+), 64 deletions(-) diff --git a/src/cpp/src/key_directory.cpp b/src/cpp/src/key_directory.cpp index b1c3e1f..dcbc3ef 100644 --- a/src/cpp/src/key_directory.cpp +++ b/src/cpp/src/key_directory.cpp @@ -1895,83 +1895,85 @@ sms_status KeyDirectory::refresh_spill_after_unlink( std::int32_t canonical_bucket, std::uint64_t removed_binding, const OperationBudget& budget) noexcept { - std::uint64_t witness{}; - std::int64_t witness_index{-1}; - auto status = find_overflow_witness( - canonical_bucket, budget, witness, witness_index); - if (status != SMS_STATUS_SUCCESS) return status; - if (witness == 0) { - sms::test_detail::reach_checkpoint( - sms::test_detail::CheckpointId::DirectoryAfterEmptySpillSummaryScan); - } - checkpoint( - DirectoryCheckpoint::after_empty_overflow_scan, - removed_binding, - witness); - - std::uint64_t desired{}; - const auto encoded = witness == 0 - ? SpillSummary::try_encode_empty(removed_binding, desired) - : SpillSummary::try_encode_present(witness, desired); - if (!encoded) return SMS_STATUS_CORRUPT_STORE; auto* const word = spill_word(canonical_bucket); + auto* const mutation = mutation_word(canonical_bucket); + if (word == nullptr || mutation == nullptr) return SMS_STATUS_CORRUPT_STORE; for (std::int32_t attempt = 0;; ++attempt) { const auto bound = budget.check_periodic(attempt); if (bound != SMS_STATUS_SUCCESS) return bound; - if (witness != 0 && - (witness_index < 0 || - MappedAtomic64::load_acquire(*overflow_word(witness_index)) != - witness)) { - status = find_overflow_witness( - canonical_bucket, budget, witness, witness_index); - if (status != SMS_STATUS_SUCCESS) return status; - if (!(witness == 0 - ? SpillSummary::try_encode_empty(removed_binding, desired) - : SpillSummary::try_encode_present(witness, desired))) { - return SMS_STATUS_CORRUPT_STORE; - } - continue; + if (MappedAtomic64::load_acquire(*mutation) != removed_binding) { + return SMS_STATUS_SUCCESS; } - auto observed = MappedAtomic64::load_acquire(*word); + + // Pair each scan with the exact summary version it observed. A helper + // can finish this unlink and publish another overflow value while this + // helper is paused; its newer summary must never be cleared using our + // earlier empty scan. + const auto captured = MappedAtomic64::load_acquire(*word); SpillSummary summary{}; - if (!decode_summary_semantic(observed, summary)) { - if (MappedAtomic64::load_acquire(*word) == observed) { + if (!decode_summary_semantic(captured, summary)) { + if (MappedAtomic64::load_acquire(*word) == captured) { return SMS_STATUS_CORRUPT_STORE; } continue; } - auto expected = observed; - if (MappedAtomic64::compare_exchange(*word, expected, desired)) { - if (witness != 0 && - MappedAtomic64::load_acquire(*overflow_word(witness_index)) != - witness) { - status = find_overflow_witness( - canonical_bucket, budget, witness, witness_index); - if (status != SMS_STATUS_SUCCESS) return status; - if (!(witness == 0 - ? SpillSummary::try_encode_empty( - removed_binding, desired) - : SpillSummary::try_encode_present( - witness, desired))) { - return SMS_STATUS_CORRUPT_STORE; - } - continue; - } - if (witness == 0) { - sms::test_detail::reach_checkpoint( - sms::test_detail::CheckpointId::DirectoryAfterSpillSummaryClear); - } else { - sms::test_detail::reach_checkpoint( - sms::test_detail::CheckpointId::DirectoryAfterSpillSummaryPublication); - } + std::uint64_t witness{}; + std::int64_t witness_index{-1}; + const auto status = find_overflow_witness( + canonical_bucket, budget, witness, witness_index); + if (status != SMS_STATUS_SUCCESS) return status; + if (witness == 0) { + sms::test_detail::reach_checkpoint( + sms::test_detail::CheckpointId::DirectoryAfterEmptySpillSummaryScan); + } + checkpoint( + DirectoryCheckpoint::after_empty_overflow_scan, + removed_binding, + witness); + if (MappedAtomic64::load_acquire(*mutation) != removed_binding) { + return SMS_STATUS_SUCCESS; + } + if (witness != 0 && + (witness_index < 0 || + MappedAtomic64::load_acquire(*overflow_word(witness_index)) != witness)) { + continue; + } + + std::uint64_t desired{}; + if (!(witness == 0 + ? SpillSummary::try_encode_empty(removed_binding, desired) + : SpillSummary::try_encode_present(witness, desired))) { + return SMS_STATUS_CORRUPT_STORE; + } + if (witness == 0) { checkpoint( - witness == 0 - ? DirectoryCheckpoint::after_spill_empty_cas - : DirectoryCheckpoint::after_spill_present, + DirectoryCheckpoint::before_spill_empty_cas, removed_binding, - desired); - return SMS_STATUS_SUCCESS; + witness); + } + auto expected = captured; + if (!MappedAtomic64::compare_exchange(*word, expected, desired)) { + // A failed CAS invalidates the scan as well as its summary version. + continue; + } + if (witness != 0 && + MappedAtomic64::load_acquire(*overflow_word(witness_index)) != witness) { + continue; } + if (witness == 0) { + sms::test_detail::reach_checkpoint( + sms::test_detail::CheckpointId::DirectoryAfterSpillSummaryClear); + } else { + sms::test_detail::reach_checkpoint( + sms::test_detail::CheckpointId::DirectoryAfterSpillSummaryPublication); + } + checkpoint( + witness == 0 + ? DirectoryCheckpoint::after_spill_empty_cas + : DirectoryCheckpoint::after_spill_present, + removed_binding, + desired); + return SMS_STATUS_SUCCESS; } } diff --git a/src/cpp/src/key_directory.hpp b/src/cpp/src/key_directory.hpp index d1fa0a0..9119531 100644 --- a/src/cpp/src/key_directory.hpp +++ b/src/cpp/src/key_directory.hpp @@ -46,6 +46,7 @@ enum class DirectoryCheckpoint : std::int32_t { before_mutation_release = 17, before_reserved_publication = 18, after_reserved_publication = 19, + before_spill_empty_cas = 20, }; struct DirectoryHooks { diff --git a/tests/cpp/directory_schedule_tests.cpp b/tests/cpp/directory_schedule_tests.cpp index 8c2f5b1..b8c5225 100644 --- a/tests/cpp/directory_schedule_tests.cpp +++ b/tests/cpp/directory_schedule_tests.cpp @@ -2,6 +2,7 @@ #include "test_support.hpp" #include +#include namespace { @@ -168,6 +169,53 @@ std::int32_t slot_state(Fixture& fixture, std::int32_t index) { return control.state; } +struct SpillReplacementContext { + Fixture* fixture{}; + DirectoryCheckpoint pause_point{}; + std::uint64_t removed_binding{}; + std::uint64_t replacement_binding{}; + bool armed{}; + sms_status unlink_status{SMS_STATUS_STORE_BUSY}; + sms_status insert_status{SMS_STATUS_STORE_BUSY}; + sms_status lookup_status{SMS_STATUS_NOT_FOUND}; + DirectoryLocation replacement_location{}; +}; + +void replace_after_empty_scan( + void* raw_context, + DirectoryCheckpoint checkpoint, + std::uint64_t binding, + std::uint64_t witness) noexcept { + auto& context = *static_cast(raw_context); + if (!context.armed || + checkpoint != context.pause_point || + binding != context.removed_binding || witness != 0) { + return; + } + context.armed = false; + auto& fixture = *context.fixture; + // The first helper is paused with an empty scan. A second helper finishes + // the unlink and releases the canonical mutation, then an owner publishes + // another overflow entry in the same bucket before the first resumes. + context.unlink_status = fixture.directory().try_unlink( + binding, OperationBudget::unbounded_scan()); + context.replacement_binding = fixture.seed_slot( + 17, "replacement-overflow", 101, 1, 1); + context.insert_status = fixture.directory().try_insert( + bytes("replacement-overflow"), + 101, + context.replacement_binding, + OperationBudget::unbounded_scan(), + context.replacement_location); + fixture.set_slot_state(17, 1, 3); + DirectoryEntry entry{}; + context.lookup_status = fixture.directory().try_lookup( + bytes("replacement-overflow"), + 101, + OperationBudget::unbounded_scan(), + entry); +} + } // namespace int main() { @@ -451,5 +499,56 @@ int main() { SMS_CHECK(summary.binding() == binding); } + for (const auto pause_point : {DirectoryCheckpoint::after_empty_overflow_scan, + DirectoryCheckpoint::before_spill_empty_cas}) { + SpillReplacementContext context{}; + context.pause_point = pause_point; + Fixture fixture( + 32, + 128, + sms::detail::DirectoryHooks{&context, &replace_after_empty_scan}); + context.fixture = &fixture; + for (std::int32_t index = 0; index < 17; ++index) { + const auto key = "spill-initial-" + std::to_string(index); + const auto binding = fixture.seed_slot(index, key, 101, 1, 1); + DirectoryLocation location{}; + SMS_CHECK(fixture.directory().try_insert( + bytes(key), 101, binding, + OperationBudget::unbounded_scan(), location) == + SMS_STATUS_SUCCESS); + fixture.set_slot_state(index, 1, 3); + if (index == 16) { + SMS_CHECK(location.kind == sms::detail::directory_target_overflow); + context.removed_binding = binding; + } + } + fixture.set_slot_state(16, 1, 6); + context.armed = true; + SMS_CHECK(fixture.directory().try_unlink( + context.removed_binding, + OperationBudget::unbounded_scan()) == SMS_STATUS_SUCCESS); + SMS_CHECK(!context.armed); + SMS_CHECK(context.unlink_status == SMS_STATUS_SUCCESS); + SMS_CHECK(context.insert_status == SMS_STATUS_SUCCESS); + SMS_CHECK(context.lookup_status == SMS_STATUS_SUCCESS); + SMS_CHECK(context.replacement_location.kind == + sms::detail::directory_target_overflow); + DirectoryEntry found{}; + SMS_CHECK(fixture.directory().try_lookup( + bytes("replacement-overflow"), 101, + OperationBudget::unbounded_scan(), found) == + SMS_STATUS_SUCCESS); + SMS_CHECK(found.binding == context.replacement_binding); + std::int32_t canonical{}; + std::int32_t alternate{}; + fixture.directory().buckets_for_hash(101, canonical, alternate); + sms::detail::SpillSummary summary{}; + SMS_CHECK(sms::detail::SpillSummary::try_decode( + fixture.directory().read_spill_summary(canonical), summary)); + SMS_CHECK(summary.is_present); + SMS_CHECK(summary.binding() == context.replacement_binding); + SMS_CHECK(fixture.directory().read_mutation(canonical) == 0); + } + return 0; }