Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 66 additions & 64 deletions src/cpp/src/key_directory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1895,83 +1895,85 @@ sms_status KeyDirectory::refresh_spill_after_unlink(
std::int32_t canonical_bucket,
std::uint64_t removed_binding,
const OperationBudget& budget) noexcept {
std::uint64_t witness{};
std::int64_t witness_index{-1};
auto status = find_overflow_witness(
canonical_bucket, budget, witness, witness_index);
if (status != SMS_STATUS_SUCCESS) return status;
if (witness == 0) {
sms::test_detail::reach_checkpoint(
sms::test_detail::CheckpointId::DirectoryAfterEmptySpillSummaryScan);
}
checkpoint(
DirectoryCheckpoint::after_empty_overflow_scan,
removed_binding,
witness);

std::uint64_t desired{};
const auto encoded = witness == 0
? SpillSummary::try_encode_empty(removed_binding, desired)
: SpillSummary::try_encode_present(witness, desired);
if (!encoded) return SMS_STATUS_CORRUPT_STORE;
auto* const word = spill_word(canonical_bucket);
auto* const mutation = mutation_word(canonical_bucket);
if (word == nullptr || mutation == nullptr) return SMS_STATUS_CORRUPT_STORE;
for (std::int32_t attempt = 0;; ++attempt) {
const auto bound = budget.check_periodic(attempt);
if (bound != SMS_STATUS_SUCCESS) return bound;
if (witness != 0 &&
(witness_index < 0 ||
MappedAtomic64::load_acquire(*overflow_word(witness_index)) !=
witness)) {
status = find_overflow_witness(
canonical_bucket, budget, witness, witness_index);
if (status != SMS_STATUS_SUCCESS) return status;
if (!(witness == 0
? SpillSummary::try_encode_empty(removed_binding, desired)
: SpillSummary::try_encode_present(witness, desired))) {
return SMS_STATUS_CORRUPT_STORE;
}
continue;
if (MappedAtomic64::load_acquire(*mutation) != removed_binding) {
return SMS_STATUS_SUCCESS;
}
auto observed = MappedAtomic64::load_acquire(*word);

// Pair each scan with the exact summary version it observed. A helper
// can finish this unlink and publish another overflow value while this
// helper is paused; its newer summary must never be cleared using our
// earlier empty scan.
const auto captured = MappedAtomic64::load_acquire(*word);
SpillSummary summary{};
if (!decode_summary_semantic(observed, summary)) {
if (MappedAtomic64::load_acquire(*word) == observed) {
if (!decode_summary_semantic(captured, summary)) {
if (MappedAtomic64::load_acquire(*word) == captured) {
return SMS_STATUS_CORRUPT_STORE;
}
continue;
}
auto expected = observed;
if (MappedAtomic64::compare_exchange(*word, expected, desired)) {
if (witness != 0 &&
MappedAtomic64::load_acquire(*overflow_word(witness_index)) !=
witness) {
status = find_overflow_witness(
canonical_bucket, budget, witness, witness_index);
if (status != SMS_STATUS_SUCCESS) return status;
if (!(witness == 0
? SpillSummary::try_encode_empty(
removed_binding, desired)
: SpillSummary::try_encode_present(
witness, desired))) {
return SMS_STATUS_CORRUPT_STORE;
}
continue;
}
if (witness == 0) {
sms::test_detail::reach_checkpoint(
sms::test_detail::CheckpointId::DirectoryAfterSpillSummaryClear);
} else {
sms::test_detail::reach_checkpoint(
sms::test_detail::CheckpointId::DirectoryAfterSpillSummaryPublication);
}
std::uint64_t witness{};
std::int64_t witness_index{-1};
const auto status = find_overflow_witness(
canonical_bucket, budget, witness, witness_index);
if (status != SMS_STATUS_SUCCESS) return status;
if (witness == 0) {
sms::test_detail::reach_checkpoint(
sms::test_detail::CheckpointId::DirectoryAfterEmptySpillSummaryScan);
}
checkpoint(
DirectoryCheckpoint::after_empty_overflow_scan,
removed_binding,
witness);
if (MappedAtomic64::load_acquire(*mutation) != removed_binding) {
return SMS_STATUS_SUCCESS;
}
if (witness != 0 &&
(witness_index < 0 ||
MappedAtomic64::load_acquire(*overflow_word(witness_index)) != witness)) {
continue;
}

std::uint64_t desired{};
if (!(witness == 0
? SpillSummary::try_encode_empty(removed_binding, desired)
: SpillSummary::try_encode_present(witness, desired))) {
return SMS_STATUS_CORRUPT_STORE;
}
if (witness == 0) {
checkpoint(
witness == 0
? DirectoryCheckpoint::after_spill_empty_cas
: DirectoryCheckpoint::after_spill_present,
DirectoryCheckpoint::before_spill_empty_cas,
removed_binding,
desired);
return SMS_STATUS_SUCCESS;
witness);
}
auto expected = captured;
if (!MappedAtomic64::compare_exchange(*word, expected, desired)) {
// A failed CAS invalidates the scan as well as its summary version.
continue;
}
if (witness != 0 &&
MappedAtomic64::load_acquire(*overflow_word(witness_index)) != witness) {
continue;
}
if (witness == 0) {
sms::test_detail::reach_checkpoint(
sms::test_detail::CheckpointId::DirectoryAfterSpillSummaryClear);
} else {
sms::test_detail::reach_checkpoint(
sms::test_detail::CheckpointId::DirectoryAfterSpillSummaryPublication);
}
checkpoint(
witness == 0
? DirectoryCheckpoint::after_spill_empty_cas
: DirectoryCheckpoint::after_spill_present,
removed_binding,
desired);
return SMS_STATUS_SUCCESS;
}
}

Expand Down
1 change: 1 addition & 0 deletions src/cpp/src/key_directory.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ enum class DirectoryCheckpoint : std::int32_t {
before_mutation_release = 17,
before_reserved_publication = 18,
after_reserved_publication = 19,
before_spill_empty_cas = 20,
};

struct DirectoryHooks {
Expand Down
99 changes: 99 additions & 0 deletions tests/cpp/directory_schedule_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include "test_support.hpp"

#include <cstdint>
#include <string>

namespace {

Expand Down Expand Up @@ -168,6 +169,53 @@ std::int32_t slot_state(Fixture& fixture, std::int32_t index) {
return control.state;
}

struct SpillReplacementContext {
Fixture* fixture{};
DirectoryCheckpoint pause_point{};
std::uint64_t removed_binding{};
std::uint64_t replacement_binding{};
bool armed{};
sms_status unlink_status{SMS_STATUS_STORE_BUSY};
sms_status insert_status{SMS_STATUS_STORE_BUSY};
sms_status lookup_status{SMS_STATUS_NOT_FOUND};
DirectoryLocation replacement_location{};
};

void replace_after_empty_scan(
void* raw_context,
DirectoryCheckpoint checkpoint,
std::uint64_t binding,
std::uint64_t witness) noexcept {
auto& context = *static_cast<SpillReplacementContext*>(raw_context);
if (!context.armed ||
checkpoint != context.pause_point ||
binding != context.removed_binding || witness != 0) {
return;
}
context.armed = false;
auto& fixture = *context.fixture;
// The first helper is paused with an empty scan. A second helper finishes
// the unlink and releases the canonical mutation, then an owner publishes
// another overflow entry in the same bucket before the first resumes.
context.unlink_status = fixture.directory().try_unlink(
binding, OperationBudget::unbounded_scan());
context.replacement_binding = fixture.seed_slot(
17, "replacement-overflow", 101, 1, 1);
context.insert_status = fixture.directory().try_insert(
bytes("replacement-overflow"),
101,
context.replacement_binding,
OperationBudget::unbounded_scan(),
context.replacement_location);
fixture.set_slot_state(17, 1, 3);
DirectoryEntry entry{};
context.lookup_status = fixture.directory().try_lookup(
bytes("replacement-overflow"),
101,
OperationBudget::unbounded_scan(),
entry);
}

} // namespace

int main() {
Expand Down Expand Up @@ -451,5 +499,56 @@ int main() {
SMS_CHECK(summary.binding() == binding);
}

for (const auto pause_point : {DirectoryCheckpoint::after_empty_overflow_scan,
DirectoryCheckpoint::before_spill_empty_cas}) {
SpillReplacementContext context{};
context.pause_point = pause_point;
Fixture fixture(
32,
128,
sms::detail::DirectoryHooks{&context, &replace_after_empty_scan});
context.fixture = &fixture;
for (std::int32_t index = 0; index < 17; ++index) {
const auto key = "spill-initial-" + std::to_string(index);
const auto binding = fixture.seed_slot(index, key, 101, 1, 1);
DirectoryLocation location{};
SMS_CHECK(fixture.directory().try_insert(
bytes(key), 101, binding,
OperationBudget::unbounded_scan(), location) ==
SMS_STATUS_SUCCESS);
fixture.set_slot_state(index, 1, 3);
if (index == 16) {
SMS_CHECK(location.kind == sms::detail::directory_target_overflow);
context.removed_binding = binding;
}
}
fixture.set_slot_state(16, 1, 6);
context.armed = true;
SMS_CHECK(fixture.directory().try_unlink(
context.removed_binding,
OperationBudget::unbounded_scan()) == SMS_STATUS_SUCCESS);
SMS_CHECK(!context.armed);
SMS_CHECK(context.unlink_status == SMS_STATUS_SUCCESS);
SMS_CHECK(context.insert_status == SMS_STATUS_SUCCESS);
SMS_CHECK(context.lookup_status == SMS_STATUS_SUCCESS);
SMS_CHECK(context.replacement_location.kind ==
sms::detail::directory_target_overflow);
DirectoryEntry found{};
SMS_CHECK(fixture.directory().try_lookup(
bytes("replacement-overflow"), 101,
OperationBudget::unbounded_scan(), found) ==
SMS_STATUS_SUCCESS);
SMS_CHECK(found.binding == context.replacement_binding);
std::int32_t canonical{};
std::int32_t alternate{};
fixture.directory().buckets_for_hash(101, canonical, alternate);
sms::detail::SpillSummary summary{};
SMS_CHECK(sms::detail::SpillSummary::try_decode(
fixture.directory().read_spill_summary(canonical), summary));
SMS_CHECK(summary.is_present);
SMS_CHECK(summary.binding() == context.replacement_binding);
SMS_CHECK(fixture.directory().read_mutation(canonical) == 0);
}

return 0;
}
Loading