From f8c116dbbf796a158fff032833fa752559048d8e Mon Sep 17 00:00:00 2001 From: Tryanks Date: Tue, 6 Oct 2026 14:01:40 +0800 Subject: [PATCH 1/4] feat: keep thread metadata and event logs in a Turso database sessions.json and every .jsonl move into tcode.db, owned by SessionStore. Metadata rows hold each project's and session's JSON as before; event rows hold each raw log line, delimiter included, so export is byte-identical. One writer connection commits the store writer's queued writes as one transaction, with synchronous=FULL and, on Apple, fullfsync, so an acknowledged write survives a crash; a failed commit is reported for every write in it and a later flush cannot certify it. An imported or forked thread commits its events and metadata together. The first start builds tcode.db.migrating from the index and every log in the data directory, verifies each thread byte for byte and the metadata field for field, marks the database complete, checkpoints, syncs and renames it, then moves the sources into legacy/. An interrupted migration starts over; a complete database is never deleted or overwritten, and a corrupt one is reported with the sqlite3 CLI as the salvage path. A lock file owned for the host's lifetime keeps a second host out of the data directory; a relaunch waits for the outgoing one. Closes #520 Closes #523 --- Cargo.lock | 900 +++++++++++++-- crates/app/src/main.rs | 71 +- crates/core/src/project.rs | 5 +- crates/runtime/src/app/approvals.rs | 2 +- crates/runtime/src/app/command_validation.rs | 4 +- crates/runtime/src/app/events.rs | 29 +- crates/runtime/src/app/history.rs | 57 +- crates/runtime/src/app/lifecycle.rs | 4 +- crates/runtime/src/app/mod.rs | 96 +- crates/runtime/src/app/orchestrate.rs | 77 +- crates/runtime/src/app/sessions.rs | 82 +- crates/runtime/src/app/snapshots.rs | 8 +- crates/runtime/src/app/store_write.rs | 369 +++++-- crates/runtime/src/app/test_support.rs | 2 +- crates/runtime/src/app/tests.rs | 396 ++++++- crates/runtime/src/pipe.rs | 189 +++- crates/services/Cargo.toml | 4 + crates/services/src/export.rs | 7 +- crates/services/src/import/mod.rs | 24 +- crates/services/src/import/tests.rs | 14 +- crates/services/src/relaunch.rs | 6 + crates/services/src/session_search.rs | 68 +- crates/services/src/store.rs | 661 ----------- crates/services/src/store/db.rs | 371 +++++++ crates/services/src/store/migrate.rs | 577 ++++++++++ crates/services/src/store/mod.rs | 1025 ++++++++++++++++++ crates/services/src/store/tests.rs | 819 ++++++++++++++ locales/en.yml | 1 + locales/zh-CN.yml | 1 + 29 files changed, 4797 insertions(+), 1072 deletions(-) delete mode 100644 crates/services/src/store.rs create mode 100644 crates/services/src/store/db.rs create mode 100644 crates/services/src/store/migrate.rs create mode 100644 crates/services/src/store/mod.rs create mode 100644 crates/services/src/store/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 7664f492b..6ec093588 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,6 +117,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "aegis" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41585f248ed9406567ae6d61ca2a55ed3dc7348e65203a01229881253acb95e3" +dependencies = [ + "cc", + "softaes", +] + [[package]] name = "aes" version = "0.8.4" @@ -173,7 +183,7 @@ dependencies = [ "futures", "futures-concurrency", "rustc-hash 2.1.3", - "rustix", + "rustix 1.1.4", "schemars 1.2.2", "serde", "serde_json", @@ -205,7 +215,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "strum", + "strum 0.28.0", "tracing", ] @@ -264,6 +274,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "allocator-api2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880" + [[package]] name = "android-activity" version = "0.6.1" @@ -329,7 +345,7 @@ checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1" dependencies = [ "anstyle", "memchr", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -382,6 +398,22 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "antithesis_sdk" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8ed6751597568a284314fd9e2db420665898600109dedc7d8d9b588f00ae9c4" +dependencies = [ + "libc", + "libloading", + "linkme", + "once_cell", + "rand 0.8.6", + "rustc_version_runtime", + "serde", + "serde_json", +] + [[package]] name = "anyhow" version = "1.0.103" @@ -417,6 +449,26 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "aristo" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcdeefa800110050103e2459a2f8dbdbd560ad046e30f1f87e24ce19c2cb8bde" +dependencies = [ + "aristo-macros", +] + +[[package]] +name = "aristo-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64a66d21a80182b35b1741997a6d2456911f54b7eb1918aa4e2382fc205268d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "arraydeque" version = "0.5.1" @@ -516,6 +568,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "assoc" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdc70193dadb9d7287fa4b633f15f90c876915b31f6af17da307fc59c9859a8" + [[package]] name = "async-broadcast" version = "0.7.2" @@ -590,7 +648,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix", + "rustix 1.1.4", "slab", "windows-sys 0.61.2", ] @@ -632,7 +690,7 @@ dependencies = [ "cfg-if", "event-listener", "futures-lite", - "rustix", + "rustix 1.1.4", ] [[package]] @@ -658,7 +716,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix", + "rustix 1.1.4", "signal-hook-registry", "slab", "windows-sys 0.61.2", @@ -956,6 +1014,19 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bincode" version = "1.3.3" @@ -965,6 +1036,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools 0.11.0", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex 1.3.0", + "syn 2.0.118", + "which 4.4.2", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -1045,6 +1139,18 @@ dependencies = [ "no_std_io2", ] +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake3" version = "1.8.7" @@ -1132,6 +1238,15 @@ dependencies = [ "cfg_aliases 0.2.2", ] +[[package]] +name = "branches" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7436b21fd6195415058eb41c9090e156b556bc7e0377bb57c5c026a421521525" +dependencies = [ + "rustc_version", +] + [[package]] name = "bs58" version = "0.5.1" @@ -1165,9 +1280,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" dependencies = [ "bytemuck_derive", ] @@ -1243,7 +1358,7 @@ checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7" dependencies = [ "bitflags 2.13.2", "polling", - "rustix", + "rustix 1.1.4", "slab", "tracing", ] @@ -1255,7 +1370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138efcf0940a02ebf0cc8d1eff41a1682a46b431630f4c52450d6265876021fa" dependencies = [ "calloop", - "rustix", + "rustix 1.1.4", "wayland-backend", "wayland-client", ] @@ -1295,9 +1410,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.66" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "jobserver", @@ -1348,6 +1463,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "cfg_block" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18758054972164c3264f7c8386f5fc6da6114cb46b619fd365d4e3b2dc3ae487" + [[package]] name = "cgl" version = "0.3.2" @@ -1563,7 +1684,7 @@ checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" dependencies = [ "serde", "termcolor", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -1935,6 +2056,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -2035,6 +2165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -2826,6 +2957,12 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fancy-regex" version = "0.16.2" @@ -2837,6 +2974,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fastbloom" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" +dependencies = [ + "getrandom 0.3.4", + "libm", + "siphasher 1.0.3", +] + [[package]] name = "fastrand" version = "2.4.1" @@ -2900,9 +3048,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "fixedbitset" @@ -3075,6 +3223,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futf" version = "0.1.5" @@ -3306,6 +3460,21 @@ dependencies = [ "x11", ] +[[package]] +name = "genawaiter" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0" +dependencies = [ + "genawaiter-macro", +] + +[[package]] +name = "genawaiter-macro" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" + [[package]] name = "generator" version = "0.8.10" @@ -3337,7 +3506,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "rustix", + "rustix 1.1.4", "windows-link 0.2.1", ] @@ -3627,7 +3796,7 @@ dependencies = [ "async-channel", "async-task", "backtrace", - "bindgen", + "bindgen 0.72.1", "bitflags 2.13.2", "chrono", "collections", @@ -3670,7 +3839,7 @@ dependencies = [ "slotmap", "smallvec", "spin 0.10.0", - "strum", + "strum 0.28.0", "sum_tree", "taffy", "thiserror 2.0.19", @@ -3959,7 +4128,7 @@ dependencies = [ "raw-window-handle", "smallvec", "smol", - "strum", + "strum 0.28.0", "url", "uuid", "wayland-backend", @@ -4017,7 +4186,7 @@ dependencies = [ "raw-window-handle", "semver", "smallvec", - "strum", + "strum 0.28.0", "uuid", "zed-font-kit 0.14.1-zed (git+https://github.com/zed-industries/font-kit?rev=94b0f28166665e8fd2f53ff6d268a14955c82269)", ] @@ -4063,7 +4232,7 @@ source = "git+https://github.com/longbridge/gpui-fast#1b381adb68a6e5a4550f73b0cd dependencies = [ "anyhow", "log", - "which", + "which 8.0.6", ] [[package]] @@ -4290,7 +4459,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "allocator-api2", + "allocator-api2 0.2.21", "equivalent", "foldhash 0.2.0", ] @@ -4301,7 +4470,7 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ - "allocator-api2", + "allocator-api2 0.2.21", "equivalent", "foldhash 0.2.0", ] @@ -4373,6 +4542,15 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "html5ever" version = "0.27.0" @@ -4571,11 +4749,36 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collator" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08984ed58ac439ebf3e13d2cf26b0c46a60afcd21721c1d14087c0b240344dda" +dependencies = [ + "icu_collator_data", + "icu_collections", + "icu_locale_core", + "icu_locale_fallback", + "icu_normalizer", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_collator_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d7e54efdddeb1208c08dd5d32b53a879ac00d2d3d051b2255fd26821d16368" + [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -4585,45 +4788,91 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_locale" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "785f595c61ef57169a467eeed7b4b6936a66f6cacbb116a96ee86da983251bf4" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_locale_data", + "icu_locale_fallback", + "icu_provider", + "tinystr", + "zerovec", +] + [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d37d91460e362a5cf58907cd7ce871411775ee6294e49a5cc8e6cec16dfa75ea" + +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", "icu_properties", "icu_provider", "smallvec", + "utf16_iter", + "utf8_iter", + "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -4634,18 +4883,20 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -4843,6 +5094,15 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "intrusive-collections" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189d0897e4cbe8c75efedf3502c18c887b05046e59d28404d4d8e46cbc4d1e86" +dependencies = [ + "memoffset", +] + [[package]] name = "inventory" version = "0.3.24" @@ -4864,6 +5124,17 @@ dependencies = [ "leaky-cow", ] +[[package]] +name = "io-uring" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "libc", +] + [[package]] name = "ipconfig" version = "0.3.4" @@ -4926,7 +5197,7 @@ dependencies = [ "rustls-pki-types", "serde", "smallvec", - "strum", + "strum 0.28.0", "time", "tokio", "tokio-stream", @@ -4972,7 +5243,7 @@ dependencies = [ "rand 0.10.2", "rustls", "simple-dns", - "strum", + "strum 0.28.0", "tokio", "tracing", "url", @@ -5055,7 +5326,7 @@ dependencies = [ "serde_json", "sha1", "simdutf8", - "strum", + "strum 0.28.0", "time", "tokio", "tokio-rustls", @@ -5329,6 +5600,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lb-wry" version = "0.53.3" @@ -5455,12 +5732,38 @@ version = "0.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9" +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linktime-proc-macro" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -5779,7 +6082,7 @@ version = "0.1.0" source = "git+https://github.com/longbridge/gpui-fast#1b381adb68a6e5a4550f73b0cd29a231ce6c7281" dependencies = [ "anyhow", - "bindgen", + "bindgen 0.72.1", "core-foundation 0.10.1", "core-video", "foreign-types", @@ -5843,6 +6146,28 @@ dependencies = [ "paste", ] +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "mime" version = "0.3.17" @@ -7120,6 +7445,21 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + +[[package]] +name = "pack1" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b7bb0ecf2e447b1f20ee94ee79ef6eed1e9d4b3c36ce1903b9dea3bf205523" +dependencies = [ + "bytemuck", +] + [[package]] name = "page_size" version = "0.6.0" @@ -7615,7 +7955,7 @@ dependencies = [ "concurrent-queue", "hermit-abi", "pin-project-lite", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -7767,6 +8107,8 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -7939,15 +8281,38 @@ dependencies = [ ] [[package]] -name = "pulp" -version = "0.22.3" +name = "prost" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ - "bytemuck", - "cfg-if", - "libm", - "num-complex", + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "pulp" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", "paste", "pulp-wasm-simd-flag", "raw-cpuid", @@ -8098,6 +8463,12 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.7.3" @@ -8225,6 +8596,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand_pcg" version = "0.10.2" @@ -8255,6 +8635,15 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + [[package]] name = "rav1e" version = "0.8.1" @@ -8700,6 +9089,16 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "roaring" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" +dependencies = [ + "bytemuck", + "byteorder", +] + [[package]] name = "ropey" version = "2.0.0-beta.1" @@ -8847,6 +9246,16 @@ dependencies = [ "semver", ] +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -8856,6 +9265,19 @@ dependencies = [ "nom 7.1.3", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -8865,7 +9287,7 @@ dependencies = [ "bitflags 2.13.2", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -9505,6 +9927,26 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "shuttle" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab17edba38d63047f46780cf7360acf7467fec2c048928689a5c1dd1c2b4e31" +dependencies = [ + "assoc", + "bitvec", + "cfg-if", + "generator", + "hex", + "owo-colors", + "rand 0.8.6", + "rand_core 0.6.4", + "rand_pcg 0.3.1", + "scoped-tls", + "smallvec", + "tracing", +] + [[package]] name = "signal-hook" version = "0.4.4" @@ -9593,6 +10035,15 @@ dependencies = [ "log", ] +[[package]] +name = "simsimd" +version = "6.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fb3bc3cdce07a7d7d4caa4c54f8aa967f6be41690482b54b24100a2253fa70" +dependencies = [ + "cc", +] + [[package]] name = "siphasher" version = "0.3.11" @@ -9684,6 +10135,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "softaes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4fdfe13e1c42e45f497416afb8dc4e783bf0e25b7be9077c1c77c5fb14cb998" + [[package]] name = "sorted-index-buffer" version = "0.2.1" @@ -9835,13 +10292,35 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", +] + [[package]] name = "strum" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "strum_macros", + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.118", ] [[package]] @@ -9979,6 +10458,12 @@ dependencies = [ "zeno", ] +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -10153,6 +10638,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.46" @@ -10329,6 +10820,7 @@ dependencies = [ "chrono", "dirs 7.0.0", "flate2", + "futures-lite", "image", "log", "serde", @@ -10337,6 +10829,7 @@ dependencies = [ "tar", "tcode-core", "tcode-protocol", + "turso", "ureq", "uuid", "zip", @@ -10461,7 +10954,7 @@ dependencies = [ "tcode-voice", "tempfile", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", "url", "web-time", "webview2-com", @@ -10517,7 +11010,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -10683,11 +11176,12 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -11027,6 +11521,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.19", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.31" @@ -11119,6 +11626,209 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "turso" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de8cbeace6385bf3fcb346d4aace8d9ba0de5cffe5e5a0ed210beebd671c208" +dependencies = [ + "thiserror 2.0.19", + "tracing", + "tracing-subscriber", + "turso_core", + "turso_sdk_kit", + "turso_sync_sdk_kit", +] + +[[package]] +name = "turso_core" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58e5f09aaead5dfc87c5a22a2b3e95fb85e2d666e27e5f4a2474c42198af20ec" +dependencies = [ + "aegis", + "aes", + "aes-gcm", + "allocator-api2 0.4.0", + "antithesis_sdk", + "arc-swap", + "aristo", + "bigdecimal", + "bitflags 2.13.2", + "branches", + "bumpalo", + "bytemuck", + "cfg_aliases 0.2.2", + "cfg_block", + "chrono", + "crc32c", + "crossbeam-epoch", + "crossbeam-utils", + "either", + "fallible-iterator", + "fastbloom", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "icu_collator", + "icu_locale", + "intrusive-collections", + "io-uring", + "libc", + "libloading", + "libm", + "loom", + "miette", + "num-bigint", + "num-traits", + "pack1", + "parking_lot", + "pastey 0.2.3", + "polling", + "rand 0.9.4", + "rapidhash", + "regex", + "regex-syntax", + "roaring", + "rustc-hash 2.1.3", + "rustix 1.1.4", + "ryu", + "serde_json", + "shuttle", + "simdutf8", + "simsimd", + "smallvec", + "strum 0.26.3", + "strum_macros 0.26.4", + "tempfile", + "thiserror 2.0.19", + "tracing", + "tracing-subscriber", + "turso_ext", + "turso_macros", + "turso_parser", + "twox-hash", + "uncased", + "uuid", + "windows-sys 0.61.2", +] + +[[package]] +name = "turso_ext" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3902472c63d19191d5982d28542595d1babef787a8482175422ecf7f28100261" +dependencies = [ + "chrono", + "getrandom 0.4.3", + "turso_macros", +] + +[[package]] +name = "turso_macros" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63bd5f0f7186707a8b4cf54bbaddff9056c218823befab15508ef6e7502c2340" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "turso_parser" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b6d3428f16fff69a13bf7f966628d9fc8e0d097fec6cd84ce1af3d5a9be6b1" +dependencies = [ + "bitflags 2.13.2", + "memchr", + "miette", + "strum 0.26.3", + "strum_macros 0.26.4", + "thiserror 2.0.19", + "turso_macros", +] + +[[package]] +name = "turso_sdk_kit" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f96fb1bf669ebd868caf17d865cfa7dc94ad5b2e840402bb28525be09767fc" +dependencies = [ + "bindgen 0.69.5", + "env_logger", + "parking_lot", + "tracing", + "tracing-appender", + "tracing-subscriber", + "turso_core", + "turso_ext", + "turso_sdk_kit_macros", +] + +[[package]] +name = "turso_sdk_kit_macros" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "592f57f212b0ce58c876ab760ef84b219719ee2ade738b53f180db87c0232432" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "turso_sync_engine" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ef57fe72ac7905ad0cfe1cb3f1f24ee1d671176c57a5151709ba9bbf4e1f050" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc32c", + "genawaiter", + "http", + "libc", + "prost", + "roaring", + "serde", + "serde_json", + "thiserror 2.0.19", + "tracing", + "turso_core", + "turso_parser", + "uuid", +] + +[[package]] +name = "turso_sync_sdk_kit" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b83c90ebfc04dc00d1c98cf030510b0d132ec819c355b9ed64d6b767f13db686" +dependencies = [ + "bindgen 0.69.5", + "env_logger", + "genawaiter", + "parking_lot", + "tracing", + "tracing-appender", + "tracing-subscriber", + "turso_core", + "turso_sdk_kit", + "turso_sdk_kit_macros", + "turso_sync_engine", +] + +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" +dependencies = [ + "rand 0.10.2", +] + [[package]] name = "typeid" version = "1.0.3" @@ -11178,6 +11888,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicase" version = "2.9.0" @@ -11244,6 +11963,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1d386ff53b415b7fe27b50bb44679e2cc4660272694b7b6f3326d8480823a94" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.2" @@ -11334,6 +12059,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -11610,7 +12341,7 @@ dependencies = [ "cc", "downcast-rs", "log", - "rustix", + "rustix 1.1.4", "scoped-tls", "smallvec", "wayland-sys", @@ -11623,7 +12354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144" dependencies = [ "bitflags 2.13.2", - "rustix", + "rustix 1.1.4", "wayland-backend", "wayland-scanner", ] @@ -11634,7 +12365,7 @@ version = "0.31.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a52d18780be9b1314328a3de5f930b73d2200112e3849ca6cb11822793fb34d" dependencies = [ - "rustix", + "rustix 1.1.4", "wayland-client", "xcursor", ] @@ -12011,6 +12742,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "which" version = "8.0.6" @@ -12805,11 +13548,17 @@ dependencies = [ "windows-core 0.62.2", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "ws_stream_wasm" @@ -12830,6 +13579,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x11" version = "2.21.0" @@ -12870,7 +13628,7 @@ dependencies = [ "as-raw-xcb-connection", "gethostname", "libc", - "rustix", + "rustix 1.1.4", "x11rb-protocol", "xcursor", ] @@ -12906,7 +13664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix", + "rustix 1.1.4", ] [[package]] @@ -13071,7 +13829,7 @@ dependencies = [ "hex", "libc", "ordered-stream", - "rustix", + "rustix 1.1.4", "serde", "serde_repr", "tracing", @@ -13348,21 +14106,23 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -13370,13 +14130,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.5", ] [[package]] diff --git a/crates/app/src/main.rs b/crates/app/src/main.rs index 83dcd71b8..f3648b510 100644 --- a/crates/app/src/main.rs +++ b/crates/app/src/main.rs @@ -183,7 +183,7 @@ fn pair_command(args: &[String], client_host: &NativeClientHost) -> Result (SpawnedHost, HostMux) { +fn start_local(store: SessionStore) -> std::io::Result<(SpawnedHost, HostMux)> { let mut host_services = HostServices { background_startup_probes: true, ai_title_generation: true, @@ -203,9 +203,61 @@ fn start_local(store: SessionStore) -> (SpawnedHost, HostMux) { } Err(error) => log::warn!("MCP host failed to bind: {error}"), } - let host = spawn_host(store, host_services).expect("failed to start tcode host thread"); + let host = spawn_host(store, host_services)?; let mux = HostMux::new(host.to_host.clone(), host.from_host.clone()); - (host, mux) + Ok((host, mux)) +} + +/// The local host did not start — most often because another Tcode already +/// owns the data directory. Say why on stderr and in a dialog, then exit +/// non-zero. +fn exit_with_startup_failure(error: std::io::Error) -> ! { + struct StartupFailure; + impl gpui::Render for StartupFailure { + fn render( + &mut self, + _: &mut gpui::Window, + _: &mut gpui::Context, + ) -> impl gpui::IntoElement { + gpui::div() + } + } + + eprintln!("tcode: {error}"); + let detail = error.to_string(); + gpui_platform::application().run(move |cx| { + // gpui ends a quit with status 0; this observer is the last code that + // runs before it. + cx.on_app_quit(|_| async { std::process::exit(1) }).detach(); + let options = WindowOptions { + window_bounds: Some(WindowBounds::centered(size(px(420.), px(160.)), cx)), + ..Default::default() + }; + let opened = cx.open_window(options, |_, cx| { + gpui::AppContext::new(cx, |_| StartupFailure) + }); + let Ok(window) = opened else { + cx.quit(); + return; + }; + cx.activate(true); + let _ = window.update(cx, |_, window, cx| { + let quit = tcode_ui::tr!("quit.confirm"); + let answer = window.prompt( + gpui::PromptLevel::Critical, + &tcode_ui::tr!("quit.startup_failed"), + Some(&detail), + &[quit.as_ref()], + cx, + ); + cx.spawn(async move |_, cx| { + let _ = answer.await; + cx.update(|cx| cx.quit()); + }) + .detach(); + }); + }); + std::process::exit(1) } struct LocalKernel { @@ -216,18 +268,18 @@ struct LocalKernel { } impl LocalKernel { - fn start(store: SessionStore) -> Self { - let (host, mux) = start_local(store); + fn start(store: SessionStore) -> std::io::Result { + let (host, mux) = start_local(store)?; let connection = mux.attach(); let control_link = HostLink::new(connection.to_host, connection.from_host); let pump_link = control_link.clone(); let control_pump = smol::spawn(async move { pump_link.pump().await }); - Self { + Ok(Self { host, mux, control_link, _control_pump: control_pump, - } + }) } /// A window's link to the local kernel. The mux keeps the kernel alive @@ -356,7 +408,10 @@ fn main() { }; // Kernel ownership is process composition, not a property of whichever // host the window currently views. - let kernel = Rc::new(LocalKernel::start(store)); + let kernel = match LocalKernel::start(store) { + Ok(kernel) => Rc::new(kernel), + Err(error) => exit_with_startup_failure(error), + }; let local_settings = kernel.settings(); gpui_platform::application() diff --git a/crates/core/src/project.rs b/crates/core/src/project.rs index e6b4ad2a8..b726f1816 100644 --- a/crates/core/src/project.rs +++ b/crates/core/src/project.rs @@ -470,8 +470,9 @@ pub fn order_sessions_with_children(sessions: Vec) -> Vec`; the store loader tolerates both. +/// Every project and session in the store. Also the shape of the legacy +/// `sessions.json` the store migrates from; older files were a bare +/// `Vec`, and the migration tolerates both. #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct IndexFile { #[serde(default)] diff --git a/crates/runtime/src/app/approvals.rs b/crates/runtime/src/app/approvals.rs index 736477b72..ba8fbe9e2 100644 --- a/crates/runtime/src/app/approvals.rs +++ b/crates/runtime/src/app/approvals.rs @@ -100,7 +100,7 @@ mod tests { fn state(name: &str) -> (TestStore, AppState) { let store = TestStore::new(name); - let state = AppState::new((*store).clone()); + let state = AppState::new((*store).clone()).unwrap(); (store, state) } diff --git a/crates/runtime/src/app/command_validation.rs b/crates/runtime/src/app/command_validation.rs index 2e35b6bcf..7fad0bcee 100644 --- a/crates/runtime/src/app/command_validation.rs +++ b/crates/runtime/src/app/command_validation.rs @@ -298,7 +298,7 @@ mod tests { #[test] fn writes_blocked_by_host_state_receive_rejected_acks() { let store = TestStore::new("write-preconditions"); - let mut state = AppState::new((*store).clone()); + let mut state = AppState::new((*store).clone()).unwrap(); let mut context = TestAppContext::default(); let mut cx = context.host_cx(); let id = state.start_draft("fixture".into(), std::env::temp_dir(), &mut cx); @@ -417,7 +417,7 @@ mod tests { #[test] fn provider_channel_rejection_is_not_acknowledged_as_success() { let store = TestStore::new("provider-write-rejection"); - let mut state = AppState::new((*store).clone()); + let mut state = AppState::new((*store).clone()).unwrap(); let mut context = TestAppContext::default(); let mut cx = context.host_cx(); let id = state.start_draft("fixture".into(), std::env::temp_dir(), &mut cx); diff --git a/crates/runtime/src/app/events.rs b/crates/runtime/src/app/events.rs index 5bf72eec9..73bfbac99 100644 --- a/crates/runtime/src/app/events.rs +++ b/crates/runtime/src/app/events.rs @@ -95,7 +95,7 @@ impl AppState { // Provider commands/skills are session metadata for the composer menus — // stored on the live session and in a per-provider cache, never folded - // into the timeline or the persisted JSONL log. Parked sessions still + // into the timeline or the persisted event log. Parked sessions still // receive provider updates, so update/cache those too. AgentEvent::ProviderCommands { commands } => { let cache_key = self.resident_mut(session_id).map(|resident| { @@ -499,7 +499,7 @@ impl AppState { } } - /// Append to JSONL + fold into the matching active or background timeline. + /// Append to the event log + fold into the matching active or background timeline. /// The same wall-clock timestamp is persisted and folded exactly once so /// the on-disk log and the loaded timeline agree. pub(super) fn record_event(&mut self, session_id: &str, event: &AgentEvent, cx: &mut HostCx) { @@ -519,10 +519,17 @@ impl AppState { ServerEvent::SessionEvent(history::wire_record(&record).into_owned()), cx, ); - self.event_records - .entry(session_id.to_string()) - .or_insert_with(|| SessionLog::load(&self.store, session_id)) - .push(record); + // A log that cannot be read is not cached: an empty stand-in would + // replace the conversation. The append is still queued. + match self.event_records.entry(session_id.to_string()) { + std::collections::hash_map::Entry::Occupied(mut log) => log.get_mut().push(record), + std::collections::hash_map::Entry::Vacant(slot) => { + match SessionLog::load(&self.store, session_id) { + Ok(log) => slot.insert(log).push(record), + Err(error) => log::error!("cannot load the event log of {session_id}: {error}"), + } + } + } self.enqueue_store_write( StoreWrite::AppendEvent { id: session_id.to_string(), @@ -645,9 +652,13 @@ impl AppState { attachments, )); } - let timeline = Timeline::fold_events( - records.unwrap_or_else(|| store.read_events(&read_id)), - ); + let records = match records { + Some(records) => records, + None => store + .read_events(&read_id) + .map_err(|error| RuntimeError::External(error.to_string()))?, + }; + let timeline = Timeline::fold_events(records); let (source, paths) = title_regeneration_context(&timeline); if source.is_empty() { return Err(RuntimeError::TitleGenerationEmpty); diff --git a/crates/runtime/src/app/history.rs b/crates/runtime/src/app/history.rs index 67b949e02..143776c38 100644 --- a/crates/runtime/src/app/history.rs +++ b/crates/runtime/src/app/history.rs @@ -207,8 +207,8 @@ fn wire_window( } /// The complete event log of one session, held in memory while the session -/// is resident so history windows cost the page rather than a re-parse of the -/// JSONL, plus the fold that decides where each window may start. +/// is resident so history windows cost the page rather than a re-read of the +/// stored log, plus the fold that decides where each window may start. /// /// Memory policy: [`AppState::event_records`] holds a log for every live or /// parked session (bounded by the resident LRU) and for nothing else. A log @@ -216,7 +216,7 @@ fn wire_window( /// appends in this process, and is dropped once the session leaves residency /// and the store writer has flushed every append queued from it /// ([`AppState::release_stale_session_logs`]): until then the log, not the -/// JSONL, is the whole conversation. +/// store, is the whole conversation. #[derive(Clone)] pub(super) struct SessionLog { records: Vec, @@ -231,8 +231,8 @@ pub(super) struct SessionLog { } impl SessionLog { - pub(super) fn load(store: &SessionStore, session_id: &str) -> Self { - Self::from_records(store.read_events(session_id)) + pub(super) fn load(store: &SessionStore, session_id: &str) -> std::io::Result { + Ok(Self::from_records(store.read_events(session_id)?)) } pub(super) fn from_records(records: impl IntoIterator) -> Self { @@ -284,21 +284,31 @@ impl AppState { /// The session's log, cached for the resident session it belongs to. A /// non-resident session is read cold and not retained, so paging it never /// grows the cache. - fn history_log(&mut self, session_id: &str) -> std::borrow::Cow<'_, SessionLog> { + fn history_log( + &mut self, + session_id: &str, + ) -> Result, tcode_protocol::ProtocolError> { if !self.event_records.contains_key(session_id) { - let log = SessionLog::load(&self.store, session_id); + let log = SessionLog::load(&self.store, session_id).map_err(|error| { + tcode_protocol::ProtocolError { + code: "history_unavailable".into(), + message: format!("could not read the history of {session_id}: {error}"), + } + })?; if self.resident(session_id).is_none() { - return std::borrow::Cow::Owned(log); + return Ok(std::borrow::Cow::Owned(log)); } self.event_records.insert(session_id.to_string(), log); } - std::borrow::Cow::Borrowed(&self.event_records[session_id]) + Ok(std::borrow::Cow::Borrowed(&self.event_records[session_id])) } /// Queue a store-writer barrier for every cached log whose session left - /// residency, and drop the log once the barrier echoes: a cold read after - /// that sees every append the log had accepted. Appends that race the - /// barrier re-arm it. + /// residency, and drop the log once the barrier confirms every earlier + /// write committed: a cold read after that sees every append the log had + /// accepted. Appends that race the barrier re-arm it. A barrier that + /// reports a failed write, or a writer that is gone, keeps the log: it is + /// then the only complete copy of the conversation. pub(super) fn release_stale_session_logs(&mut self, cx: &mut HostCx) { let stale: Vec<(String, usize)> = self .event_records @@ -315,13 +325,21 @@ impl AppState { self.enqueue_store_write(StoreWrite::Flush(completion), cx); let host_cx = cx.clone(); HostCx::spawn_detached(cx, async move { - // A stopped writer has dropped its queue as well; nothing - // later can still land in the JSONL. - let _ = completed.recv().await; + let flushed_ok = match completed.recv().await { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(error), + Err(_) => Err("the session store writer has stopped".to_owned()), + }; host_cx.enqueue(move |state, cx| { let Some(log) = state.event_records.get_mut(&session_id) else { return; }; + if let Err(error) = flushed_ok { + // The barrier stays armed, so the log is never offered + // for release again. + log::warn!("keeping the in-memory history of {session_id}: {error}"); + return; + } log.release_barrier = None; let appended_since = log.records.len() != flushed; if state.resident(&session_id).is_some() { @@ -344,7 +362,10 @@ impl AppState { let Topic::SessionEvents { session_id } = &subscription.topic else { unreachable!() }; - let log = self.history_log(session_id); + let log = match self.history_log(session_id) { + Ok(log) => log, + Err(error) => return ServerEvent::SessionHistoryError(error), + }; let records = log.records(); let total = records.len(); let total_turns = log.fold().turns.len() as u64; @@ -388,7 +409,7 @@ impl AppState { before: u64, limit: u32, ) -> Result { - let log = self.history_log(session_id); + let log = self.history_log(session_id)?; let records = log.records(); let end = before.min(records.len() as u64) as usize; let count = (limit as usize).clamp(1, SESSION_HISTORY_RECORDS); @@ -418,7 +439,7 @@ impl AppState { session_id: &str, item_id: &str, ) -> Result { - let log = self.history_log(session_id); + let log = self.history_log(session_id)?; let output = log .fold() .entries diff --git a/crates/runtime/src/app/lifecycle.rs b/crates/runtime/src/app/lifecycle.rs index 2d01492d7..51a1cd982 100644 --- a/crates/runtime/src/app/lifecycle.rs +++ b/crates/runtime/src/app/lifecycle.rs @@ -325,8 +325,8 @@ impl AppState { cx, ); // Reflect the upsert in memory instead of reloading the whole index - // from disk: `persist_meta` runs on every turn, - // where re-reading and re-parsing a large sessions.json stalls the UI. + // from the store: `persist_meta` runs on every turn, where re-reading + // and re-parsing every session's meta would stall the mailbox. // `sessions` stays newest-first, matching `load_index`'s order. self.upsert_session_in_memory(meta.clone()); } diff --git a/crates/runtime/src/app/mod.rs b/crates/runtime/src/app/mod.rs index c33c2c649..62cbf4d20 100644 --- a/crates/runtime/src/app/mod.rs +++ b/crates/runtime/src/app/mod.rs @@ -260,7 +260,7 @@ use providers::{ }; pub use sessions::ResidentSessions; pub(crate) use snapshots::DomainDiff; -use store_write::{StoreWrite, run_store_write}; +use store_write::{StoreWrite, StoreWriteFailure, StoreWriter}; /// The result of a provider version check. #[derive(Debug, Clone, Default)] @@ -310,8 +310,12 @@ pub struct AppState { settings_store: SettingsStore, store_writes: smol::channel::Sender, store_write_receiver: Option>, - store_write_failures: smol::channel::Sender>, - store_write_failure_receiver: Option>>, + store_write_failures: smol::channel::Sender, + store_write_failure_receiver: Option>, + /// The store-writer thread, joined by [`AppState::close_store`]. + store_writer: Option>, + /// Set once the store failed and every thread was stopped. + store_failed: bool, pub sessions: Vec, pub projects: Vec, pub residents: ResidentSessions, @@ -436,16 +440,18 @@ pub(crate) fn startup_collapsed_threads(sessions: &[SessionMeta]) -> Vec } impl AppState { - pub fn new(store: SessionStore) -> Self { + pub fn new(store: SessionStore) -> std::io::Result { Self::with_ai_titles(store, false) } - pub(crate) fn with_ai_titles(store: SessionStore, ai_title_generation_enabled: bool) -> Self { - // Load + migrate once and persist so derived project ids stay stable. - let file = store.read_file(); - if let Err(err) = store.persist_index(&file) { - log::warn!("failed to persist migrated session index: {err}"); - } + /// Take ownership of the data dir and load its threads. Fails while + /// another host owns the directory, or when its database cannot be opened. + pub(crate) fn with_ai_titles( + store: SessionStore, + ai_title_generation_enabled: bool, + ) -> std::io::Result { + store.open()?; + let file = store.read_file()?; let mut sessions = file.sessions; sessions.sort_by_key(|b| std::cmp::Reverse(b.updated_at)); let projects = file.projects; @@ -488,13 +494,15 @@ impl AppState { let (store_writes, store_write_receiver) = smol::channel::unbounded(); let (store_write_failures, store_write_failure_receiver) = smol::channel::unbounded(); let session_search = Arc::new(std::sync::Mutex::new(SessionSearch::new(store.clone()))); - Self { + Ok(Self { store, settings_store, store_writes, store_write_receiver: Some(store_write_receiver), store_write_failures, store_write_failure_receiver: Some(store_write_failure_receiver), + store_writer: None, + store_failed: false, sessions, projects, residents: ResidentSessions::default(), @@ -540,7 +548,7 @@ impl AppState { external_imports: HashMap::new(), next_import_run_id: 1, session_search, - } + }) } #[cfg(any(test, feature = "test-support"))] @@ -550,27 +558,32 @@ impl AppState { fn start_store_writer(&mut self, cx: &mut HostCx) { if let Some(writes) = self.store_write_receiver.take() { - let store = self.store.clone(); - let settings_store = self.settings_store.clone(); - let terminal_preferences_path = self.terminal_preferences_path.clone(); - let failures = self.store_write_failures.clone(); - HostCx::spawn_detached(cx, async move { - while let Ok(write) = writes.recv().await { - if let Some(failure) = - run_store_write(&store, &settings_store, &terminal_preferences_path, write) - { - let _ = failures.send(failure).await; - } + let writer = StoreWriter::new( + self.store.clone(), + self.settings_store.clone(), + self.terminal_preferences_path.clone(), + self.store_write_failures.clone(), + ); + // Its own thread: every commit blocks on the database's fsync. + match std::thread::Builder::new() + .name("tcode-store-writer".into()) + .spawn(move || writer.run(writes)) + { + Ok(handle) => self.store_writer = Some(handle), + Err(error) => { + log::error!("could not start the session store writer: {error}"); + self.store_writes.close(); } - }); + } } if let Some(failures) = self.store_write_failure_receiver.take() { let host_cx = cx.clone(); HostCx::spawn_detached(cx, async move { while let Ok(failure) = failures.recv().await { host_cx.enqueue(move |state, cx| match failure { - Ok(error) => state.report_error(error, cx), - Err(message) => log::warn!("{message}"), + StoreWriteFailure::Error(error) => state.report_error(error, cx), + StoreWriteFailure::Warning(message) => log::warn!("{message}"), + StoreWriteFailure::StoreFailed(reason) => state.store_failed(&reason, cx), }); } }); @@ -579,9 +592,38 @@ impl AppState { fn enqueue_store_write(&mut self, write: StoreWrite, cx: &mut HostCx) { self.start_store_writer(cx); - if self.store_writes.try_send(write).is_err() { + if let Err(rejected) = self.store_writes.try_send(write) { log::error!("session store writer stopped before accepting a write"); + if let Some(StoreWriteFailure::Error(error)) = rejected + .into_inner() + .reject("the session store writer has stopped") + { + self.report_error(error, cx); + } + } + } + + /// The store stopped serving: stop every provider so nothing keeps + /// producing events that can no longer be saved. The failed writes were + /// reported with the reason. + fn store_failed(&mut self, reason: &str, cx: &mut HostCx) { + if std::mem::replace(&mut self.store_failed, true) { + return; + } + log::error!("session store failed ({reason}); stopping every thread"); + self.shutdown_all(cx); + } + + /// Drain the store writer, then checkpoint and close the database. Only + /// after the host loop has stopped, so nothing enqueues behind it. + pub(crate) fn close_store(&mut self) -> std::io::Result<()> { + self.store_writes.close(); + if let Some(writer) = self.store_writer.take() + && writer.join().is_err() + { + log::error!("the session store writer panicked"); } + self.store.close() } fn enqueue_settings(&mut self, settings: &Settings, cx: &mut HostCx) { diff --git a/crates/runtime/src/app/orchestrate.rs b/crates/runtime/src/app/orchestrate.rs index 10b94465b..db631f61e 100644 --- a/crates/runtime/src/app/orchestrate.rs +++ b/crates/runtime/src/app/orchestrate.rs @@ -875,18 +875,23 @@ impl AppState { unloaded .into_iter() .map(|id| { - let timeline = Timeline::fold_events(store.read_events(&id)); - (id, timeline) + let events = store + .read_events(&id) + .map_err(|error| format!("could not read thread {id}: {error}"))?; + Ok((id, Timeline::fold_events(events))) }) - .collect::>() + .collect::, String>>() }) .await; - let result = host_cx - .enqueue_and_wait(move |state, _| { - state.orchestrate_status_json(&children, &timelines) - }) - .await - .map_err(|_| "tcode orchestrator is not available".to_string()); + let result = match timelines { + Ok(timelines) => host_cx + .enqueue_and_wait(move |state, _| { + state.orchestrate_status_json(&children, &timelines) + }) + .await + .map_err(|_| "tcode orchestrator is not available".to_string()), + Err(error) => Err(error), + }; let _ = reply.send(result).await; }); } @@ -915,15 +920,18 @@ impl AppState { HostCx::spawn_detached(cx, async move { let read_id = thread_id.clone(); let timeline = host_cx - .unblock(move || Timeline::fold_events(store.read_events(&read_id))) + .unblock(move || store.read_events(&read_id).map(Timeline::fold_events)) .await; - let result = host_cx - .enqueue_and_wait(move |state, _| { - let timeline = state.loaded_child_timeline(&thread_id).unwrap_or(&timeline); - state.orchestrate_result_json(&meta, timeline) - }) - .await - .unwrap_or_else(|_| Err("tcode orchestrator is not available".to_string())); + let result = match timeline { + Ok(timeline) => host_cx + .enqueue_and_wait(move |state, _| { + let timeline = state.loaded_child_timeline(&thread_id).unwrap_or(&timeline); + state.orchestrate_result_json(&meta, timeline) + }) + .await + .unwrap_or_else(|_| Err("tcode orchestrator is not available".to_string())), + Err(error) => Err(format!("could not read thread {thread_id}: {error}")), + }; let _ = reply.send(result).await; }); } @@ -1047,14 +1055,43 @@ impl AppState { // attention. let auto_archive = child.archive_on_complete && matches!(status, TurnStatus::Completed); let result_max_chars = child.result_max_chars; + // The cached log holds appends the store writer may not have committed + // yet; without one, the store is read only after a barrier. + let source = match self.event_records.get(&child_id) { + Some(log) => Ok(log.fold().clone()), + None => Err(self.store_write_barrier(cx)), + }; let store = self.store.clone(); let host_cx = cx.clone(); HostCx::spawn_detached(cx, async move { let read_id = child_id.clone(); - let timeline = host_cx - .unblock(move || Timeline::fold_events(store.read_events(&read_id))) - .await; + let timeline = match source { + Ok(timeline) => Ok(timeline), + Err(barrier) => match barrier.recv().await { + Ok(Ok(())) => { + host_cx + .unblock(move || store.read_events(&read_id).map(Timeline::fold_events)) + .await + } + Ok(Err(error)) => Err(std::io::Error::other(error)), + Err(_) => Err(std::io::Error::other( + "the session store writer has stopped", + )), + }, + }; host_cx.enqueue(move |state, cx| { + let timeline = match timeline { + Ok(timeline) => timeline, + Err(error) => { + state.report_error( + RuntimeError::External(format!( + "could not read thread {child_id} to report its completion: {error}" + )), + cx, + ); + return; + } + }; let child_still_exists = state.sessions.iter().any(|meta| { meta.id == child_id && meta.parent_session_id.as_deref() == Some(parent_id.as_str()) diff --git a/crates/runtime/src/app/sessions.rs b/crates/runtime/src/app/sessions.rs index eb4355575..6dfc96db9 100644 --- a/crates/runtime/src/app/sessions.rs +++ b/crates/runtime/src/app/sessions.rs @@ -402,7 +402,13 @@ impl AppState { /// Reload sessions written by the external-history importer and expand its /// project group. fn finish_external_import(&mut self, project_id: &str, cx: &mut HostCx) { - self.sessions = self.store.load_index(); + match self.store.load_index() { + Ok(sessions) => self.sessions = sessions, + Err(error) => self.report_error( + RuntimeError::External(format!("could not reload the imported threads: {error}")), + cx, + ), + } if self .settings .collapsed_projects @@ -437,10 +443,17 @@ impl AppState { let store = self.store.clone(); let host_cx = cx.clone(); cx.spawn_background(async move { - barrier.recv().await.map_err(|error| ProtocolError { - code: "store_barrier_closed".into(), - message: format!("session-store flush failed: {error}"), - })?; + barrier + .recv() + .await + .map_err(|error| ProtocolError { + code: "store_barrier_closed".into(), + message: format!("session-store flush failed: {error}"), + })? + .map_err(|message| ProtocolError { + code: "store_flush_failed".into(), + message, + })?; let suggested_name = export::export_file_name(&meta.title, format); let bytes = host_cx .unblock(move || export::render_thread(&store, &meta, format)) @@ -923,9 +936,9 @@ impl AppState { let fork_id = fork.id.clone(); let (completion, completed) = smol::channel::bounded(1); self.enqueue_store_write( - StoreWrite::CloneEvents { + StoreWrite::Fork { src: source.id, - dst: fork.id.clone(), + meta: Box::new(fork.clone()), completion, }, cx, @@ -938,13 +951,6 @@ impl AppState { .unwrap_or_else(|_| Err("session store writer stopped".into())); host_cx.enqueue(move |state, cx| match result { Ok(()) => { - state.enqueue_store_write( - StoreWrite::UpsertMeta { - meta: Box::new(fork.clone()), - initial: true, - }, - cx, - ); state.upsert_session_in_memory(fork.clone()); state.select_session(&fork.id, cx); if let Some(snapshot) = @@ -959,7 +965,7 @@ impl AppState { } } Err(error) => { - state.report_error(RuntimeError::PersistEvent { error }, cx); + state.report_error(RuntimeError::PersistSession { error }, cx); } }); }); @@ -967,7 +973,7 @@ impl AppState { } /// Permanently delete a thread: stop the provider, close its terminal, - /// delete meta + JSONL, and (when `remove_worktree`) remove the git worktree + /// delete meta + event log, and (when `remove_worktree`) remove the git worktree /// it was the last user of. pub fn delete_session(&mut self, session_id: &str, remove_worktree: bool, cx: &mut HostCx) { self.clear_approvals(session_id); @@ -1448,19 +1454,32 @@ impl AppState { }; // A cached log is the whole conversation, including appends whose // disk writes are still queued, so the timeline derives from it and - // never from a JSONL that can lag the store writer. A session opened + // never from the store, which lags the writes still queued. A session opened // for a client loads its log here, on the mailbox, because the // snapshot that answers the subscription needs it in the same turn; // a background session parses off the mailbox and caches on completion. - let cached = match target { - TimelineLoadTarget::Active { .. } => Some( - self.event_records - .entry(session_id.clone()) - .or_insert_with(|| SessionLog::load(&self.store, &session_id)), - ), - TimelineLoadTarget::Background => self.event_records.get_mut(&session_id), + if matches!(target, TimelineLoadTarget::Active { .. }) + && !self.event_records.contains_key(&session_id) + { + match SessionLog::load(&self.store, &session_id) { + Ok(log) => { + self.event_records.insert(session_id.clone(), log); + } + Err(error) => { + self.report_error( + RuntimeError::External(format!( + "could not load thread {session_id}: {error}" + )), + cx, + ); + return; + } + } } - .map(|log| (log.records().len(), log.fold().clone())); + let cached = self + .event_records + .get(&session_id) + .map(|log| (log.records().len(), log.fold().clone())); let store = self.store.clone(); let host_cx = cx.clone(); HostCx::spawn_detached(cx, async move { @@ -1468,10 +1487,13 @@ impl AppState { let (timeline, folded, loaded, git_branch) = { let (mut timeline, folded, loaded) = match cached { Some((folded, fold)) => (fold, folded, None), - None => { - let log = SessionLog::load(&store, &read_id); - (log.fold().clone(), log.records().len(), Some(log)) - } + None => match SessionLog::load(&store, &read_id) { + Ok(log) => (log.fold().clone(), log.records().len(), Some(log)), + Err(error) => { + log::error!("could not load thread {read_id}: {error}"); + return; + } + }, }; let (mark_idle, load_branch) = match target { TimelineLoadTarget::Active { mark_idle } => (mark_idle, true), @@ -1524,7 +1546,7 @@ impl AppState { } /// Adopt a subscribed session, including an uncommitted parked draft. Stored - /// sessions replay their JSONL; providers start lazily on the next send. + /// sessions replay their event log; providers start lazily on the next send. pub fn select_session(&mut self, session_id: &str, cx: &mut HostCx) { if self.residents.live.contains_key(session_id) { return; diff --git a/crates/runtime/src/app/snapshots.rs b/crates/runtime/src/app/snapshots.rs index 975e02a35..ed23b7503 100644 --- a/crates/runtime/src/app/snapshots.rs +++ b/crates/runtime/src/app/snapshots.rs @@ -459,8 +459,12 @@ impl AppState { } /// Enqueue a FIFO barrier used by the application quit hook. The returned - /// receiver resolves only after every earlier store write has completed. - pub fn store_write_barrier(&mut self, cx: &mut HostCx) -> smol::channel::Receiver<()> { + /// receiver resolves only after every earlier store write has committed, + /// with the store failure that lost one if any did. + pub fn store_write_barrier( + &mut self, + cx: &mut HostCx, + ) -> smol::channel::Receiver> { let (completion, barrier) = smol::channel::bounded(1); self.enqueue_store_write(StoreWrite::Flush(completion), cx); barrier diff --git a/crates/runtime/src/app/store_write.rs b/crates/runtime/src/app/store_write.rs index 9d16c1c81..a6dbfc112 100644 --- a/crates/runtime/src/app/store_write.rs +++ b/crates/runtime/src/app/store_write.rs @@ -1,4 +1,11 @@ use super::*; +use tcode_services::store::Mutation; + +/// A drain of the queue is committed in transactions of at most this many +/// writes and about this many bytes; streamed deltas then share a commit +/// instead of each paying for one. +const MAX_BATCH_WRITES: usize = 256; +const MAX_BATCH_BYTES: usize = 4 << 20; pub(super) enum StoreWrite { AppendEvent { @@ -13,9 +20,11 @@ pub(super) enum StoreWrite { UpsertProject(Project), RemoveSession(String), RemoveProject(String), - CloneEvents { + /// The source's event log copied to a new thread, committed together with + /// the new thread's meta. + Fork { src: String, - dst: String, + meta: Box, completion: smol::channel::Sender>, }, SaveCommands { @@ -31,7 +40,102 @@ pub(super) enum StoreWrite { value: Option, }, ClearProfileSecrets(String), - Flush(smol::channel::Sender<()>), + /// Answered once every write queued before it has committed, or with the + /// first store failure since the writer started: a write that failed is + /// never certified by a later flush. + Flush(smol::channel::Sender>), +} + +/// What the writer reports back to the host. +pub(super) enum StoreWriteFailure { + Error(RuntimeError), + Warning(String), + /// The store stopped serving after a panic or a broken connection; the + /// failed writes are reported separately. + StoreFailed(String), +} + +/// One store write inside a batch: how its failure is reported, and who waits +/// for its commit. +struct Member { + failure: fn(String) -> RuntimeError, + completion: Option>>, +} + +#[derive(Default)] +struct Batch { + mutations: Vec, + members: Vec, + bytes: usize, +} + +impl StoreWrite { + /// The store changes a write makes, or `None` for a write that is not a + /// database change. + fn mutations(&self) -> Option, String>> { + Some(Ok(match self { + StoreWrite::AppendEvent { id, ts, event } => { + match Mutation::append_event(id, *ts, event) { + Ok(mutation) => vec![mutation], + Err(error) => return Some(Err(error.to_string())), + } + } + StoreWrite::UpsertMeta { meta, .. } => vec![Mutation::upsert_meta((**meta).clone())], + StoreWrite::UpsertProject(project) => vec![Mutation::upsert_project(project.clone())], + StoreWrite::RemoveSession(id) => vec![Mutation::remove_session(id)], + StoreWrite::RemoveProject(id) => vec![Mutation::remove_project(id)], + StoreWrite::Fork { src, meta, .. } => vec![ + Mutation::clone_events(src, &meta.id), + Mutation::upsert_meta((**meta).clone()), + ], + _ => return None, + })) + } + + fn member(self) -> Member { + let (failure, completion): (fn(String) -> RuntimeError, _) = match self { + StoreWrite::AppendEvent { .. } => (|error| RuntimeError::PersistEvent { error }, None), + StoreWrite::UpsertMeta { initial: true, .. } => { + (|error| RuntimeError::PersistSession { error }, None) + } + StoreWrite::UpsertMeta { initial: false, .. } => { + (|error| RuntimeError::PersistSessionIndex { error }, None) + } + StoreWrite::UpsertProject(_) => (|error| RuntimeError::PersistProject { error }, None), + StoreWrite::RemoveSession(_) => (|error| RuntimeError::DeleteSession { error }, None), + StoreWrite::RemoveProject(_) => (|error| RuntimeError::DeleteProject { error }, None), + StoreWrite::Fork { completion, .. } => ( + |error| RuntimeError::PersistSession { error }, + Some(completion), + ), + StoreWrite::Flush(completion) => ( + |error| RuntimeError::PersistEvent { error }, + Some(completion), + ), + StoreWrite::SaveCommands { .. } + | StoreWrite::WriteTerminalUi(_) + | StoreWrite::WriteSettings(_) + | StoreWrite::SetProfileSecret { .. } + | StoreWrite::ClearProfileSecrets(_) => { + (|error| RuntimeError::PersistSettings { error }, None) + } + }; + Member { + failure, + completion, + } + } + + /// Fail a write the writer never accepted: its waiter hears why, and + /// anything else is reported like a failed write. + pub(super) fn reject(self, reason: &str) -> Option { + let reports = !matches!(self, StoreWrite::Flush(_) | StoreWrite::Fork { .. }); + let member = self.member(); + if let Some(completion) = member.completion { + let _ = completion.try_send(Err(reason.to_owned())); + } + reports.then(|| StoreWriteFailure::Error((member.failure)(reason.to_owned()))) + } } pub(super) fn atomic_write(path: PathBuf, bytes: Vec) -> std::io::Result<()> { @@ -40,104 +144,177 @@ pub(super) fn atomic_write(path: PathBuf, bytes: Vec) -> std::io::Result<()> fs::rename(tmp, path) } -pub(super) fn run_store_write( - store: &SessionStore, - settings_store: &SettingsStore, - terminal_preferences_path: &Path, - write: StoreWrite, -) -> Option> { - match write { - StoreWrite::AppendEvent { id, ts, event } => { - store.append_event(&id, ts, &event).err().map(|err| { - Ok(RuntimeError::PersistEvent { - error: err.to_string(), - }) - }) +pub(super) struct StoreWriter { + pub(super) store: SessionStore, + pub(super) settings_store: SettingsStore, + pub(super) terminal_preferences_path: PathBuf, + pub(super) failures: smol::channel::Sender, + /// The first failed database write, until the writer stops. + unresolved: Option, +} + +impl StoreWriter { + pub(super) fn new( + store: SessionStore, + settings_store: SettingsStore, + terminal_preferences_path: PathBuf, + failures: smol::channel::Sender, + ) -> Self { + Self { + store, + settings_store, + terminal_preferences_path, + failures, + unresolved: None, + } + } + + /// Serve `writes` until every sender is gone and the queue is drained. + pub(super) fn run(mut self, writes: smol::channel::Receiver) { + while let Ok(first) = writes.recv_blocking() { + let mut queue = vec![first]; + while queue.len() < MAX_BATCH_WRITES { + match writes.try_recv() { + Ok(write) => queue.push(write), + Err(_) => break, + } + } + self.process(queue); } - StoreWrite::UpsertMeta { meta, initial } => store.upsert_meta(&meta).err().map(|err| { - if initial { - Ok(RuntimeError::PersistSession { - error: err.to_string(), - }) - } else { - Ok(RuntimeError::PersistSessionIndex { - error: err.to_string(), - }) + } + + /// Database writes accumulate into one transaction; any other write is an + /// ordered boundary that commits what came before it first. + fn process(&mut self, queue: Vec) { + let mut batch = Batch::default(); + for write in queue { + match write.mutations() { + Some(Ok(mutations)) => { + batch.bytes += mutations.iter().map(Mutation::payload_len).sum::(); + batch.mutations.extend(mutations); + batch.members.push(write.member()); + if batch.bytes >= MAX_BATCH_BYTES { + self.commit(&mut batch); + } + } + Some(Err(error)) => { + let member = write.member(); + self.unresolved.get_or_insert_with(|| error.clone()); + self.report(StoreWriteFailure::Error((member.failure)(error))); + } + None => { + self.commit(&mut batch); + if let Some(failure) = self.run_file_write(write) { + self.report(failure); + } + } } - }), - StoreWrite::UpsertProject(project) => store.upsert_project(&project).err().map(|err| { - Ok(RuntimeError::PersistProject { - error: err.to_string(), - }) - }), - StoreWrite::RemoveSession(id) => store.remove_session(&id).err().map(|err| { - Ok(RuntimeError::DeleteSession { - error: err.to_string(), - }) - }), - StoreWrite::RemoveProject(id) => store.remove_project(&id).err().map(|err| { - Ok(RuntimeError::DeleteProject { + } + self.commit(&mut batch); + } + + fn commit(&mut self, batch: &mut Batch) { + let Batch { + mutations, members, .. + } = std::mem::take(batch); + if members.is_empty() { + return; + } + match self.store.apply(&mutations) { + Ok(()) => { + for member in members { + if let Some(completion) = member.completion { + let _ = completion.try_send(Ok(())); + } + } + } + // The whole batch failed together; it is reported, not replayed. + Err(error) => { + let error = error.to_string(); + self.unresolved.get_or_insert_with(|| error.clone()); + for member in members { + match member.completion { + // The waiter reports its own failure. + Some(completion) => { + let _ = completion.try_send(Err(error.clone())); + } + None => { + self.report(StoreWriteFailure::Error((member.failure)(error.clone()))) + } + } + } + if self.store.is_failed() { + self.report(StoreWriteFailure::StoreFailed(error)); + } + } + } + } + + fn report(&self, failure: StoreWriteFailure) { + let _ = self.failures.send_blocking(failure); + } + + fn run_file_write(&mut self, write: StoreWrite) -> Option { + let settings_failure = |err: std::io::Error| { + StoreWriteFailure::Error(RuntimeError::PersistSettings { error: err.to_string(), }) - }), - StoreWrite::CloneEvents { - src, - dst, - completion, - } => { - let result = store - .clone_events(&src, &dst) - .map_err(|err| err.to_string()); - let _ = completion.try_send(result); - None - } - StoreWrite::SaveCommands { - provider, - acp_agent_id, - commands, - } => store - .save_commands(provider, acp_agent_id.as_deref(), &commands) - .err() - .map(|err| { - Err(format!( - "failed to persist {provider:?} command cache: {err}" - )) - }), - StoreWrite::WriteTerminalUi(bytes) => { - atomic_write(terminal_preferences_path.to_path_buf(), bytes) + }; + match write { + StoreWrite::SaveCommands { + provider, + acp_agent_id, + commands, + } => self + .store + .save_commands(provider, acp_agent_id.as_deref(), &commands) .err() - .map(|err| Err(format!("failed to persist terminal UI state: {err}"))) - } - StoreWrite::WriteSettings(bytes) => atomic_write(store.root().join("settings.json"), bytes) - .err() - .map(|err| { - Ok(RuntimeError::PersistSettings { - error: err.to_string(), - }) - }), - StoreWrite::SetProfileSecret { - profile_id, - key, - value, - } => settings_store - .set_profile_secret(&profile_id, &key, value.as_deref()) - .err() - .map(|err| { - Ok(RuntimeError::PersistSettings { - error: err.to_string(), - }) - }), - StoreWrite::ClearProfileSecrets(profile_id) => settings_store - .clear_profile_secrets(&profile_id) - .err() - .map(|err| { - Ok(RuntimeError::PersistSettings { - error: err.to_string(), - }) - }), - StoreWrite::Flush(completion) => { - let _ = completion.try_send(()); - None + .map(|err| { + StoreWriteFailure::Warning(format!( + "failed to persist {provider:?} command cache: {err}" + )) + }), + StoreWrite::WriteTerminalUi(bytes) => { + atomic_write(self.terminal_preferences_path.clone(), bytes) + .err() + .map(|err| { + StoreWriteFailure::Warning(format!( + "failed to persist terminal UI state: {err}" + )) + }) + } + StoreWrite::WriteSettings(bytes) => { + atomic_write(self.store.root().join("settings.json"), bytes) + .err() + .map(settings_failure) + } + StoreWrite::SetProfileSecret { + profile_id, + key, + value, + } => self + .settings_store + .set_profile_secret(&profile_id, &key, value.as_deref()) + .err() + .map(settings_failure), + StoreWrite::ClearProfileSecrets(profile_id) => self + .settings_store + .clear_profile_secrets(&profile_id) + .err() + .map(settings_failure), + StoreWrite::Flush(completion) => { + let _ = completion.try_send(match &self.unresolved { + None => Ok(()), + Some(error) => Err(format!("an earlier session-store write failed: {error}")), + }); + None + } + StoreWrite::AppendEvent { .. } + | StoreWrite::UpsertMeta { .. } + | StoreWrite::UpsertProject(_) + | StoreWrite::RemoveSession(_) + | StoreWrite::RemoveProject(_) + | StoreWrite::Fork { .. } => unreachable!("database writes are batched"), } } } diff --git a/crates/runtime/src/app/test_support.rs b/crates/runtime/src/app/test_support.rs index e4f8857a5..a05eda36c 100644 --- a/crates/runtime/src/app/test_support.rs +++ b/crates/runtime/src/app/test_support.rs @@ -241,7 +241,7 @@ impl std::ops::DerefMut for TestClientState { impl TestClientState { pub(super) fn new(store: SessionStore) -> Self { Self { - host: AppState::new(store), + host: AppState::new(store).unwrap(), selected: None, } } diff --git a/crates/runtime/src/app/tests.rs b/crates/runtime/src/app/tests.rs index 4707b0c6d..f9e587ea8 100644 --- a/crates/runtime/src/app/tests.rs +++ b/crates/runtime/src/app/tests.rs @@ -174,6 +174,7 @@ fn provider_native_subagent_events_create_and_feed_read_only_mirror_session() { let persisted = state .store .load_index() + .unwrap() .into_iter() .find(|meta| meta.id == mirror.id) .unwrap(); @@ -183,7 +184,7 @@ fn provider_native_subagent_events_create_and_feed_read_only_mirror_session() { serde_json::json!("low") ); assert_eq!(persisted.interaction_mode, InteractionMode::Plan); - let mirror_events = state.store.read_events(&mirror.id); + let mirror_events = state.store.read_events(&mirror.id).unwrap(); assert!(mirror_events.iter().any(|stored| matches!( &stored.event, AgentEvent::ItemCompleted(ThreadItem { @@ -192,7 +193,7 @@ fn provider_native_subagent_events_create_and_feed_read_only_mirror_session() { .. }) if id == "child-answer" ))); - let parent_events = state.store.read_events("parent"); + let parent_events = state.store.read_events("parent").unwrap(); assert!(parent_events.iter().any(|stored| matches!( &stored.event, AgentEvent::ItemCompleted(ThreadItem { @@ -311,7 +312,7 @@ fn assert_native_mirror_turn_lifecycle(evict: bool, late: bool, end: MirrorEnd, TurnStatus::Completed }; state.update(cx, |state, cx| { - let events = state.store.read_events(&mirror_id); + let events = state.store.read_events(&mirror_id).unwrap(); let mut open = false; let mut boundaries = 0; let mut items = 0; @@ -614,7 +615,7 @@ fn nested_subagent_items_open_a_titled_mirror_under_the_child_mirror_and_close_i }); cx.run_until_parked(); state.update(cx, |state, _| { - let events = state.store.read_events(&grandchild_id); + let events = state.store.read_events(&grandchild_id).unwrap(); assert!(matches!( &events.first().unwrap().event, AgentEvent::TurnStarted { turn_id } if turn_id == "toolu_grandchild" @@ -627,22 +628,18 @@ fn nested_subagent_items_open_a_titled_mirror_under_the_child_mirror_and_close_i } )); assert!(matches!( - &state.store.read_events(&second_id).last().unwrap().event, + &state.store.read_events(&second_id).unwrap().last().unwrap().event, AgentEvent::TurnCompleted { turn_id, status: TurnStatus::Interrupted, .. } if turn_id == "toolu_grandchild_2" )); - assert!( - state - .store - .read_events(&child_id) - .iter() - .all(|stored| match &stored.event { - AgentEvent::ItemStarted(item) - | AgentEvent::ItemUpdated(item) - | AgentEvent::ItemCompleted(item) => item.parent_item_id.is_none(), - _ => true, - }) - ); + assert!(state.store.read_events(&child_id).unwrap().iter().all( + |stored| match &stored.event { + AgentEvent::ItemStarted(item) + | AgentEvent::ItemUpdated(item) + | AgentEvent::ItemCompleted(item) => item.parent_item_id.is_none(), + _ => true, + } + )); }); } @@ -687,7 +684,7 @@ fn unknown_parent_mirror_closes_when_the_parent_session_closes() { }); cx.run_until_parked(); state.update(cx, |state, _| { - let events = state.store.read_events(&mirror_id); + let events = state.store.read_events(&mirror_id).unwrap(); assert!(matches!( &events.last().unwrap().event, AgentEvent::TurnCompleted { turn_id, status: TurnStatus::Interrupted, .. } @@ -753,7 +750,7 @@ fn loading_a_mirror_with_an_open_turn_and_no_live_parent_ends_it() { mirror.timeline.last_turn_status, Some(TurnStatus::Interrupted) ); - let events = state.store.read_events("mirror"); + let events = state.store.read_events("mirror").unwrap(); assert_eq!(events.len(), 3); assert!(matches!( &events[2].event, @@ -773,7 +770,7 @@ fn loading_a_mirror_with_an_open_turn_and_no_live_parent_ends_it() { }); cx.run_until_parked(); state.update(cx, |state, _| { - assert_eq!(state.store.read_events("mirror").len(), 3); + assert_eq!(state.store.read_events("mirror").unwrap().len(), 3); }); } @@ -1075,6 +1072,7 @@ fn scripted_provider_connects_command_launch_and_agent_event_paths() { !state .store .load_index() + .unwrap() .iter() .any(|meta| meta.id == session_id) ); @@ -1143,6 +1141,7 @@ fn scripted_provider_connects_command_launch_and_agent_event_paths() { state .store .load_index() + .unwrap() .into_iter() .find(|meta| meta.id == session_id) .unwrap(), @@ -1324,7 +1323,7 @@ fn archive_and_unarchive_apply_exact_timestamp_cascades() { ); }); cx.run_until_parked(); - let persisted = store.load_index(); + let persisted = store.load_index().unwrap(); assert!( persisted .iter() @@ -1666,7 +1665,11 @@ fn title_regeneration_uses_stored_history_and_preserves_intervening_changes() { .find(|meta| meta.id == id) .map(|meta| meta.title.as_str()); assert_eq!(title, expected, "{outcome}"); - let persisted = store.load_index().into_iter().find(|meta| meta.id == id); + let persisted = store + .load_index() + .unwrap() + .into_iter() + .find(|meta| meta.id == id); assert_eq!(persisted.as_ref().map(|meta| meta.title.as_str()), expected); if !matches!(outcome, "renamed" | "deleted") { assert_eq!(state.sessions[0].updated_at, 123, "{outcome}"); @@ -2416,7 +2419,7 @@ fn orchestrate_turn_records_context_and_runs_with_collaboration_disabled() { }); cx.run_until_parked(); state.update(cx, |state, _| { - let events = state.store.read_events("orchestrator"); + let events = state.store.read_events("orchestrator").unwrap(); let recorded = events .iter() .find_map(|stored| match &stored.event { @@ -2907,7 +2910,7 @@ fn draft_defaults_follow_project_history_then_global_history_without_persisting_ vec![] }; assert_eq!(draft.meta.option_selections, expected, "{name}"); - assert!(state.store.load_index().is_empty(), "{name}"); + assert!(state.store.load_index().unwrap().is_empty(), "{name}"); }); } } @@ -2952,7 +2955,7 @@ fn draft_model_selection_switches_to_the_rows_explicit_provider() { assert_eq!(draft.meta.model.as_deref(), Some("claude-fable-5-1")); assert!(draft.meta.acp_agent_id.is_none()); assert!(draft.meta.option_selections.is_empty()); - assert!(state.store.load_index().is_empty()); + assert!(state.store.load_index().unwrap().is_empty()); }); } @@ -4152,7 +4155,7 @@ fn steering_parked_orchestrator_callback_uses_recorded_id() { }); cx.run_until_parked(); state.update(cx, |state, _| { - let timeline = Timeline::fold_events(state.store.read_events("parent")); + let timeline = Timeline::fold_events(state.store.read_events("parent").unwrap()); assert!(timeline.entries.iter().any(|entry| matches!( &entry.content, EntryContent::Steer { @@ -5285,15 +5288,22 @@ fn unaccepted_send_survives_eof_and_is_delivered_once_after_resume() { let active = state.selected_session().unwrap(); assert_eq!(active.queue.len(), 1); assert_eq!(active.delivery_in_flight, Some(delivery_id)); - assert!(!state.store.read_events(&session_id).iter().any(|stored| { - matches!( - &stored.event, - AgentEvent::ItemCompleted(ThreadItem { - content: ItemContent::UserMessage { text, .. }, - .. - }) if text == "survive the eof race" - ) - })); + assert!( + !state + .store + .read_events(&session_id) + .unwrap() + .iter() + .any(|stored| { + matches!( + &stored.event, + AgentEvent::ItemCompleted(ThreadItem { + content: ItemContent::UserMessage { text, .. }, + .. + }) if text == "survive the eof race" + ) + }) + ); // EOF wins before the first actor writes, so no TurnAccepted exists. state.on_event( @@ -5343,6 +5353,7 @@ fn unaccepted_send_survives_eof_and_is_delivered_once_after_resume() { let delivered = state .store .read_events(&session_id) + .unwrap() .iter() .filter(|stored| { matches!( @@ -5449,7 +5460,7 @@ fn select_session_readopts_idle_resident_without_changing_recency_or_shutdown() assert!(actor.try_recv().is_err(), "re-adoption sent Shutdown"); }); cx.run_until_parked(); - assert_eq!(test_store.load_index()[0].updated_at, updated_at); + assert_eq!(test_store.load_index().unwrap()[0].updated_at, updated_at); } #[test] @@ -5666,7 +5677,7 @@ fn fork_thread_clones_timeline_and_provider_cursor() { serde_json::json!({"thread_id": "native-empty-source"}), )); store.upsert_meta(&empty_source).unwrap(); - assert!(!root.join(format!("{}.jsonl", empty_source.id)).exists()); + assert!(store.read_event_log(&empty_source.id).unwrap().is_empty()); let state = cx.new_entity(TestClientState::new(store)); state.update(cx, |state, cx| state.fork_thread(&source.id, cx)); @@ -5688,10 +5699,10 @@ fn fork_thread_clones_timeline_and_provider_cursor() { assert_eq!(fork.cwd, source.cwd); assert_eq!(fork.worktree, None); assert!(!active.timeline.turn_running); - assert_eq!(state.store.read_events(&fork.id).len(), 1); + assert_eq!(state.store.read_events(&fork.id).unwrap().len(), 1); assert_eq!( - std::fs::read(root.join(format!("{}.jsonl", fork.id))).unwrap(), - std::fs::read(root.join(format!("{}.jsonl", source.id))).unwrap() + state.store.read_event_log(&fork.id).unwrap(), + state.store.read_event_log(&source.id).unwrap() ); }); @@ -5718,11 +5729,18 @@ fn fork_thread_clones_timeline_and_provider_cursor() { state .store .load_index() + .unwrap() .iter() .any(|meta| meta.id == fork.id) ); - assert!(!root.join(format!("{}.jsonl", empty_source.id)).exists()); - assert!(!root.join(format!("{}.jsonl", fork.id)).exists()); + assert!( + state + .store + .read_event_log(&empty_source.id) + .unwrap() + .is_empty() + ); + assert!(state.store.read_event_log(&fork.id).unwrap().is_empty()); }); } @@ -5739,6 +5757,14 @@ fn store_writer_flush_persists_ordered_events_metadata_and_secrets_for_reopening state.record_event(&id, &persisted_assistant_event("first"), cx); state.set_profile_secret("profile", "ANTHROPIC_API_KEY", Some("writer-secret"), cx); state.record_event(&id, &persisted_assistant_event("second"), cx); + // More than one batch's worth, so the flush waits for several commits. + for index in 0..600 { + state.record_event( + &id, + &persisted_assistant_event(&format!("bulk {index}")), + cx, + ); + } }); cx.run_until_parked(); @@ -5746,13 +5772,14 @@ fn store_writer_flush_persists_ordered_events_metadata_and_secrets_for_reopening assert_eq!( fresh .load_index() + .unwrap() .iter() .find(|meta| meta.id == id) .unwrap() .title, "persisted by writer" ); - let events = fresh.read_events(&id); + let events = fresh.read_events(&id).unwrap(); let texts: Vec<_> = events .iter() .filter_map(|stored| match &stored.event { @@ -5763,7 +5790,9 @@ fn store_writer_flush_persists_ordered_events_metadata_and_secrets_for_reopening _ => None, }) .collect(); - assert_eq!(texts, ["first", "second"]); + let bulk: Vec<_> = (0..600).map(|index| format!("bulk {index}")).collect(); + assert_eq!(texts[..2], ["first", "second"]); + assert_eq!(texts[2..], bulk); let settings = SettingsStore::new(test_store.root().clone()); assert_eq!( settings @@ -5774,6 +5803,75 @@ fn store_writer_flush_persists_ordered_events_metadata_and_secrets_for_reopening ); } +/// A write the store cannot commit is reported once per write, every later +/// flush says the store lost something, and the in-memory history that holds +/// the unsaved records is kept instead of being released. +#[test] +fn failed_store_writes_are_reported_and_keep_the_unsaved_history() { + let cx = &mut TestAppContext::default(); + let test_store = TestStore::new("tcode-writer-failure"); + let state = cx.new_entity(TestClientState::new((*test_store).clone())); + let (commands, _receiver) = smol::channel::unbounded(); + let parked = live_session(ProviderKind::Codex, commands); + let id = parked.meta.id.clone(); + state.update(cx, |state, cx| { + state.residents.parked.insert(id.clone(), parked); + state.record_event(&id, &persisted_assistant_event("saved"), cx); + }); + cx.run_until_parked(); + cx.drain_outgoing(); + + // The database stops accepting writes underneath the host. + test_store.close().unwrap(); + state.update(cx, |state, cx| { + for text in ["unsaved one", "unsaved two"] { + state.record_event(&id, &persisted_assistant_event(text), cx); + } + }); + cx.run_until_parked(); + let failures = cx + .drain_outgoing() + .into_iter() + .filter(|message| { + matches!( + message, + HostMessage::Event(EventEnvelope { + event: ServerEvent::Runtime(RuntimeEvent::Error( + RuntimeError::PersistEvent { .. } + )), + .. + }) + ) + }) + .count(); + assert_eq!(failures, 2); + + // The session leaves residency; its log is the only copy of the unsaved + // records, so the release barrier's failure keeps it. + state.update(cx, |state, cx| { + state.residents.parked.remove(&id); + state.release_stale_session_logs(cx); + }); + cx.run_until_parked(); + let barrier = state.update(cx, |state, cx| state.store_write_barrier(cx)); + cx.run_until_parked(); + assert!(barrier.try_recv().unwrap().is_err()); + state.read(|state| { + let texts: Vec<_> = state.event_records[&id] + .records() + .iter() + .filter_map(|record| match &record.event { + AgentEvent::ItemCompleted(ThreadItem { + content: ItemContent::AssistantMessage { text }, + .. + }) => Some(text.as_str()), + _ => None, + }) + .collect(); + assert_eq!(texts, ["saved", "unsaved one", "unsaved two"]); + }); +} + /// A project's unsent draft is keyed by the project, so deleting the project /// drops the draft's parked terminal workspace and its persisted terminal /// preferences instead of leaving them behind forever. @@ -6210,7 +6308,7 @@ fn stop_then_new_thread_keeps_the_first_message_visible() { cx.run_until_parked(); state.update(cx, |state, _| { // And it is durable: a replay of the JSONL shows the same thing. - let replayed = Timeline::fold_events(state.store.read_events(&id_b)); + let replayed = Timeline::fold_events(state.store.read_events(&id_b).unwrap()); assert!(replayed.entries.iter().any( |e| matches!(&e.content, EntryContent::Item(ItemContent::UserMessage { text, .. }) if text == "second message") )); @@ -6232,7 +6330,7 @@ fn submitted_queue_head_cannot_leak_delivery_after_turn_completion() { state.update(cx, |state, cx| { state.store.upsert_meta(&session.meta).unwrap(); - state.sessions = state.store.load_index(); + state.sessions = state.store.load_index().unwrap(); state.install_selected(session); state.send_turn(&id, "finish this task".into(), Vec::new(), cx); let delivery_id = match commands.try_recv() { @@ -6321,6 +6419,7 @@ fn event_stream_end_closes_only_its_own_live_provider_in_any_residency() { cx.run_until_parked(); let closes = store .read_events("session") + .unwrap() .iter() .filter(|stored| matches!(stored.event, AgentEvent::SessionClosed { .. })) .count(); @@ -6431,7 +6530,7 @@ fn switching_threads_parks_a_working_session_instead_of_killing_it() { // A live session with a running turn (the overnight workflow). state.store.upsert_meta(&session.meta).unwrap(); - state.sessions = state.store.load_index(); + state.sessions = state.store.load_index().unwrap(); state.install_selected(session); state.send_turn(&id_a, "run the long migration".into(), Vec::new(), cx); state.send_turn(&id_a, "queued follow-up".into(), Vec::new(), cx); @@ -6559,7 +6658,7 @@ fn drained_parked_session_stays_resident() { state.update(cx, |state, cx| { state.store.upsert_meta(&session.meta).unwrap(); - state.sessions = state.store.load_index(); + state.sessions = state.store.load_index().unwrap(); state.install_selected(session); state.send_turn(&id, "one last thing".into(), Vec::new(), cx); let delivery_id = match commands.try_recv() { @@ -7410,7 +7509,7 @@ fn persist_streamed_turns(store: &SessionStore, id: &str, turns: u64) -> Vec> 7; + seed ^= seed << 17; + seed % range + }; + let (mut checkpointed_runs, mut acknowledged, mut unacknowledged) = (0, 0, 0); + for run in 0..runs { + let root = std::env::temp_dir().join(format!("tcode-crash-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&root).unwrap(); + let mut child = tcode_services::process::command(std::env::current_exe().unwrap()) + .args([ + "--exact", + "app::tests::store_writer_crash_child", + "--ignored", + "--nocapture", + ]) + .env(CRASH_CHILD_DIR, &root) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let stdout = std::io::BufReader::new(child.stdout.take().unwrap()); + let (lines, received) = std::sync::mpsc::channel(); + let reader = std::thread::spawn(move || { + for line in stdout.lines().map_while(Result::ok) { + if let Some((_, report)) = line.split_once(CRASH_ACK) { + let _ = lines.send(report.to_owned()); + } + } + }); + let session = received + .recv_timeout(Duration::from_secs(30)) + .unwrap() + .strip_prefix("session ") + .unwrap() + .to_owned(); + std::thread::sleep(Duration::from_millis(200 + random(2_800))); + child.kill().unwrap(); + child.wait().unwrap(); + reader.join().unwrap(); + let acked = received + .try_iter() + .filter_map(|report| report.parse::().ok()) + .max(); + + // Preserve the files exactly as the kill left them, before anything + // opens them again. + let preserved = root.join("preserved"); + std::fs::create_dir_all(&preserved).unwrap(); + for name in ["tcode.db", "tcode.db-wal"] { + if root.join(name).exists() { + std::fs::copy(root.join(name), preserved.join(name)).unwrap(); + } + } + let main_file = std::fs::metadata(root.join("tcode.db")).unwrap().len(); + // A fresh database is a handful of pages; anything larger was + // written there by a checkpoint while records streamed. + if main_file > 64 * 1024 { + checkpointed_runs += 1; + } + + let store = SessionStore::open_at(root.clone()).unwrap(); + let log = store.read_event_log(&session).unwrap(); + let rows = log + .split_inclusive(|byte| *byte == b'\n') + .collect::>(); + assert!( + rows.iter().all(|row| row.ends_with(b"\n")), + "run {run}: torn row" + ); + let events = store.read_events(&session).unwrap(); + assert_eq!(events.len(), rows.len(), "run {run}: a row does not parse"); + let indices: Vec = events + .iter() + .map(|stored| match &stored.event { + AgentEvent::ItemCompleted(ThreadItem { + content: ItemContent::AssistantMessage { text }, + .. + }) => text.split(' ').next().unwrap().parse().unwrap(), + other => panic!("run {run}: unexpected record {other:?}"), + }) + .collect(); + let expected: Vec = (0..indices.len() as u64).collect(); + assert_eq!( + indices, expected, + "run {run}: records are not a gapless prefix" + ); + if let Some(acked) = acked { + assert!( + indices.len() as u64 > acked, + "run {run}: acknowledged record {acked} lost ({} survived)", + indices.len() + ); + acknowledged += acked + 1; + unacknowledged += indices.len() as u64 - (acked + 1); + } + store.close().unwrap(); + drop(store); + + let check = tcode_services::process::command("sqlite3") + .arg(preserved.join("tcode.db")) + .arg("PRAGMA integrity_check;") + .output() + .unwrap(); + assert_eq!( + String::from_utf8_lossy(&check.stdout).trim(), + "ok", + "run {run}: sqlite3 integrity_check: {}", + String::from_utf8_lossy(&check.stderr) + ); + println!( + "run {run}: acked {acked:?}, survived {}, main file {main_file} bytes", + indices.len() + ); + std::fs::remove_dir_all(&root).unwrap(); + } + println!( + "{runs} kills: {acknowledged} acknowledged records all survived, {unacknowledged} \ + committed but unacknowledged, {checkpointed_runs} runs crossed an auto-checkpoint" + ); + assert!(checkpointed_runs > 0, "no run crossed an auto-checkpoint"); +} diff --git a/crates/runtime/src/pipe.rs b/crates/runtime/src/pipe.rs index 1f90717fc..a469e7923 100644 --- a/crates/runtime/src/pipe.rs +++ b/crates/runtime/src/pipe.rs @@ -120,7 +120,13 @@ pub fn spawn_host(store: SessionStore, mut services: HostServices) -> std::io::R std::thread::Builder::new() .name("tcode-host".into()) .spawn(move || { - let mut state = AppState::with_ai_titles(store, services.ai_title_generation); + let mut state = match AppState::with_ai_titles(store, services.ai_title_generation) { + Ok(state) => state, + Err(error) => { + let _ = ready_tx.send(Err(error)); + return; + } + }; if let Some((url, tokens)) = preview_registration { state.attach_preview_mcp(url, tokens); } @@ -143,8 +149,11 @@ pub fn spawn_host(store: SessionStore, mut services: HostServices) -> std::io::R state.refresh_provider_status(&mut cx); } state.sync_terminal_handles(); - let _ = ready_tx.send(()); - smol::block_on(host_loop(state, cx, client_rx, mailbox_rx)); + let _ = ready_tx.send(Ok(())); + let mut state = smol::block_on(host_loop(state, cx, client_rx, mailbox_rx)); + if let Err(error) = state.close_store() { + log::error!("could not close the session store cleanly: {error}"); + } let _ = stopped_tx.send_blocking(()); })?; ready_rx.recv().map_err(|error| { @@ -152,7 +161,7 @@ pub fn spawn_host(store: SessionStore, mut services: HostServices) -> std::io::R std::io::ErrorKind::BrokenPipe, format!("host failed during startup: {error}"), ) - })?; + })??; Ok(SpawnedHost { to_host: client_tx, @@ -169,7 +178,7 @@ async fn host_loop( mut cx: HostCx, client: smol::channel::Receiver, mailbox: smol::channel::Receiver, -) { +) -> AppState { let mut domain_diff = DomainDiff::new(&state); loop { enum Input { @@ -200,6 +209,7 @@ async fn host_loop( state.reap_terminal_projections(); domain_diff.emit_changes(&state, &mut cx); } + state } fn malformed_message_id(line: &str) -> Option { @@ -247,14 +257,17 @@ pub(crate) fn handle_client_message(state: &mut AppState, cx: &mut HostCx, messa CommandOutcome::StoreBarrier(barrier) => { let response_cx = cx.clone(); cx.spawn_detached(async move { - let result = barrier - .recv() - .await - .map(|()| CommandResponse::Unit) - .map_err(|error| ProtocolError { + let result = match barrier.recv().await { + Ok(Ok(())) => Ok(CommandResponse::Unit), + Ok(Err(message)) => Err(ProtocolError { + code: "store_flush_failed".into(), + message, + }), + Err(error) => Err(ProtocolError { code: "store_barrier_closed".into(), message: error.to_string(), - }); + }), + }; response_cx.send_message(HostMessage::Ack { id, result }); }); } @@ -293,7 +306,7 @@ pub(crate) fn handle_client_message(state: &mut AppState, cx: &mut HostCx, messa enum CommandOutcome { Immediate(Result), - StoreBarrier(smol::channel::Receiver<()>), + StoreBarrier(smol::channel::Receiver>), } fn dispatch_command(app: &mut AppState, cx: &mut HostCx, command: Command) -> CommandOutcome { @@ -1316,13 +1329,15 @@ mod tests { let first = project.icon_path.clone().unwrap(); assert!(first.starts_with(root.join("project-icons"))); host.shutdown_blocking().unwrap(); + // Each restart opens the data dir afresh, as a new process would. + let reopen = || SessionStore::open_at(root.clone()).unwrap(); assert_eq!( - store.read_file().projects[0].icon_path.as_ref(), + reopen().read_file().unwrap().projects[0].icon_path.as_ref(), Some(&first) ); // Removing the source must not break the saved custom copy. std::fs::remove_file(&logo).unwrap(); - let host = spawn_host(store.clone(), HostServices::default()).unwrap(); + let host = spawn_host(reopen(), HostServices::default()).unwrap(); let link = host.link(); assert!(matches!( smol::block_on(link.query(Query::ReadProjectIcon { @@ -1338,11 +1353,14 @@ mod tests { }) .unwrap(); host.shutdown_blocking().unwrap(); - let second = store.read_file().projects[0].icon_path.clone().unwrap(); + let second = reopen().read_file().unwrap().projects[0] + .icon_path + .clone() + .unwrap(); assert_ne!(first, second); assert!(!first.exists()); std::fs::write(&logo, include_bytes!("../../../assets/icons/app/tcode.png")).unwrap(); - let host = spawn_host(store.clone(), HostServices::default()).unwrap(); + let host = spawn_host(reopen(), HostServices::default()).unwrap(); let link = host.link(); link.command_blocking(Command::SetProjectIcon { project_id: "p".into(), @@ -1358,12 +1376,149 @@ mod tests { QueryResponse::FileBytes(_) )); host.shutdown_blocking().unwrap(); - assert!(store.read_file().projects[0].icon_path.is_none()); + assert!( + reopen().read_file().unwrap().projects[0] + .icon_path + .is_none() + ); assert!(!second.exists()); assert!(logo.exists()); std::fs::remove_dir_all(root).unwrap(); } + /// A host started on a data dir an older build wrote moves the JSON index + /// and the JSONL logs into its database once, and serves every thread's + /// timeline exactly as the logs fold, before and after a restart. + #[test] + fn a_host_on_legacy_files_migrates_once_and_serves_the_same_timelines() { + use agent::{AgentEvent, ItemContent, ThreadItem, TurnStatus}; + use tcode_core::session::{StoredEvent, Timeline}; + + let data_root = + std::env::temp_dir().join(format!("tcode-host-migration-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&data_root).unwrap(); + let item = |id: &str, content| { + AgentEvent::ItemCompleted(ThreadItem { + id: id.into(), + parent_item_id: None, + content, + }) + }; + let events = [ + ( + None, + AgentEvent::TurnStarted { + turn_id: "t1".into(), + }, + ), + ( + None, + item( + "user", + ItemContent::UserMessage { + text: "What changed?".into(), + context_len: None, + attachments: vec![], + }, + ), + ), + ( + Some(1_000), + item( + "assistant", + ItemContent::AssistantMessage { + text: "The parser.".into(), + }, + ), + ), + ( + Some(2_000), + AgentEvent::TurnCompleted { + turn_id: "t1".into(), + status: TurnStatus::Completed, + usage: None, + }, + ), + ( + Some(3_000), + AgentEvent::TurnStarted { + turn_id: "t2".into(), + }, + ), + ]; + let line = |(ts, event): &(Option, AgentEvent)| match ts { + None => serde_json::to_string(event).unwrap(), + Some(ts) => serde_json::json!({"ts": ts, "event": event}).to_string(), + }; + // Legacy bare events, a CRLF line, a blank and an unparseable line, + // and a last line without its newline. + let log = format!( + "{}\n{}\n\n{{torn\n{}\r\n{}\n{}", + line(&events[0]), + line(&events[1]), + line(&events[2]), + line(&events[3]), + line(&events[4]), + ); + std::fs::write(data_root.join("legacy-thread.jsonl"), &log).unwrap(); + std::fs::write( + data_root.join("sessions.json"), + serde_json::json!({ + "projects": [{"id": "p", "name": "Project", "root": "/work", "created_at": 1}], + "sessions": [{ + "id": "legacy-thread", "title": "Legacy", "provider": "codex", + "cwd": "/work", "project_id": "p", "created_at": 1, "updated_at": 2 + }] + }) + .to_string(), + ) + .unwrap(); + let expected = format!( + "{:?}", + Timeline::fold_events(events.iter().map(|(ts, event)| StoredEvent { + ts: *ts, + event: event.clone(), + elided: None, + })) + ); + + for start in ["migrating", "restarted"] { + let store = SessionStore::open_at(data_root.clone()).unwrap(); + let host = spawn_host(store, HostServices::default()).unwrap(); + let link = host.link(); + let stream = link.events(); + let topic = Topic::SessionEvents { + session_id: "legacy-thread".into(), + }; + link.subscribe(Subscription { + after: None, + topic: topic.clone(), + }) + .unwrap(); + let snapshot = next_event(&stream, |event| { + event.topic == topic && matches!(event.event, ServerEvent::SessionSnapshot { .. }) + }); + let ServerEvent::SessionSnapshot { records, total, .. } = snapshot.event else { + unreachable!("filtered to snapshots") + }; + assert_eq!(total, 5, "{start}"); + assert_eq!( + format!("{:?}", Timeline::fold_events(records)), + expected, + "{start}" + ); + host.shutdown_blocking().unwrap(); + assert!(!data_root.join("sessions.json").exists(), "{start}"); + assert!(!data_root.join("legacy-thread.jsonl").exists(), "{start}"); + assert_eq!( + std::fs::read_to_string(data_root.join("legacy/legacy-thread.jsonl")).unwrap(), + log, + "{start}" + ); + } + std::fs::remove_dir_all(data_root).unwrap(); + } + #[test] fn dedicated_host_round_trips_commands_queries_events_and_quit_barrier() { let data_root = diff --git a/crates/services/Cargo.toml b/crates/services/Cargo.toml index 28b627235..c5a04bc33 100644 --- a/crates/services/Cargo.toml +++ b/crates/services/Cargo.toml @@ -14,12 +14,16 @@ base64 = "0.23" chrono = "0.4" dirs = "7" flate2 = "1" +futures-lite = "2" log = "0.4" image = { version = "0.25.10", default-features = false, features = ["png", "jpeg", "gif", "webp", "bmp", "tiff", "ico"] } serde = { version = "1", features = ["derive"] } serde_json = "1" smol = "2" tar = "0.4" +# Default features add a global allocator (mimalloc) and full-text search +# (tantivy, zstd); the store needs neither. +turso = { version = "=0.8.1", default-features = false } ureq = { version = "2", default-features = false, features = ["tls", "gzip"] } zip = { version = "2", default-features = false, features = ["deflate-flate2", "flate2"] } diff --git a/crates/services/src/export.rs b/crates/services/src/export.rs index 98f09d5e4..e69aa1a62 100644 --- a/crates/services/src/export.rs +++ b/crates/services/src/export.rs @@ -392,7 +392,7 @@ mod tests { ImportOutcome::Imported ); - let restored = destination.load_index().pop().unwrap(); + let restored = destination.load_index().unwrap().pop().unwrap(); assert_ne!(restored.id, meta.id); assert_eq!(restored.cwd, project.root); assert_eq!(restored.project_id.as_deref(), Some(project.id.as_str())); @@ -404,8 +404,9 @@ mod tests { destination.read_event_log(&restored.id).unwrap(), source.read_event_log(&meta.id).unwrap() ); - let source_timeline = Timeline::fold_events(source.read_events(&meta.id)); - let restored_timeline = Timeline::fold_events(destination.read_events(&restored.id)); + let source_timeline = Timeline::fold_events(source.read_events(&meta.id).unwrap()); + let restored_timeline = + Timeline::fold_events(destination.read_events(&restored.id).unwrap()); let summarize = |timeline: &Timeline| { timeline .entries diff --git a/crates/services/src/import/mod.rs b/crates/services/src/import/mod.rs index 59a33c448..2954c6938 100644 --- a/crates/services/src/import/mod.rs +++ b/crates/services/src/import/mod.rs @@ -11,7 +11,7 @@ use agent::{AgentEvent, ProviderKind, ResumeCursor, ThreadItem, TurnStatus}; use serde_json::json; use crate::export::{ReadExportError, TcodeThreadExport, read_tcode_export}; -use crate::store::SessionStore; +use crate::store::{Mutation, SessionStore}; use tcode_core::project::{Project, SessionMeta}; pub use tcode_protocol::{ExternalThread, RecentDir, SourceTool}; @@ -233,12 +233,19 @@ pub fn import_thread( )); } + // The transcript and its meta commit together, so a failure never leaves + // an orphan log behind. + let mut mutations = Vec::with_capacity(events.len() + 1); for (ts, event) in &events { - if let Err(err) = store.append_event(&meta.id, *ts, event) { - return ImportOutcome::Failed(format!("failed to write imported events: {err}")); + match Mutation::append_event(&meta.id, *ts, event) { + Ok(mutation) => mutations.push(mutation), + Err(err) => { + return ImportOutcome::Failed(format!("failed to encode imported events: {err}")); + } } } - if let Err(err) = store.upsert_meta(&meta) { + mutations.push(Mutation::upsert_meta(meta)); + if let Err(err) = store.apply(&mutations) { return ImportOutcome::Failed(format!("failed to write imported session: {err}")); } existing.insert(converted.external_id); @@ -269,10 +276,11 @@ fn import_tcode_thread( meta.parent_session_id = None; meta.imported_from = Some(external_id.clone()); - if let Err(error) = store.write_event_log(&meta.id, &export.event_log) { - return ImportOutcome::Failed(format!("failed to write imported events: {error}")); - } - if let Err(error) = store.upsert_meta(&meta) { + let mutations = [ + Mutation::replace_event_log(&meta.id, export.event_log), + Mutation::upsert_meta(meta), + ]; + if let Err(error) = store.apply(&mutations) { return ImportOutcome::Failed(format!("failed to write imported session: {error}")); } existing.insert(external_id); diff --git a/crates/services/src/import/tests.rs b/crates/services/src/import/tests.rs index f5f4f3875..e93dff8f0 100644 --- a/crates/services/src/import/tests.rs +++ b/crates/services/src/import/tests.rs @@ -267,7 +267,7 @@ fn import_is_idempotent_and_replays_into_timeline() { title_hint: None, last_active_ms: 0, }; - let mut existing = existing_external_ids(&store.load_index()); + let mut existing = existing_external_ids(&store.load_index().unwrap()); assert_eq!( import_thread(&store, &project, &thread, &mut existing), ImportOutcome::Imported @@ -277,7 +277,7 @@ fn import_is_idempotent_and_replays_into_timeline() { ImportOutcome::SkippedDuplicate ); - let index = store.load_index(); + let index = store.load_index().unwrap(); assert_eq!(index.len(), 1); let meta = &index[0]; assert_eq!(meta.project_id.as_deref(), Some("project-1")); @@ -286,7 +286,7 @@ fn import_is_idempotent_and_replays_into_timeline() { meta.resume_cursor.as_ref().unwrap().0["session_id"], "session-1" ); - let timeline = Timeline::fold_events(store.read_events(&meta.id)); + let timeline = Timeline::fold_events(store.read_events(&meta.id).unwrap()); assert!(timeline.entries.iter().any( |entry| matches!(&entry.content, EntryContent::Item(ItemContent::UserMessage { text, .. }) if text == "Imported question") )); @@ -295,12 +295,12 @@ fn import_is_idempotent_and_replays_into_timeline() { let mut imported = meta.clone(); imported.resume_cursor = None; store.upsert_meta(&imported).unwrap(); - let mut existing = existing_external_ids(&store.load_index()); + let mut existing = existing_external_ids(&store.load_index().unwrap()); assert_eq!( import_thread(&store, &project, &thread, &mut existing), ImportOutcome::SkippedDuplicate ); - assert_eq!(store.load_index().len(), 1); + assert_eq!(store.load_index().unwrap().len(), 1); for (index, (provider, source, external_id, cursor, lines)) in [ (ProviderKind::ClaudeCode, SourceTool::ClaudeCode, "claude:native-claude", @@ -321,9 +321,9 @@ fn import_is_idempotent_and_replays_into_timeline() { let thread = ExternalThread { source, file, external_id: external_id.into(), title_hint: None, last_active_ms: 0, }; - let mut existing = existing_external_ids(&store.load_index()); + let mut existing = existing_external_ids(&store.load_index().unwrap()); assert_eq!(import_thread(&store, &project, &thread, &mut existing), ImportOutcome::SkippedDuplicate, "{external_id}"); - assert_eq!(store.load_index().len(), index + 2); + assert_eq!(store.load_index().unwrap().len(), index + 2); } } diff --git a/crates/services/src/relaunch.rs b/crates/services/src/relaunch.rs index cce8a3420..ed1f49edc 100644 --- a/crates/services/src/relaunch.rs +++ b/crates/services/src/relaunch.rs @@ -72,6 +72,12 @@ pub fn clear(data_dir: &Path) -> std::io::Result<()> { } } +/// Whether a marker is waiting, without consuming it: the next instance is +/// already starting while the one that wrote it is still closing. +pub fn pending(data_dir: &Path) -> bool { + marker_path(data_dir).is_file() +} + /// Read the marker and delete it (consume-once). Returns `None` when absent or /// unparsable; the file is removed either way so a corrupt marker can't wedge /// every future launch into a relaunch loop. diff --git a/crates/services/src/session_search.rs b/crates/services/src/session_search.rs index 73bd30491..5a4a656bd 100644 --- a/crates/services/src/session_search.rs +++ b/crates/services/src/session_search.rs @@ -1,12 +1,11 @@ //! Lazy full-text search over persisted session event logs. //! -//! The index is deliberately in-memory: each session is parsed only when its -//! JSONL file's length or modification time changes. This keeps the append-only -//! persistence format authoritative and avoids a second on-disk database. +//! The index is deliberately in-memory: each session is parsed again only after +//! the store reports a committed change to its event log +//! ([`SessionStore::event_generation`]). The store stays authoritative and the +//! cache dies with it. use std::collections::{HashMap, HashSet}; -use std::fs; -use std::time::SystemTime; use agent::ItemContent; use tcode_core::project::SessionMeta; @@ -23,19 +22,13 @@ pub struct SearchableEntry { pub text: String, } -#[derive(Debug, Clone, PartialEq, Eq)] -struct FileFingerprint { - len: u64, - modified: Option, -} - #[derive(Debug, Clone)] struct CachedSession { - fingerprint: FileFingerprint, + generation: u64, entries: Vec, } -/// Incremental, file-freshness-based content index for one [`SessionStore`]. +/// Incremental content index for one [`SessionStore`]. pub struct SessionSearch { store: SessionStore, cache: HashMap, @@ -91,34 +84,29 @@ impl SessionSearch { } fn refresh(&mut self, meta: &SessionMeta) { - let path = self.store.root().join(format!("{}.jsonl", meta.id)); - let fingerprint = match fs::metadata(path) { - Ok(metadata) => FileFingerprint { - len: metadata.len(), - modified: metadata.modified().ok(), - }, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => FileFingerprint { - len: 0, - modified: None, - }, - Err(error) => { - log::warn!("cannot inspect session log {}: {error}", meta.id); - return; - } - }; + // Read before the events: a write that commits during the read moves + // the generation on, so the next search parses again. + let generation = self.store.event_generation(&meta.id); if self .cache .get(&meta.id) - .is_some_and(|cached| cached.fingerprint == fingerprint) + .is_some_and(|cached| cached.generation == generation) { return; } - let entries = extract_searchable_entries(&self.store.read_events(&meta.id)); + let events = match self.store.read_events(&meta.id) { + Ok(events) => events, + Err(error) => { + log::warn!("cannot search session {}: {error}", meta.id); + self.cache.remove(&meta.id); + return; + } + }; self.cache.insert( meta.id.clone(), CachedSession { - fingerprint, - entries, + generation, + entries: extract_searchable_entries(&events), }, ); } @@ -249,6 +237,8 @@ mod tests { use serde_json::json; use tcode_core::project::SessionMeta; + use crate::store::Mutation; + use super::*; fn completed(id: &str, content: ItemContent) -> StoredEvent { @@ -380,8 +370,18 @@ mod tests { assert_eq!(updated.len(), 5); assert_eq!(updated[4].entry_id, "next"); assert_eq!(updated[4].turn, 1); - fs::remove_file(root.join(format!("{}.jsonl", meta.id))).unwrap(); + // A replaced log of the same shape (line count, length) is still seen + // as changed. + let replaced = String::from_utf8(store.read_event_log(&meta.id).unwrap()) + .unwrap() + .replace("auth.rs", "xyzw.rs"); + store + .apply(&[Mutation::replace_event_log(&meta.id, replaced.into_bytes())]) + .unwrap(); assert!(search.search(sessions, "auth.rs", 10).is_empty()); + assert_eq!(search.search(sessions, "xyzw.rs", 10).len(), 5); + store.remove_session(&meta.id).unwrap(); + assert!(search.search(sessions, "xyzw.rs", 10).is_empty()); for (text, query, expected) in [ (" 中文\nCAFÉ\t😀 ", " café ", Some("中文 CAFÉ 😀")), ("İstanbul", "i\u{307}stan", Some("İstanbul")), @@ -430,6 +430,6 @@ mod tests { assert!(hits[0].snippet.contains("AUTH.rs")); assert!(hits[0].snippet.chars().count() <= 142); assert!(hits[0].snippet.starts_with('…') && hits[0].snippet.ends_with('…')); - fs::remove_dir_all(root).unwrap(); + std::fs::remove_dir_all(root).unwrap(); } } diff --git a/crates/services/src/store.rs b/crates/services/src/store.rs deleted file mode 100644 index 3e9498d14..000000000 --- a/crates/services/src/store.rs +++ /dev/null @@ -1,661 +0,0 @@ -//! On-disk persistence for tcode sessions. -//! -//! Layout (under the platform data dir, e.g. `~/Library/Application Support/tcode/`): -//! * `sessions.json` — an [`IndexFile`] containing projects and sessions. -//! * `.jsonl` — append-only `{ ts, event }` records. -//! -//! Replay accepts timestamped records and legacy bare [`AgentEvent`] lines, -//! then folds [`StoredEvent`]s into a [`tcode_core::session::Timeline`]. - -use std::fs::{self, File, OpenOptions}; -use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; -use std::path::PathBuf; - -use agent::{AgentEvent, ModelSpec, ProviderCommand, ProviderKind}; -use serde::{Deserialize, Serialize}; - -use tcode_core::project::{IndexFile, Project, SessionMeta, migrate_index}; -use tcode_core::session::StoredEvent; - -/// On-disk envelope wrapping each event with its record time. Kept private: -/// callers deal in [`StoredEvent`] (which tolerates the legacy bare form). -#[derive(Serialize, Deserialize)] -struct EventEnvelope { - ts: u64, - event: AgentEvent, -} - -/// Cheap, cloneable handle to the on-disk data directory. -#[derive(Debug, Clone)] -pub struct SessionStore { - root: PathBuf, - /// Full event-log parses, shared by every clone of this store so tests can - /// prove a caller served history from memory. - #[cfg(any(test, feature = "test-support"))] - event_reads: std::sync::Arc, -} - -impl SessionStore { - /// Open (creating if needed) the store under the platform data dir, or under - /// `TCODE_DATA_DIR` when it is set — which gives a throwaway profile (its own - /// sessions, settings and installed ACP agents) for demos and screenshots. - pub fn open_default() -> std::io::Result { - let root = match std::env::var_os("TCODE_DATA_DIR") { - Some(dir) => PathBuf::from(dir), - None => dirs::data_dir() - .unwrap_or_else(|| PathBuf::from(".")) - .join("tcode"), - }; - Self::open_at(root) - } - - pub fn open_at(root: PathBuf) -> std::io::Result { - fs::create_dir_all(&root)?; - Ok(Self { - root, - #[cfg(any(test, feature = "test-support"))] - event_reads: Default::default(), - }) - } - - /// How many times [`SessionStore::read_events`] parsed a log through this - /// store or any of its clones. - #[cfg(any(test, feature = "test-support"))] - pub fn event_reads(&self) -> usize { - self.event_reads.load(std::sync::atomic::Ordering::Relaxed) - } - - pub fn root(&self) -> &PathBuf { - &self.root - } - - fn index_path(&self) -> PathBuf { - self.root.join("sessions.json") - } - - fn events_path(&self, id: &str) -> PathBuf { - self.root.join(format!("{id}.jsonl")) - } - - /// Read the native event-log bytes without parsing or normalizing them. - pub fn read_event_log(&self, id: &str) -> std::io::Result> { - match fs::read(self.events_path(id)) { - Ok(bytes) => Ok(bytes), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), - Err(error) => Err(error), - } - } - - /// Atomically install a complete native event log for a session. - pub fn write_event_log(&self, id: &str, bytes: &[u8]) -> std::io::Result<()> { - let destination = self.events_path(id); - let temporary = destination.with_extension("jsonl.tmp"); - fs::write(&temporary, bytes)?; - fs::rename(temporary, destination) - } - - fn models_path(&self, provider: ProviderKind) -> PathBuf { - let name = match provider { - ProviderKind::Codex => "codex", - ProviderKind::ClaudeCode => "claude", - ProviderKind::Pi => "pi", - ProviderKind::OpenCode => "opencode", - ProviderKind::Cursor => "cursor", - ProviderKind::Grok => "grok", - // ACP agents publish their models over the wire at session start - // (`AgentEvent::ProviderOptions`), so there is no catalog to cache. - ProviderKind::Acp => "acp", - }; - self.root.join(format!("models-{name}.json")) - } - - fn commands_path(&self, provider: ProviderKind, acp_agent_id: Option<&str>) -> Option { - let name = match provider { - ProviderKind::Codex => "codex".to_string(), - ProviderKind::ClaudeCode => "claude".to_string(), - ProviderKind::Pi => "pi".to_string(), - ProviderKind::OpenCode => "opencode".to_string(), - ProviderKind::Cursor => "cursor".to_string(), - ProviderKind::Grok => "grok".to_string(), - ProviderKind::Acp => { - let id = acp_agent_id?; - // Registry ids are external input and may contain path separators. - // Hex keeps the filename reversible and collision-free without - // allowing an id to escape the data directory. - let encoded = id - .as_bytes() - .iter() - .map(|byte| format!("{byte:02x}")) - .collect::(); - format!("acp-{encoded}") - } - }; - Some(self.root.join(format!("commands-{name}.json"))) - } - - /// Load the last-fetched model catalog for `provider` so the picker is - /// instant offline. Empty when never fetched / unreadable. - pub fn load_models(&self, provider: ProviderKind) -> Vec { - let Ok(bytes) = fs::read(self.models_path(provider)) else { - return Vec::new(); - }; - serde_json::from_slice(&bytes).unwrap_or_default() - } - - /// Persist the freshly fetched model catalog for `provider`. - pub fn save_models(&self, provider: ProviderKind, models: &[ModelSpec]) -> std::io::Result<()> { - let path = self.models_path(provider); - let tmp = path.with_extension("json.tmp"); - let data = serde_json::to_vec_pretty(models) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - fs::write(&tmp, data)?; - fs::rename(&tmp, path) - } - - /// Load the most recently reported command/skill list for a native provider - /// or one specific ACP agent. Empty when missing, unreadable, or when an ACP - /// agent id was not supplied. - pub fn load_commands( - &self, - provider: ProviderKind, - acp_agent_id: Option<&str>, - ) -> Vec { - let Some(path) = self.commands_path(provider, acp_agent_id) else { - return Vec::new(); - }; - let Ok(bytes) = fs::read(path) else { - return Vec::new(); - }; - serde_json::from_slice(&bytes).unwrap_or_default() - } - - /// Atomically persist the complete command/skill list reported by a native - /// provider or one specific ACP agent. Empty lists are meaningful: they - /// replace a stale non-empty cache. - pub fn save_commands( - &self, - provider: ProviderKind, - acp_agent_id: Option<&str>, - commands: &[ProviderCommand], - ) -> std::io::Result<()> { - let path = self.commands_path(provider, acp_agent_id).ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "ACP command cache requires an agent id", - ) - })?; - let tmp = path.with_extension("json.tmp"); - let data = serde_json::to_vec_pretty(commands) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - fs::write(&tmp, data)?; - fs::rename(&tmp, path) - } - - /// Load the whole index file (projects + sessions), tolerating the old - /// bare-array schema and deriving implicit projects for orphan sessions. - pub fn read_file(&self) -> IndexFile { - let path = self.index_path(); - let Ok(bytes) = fs::read(&path) else { - return IndexFile::default(); - }; - // Current schema is an object; the legacy schema was a bare array. - let parsed = serde_json::from_slice::(&bytes).or_else(|_| { - serde_json::from_slice::>(&bytes).map(|sessions| IndexFile { - projects: Vec::new(), - sessions, - }) - }); - match parsed { - Ok(file) => migrate_index(file), - Err(err) => { - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|duration| duration.as_nanos()) - .unwrap_or(0); - let corrupt_path = self.root.join(format!("sessions.json.corrupt-{timestamp}")); - match fs::rename(&path, &corrupt_path) { - Ok(()) => log::warn!( - "failed to parse sessions.json: {err}; preserved it as {}", - corrupt_path.display() - ), - Err(rename_err) => log::warn!( - "failed to parse sessions.json: {err}; failed to preserve corrupt index: {rename_err}" - ), - } - IndexFile::default() - } - } - } - - /// Load the session index (newest first). Empty if missing / unreadable. - pub fn load_index(&self) -> Vec { - let mut metas = self.read_file().sessions; - metas.sort_by_key(|b| std::cmp::Reverse(b.updated_at)); - metas - } - - /// Insert or replace a meta in the index (by id), then persist. - pub fn upsert_meta(&self, meta: &SessionMeta) -> std::io::Result<()> { - let mut file = self.read_file(); - if let Some(existing) = file.sessions.iter_mut().find(|m| m.id == meta.id) { - *existing = meta.clone(); - } else { - file.sessions.push(meta.clone()); - } - self.persist_index(&file) - } - - /// Insert or replace a project, then remove its previous managed icon. - pub fn upsert_project(&self, project: &Project) -> std::io::Result<()> { - let mut file = self.read_file(); - let mut previous = None; - if let Some(existing) = file.projects.iter_mut().find(|p| p.id == project.id) { - if existing.icon_path != project.icon_path { - previous = existing.icon_path.clone(); - } - *existing = project.clone(); - } else { - file.projects.push(project.clone()); - } - self.persist_index(&file)?; - self.remove_project_icon(previous); - Ok(()) - } - - /// Remove a project and its managed icon. Sessions are removed separately. - pub fn remove_project(&self, id: &str) -> std::io::Result<()> { - let mut file = self.read_file(); - let icon = file - .projects - .iter() - .find(|p| p.id == id) - .and_then(|p| p.icon_path.clone()); - file.projects.retain(|project| project.id != id); - self.persist_index(&file)?; - self.remove_project_icon(icon); - Ok(()) - } - - fn remove_project_icon(&self, path: Option) { - if let Some(path) = path - && path.parent() == Some(self.root.join("project-icons").as_path()) - && let Err(error) = fs::remove_file(&path) - && error.kind() != std::io::ErrorKind::NotFound - { - log::warn!("could not remove project icon {}: {error}", path.display()); - } - } - - /// Persist a whole index file atomically (also flushes migration on startup). - pub fn persist_index(&self, file: &IndexFile) -> std::io::Result<()> { - let tmp = self.index_path().with_extension("json.tmp"); - let data = serde_json::to_vec_pretty(file) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - fs::write(&tmp, data)?; - fs::rename(&tmp, self.index_path()) - } - - /// Append one event to the session's JSONL log, wrapped in a timestamped - /// envelope (`{"ts": , "event": {…}}`). - pub fn append_event(&self, id: &str, ts: u64, event: &AgentEvent) -> std::io::Result<()> { - let envelope = EventEnvelope { - ts, - event: event.clone(), - }; - let line = serde_json::to_string(&envelope) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - let mut file: File = OpenOptions::new() - .create(true) - .read(true) - .append(true) - .open(self.events_path(id))?; - let len = file.metadata()?.len(); - if len > 0 { - file.seek(SeekFrom::End(-1))?; - let mut last = [0_u8; 1]; - file.read_exact(&mut last)?; - if last[0] != b'\n' { - file.write_all(b"\n")?; - } - } - file.write_all(line.as_bytes())?; - file.write_all(b"\n") - } - - /// Read and parse every persisted event for a session (skipping bad lines). - /// - /// Each line is tolerantly parsed as either a timestamped envelope - /// (`{"ts":…,"event":…}`) or a legacy bare event (`{"type":…}`), so logs - /// written before the envelope format still replay (with `ts == None`). - pub fn read_events(&self, id: &str) -> Vec { - #[cfg(any(test, feature = "test-support"))] - self.event_reads - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let path = self.events_path(id); - let Ok(file) = File::open(&path) else { - return Vec::new(); - }; - let mut events = Vec::new(); - for line in BufReader::new(file).lines() { - let Ok(line) = line else { break }; - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - match parse_stored_line(trimmed) { - Ok(stored) => events.push(stored), - Err(err) => log::warn!("skipping unparseable event in {id}.jsonl: {err}"), - } - } - events - } - - /// Atomically clone one session's append-only event log. A missing source - /// is an empty transcript and therefore succeeds without creating a file. - pub fn clone_events(&self, src_id: &str, dst_id: &str) -> std::io::Result<()> { - let src = self.events_path(src_id); - let data = match fs::read(src) { - Ok(data) => data, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(err) => return Err(err), - }; - let dst = self.events_path(dst_id); - let tmp = dst.with_extension("jsonl.tmp"); - fs::write(&tmp, data)?; - fs::rename(tmp, dst) - } - - /// Remove a session from the index and delete its event log. - pub fn remove_session(&self, id: &str) -> std::io::Result<()> { - let mut file = self.read_file(); - file.sessions.retain(|meta| meta.id != id); - self.persist_index(&file)?; - match fs::remove_file(self.events_path(id)) { - Ok(()) => Ok(()), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(err) => Err(err), - } - } -} - -/// Parse one JSONL line into a [`StoredEvent`], accepting both the timestamped -/// envelope and the legacy bare-event form. Envelope is tried first; a bare -/// event lacks the `ts`/`event` keys so it can't masquerade as one, and an -/// envelope lacks the top-level `type` tag so it can't parse as a bare event. -pub(crate) fn parse_stored_line(line: &str) -> Result { - match serde_json::from_str::(line) { - Ok(envelope) => Ok(StoredEvent { - ts: Some(envelope.ts), - event: envelope.event, - elided: None, - }), - Err(_envelope_err) => match serde_json::from_str::(line) { - Ok(event) => Ok(StoredEvent { - ts: None, - event, - elided: None, - }), - // Both forms failed: the line is genuinely corrupt. The bare-event - // error is the more informative one (the envelope attempt always - // fails on a bare event merely because `ts` is missing). - Err(bare_err) => Err(bare_err), - }, - } -} - -pub use tcode_core::project::now_secs; - -/// Current wall-clock time in unix milliseconds (used for event envelopes). -pub fn now_millis() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) -} - -#[cfg(test)] -mod tests { - use super::*; - use agent::{ProviderCommandKind, TurnStatus}; - - fn temp_root() -> PathBuf { - let mut p = std::env::temp_dir(); - p.push(format!("tcode-store-test-{}", uuid::Uuid::new_v4())); - p - } - - #[test] - fn removing_project_cleans_only_managed_icons_after_persisting() { - let root = - std::env::temp_dir().join(format!("tcode-icon-cleanup-{}", uuid::Uuid::new_v4())); - let store = SessionStore::open_at(root.clone()).unwrap(); - fs::create_dir(root.join("project-icons")).unwrap(); - let mut project = Project::from_root(root.join("project")); - for (path, managed) in [ - (root.join("project-icons/icon.png"), true), - (root.join("original.png"), false), - ] { - fs::write(&path, b"stored image").unwrap(); - project.icon_path = Some(path.clone()); - store.upsert_project(&project).unwrap(); - // A failed index replacement must retain the image still referenced on disk. - fs::create_dir(root.join("sessions.json.tmp")).unwrap(); - assert!(store.remove_project(&project.id).is_err()); - assert!(path.exists()); - assert_eq!( - store.read_file().projects[0].icon_path.as_ref(), - Some(&path) - ); - fs::remove_dir(root.join("sessions.json.tmp")).unwrap(); - store.remove_project(&project.id).unwrap(); - assert!(store.read_file().projects.is_empty()); - assert_eq!(path.exists(), !managed); - } - fs::remove_dir_all(root).unwrap(); - } - - #[test] - fn session_index_upserts_orders_and_removes_only_the_selected_session() { - let store = SessionStore::open_at(temp_root()).unwrap(); - let mut a = SessionMeta::new(ProviderKind::Codex, PathBuf::from("/a"), None); - a.updated_at = 100; - let mut b = SessionMeta::new(ProviderKind::ClaudeCode, PathBuf::from("/b"), None); - b.updated_at = 200; - store.upsert_meta(&a).unwrap(); - store.upsert_meta(&b).unwrap(); - - let index = store.load_index(); - assert_eq!(index.len(), 2); - // newest first - assert_eq!(index[0].id, b.id); - assert_eq!(index[1].id, a.id); - - // upsert replaces - let mut a2 = a.clone(); - a2.title = "renamed".into(); - store.upsert_meta(&a2).unwrap(); - let index = store.load_index(); - assert_eq!(index.len(), 2); - assert_eq!( - index.iter().find(|m| m.id == a.id).unwrap().title, - "renamed" - ); - store - .append_event( - &a.id, - 1, - &AgentEvent::TurnStarted { - turn_id: "turn-a".into(), - }, - ) - .unwrap(); - store - .append_event( - &b.id, - 2, - &AgentEvent::TurnStarted { - turn_id: "turn-b".into(), - }, - ) - .unwrap(); - store.remove_session(&a.id).unwrap(); - store.remove_session(&a.id).unwrap(); - let remaining = store.load_index(); - assert_eq!(remaining.len(), 1); - assert_eq!(remaining[0].id, b.id); - assert!(!store.events_path(&a.id).exists()); - assert!(store.events_path(&b.id).exists()); - let _ = fs::remove_dir_all(store.root()); - } - - #[test] - fn command_cache_roundtrips_per_provider_and_acp_agent() { - let root = temp_root(); - let store = SessionStore::open_at(root.clone()).unwrap(); - let native = vec![ProviderCommand { - name: "review".into(), - description: Some("Review the current changes".into()), - kind: ProviderCommandKind::Command, - }]; - let acp = vec![ProviderCommand { - name: "browser".into(), - description: None, - kind: ProviderCommandKind::Skill, - }]; - store - .save_commands(ProviderKind::ClaudeCode, None, &native) - .unwrap(); - store - .save_commands(ProviderKind::Acp, Some("vendor/agent"), &acp) - .unwrap(); - - // Reopen the store to prove the values come from disk, not memory. - let reopened = SessionStore::open_at(root.clone()).unwrap(); - assert_eq!( - reopened.load_commands(ProviderKind::ClaudeCode, None), - native - ); - assert_eq!( - reopened.load_commands(ProviderKind::Acp, Some("vendor/agent")), - acp - ); - assert!( - reopened - .load_commands(ProviderKind::Acp, Some("different-agent")) - .is_empty() - ); - assert!(root.join("commands-claude.json").is_file()); - assert!( - root.join("commands-acp-76656e646f722f6167656e74.json") - .is_file() - ); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn append_recovers_a_mixed_legacy_log_and_writes_the_current_envelope() { - let store = SessionStore::open_at(temp_root()).unwrap(); - let id = "mixed"; - fs::write(store.events_path(id), concat!( - "{\"type\":\"turn_started\",\"turn_id\":\"legacy\"}\n", - "{\"ts\":2000,\"event\":{\"type\":\"turn_completed\",\"turn_id\":\"legacy\",\"status\":\"completed\",\"usage\":null}}\n", - "\n{not valid json}\n{\"type\":\"turn_started" - )).unwrap(); - store - .append_event( - id, - 3000, - &AgentEvent::TurnStarted { - turn_id: "next".into(), - }, - ) - .unwrap(); - let reopened = SessionStore::open_at(store.root().clone()).unwrap(); - let events = reopened.read_events(id); - assert_eq!(events.len(), 3); - assert_eq!( - events.iter().map(|event| event.ts).collect::>(), - [None, Some(2000), Some(3000)] - ); - assert!( - matches!(&events[0].event, AgentEvent::TurnStarted { turn_id } if turn_id == "legacy") - ); - assert!( - matches!(&events[1].event, AgentEvent::TurnCompleted { turn_id, status: TurnStatus::Completed, .. } if turn_id == "legacy") - ); - assert!( - matches!(&events[2].event, AgentEvent::TurnStarted { turn_id } if turn_id == "next") - ); - let raw = fs::read_to_string(store.events_path(id)).unwrap(); - assert_eq!( - raw.lines().last().unwrap(), - r#"{"ts":3000,"event":{"type":"turn_started","turn_id":"next"}}"# - ); - fs::remove_dir_all(store.root()).unwrap(); - } - - #[test] - fn corrupt_index_is_preserved_before_returning_empty() { - let store = SessionStore::open_at(temp_root()).unwrap(); - let corrupt_bytes = b"not valid session json"; - fs::write(store.index_path(), corrupt_bytes).unwrap(); - - assert!(store.load_index().is_empty()); - assert!(!store.index_path().exists()); - let backups: Vec<_> = fs::read_dir(store.root()) - .unwrap() - .filter_map(Result::ok) - .filter(|entry| { - entry - .file_name() - .to_string_lossy() - .starts_with("sessions.json.corrupt-") - }) - .collect(); - assert_eq!(backups.len(), 1); - assert_eq!(fs::read(backups[0].path()).unwrap(), corrupt_bytes); - let _ = fs::remove_dir_all(store.root()); - } - - #[test] - fn legacy_bare_array_index_loads_and_derives_projects() { - let store = SessionStore::open_at(temp_root()).unwrap(); - // Old-format file: a bare JSON array with no project_id fields. - let legacy = serde_json::json!([ - { - "id": "s1", "title": "One", "provider": "claude_code", - "cwd": "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/work/alpha", "created_at": 1, "updated_at": 10 - }, - { - "id": "s2", "title": "Two", "provider": "codex", - "cwd": "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/work/alpha", "created_at": 2, "updated_at": 20 - }, - { - "id": "s3", "title": "Three", "provider": "codex", - "cwd": "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/work/beta", "created_at": 3, "updated_at": 30 - } - ]); - fs::write(store.index_path(), legacy.to_string()).unwrap(); - - let file = store.read_file(); - // Two distinct roots -> two derived projects, deduped by root. - assert_eq!(file.projects.len(), 2); - let alpha = file - .projects - .iter() - .find(|p| p.root == std::path::Path::new("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/work/alpha")) - .unwrap(); - assert_eq!(alpha.name, "alpha"); - // Both alpha sessions share the same derived project. - let s1 = file.sessions.iter().find(|s| s.id == "s1").unwrap(); - let s2 = file.sessions.iter().find(|s| s.id == "s2").unwrap(); - assert_eq!(s1.project_id, Some(alpha.id.clone())); - assert_eq!(s2.project_id, s1.project_id); - let s3 = file.sessions.iter().find(|s| s.id == "s3").unwrap(); - assert_ne!(s3.project_id, s1.project_id); - let migrated_again = migrate_index(file.clone()); - assert_eq!(migrated_again.projects, file.projects); - assert_eq!(migrated_again.sessions, file.sessions); - let _ = fs::remove_dir_all(store.root()); - } -} diff --git a/crates/services/src/store/db.rs b/crates/services/src/store/db.rs new file mode 100644 index 000000000..a50c38da8 --- /dev/null +++ b/crates/services/src/store/db.rs @@ -0,0 +1,371 @@ +//! The Turso database file behind [`super::SessionStore`]: opening it, the +//! single writer connection, reader connections, and the checkpoint that +//! leaves the main file self-contained. +//! +//! Turso's API is async but drives its own I/O, so every call is run to +//! completion with `block_on` on the calling (blocking) thread. + +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use futures_lite::future::block_on; +use turso::{Builder, Connection, Database, Row, Value}; + +/// `PRAGMA user_version` of a completed store. A staged database keeps 0 +/// until its last write, so a file at the final name with 0 was never +/// finished, and a larger value was written by a newer Tcode. +pub(super) const SCHEMA_VERSION: i64 = 1; + +const SCHEMA: &str = " +CREATE TABLE projects (id TEXT PRIMARY KEY, body BLOB NOT NULL); +CREATE TABLE sessions (id TEXT PRIMARY KEY, body BLOB NOT NULL); +CREATE TABLE events ( + session_id TEXT NOT NULL, + position INTEGER NOT NULL, + line BLOB NOT NULL, + PRIMARY KEY (session_id, position) +); +"; + +/// A failure after which the connection or database can no longer be +/// trusted; the store stops serving every handle once it sees one. +#[derive(Debug)] +pub(super) struct Broken(pub(super) String); + +impl std::fmt::Display for Broken { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for Broken {} + +pub(super) fn is_broken(error: &io::Error) -> bool { + error.get_ref().is_some_and(|inner| inner.is::()) +} + +pub(super) struct Db { + path: PathBuf, + database: Database, + /// Every write goes through this one connection, one transaction at a + /// time. Turso's `busy_timeout` spins the CPU, so a second writer is never + /// allowed to contend instead. + writer: Mutex, + /// Idle reader connections. Readers see the last committed snapshot and + /// never wait for the writer. + readers: Mutex>, +} + +impl Db { + /// Open an existing database file, or create an empty one with the schema + /// when `create` is set (only ever on the staging path). + pub(super) fn open(path: &Path, create: bool) -> io::Result { + if !create && !path.is_file() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("{} does not exist", path.display()), + )); + } + let location = path.to_str().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} is not a UTF-8 path", path.display()), + ) + })?; + let database = block_on(Builder::new_local(location).build()) + .map_err(|error| open_error(path, error))?; + let writer = database + .connect() + .map_err(|error| sql_error(path, "connect", error))?; + let db = Self { + path: path.to_path_buf(), + database, + writer: Mutex::new(writer), + readers: Mutex::new(Vec::new()), + }; + { + let writer = db.writer()?; + // FULL syncs the WAL on every commit; on Apple platforms only + // F_FULLFSYNC (`fullfsync`) also flushes the drive's write cache, + // so a committed batch survives power loss. Both are read back + // because Turso accepts and ignores values it does not implement. + db.execute(&writer, "PRAGMA synchronous = FULL", ())?; + db.execute(&writer, "PRAGMA fullfsync = ON", ())?; + let synchronous = db.query_integer(&writer, "PRAGMA synchronous")?; + if synchronous != Some(2) { + return Err(io::Error::other(format!( + "{}: PRAGMA synchronous is {synchronous:?}, expected FULL", + path.display() + ))); + } + #[cfg(target_vendor = "apple")] + if db.query_integer(&writer, "PRAGMA fullfsync")? != Some(1) { + return Err(io::Error::other(format!( + "{}: PRAGMA fullfsync did not take effect", + path.display() + ))); + } + if create { + block_on(writer.execute_batch(SCHEMA)) + .map_err(|error| sql_error(path, "create schema", error))?; + } + } + Ok(db) + } + + fn writer(&self) -> io::Result> { + self.writer + .lock() + .map_err(|_| io::Error::other(Broken("the store writer panicked".into()))) + } + + /// Run `body` in one IMMEDIATE transaction on the writer connection and + /// commit it. Any failure rolls back explicitly; a connection whose + /// rollback cannot be confirmed is reported [`Broken`] and never reused. + pub(super) fn write( + &self, + body: impl FnOnce(&Self, &Connection) -> io::Result, + ) -> io::Result { + let writer = self.writer()?; + self.execute(&writer, "BEGIN IMMEDIATE", ())?; + let result = body(self, &writer) + .and_then(|value| self.execute(&writer, "COMMIT", ()).map(|_| value)); + if let Err(error) = result { + self.roll_back(&writer)?; + return Err(error); + } + result + } + + fn roll_back(&self, writer: &Connection) -> io::Result<()> { + let autocommit = |connection: &Connection| { + connection + .is_autocommit() + .map_err(|error| sql_error(&self.path, "inspect transaction", error)) + }; + if !autocommit(writer)? { + block_on(writer.execute("ROLLBACK", ())).map_err(|error| { + io::Error::other(Broken(format!( + "{}: rollback failed: {error}", + self.path.display() + ))) + })?; + } + if !autocommit(writer)? { + return Err(io::Error::other(Broken(format!( + "{}: a transaction is still open after rollback", + self.path.display() + )))); + } + Ok(()) + } + + /// Run `body` on an idle reader connection. A connection that saw an + /// error is dropped rather than returned to the pool. + pub(super) fn read( + &self, + body: impl FnOnce(&Self, &Connection) -> io::Result, + ) -> io::Result { + let pooled = self + .readers + .lock() + .ok() + .and_then(|mut readers| readers.pop()); + let connection = match pooled { + Some(connection) => connection, + None => self + .database + .connect() + .map_err(|error| sql_error(&self.path, "connect", error))?, + }; + let result = body(self, &connection); + if result.is_ok() + && let Ok(mut readers) = self.readers.lock() + { + readers.push(connection); + } + result + } + + /// Execute one statement to completion; its statement is finished before + /// this returns, so the next statement never meets an active one. + pub(super) fn execute( + &self, + connection: &Connection, + sql: &str, + params: impl turso::IntoParams, + ) -> io::Result { + block_on(async { + // Cached: migration and batches run the same few statements for + // every line. + let mut statement = connection.prepare_cached(sql).await?; + statement.execute(params).await + }) + .map_err(|error| sql_error(&self.path, sql, error)) + } + + /// Run a query and hand every row to `each`, draining the statement so it + /// is finished before this returns. + pub(super) fn query( + &self, + connection: &Connection, + sql: &str, + params: impl turso::IntoParams, + mut each: impl FnMut(&Row) -> io::Result<()>, + ) -> io::Result<()> { + block_on(async { + let mut rows = connection + .prepare_cached(sql) + .await + .map_err(|error| sql_error(&self.path, sql, error))? + .query(params) + .await + .map_err(|error| sql_error(&self.path, sql, error))?; + while let Some(row) = rows + .next() + .await + .map_err(|error| sql_error(&self.path, sql, error))? + { + each(&row)?; + } + Ok(()) + }) + } + + fn query_integer(&self, connection: &Connection, sql: &str) -> io::Result> { + let mut value = None; + self.query(connection, sql, (), |row| { + value = Some(integer(row, 0)?); + Ok(()) + })?; + Ok(value) + } + + pub(super) fn user_version(&self) -> io::Result { + let writer = self.writer()?; + Ok(self + .query_integer(&writer, "PRAGMA user_version")? + .unwrap_or(0)) + } + + /// Copy every committed frame into the main file and truncate the WAL, + /// failing unless both are confirmed: after this the main file alone is + /// the database. + pub(super) fn checkpoint(&self) -> io::Result<()> { + let writer = self.writer()?; + let mut busy = None; + self.query(&writer, "PRAGMA wal_checkpoint(TRUNCATE)", (), |row| { + busy = Some(integer(row, 0)?); + Ok(()) + })?; + if busy != Some(0) { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + format!( + "{}: checkpoint did not complete (busy = {busy:?})", + self.path.display() + ), + )); + } + let wal = wal_path(&self.path); + match std::fs::metadata(&wal) { + Ok(metadata) if metadata.len() > 0 => Err(io::Error::other(format!( + "{} still holds {} bytes after a truncating checkpoint", + wal.display(), + metadata.len() + ))), + Ok(_) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } + } +} + +pub(super) fn wal_path(path: &Path) -> PathBuf { + let mut name = path.as_os_str().to_owned(); + name.push("-wal"); + PathBuf::from(name) +} + +pub(super) fn integer(row: &Row, index: usize) -> io::Result { + match row.get_value(index) { + Ok(Value::Integer(value)) => Ok(value), + Ok(other) => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("expected an integer column, found {other:?}"), + )), + Err(error) => Err(io::Error::new( + io::ErrorKind::InvalidData, + error.to_string(), + )), + } +} + +pub(super) fn blob(row: &Row, index: usize) -> io::Result> { + match row.get_value(index) { + Ok(Value::Blob(bytes)) => Ok(bytes), + Ok(other) => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("expected a blob column, found {other:?}"), + )), + Err(error) => Err(io::Error::new( + io::ErrorKind::InvalidData, + error.to_string(), + )), + } +} + +pub(super) fn text(row: &Row, index: usize) -> io::Result { + match row.get_value(index) { + Ok(Value::Text(text)) => Ok(text), + Ok(other) => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("expected a text column, found {other:?}"), + )), + Err(error) => Err(io::Error::new( + io::ErrorKind::InvalidData, + error.to_string(), + )), + } +} + +fn open_error(path: &Path, error: turso::Error) -> io::Error { + let message = error.to_string(); + // Turso holds an exclusive lock on the file for as long as a process has + // it open; the message is the only signal its error carries. + if message.contains("locked by another process") { + return io::Error::new( + io::ErrorKind::ResourceBusy, + format!( + "{} is in use by another Tcode process: {message}", + path.display() + ), + ); + } + match error { + turso::Error::IoError(kind, operation) => io::Error::new( + kind, + format!("could not open {} ({operation}): {message}", path.display()), + ), + _ => io::Error::new( + io::ErrorKind::InvalidData, + format!( + "could not open the session database {}: {message}. The file is kept as it \ + is; it can be examined with the sqlite3 command-line tool.", + path.display() + ), + ), + } +} + +pub(super) fn sql_error(path: &Path, operation: &str, error: turso::Error) -> io::Error { + let kind = match &error { + turso::Error::IoError(kind, _) => *kind, + turso::Error::DatabaseFull(_) => io::ErrorKind::StorageFull, + turso::Error::Busy(_) | turso::Error::BusySnapshot(_) => io::ErrorKind::ResourceBusy, + turso::Error::Corrupt(_) | turso::Error::NotAdb(_) => io::ErrorKind::InvalidData, + _ => io::ErrorKind::Other, + }; + io::Error::new(kind, format!("{}: {operation}: {error}", path.display())) +} diff --git a/crates/services/src/store/migrate.rs b/crates/services/src/store/migrate.rs new file mode 100644 index 000000000..e35c6f060 --- /dev/null +++ b/crates/services/src/store/migrate.rs @@ -0,0 +1,577 @@ +//! Creating `tcode.db`, once: from the JSON index (`sessions.json`) and the +//! per-thread JSONL logs older builds wrote, or empty on a fresh install. +//! +//! The database is built and verified under a staging name and published by +//! a rename, so a file at the final name is always complete: +//! +//! 1. `tcode.db.migrating` is created with the schema and `user_version` 0. +//! 2. The index is imported, then every `*.jsonl` in the data dir (orphans +//! included) in bounded transactions; each log is read back and compared +//! byte for byte with its file, then the metadata field for field. +//! 3. `user_version` is set to 1, the WAL is checkpointed and truncated, +//! every handle is dropped, and a fresh open re-counts what was committed. +//! 4. The staged file is synced, the list of sources to archive is written to +//! `tcode.db.archive`, and the staged file is renamed to `tcode.db`; the +//! directory is synced around both. +//! 5. The sources move into `legacy/`, and the archive list is removed. +//! +//! An interrupted run before step 4's rename leaves no `tcode.db`, and the +//! next start discards the staging files and begins again from the untouched +//! sources. After the rename, the next start finishes step 5 from the list. + +use std::collections::HashSet; +use std::fs::{self, File}; +use std::io::{self, BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use tcode_core::project::{IndexFile, SessionMeta, migrate_index}; + +use super::db::{Db, SCHEMA_VERSION, blob, integer, wal_path}; +use super::{DB_FILE, insert_segments, read_index}; + +const STAGING_FILE: &str = "tcode.db.migrating"; +const ARCHIVE_LIST: &str = "tcode.db.archive"; +const LEGACY_INDEX: &str = "sessions.json"; +const LEGACY_DIR: &str = "legacy"; +/// A thread's log is committed in transactions of about this size; one +/// transaction for a multi-hundred-megabyte log holds every dirty page in +/// memory and doubles the WAL. +const CHUNK_BYTES: usize = 8 << 20; +/// At most this many rows, and about [`CHUNK_BYTES`], are compared per read +/// when verifying, so a reader never pins the WAL or memory for a whole log. +const VERIFY_ROWS: i64 = 4096; + +/// Make sure a complete `tcode.db` exists, migrating or creating it, and +/// finish an archival a previous start left half done. Called with the data +/// dir's ownership lock held. +pub(super) fn prepare(root: &Path) -> io::Result<()> { + if root.join(DB_FILE).exists() { + return archive_sources(root); + } + remove_staging(root)?; + let started = Instant::now(); + let summary = build_staging(root)?; + publish(root, &summary.sources)?; + log::info!( + "migrated {} thread(s) ({} line(s), {} blank, {} bytes) and {} session(s) in {} \ + project(s) into {} in {:.1}s", + summary.logs, + summary.lines, + summary.blank_lines, + summary.bytes, + summary.sessions, + summary.projects, + root.join(DB_FILE).display(), + started.elapsed().as_secs_f64() + ); + archive_sources(root) +} + +/// An older build that ran after the migration writes JSON again; this build +/// never reads it back. +pub(super) fn warn_about_stray_sources(root: &Path) { + let Ok(sources) = legacy_sources(root) else { + return; + }; + if !sources.is_empty() { + log::warn!( + "ignoring {} file(s) in {} left by an older Tcode build (sessions.json / *.jsonl); \ + threads are read only from {DB_FILE}", + sources.len(), + root.display() + ); + } +} + +struct Summary { + sources: Vec, + projects: usize, + sessions: usize, + logs: usize, + lines: u64, + blank_lines: u64, + bytes: u64, +} + +/// Every staging file a previous run may have left. Never the final name's. +fn remove_staging(root: &Path) -> io::Result<()> { + let staging = root.join(STAGING_FILE); + for path in [ + wal_path(&staging), + sidecar(&staging, "-shm"), + sidecar(&staging, "-journal"), + staging, + root.join(ARCHIVE_LIST), + ] { + match fs::remove_file(&path) { + Ok(()) => log::warn!("discarded {} from an interrupted migration", path.display()), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(()) +} + +fn sidecar(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_owned(); + name.push(suffix); + PathBuf::from(name) +} + +/// `sessions.json` and every `.jsonl` directly in the data dir. +fn legacy_sources(root: &Path) -> io::Result> { + let mut names = Vec::new(); + for entry in fs::read_dir(root)? { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + if name == LEGACY_INDEX + || Path::new(&name) + .extension() + .is_some_and(|ext| ext == "jsonl") + { + names.push(name); + } + } + names.sort(); + Ok(names) +} + +fn build_staging(root: &Path) -> io::Result { + let staging = root.join(STAGING_FILE); + let db = Db::open(&staging, true)?; + let index = migrate_index(read_legacy_index(root)?); + reject_duplicates(&index)?; + db.write(|db, connection| { + for project in &index.projects { + let body = serde_json::to_vec(project).map_err(io::Error::other)?; + db.execute( + connection, + "INSERT INTO projects (id, body) VALUES (?1, ?2)", + (project.id.as_str(), body), + )?; + } + for session in &index.sessions { + let body = serde_json::to_vec(session).map_err(io::Error::other)?; + db.execute( + connection, + "INSERT INTO sessions (id, body) VALUES (?1, ?2)", + (session.id.as_str(), body), + )?; + } + Ok(()) + })?; + + let sources = legacy_sources(root)?; + let mut summary = Summary { + sources: sources.clone(), + projects: index.projects.len(), + sessions: index.sessions.len(), + logs: 0, + lines: 0, + blank_lines: 0, + bytes: 0, + }; + let mut tallies = Vec::new(); + for name in &sources { + let Some(id) = name.strip_suffix(".jsonl") else { + continue; + }; + let path = root.join(name); + let tally = import_log(&db, id, &path)?; + verify_log(&db, id, &path, &tally)?; + log::info!( + "migrated {name}: {} line(s), {} blank, {} bytes, verified byte-identical", + tally.lines, + tally.blank_lines, + tally.bytes + ); + summary.logs += 1; + summary.lines += tally.lines; + summary.blank_lines += tally.blank_lines; + summary.bytes += tally.bytes; + tallies.push((id.to_owned(), tally)); + } + + let stored = db.read(read_index)?; + verify_index(&index, &stored)?; + db.write(|db, connection| { + db.execute( + connection, + &format!("PRAGMA user_version = {SCHEMA_VERSION}"), + (), + ) + .map(drop) + })?; + db.checkpoint()?; + drop(db); + + // A fresh open sees only what reached the main file. + let reopened = Db::open(&staging, false)?; + if reopened.user_version()? != SCHEMA_VERSION { + return Err(io::Error::other(format!( + "{} lost its completion marker after the checkpoint", + staging.display() + ))); + } + // Per thread, so each count is an index range scan rather than one sort + // over every row in the database. + let mut committed_total = 0; + for (id, tally) in &tallies { + let (lines, bytes) = reopened.read(|db, connection| { + let mut counts = (0, 0); + db.query( + connection, + "SELECT COUNT(*), COALESCE(SUM(length(line)), 0) FROM events WHERE session_id = ?1", + (id.as_str(),), + |row| { + counts = (integer(row, 0)? as u64, integer(row, 1)? as u64); + Ok(()) + }, + )?; + Ok(counts) + })?; + if (lines, bytes) != (tally.lines, tally.bytes) { + return Err(io::Error::other(format!( + "{}: thread {id} holds {lines} line(s) / {bytes} bytes after reopening, the \ + source {} / {}", + staging.display(), + tally.lines, + tally.bytes + ))); + } + committed_total += lines; + } + let mut stored_total = 0; + reopened.read(|db, connection| { + db.query(connection, "SELECT COUNT(*) FROM events", (), |row| { + stored_total = integer(row, 0)? as u64; + Ok(()) + }) + })?; + if stored_total != committed_total { + return Err(io::Error::other(format!( + "{}: {stored_total} event rows after reopening, {committed_total} migrated", + staging.display() + ))); + } + let stored = reopened.read(read_index)?; + verify_index(&index, &stored)?; + drop(reopened); + let wal = wal_path(&staging); + match fs::metadata(&wal) { + Ok(metadata) if metadata.len() > 0 => { + return Err(io::Error::other(format!( + "{} holds {} bytes after the staged database was closed", + wal.display(), + metadata.len() + ))); + } + Ok(_) => fs::remove_file(&wal)?, + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + sync_file(&staging)?; + Ok(summary) +} + +/// Today's tolerance: the object schema or the legacy bare array; an +/// unparseable file is preserved beside itself and the logs still migrate. +fn read_legacy_index(root: &Path) -> io::Result { + let path = root.join(LEGACY_INDEX); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(IndexFile::default()), + Err(error) => return Err(error), + }; + let parsed = serde_json::from_slice::(&bytes).or_else(|_| { + serde_json::from_slice::>(&bytes).map(|sessions| IndexFile { + projects: Vec::new(), + sessions, + }) + }); + match parsed { + Ok(file) => Ok(file), + Err(error) => { + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0); + let corrupt = root.join(format!("{LEGACY_INDEX}.corrupt-{timestamp}")); + fs::rename(&path, &corrupt)?; + log::warn!( + "failed to parse {LEGACY_INDEX}: {error}; preserved it as {} and migrated the \ + event logs without it", + corrupt.display() + ); + Ok(IndexFile::default()) + } + } +} + +/// Upserting by id would silently keep only the last of two entries. +fn reject_duplicates(index: &IndexFile) -> io::Result<()> { + let mut seen = HashSet::new(); + for id in index.projects.iter().map(|project| &project.id) { + if !seen.insert(("project", id)) { + return Err(duplicate("project", id)); + } + } + for id in index.sessions.iter().map(|session| &session.id) { + if !seen.insert(("session", id)) { + return Err(duplicate("session", id)); + } + } + Ok(()) +} + +fn duplicate(kind: &str, id: &str) -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{LEGACY_INDEX} lists {kind} {id} more than once; nothing was migrated and the file \ + is unchanged" + ), + ) +} + +fn verify_index(expected: &IndexFile, stored: &IndexFile) -> io::Result<()> { + if stored.projects != expected.projects { + return Err(io::Error::other( + "the migrated projects differ from sessions.json when read back", + )); + } + if stored.sessions != expected.sessions { + let id = expected + .sessions + .iter() + .zip(&stored.sessions) + .find(|(expected, stored)| expected != stored) + .map(|(expected, _)| expected.id.as_str()) + .unwrap_or("(count)"); + return Err(io::Error::other(format!( + "the migrated session {id} differs from sessions.json when read back" + ))); + } + Ok(()) +} + +#[derive(Default)] +struct Tally { + lines: u64, + blank_lines: u64, + bytes: u64, +} + +/// Every raw segment of `path`: each line with its `\n`, an unterminated last +/// line as it is. +fn read_segments(path: &Path, mut each: impl FnMut(Vec) -> io::Result<()>) -> io::Result<()> { + let mut reader = BufReader::with_capacity(1 << 20, File::open(path)?); + loop { + let mut segment = Vec::new(); + if reader.read_until(b'\n', &mut segment)? == 0 { + return Ok(()); + } + each(segment)?; + } +} + +fn import_log(db: &Db, id: &str, path: &Path) -> io::Result { + let mut tally = Tally::default(); + let mut pending: Vec> = Vec::new(); + let mut pending_bytes = 0; + let mut position = 0; + let flush = |pending: &mut Vec>, position: &mut i64| -> io::Result<()> { + if !pending.is_empty() { + *position = db.write(|db, connection| { + insert_segments(db, connection, id, *position, pending.iter()) + })?; + pending.clear(); + } + Ok(()) + }; + read_segments(path, |segment| { + tally.lines += 1; + tally.bytes += segment.len() as u64; + if segment.trim_ascii().is_empty() { + tally.blank_lines += 1; + } + pending_bytes += segment.len(); + pending.push(segment); + if pending_bytes >= CHUNK_BYTES { + flush(&mut pending, &mut position)?; + pending_bytes = 0; + } + Ok(()) + })?; + flush(&mut pending, &mut position)?; + Ok(tally) +} + +/// Compare the committed rows with the file again, segment for segment. +fn verify_log(db: &Db, id: &str, path: &Path, tally: &Tally) -> io::Result<()> { + let mismatch = |position: i64| { + io::Error::other(format!( + "{}: line {} differs from the migrated copy", + path.display(), + position + 1 + )) + }; + let mut page: Vec> = Vec::new(); + let mut page_start = 0_i64; + let mut position = 0_i64; + read_segments(path, |segment| { + if position - page_start == page.len() as i64 { + page_start = position; + page = db.read(|db, connection| { + // Sizes first, so a page holds about CHUNK_BYTES however long + // the lines are, and at least one line. + let mut last = position - 1; + let mut bytes = 0; + db.query( + connection, + "SELECT position, length(line) FROM events \ + WHERE session_id = ?1 AND position >= ?2 ORDER BY position LIMIT ?3", + (id, position, VERIFY_ROWS), + |row| { + if last < position || bytes < CHUNK_BYTES as i64 { + last = integer(row, 0)?; + bytes += integer(row, 1)?; + } + Ok(()) + }, + )?; + let mut rows = Vec::new(); + db.query( + connection, + "SELECT position, line FROM events \ + WHERE session_id = ?1 AND position BETWEEN ?2 AND ?3 ORDER BY position", + (id, position, last), + |row| { + let stored = integer(row, 0)?; + if stored != position + rows.len() as i64 { + return Err(mismatch(position + rows.len() as i64)); + } + rows.push(blob(row, 1)?); + Ok(()) + }, + )?; + Ok(rows) + })?; + } + let stored = page + .get((position - page_start) as usize) + .ok_or_else(|| mismatch(position))?; + if *stored != segment { + return Err(mismatch(position)); + } + position += 1; + Ok(()) + })?; + let mut extra = 0; + db.read(|db, connection| { + db.query( + connection, + "SELECT COUNT(*) FROM events WHERE session_id = ?1 AND position >= ?2", + (id, position), + |row| { + extra = integer(row, 0)?; + Ok(()) + }, + ) + })?; + if extra != 0 || position as u64 != tally.lines { + return Err(mismatch(position)); + } + Ok(()) +} + +fn publish(root: &Path, sources: &[String]) -> io::Result<()> { + let list = root.join(ARCHIVE_LIST); + if !sources.is_empty() { + let temporary = sidecar(&list, ".tmp"); + fs::write( + &temporary, + serde_json::to_vec_pretty(sources).map_err(io::Error::other)?, + )?; + sync_file(&temporary)?; + fs::rename(&temporary, &list)?; + } + sync_dir(root)?; + fs::rename(root.join(STAGING_FILE), root.join(DB_FILE))?; + sync_dir(root) +} + +/// Move the sources named by `tcode.db.archive` into `legacy/`. Safe to repeat: +/// a moved file is skipped, and a name already taken in `legacy/` keeps its +/// source where it is. +fn archive_sources(root: &Path) -> io::Result<()> { + let list = root.join(ARCHIVE_LIST); + let names: Vec = match fs::read(&list) { + Ok(bytes) => serde_json::from_slice(&bytes).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("{} is unreadable: {error}", list.display()), + ) + })?, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + let legacy = root.join(LEGACY_DIR); + fs::create_dir_all(&legacy)?; + let mut moved = 0; + for name in &names { + // The list is written by this module, but a name is still never + // allowed to leave the data dir. + if Path::new(name).file_name() != Some(std::ffi::OsStr::new(name)) { + continue; + } + let source = root.join(name); + let destination = legacy.join(name); + if !source.exists() { + continue; + } + if destination.exists() { + log::warn!( + "kept {} in place: {} already exists", + source.display(), + destination.display() + ); + continue; + } + fs::rename(&source, &destination)?; + moved += 1; + } + sync_dir(&legacy)?; + sync_dir(root)?; + fs::remove_file(&list)?; + sync_dir(root)?; + log::info!( + "moved {moved} migrated source file(s) into {}", + legacy.display() + ); + Ok(()) +} + +/// Windows flushes only a handle opened for writing. +fn sync_file(path: &Path) -> io::Result<()> { + fs::OpenOptions::new().write(true).open(path)?.sync_all() +} + +/// Make a directory's entries (a rename, a new file) durable. +#[cfg(unix)] +fn sync_dir(dir: &Path) -> io::Result<()> { + File::open(dir)?.sync_all() +} + +/// NTFS journals renames itself, and std cannot open a directory handle on +/// Windows. +#[cfg(not(unix))] +fn sync_dir(_dir: &Path) -> io::Result<()> { + Ok(()) +} diff --git a/crates/services/src/store/mod.rs b/crates/services/src/store/mod.rs new file mode 100644 index 000000000..31aad88ff --- /dev/null +++ b/crates/services/src/store/mod.rs @@ -0,0 +1,1025 @@ +//! Persistence for tcode threads. +//! +//! Projects, thread metadata and every thread's event log live in one Turso +//! database, `tcode.db`, in the platform data dir (e.g. +//! `~/Library/Application Support/tcode/`): +//! * `projects` and `sessions` hold each [`Project`] / [`SessionMeta`] as the +//! JSON serde produces for it. +//! * `events` holds each thread's log as raw byte segments, one per line +//! including its `\n`, densely numbered from 0: the bytes of a `{ ts, event }` +//! record, or whatever a migrated or imported log contained. +//! +//! Model and command caches stay as JSON files beside it. +//! +//! Replay accepts timestamped records and legacy bare [`AgentEvent`] lines, +//! then folds [`StoredEvent`]s into a [`tcode_core::session::Timeline`]. + +mod db; +mod migrate; +#[cfg(test)] +mod tests; + +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, LazyLock, Mutex, MutexGuard, Weak}; +use std::time::{Duration, Instant}; + +use agent::{AgentEvent, ModelSpec, ProviderCommand, ProviderKind}; +use serde::{Deserialize, Serialize}; + +use tcode_core::project::{IndexFile, Project, SessionMeta}; +use tcode_core::session::StoredEvent; + +use db::{Broken, Db, SCHEMA_VERSION, blob, integer, is_broken}; + +const DB_FILE: &str = "tcode.db"; +/// Held with an OS file lock for as long as a host owns the data dir, which +/// covers the migration as well as the open database. It is never deleted: +/// unlinking a lock file while another process waits on it would let two +/// owners in. +const LOCK_FILE: &str = "tcode.lock"; +/// A macOS permission relaunch starts the new instance while the old one is +/// still closing its store. +const RELAUNCH_WAIT: Duration = Duration::from_secs(15); + +/// On-disk envelope wrapping each event with its record time. Kept private: +/// callers deal in [`StoredEvent`] (which tolerates the legacy bare form). +#[derive(Serialize, Deserialize)] +struct EventEnvelope { + ts: u64, + event: AgentEvent, +} + +#[derive(Serialize)] +struct EventEnvelopeRef<'a> { + ts: u64, + event: &'a AgentEvent, +} + +/// Cheap, cloneable handle to the data directory. Every handle for the same +/// directory in this process shares one database, writer and lifecycle. +#[derive(Clone)] +pub struct SessionStore { + root: PathBuf, + shared: Arc, +} + +impl std::fmt::Debug for SessionStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SessionStore") + .field("root", &self.root) + .finish_non_exhaustive() + } +} + +struct Shared { + state: Mutex, + /// Signalled whenever an operation finishes, for [`SessionStore::close`]. + idle: Condvar, + /// Per-session event-log generation, advanced after every committed + /// change to that log; see [`SessionStore::event_generation`]. + generations: Mutex>, + next_generation: AtomicU64, + /// Full event-log parses, shared by every clone of this store so tests can + /// prove a caller served history from memory. + #[cfg(any(test, feature = "test-support"))] + event_reads: std::sync::atomic::AtomicUsize, +} + +enum State { + /// Opened on first use, so a handle that only needs the directory (a + /// subcommand, a settings file) never takes the data dir. + Unopened, + Open(Live), + /// [`SessionStore::close`] is waiting for in-flight operations. + Closing(Live), + Closed, + /// A panic or an unrecoverable connection: nothing is served again, and + /// the ownership lock stays held while any handle remains, so no other + /// process opens a database this one may have left mid-operation. + Failed { + reason: String, + _ownership: Option, + }, +} + +struct Live { + db: Arc, + ownership: File, + in_flight: usize, +} + +/// Every open handle by canonical data dir, so a second +/// [`SessionStore::open_at`] in one process shares the first one's database +/// instead of contending for its locks. +static STORES: LazyLock>>> = LazyLock::new(Default::default); + +/// One change to the store. [`SessionStore::apply`] commits a sequence of them +/// in a single transaction. +#[derive(Debug, Clone)] +pub struct Mutation(Op); + +#[derive(Debug, Clone)] +enum Op { + AppendEvent { session_id: String, line: Vec }, + ReplaceEventLog { session_id: String, bytes: Vec }, + CloneEvents { src: String, dst: String }, + UpsertMeta(Box), + UpsertProject(Box), + RemoveSession(String), + RemoveProject(String), +} + +impl Mutation { + /// Append one event, wrapped in a timestamped envelope + /// (`{"ts": , "event": {…}}`). + pub fn append_event(session_id: &str, ts: u64, event: &AgentEvent) -> io::Result { + let mut line = serde_json::to_vec(&EventEnvelopeRef { ts, event }).map_err(invalid_data)?; + line.push(b'\n'); + Ok(Self(Op::AppendEvent { + session_id: session_id.to_owned(), + line, + })) + } + + /// Replace a session's whole native event log with `bytes`, kept exactly. + pub fn replace_event_log(session_id: &str, bytes: Vec) -> Self { + Self(Op::ReplaceEventLog { + session_id: session_id.to_owned(), + bytes, + }) + } + + /// Give `dst` a copy of `src`'s event log. A missing source is an empty + /// transcript. + pub fn clone_events(src: &str, dst: &str) -> Self { + Self(Op::CloneEvents { + src: src.to_owned(), + dst: dst.to_owned(), + }) + } + + /// Insert or replace a meta (by id). + pub fn upsert_meta(meta: SessionMeta) -> Self { + Self(Op::UpsertMeta(Box::new(meta))) + } + + /// Insert or replace a project; its previous managed icon is removed once + /// the change has committed. + pub fn upsert_project(project: Project) -> Self { + Self(Op::UpsertProject(Box::new(project))) + } + + /// Remove a session's meta and its event log. + pub fn remove_session(id: &str) -> Self { + Self(Op::RemoveSession(id.to_owned())) + } + + /// Remove a project; its managed icon is removed once the change has + /// committed. Sessions are removed separately. + pub fn remove_project(id: &str) -> Self { + Self(Op::RemoveProject(id.to_owned())) + } + + /// Approximate bytes this change writes, for bounding a batch. + pub fn payload_len(&self) -> usize { + match &self.0 { + Op::AppendEvent { line, .. } => line.len(), + Op::ReplaceEventLog { bytes, .. } => bytes.len(), + _ => 0, + } + } + + /// The session whose event log this change rewrites, if any. + fn event_log(&self) -> Option<&str> { + match &self.0 { + Op::AppendEvent { session_id, .. } | Op::ReplaceEventLog { session_id, .. } => { + Some(session_id) + } + Op::CloneEvents { dst, .. } => Some(dst), + Op::RemoveSession(id) => Some(id), + Op::UpsertMeta(_) | Op::UpsertProject(_) | Op::RemoveProject(_) => None, + } + } +} + +impl SessionStore { + /// Open (creating if needed) the store under the platform data dir, or under + /// `TCODE_DATA_DIR` when it is set — which gives a throwaway profile (its own + /// sessions, settings and installed ACP agents) for demos and screenshots. + pub fn open_default() -> io::Result { + let root = match std::env::var_os("TCODE_DATA_DIR") { + Some(dir) => PathBuf::from(dir), + None => dirs::data_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join("tcode"), + }; + Self::open_at(root) + } + + /// A handle to `root`, created if needed. The database is not opened until + /// [`SessionStore::open`] or the first operation that needs it. + pub fn open_at(root: PathBuf) -> io::Result { + fs::create_dir_all(&root)?; + let key = fs::canonicalize(&root)?; + let mut stores = STORES + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + stores.retain(|_, shared| shared.strong_count() > 0); + let shared = match stores.get(&key).and_then(Weak::upgrade) { + Some(shared) if !shared.finished() => shared, + _ => { + let shared = Arc::new(Shared { + state: Mutex::new(State::Unopened), + idle: Condvar::new(), + generations: Mutex::default(), + next_generation: AtomicU64::new(1), + #[cfg(any(test, feature = "test-support"))] + event_reads: Default::default(), + }); + stores.insert(key, Arc::downgrade(&shared)); + shared + } + }; + Ok(Self { root, shared }) + } + + /// How many times [`SessionStore::read_events`] parsed a log through this + /// store or any of its clones. + #[cfg(any(test, feature = "test-support"))] + pub fn event_reads(&self) -> usize { + self.shared + .event_reads + .load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn root(&self) -> &PathBuf { + &self.root + } + + /// Take ownership of the data dir and open its database now, migrating the + /// JSON index and JSONL logs on first start. Fails with + /// [`io::ErrorKind::ResourceBusy`] while another host owns the directory. + pub fn open(&self) -> io::Result<()> { + self.run("open", |_| Ok(())) + } + + /// Whether the store stopped serving after a panic or a broken connection. + pub fn is_failed(&self) -> bool { + matches!( + *self + .shared + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()), + State::Failed { .. } + ) + } + + /// Wait for every in-flight operation, checkpoint the WAL into the main + /// file and release the database and the data dir. Every handle of this + /// store fails afterwards; a store that already failed is not + /// checkpointed. + pub fn close(&self) -> io::Result<()> { + let mut state = self.shared.lock_state()?; + loop { + match std::mem::replace(&mut *state, State::Closed) { + State::Unopened | State::Closed => return Ok(()), + State::Open(live) => *state = State::Closing(live), + State::Closing(live) if live.in_flight > 0 => { + *state = State::Closing(live); + state = self + .shared + .idle + .wait(state) + .unwrap_or_else(|poisoned| poisoned.into_inner()); + } + State::Closing(live) => { + drop(state); + let Live { db, ownership, .. } = live; + let result = match Arc::try_unwrap(db) { + Ok(db) => catch_unwind(AssertUnwindSafe(|| db.checkpoint())) + .unwrap_or_else(|panic| { + Err(io::Error::other(format!( + "session store panicked during the closing checkpoint: {}", + panic_message(panic.as_ref()) + ))) + }), + Err(_) => Err(io::Error::other( + "session store is still referenced after its last operation", + )), + }; + drop(ownership); + return result; + } + failed @ State::Failed { .. } => { + let error = failed_error(&failed); + *state = failed; + return Err(error); + } + } + } + } + + /// Run `body` against the open database. A panic inside it (Turso asserts + /// its invariants by panicking) becomes an error naming `operation` and + /// fails the store for every handle. + fn run(&self, operation: &str, body: impl FnOnce(&Db) -> io::Result) -> io::Result { + let db = self.acquire(operation)?; + let outcome = catch_unwind(AssertUnwindSafe(|| body(&db))); + drop(db); + let result = match outcome { + Ok(Err(error)) if is_broken(&error) => { + self.shared.fail(format!("{operation}: {error}")); + Err(error) + } + Ok(result) => result, + Err(panic) => { + let reason = format!( + "session store panicked during {operation}: {}", + panic_message(panic.as_ref()) + ); + log::error!("{reason}"); + self.shared.fail(reason.clone()); + Err(io::Error::other(reason)) + } + }; + self.shared.release(); + result + } + + fn acquire(&self, operation: &str) -> io::Result> { + let mut state = self.shared.lock_state()?; + if matches!(*state, State::Unopened) { + match catch_unwind(AssertUnwindSafe(|| open_live(&self.root))) { + Ok(Ok(live)) => *state = State::Open(live), + Ok(Err(error)) => return Err(error), + Err(panic) => { + let reason = format!( + "session store panicked while opening {}: {}", + self.root.display(), + panic_message(panic.as_ref()) + ); + log::error!("{reason}"); + *state = State::Failed { + reason: reason.clone(), + _ownership: None, + }; + return Err(io::Error::other(reason)); + } + } + } + match &mut *state { + State::Open(live) => { + live.in_flight += 1; + Ok(live.db.clone()) + } + State::Closing(_) | State::Closed => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + format!("cannot {operation}: the session store is closed"), + )), + failed @ State::Failed { .. } => Err(failed_error(failed)), + State::Unopened => unreachable!("opened above"), + } + } + + /// Commit `mutations` in one transaction, then run their deferred side + /// effects. Either every change is committed or none is. + pub fn apply(&self, mutations: &[Mutation]) -> io::Result<()> { + if mutations.is_empty() { + return Ok(()); + } + let icons = self.run("write", |db| { + db.write(|db, connection| { + let mut icons = Vec::new(); + for mutation in mutations { + apply_op(db, connection, &mutation.0, &mut icons)?; + } + Ok(icons) + }) + })?; + { + let mut generations = self + .shared + .generations + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + for id in mutations.iter().filter_map(Mutation::event_log) { + let generation = self.shared.next_generation.fetch_add(1, Ordering::Relaxed); + generations.insert(id.to_owned(), generation); + } + } + for icon in icons { + self.remove_project_icon(icon); + } + Ok(()) + } + + /// A value that changes whenever a committed write changes `id`'s event + /// log through this store, so a cache of something derived from the log + /// can tell it is stale. 0 means unchanged since the store was opened. + pub fn event_generation(&self, id: &str) -> u64 { + self.shared + .generations + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .get(id) + .copied() + .unwrap_or(0) + } + + /// Read a native event log back as the exact bytes it holds. + pub fn read_event_log(&self, id: &str) -> io::Result> { + self.run("read event log", |db| { + db.read(|db, connection| { + let mut bytes = Vec::new(); + db.query( + connection, + "SELECT line FROM events WHERE session_id = ?1 ORDER BY position", + (id,), + |row| { + bytes.extend_from_slice(&blob(row, 0)?); + Ok(()) + }, + )?; + Ok(bytes) + }) + }) + } + + fn models_path(&self, provider: ProviderKind) -> PathBuf { + let name = match provider { + ProviderKind::Codex => "codex", + ProviderKind::ClaudeCode => "claude", + ProviderKind::Pi => "pi", + ProviderKind::OpenCode => "opencode", + ProviderKind::Cursor => "cursor", + ProviderKind::Grok => "grok", + // ACP agents publish their models over the wire at session start + // (`AgentEvent::ProviderOptions`), so there is no catalog to cache. + ProviderKind::Acp => "acp", + }; + self.root.join(format!("models-{name}.json")) + } + + fn commands_path(&self, provider: ProviderKind, acp_agent_id: Option<&str>) -> Option { + let name = match provider { + ProviderKind::Codex => "codex".to_string(), + ProviderKind::ClaudeCode => "claude".to_string(), + ProviderKind::Pi => "pi".to_string(), + ProviderKind::OpenCode => "opencode".to_string(), + ProviderKind::Cursor => "cursor".to_string(), + ProviderKind::Grok => "grok".to_string(), + ProviderKind::Acp => { + let id = acp_agent_id?; + // Registry ids are external input and may contain path separators. + // Hex keeps the filename reversible and collision-free without + // allowing an id to escape the data directory. + let encoded = id + .as_bytes() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + format!("acp-{encoded}") + } + }; + Some(self.root.join(format!("commands-{name}.json"))) + } + + /// Load the last-fetched model catalog for `provider` so the picker is + /// instant offline. Empty when never fetched / unreadable. + pub fn load_models(&self, provider: ProviderKind) -> Vec { + let Ok(bytes) = fs::read(self.models_path(provider)) else { + return Vec::new(); + }; + serde_json::from_slice(&bytes).unwrap_or_default() + } + + /// Persist the freshly fetched model catalog for `provider`. + pub fn save_models(&self, provider: ProviderKind, models: &[ModelSpec]) -> io::Result<()> { + let path = self.models_path(provider); + let tmp = path.with_extension("json.tmp"); + let data = serde_json::to_vec_pretty(models).map_err(invalid_data)?; + fs::write(&tmp, data)?; + fs::rename(&tmp, path) + } + + /// Load the most recently reported command/skill list for a native provider + /// or one specific ACP agent. Empty when missing, unreadable, or when an ACP + /// agent id was not supplied. + pub fn load_commands( + &self, + provider: ProviderKind, + acp_agent_id: Option<&str>, + ) -> Vec { + let Some(path) = self.commands_path(provider, acp_agent_id) else { + return Vec::new(); + }; + let Ok(bytes) = fs::read(path) else { + return Vec::new(); + }; + serde_json::from_slice(&bytes).unwrap_or_default() + } + + /// Atomically persist the complete command/skill list reported by a native + /// provider or one specific ACP agent. Empty lists are meaningful: they + /// replace a stale non-empty cache. + pub fn save_commands( + &self, + provider: ProviderKind, + acp_agent_id: Option<&str>, + commands: &[ProviderCommand], + ) -> io::Result<()> { + let path = self.commands_path(provider, acp_agent_id).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "ACP command cache requires an agent id", + ) + })?; + let tmp = path.with_extension("json.tmp"); + let data = serde_json::to_vec_pretty(commands).map_err(invalid_data)?; + fs::write(&tmp, data)?; + fs::rename(&tmp, path) + } + + /// Load every project and session, in the order they were first stored. + /// A row this build cannot decode is skipped with a warning and left in + /// the database untouched. + pub fn read_file(&self) -> io::Result { + self.run("read index", |db| { + db.read(|db, connection| { + // One read transaction, so projects and sessions come from the + // same snapshot. + db.execute(connection, "BEGIN", ())?; + let file = read_index(db, connection); + db.execute(connection, "COMMIT", ())?; + file + }) + }) + } + + /// Load the session index (newest first). + pub fn load_index(&self) -> io::Result> { + let mut metas = self.read_file()?.sessions; + metas.sort_by_key(|b| std::cmp::Reverse(b.updated_at)); + Ok(metas) + } + + /// Insert or replace a meta in the index (by id). + pub fn upsert_meta(&self, meta: &SessionMeta) -> io::Result<()> { + self.apply(&[Mutation::upsert_meta(meta.clone())]) + } + + /// Insert or replace a project, then remove its previous managed icon. + pub fn upsert_project(&self, project: &Project) -> io::Result<()> { + self.apply(&[Mutation::upsert_project(project.clone())]) + } + + /// Remove a project and its managed icon. Sessions are removed separately. + pub fn remove_project(&self, id: &str) -> io::Result<()> { + self.apply(&[Mutation::remove_project(id)]) + } + + fn remove_project_icon(&self, path: Option) { + if let Some(path) = path + && path.parent() == Some(self.root.join("project-icons").as_path()) + && let Err(error) = fs::remove_file(&path) + && error.kind() != io::ErrorKind::NotFound + { + log::warn!("could not remove project icon {}: {error}", path.display()); + } + } + + /// Append one event to the session's log, wrapped in a timestamped + /// envelope (`{"ts": , "event": {…}}`). + pub fn append_event(&self, id: &str, ts: u64, event: &AgentEvent) -> io::Result<()> { + self.apply(&[Mutation::append_event(id, ts, event)?]) + } + + /// Read and parse every persisted event for a session, skipping lines that + /// do not parse. + /// + /// Each line is tolerantly parsed as either a timestamped envelope + /// (`{"ts":…,"event":…}`) or a legacy bare event (`{"type":…}`), so logs + /// written before the envelope format still replay (with `ts == None`). + pub fn read_events(&self, id: &str) -> io::Result> { + #[cfg(any(test, feature = "test-support"))] + self.shared + .event_reads + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.run("read events", |db| { + db.read(|db, connection| { + let mut events = Vec::new(); + db.query( + connection, + "SELECT position, line FROM events WHERE session_id = ?1 ORDER BY position", + (id,), + |row| { + let position = integer(row, 0)?; + let line = blob(row, 1)?; + let Ok(line) = std::str::from_utf8(&line) else { + log::warn!("skipping event {position} of {id}: not UTF-8"); + return Ok(()); + }; + let trimmed = line.trim(); + if trimmed.is_empty() { + return Ok(()); + } + match parse_stored_line(trimmed) { + Ok(stored) => events.push(stored), + Err(err) => { + log::warn!("skipping unparseable event {position} of {id}: {err}") + } + } + Ok(()) + }, + )?; + Ok(events) + }) + }) + } + + /// Remove a session from the index and delete its event log. + pub fn remove_session(&self, id: &str) -> io::Result<()> { + self.apply(&[Mutation::remove_session(id)]) + } +} + +impl Shared { + fn lock_state(&self) -> io::Result> { + // The state is replaced whole under the lock, so a poisoned guard + // still holds a consistent value. + Ok(self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner())) + } + + fn finished(&self) -> bool { + matches!( + *self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()), + State::Closed | State::Failed { .. } + ) + } + + fn release(&self) { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let State::Open(live) | State::Closing(live) = &mut *state { + live.in_flight -= 1; + } + self.idle.notify_all(); + } + + fn fail(&self, reason: String) { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let ownership = match std::mem::replace(&mut *state, State::Closed) { + State::Open(live) | State::Closing(live) => Some(live.ownership), + State::Failed { _ownership, .. } => _ownership, + State::Unopened | State::Closed => None, + }; + *state = State::Failed { + reason, + _ownership: ownership, + }; + self.idle.notify_all(); + } +} + +fn failed_error(state: &State) -> io::Error { + let State::Failed { reason, .. } = state else { + unreachable!("only called for a failed store") + }; + io::Error::other(Broken(format!( + "the session store stopped after an internal failure ({reason}); restart Tcode" + ))) +} + +fn panic_message(panic: &(dyn std::any::Any + Send)) -> String { + panic + .downcast_ref::<&str>() + .map(|message| (*message).to_owned()) + .or_else(|| panic.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".into()) +} + +/// Take the data dir, bring `tcode.db` up to date (migrating on first start) +/// and open it. +fn open_live(root: &Path) -> io::Result { + let ownership = acquire_ownership(root)?; + migrate::prepare(root)?; + let path = root.join(DB_FILE); + let db = Db::open(&path, false)?; + match db.user_version()? { + SCHEMA_VERSION => {} + 0 => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{} was never completed (user_version 0). It is kept as it is and can be \ + examined with the sqlite3 command-line tool.", + path.display() + ), + )); + } + version => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{} has schema version {version}, written by a newer Tcode; this build \ + understands version {SCHEMA_VERSION}", + path.display() + ), + )); + } + } + migrate::warn_about_stray_sources(root); + Ok(Live { + db: Arc::new(db), + ownership, + in_flight: 0, + }) +} + +fn acquire_ownership(root: &Path) -> io::Result { + let path = root.join(LOCK_FILE); + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&path)?; + let mut deadline = None; + let mut delay = Duration::from_millis(20); + loop { + match file.try_lock() { + Ok(()) => return Ok(file), + Err(fs::TryLockError::Error(error)) => return Err(error), + Err(fs::TryLockError::WouldBlock) => { + let deadline = *deadline.get_or_insert_with(|| { + // Peek only: the marker is consumed by the host after it + // owns the data dir. + crate::relaunch::pending(root).then(|| Instant::now() + RELAUNCH_WAIT) + }); + match deadline { + Some(deadline) if Instant::now() < deadline => { + std::thread::sleep(delay); + delay = (delay * 2).min(Duration::from_millis(250)); + } + _ => { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + format!( + "another Tcode host is already using the data directory {}; \ + quit it first, or set TCODE_DATA_DIR to use another directory", + root.display() + ), + )); + } + } + } + } + } +} + +fn read_index(db: &Db, connection: &turso::Connection) -> io::Result { + fn rows( + db: &Db, + connection: &turso::Connection, + table: &str, + ) -> io::Result> { + let mut values = Vec::new(); + db.query( + connection, + &format!("SELECT id, body FROM {table} ORDER BY rowid"), + (), + |row| { + let body = blob(row, 1)?; + match serde_json::from_slice(&body) { + Ok(value) => values.push(value), + Err(error) => log::warn!( + "skipping {table} row {:?} this build cannot read: {error}", + db::text(row, 0).unwrap_or_default() + ), + } + Ok(()) + }, + )?; + Ok(values) + } + Ok(IndexFile { + projects: rows(db, connection, "projects")?, + sessions: rows(db, connection, "sessions")?, + }) +} + +/// The raw segments of a native log: every line with its `\n`, and an +/// unterminated last line as it is. +fn segments(bytes: &[u8]) -> impl Iterator { + bytes.split_inclusive(|byte| *byte == b'\n') +} + +fn insert_segments( + db: &Db, + connection: &turso::Connection, + session_id: &str, + first: i64, + lines: impl IntoIterator>, +) -> io::Result { + let mut position = first; + for line in lines { + db.execute( + connection, + "INSERT INTO events (session_id, position, line) VALUES (?1, ?2, ?3)", + (session_id, position, line.as_ref()), + )?; + position += 1; + } + Ok(position) +} + +fn apply_op( + db: &Db, + connection: &turso::Connection, + op: &Op, + icons: &mut Vec>, +) -> io::Result<()> { + match op { + Op::AppendEvent { session_id, line } => { + let mut last = None; + db.query( + connection, + "SELECT position, substr(line, -1) = x'0a' FROM events \ + WHERE session_id = ?1 ORDER BY position DESC LIMIT 1", + (session_id.as_str(),), + |row| { + last = Some((integer(row, 0)?, integer(row, 1)? != 0)); + Ok(()) + }, + )?; + let next = match last { + None => 0, + Some((position, terminated)) => { + if !terminated { + // A log imported without a final newline: terminate + // its last line so the new record starts a line. + let mut unterminated = Vec::new(); + db.query( + connection, + "SELECT line FROM events WHERE session_id = ?1 AND position = ?2", + (session_id.as_str(), position), + |row| { + unterminated = blob(row, 0)?; + Ok(()) + }, + )?; + unterminated.push(b'\n'); + db.execute( + connection, + "UPDATE events SET line = ?3 WHERE session_id = ?1 AND position = ?2", + (session_id.as_str(), position, unterminated), + )?; + } + position + 1 + } + }; + insert_segments(db, connection, session_id, next, [line])?; + } + Op::ReplaceEventLog { session_id, bytes } => { + db.execute( + connection, + "DELETE FROM events WHERE session_id = ?1", + (session_id.as_str(),), + )?; + insert_segments(db, connection, session_id, 0, segments(bytes))?; + } + Op::CloneEvents { src, dst } => { + db.execute( + connection, + "DELETE FROM events WHERE session_id = ?1", + (dst.as_str(),), + )?; + db.execute( + connection, + "INSERT INTO events (session_id, position, line) \ + SELECT ?2, position, line FROM events WHERE session_id = ?1", + (src.as_str(), dst.as_str()), + )?; + } + Op::UpsertMeta(meta) => { + let body = serde_json::to_vec(meta.as_ref()).map_err(invalid_data)?; + db.execute( + connection, + "INSERT INTO sessions (id, body) VALUES (?1, ?2) \ + ON CONFLICT (id) DO UPDATE SET body = excluded.body", + (meta.id.as_str(), body), + )?; + } + Op::UpsertProject(project) => { + if let Some(previous) = stored_project(db, connection, &project.id)? + && previous.icon_path != project.icon_path + { + icons.push(previous.icon_path); + } + let body = serde_json::to_vec(project.as_ref()).map_err(invalid_data)?; + db.execute( + connection, + "INSERT INTO projects (id, body) VALUES (?1, ?2) \ + ON CONFLICT (id) DO UPDATE SET body = excluded.body", + (project.id.as_str(), body), + )?; + } + Op::RemoveSession(id) => { + db.execute( + connection, + "DELETE FROM events WHERE session_id = ?1", + (id.as_str(),), + )?; + db.execute( + connection, + "DELETE FROM sessions WHERE id = ?1", + (id.as_str(),), + )?; + } + Op::RemoveProject(id) => { + if let Some(previous) = stored_project(db, connection, id)? { + icons.push(previous.icon_path); + } + db.execute( + connection, + "DELETE FROM projects WHERE id = ?1", + (id.as_str(),), + )?; + } + } + Ok(()) +} + +fn stored_project( + db: &Db, + connection: &turso::Connection, + id: &str, +) -> io::Result> { + let mut project = None; + db.query( + connection, + "SELECT body FROM projects WHERE id = ?1", + (id,), + |row| { + project = serde_json::from_slice(&blob(row, 0)?).ok(); + Ok(()) + }, + )?; + Ok(project) +} + +fn invalid_data(error: impl Into>) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, error) +} + +/// Parse one JSONL line into a [`StoredEvent`], accepting both the timestamped +/// envelope and the legacy bare-event form. Envelope is tried first; a bare +/// event lacks the `ts`/`event` keys so it can't masquerade as one, and an +/// envelope lacks the top-level `type` tag so it can't parse as a bare event. +pub(crate) fn parse_stored_line(line: &str) -> Result { + match serde_json::from_str::(line) { + Ok(envelope) => Ok(StoredEvent { + ts: Some(envelope.ts), + event: envelope.event, + elided: None, + }), + Err(_envelope_err) => match serde_json::from_str::(line) { + Ok(event) => Ok(StoredEvent { + ts: None, + event, + elided: None, + }), + // Both forms failed: the line is genuinely corrupt. The bare-event + // error is the more informative one (the envelope attempt always + // fails on a bare event merely because `ts` is missing). + Err(bare_err) => Err(bare_err), + }, + } +} + +pub use tcode_core::project::now_secs; + +/// Current wall-clock time in unix milliseconds (used for event envelopes). +pub fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} diff --git a/crates/services/src/store/tests.rs b/crates/services/src/store/tests.rs new file mode 100644 index 000000000..e7943a97c --- /dev/null +++ b/crates/services/src/store/tests.rs @@ -0,0 +1,819 @@ +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::Path; + +use agent::{ApprovalMode, ProviderCommand, ProviderCommandKind, TurnStatus}; + +use super::*; + +/// A data dir that is removed when the test ends. +struct DataDir(PathBuf); + +impl DataDir { + fn new() -> Self { + let path = std::env::temp_dir().join(format!("tcode-store-test-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(&path).unwrap(); + Self(path) + } + + fn path(&self) -> &Path { + &self.0 + } + + fn store(&self) -> SessionStore { + SessionStore::open_at(self.0.clone()).unwrap() + } +} + +impl Drop for DataDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +/// A log as older builds wrote it, with every irregularity a real one can +/// hold: a legacy bare event, CRLF, a blank line, an unparseable line, a line +/// that is not UTF-8, and a torn last line without its newline. +const MIXED_LOG: &[u8] = b"{\"type\":\"turn_started\",\"turn_id\":\"legacy\"}\n\ +{\"ts\":2000,\"event\":{\"type\":\"turn_completed\",\"turn_id\":\"legacy\",\"status\":\"completed\",\"usage\":null}}\r\n\ +\n\ +{not valid json}\n\ +\xff\xfe{\"type\":\"turn_started\",\"turn_id\":\"not-utf8\"}\n\ +{\"ts\":2500,\"event\":{\"type\":\"turn_started\",\"turn_id\":\"second\"}}\n\ +{\"type\":\"turn_started"; + +const ORPHAN_LOG: &[u8] = + b"{\"ts\":1,\"event\":{\"type\":\"turn_started\",\"turn_id\":\"orphan\"}}\n"; + +/// `sessions.json` in the current object schema, with one session that +/// carries fields older builds wrote and this one dropped, and one that has +/// no project yet. +fn legacy_index() -> serde_json::Value { + serde_json::json!({ + "projects": [ + {"id": "p1", "name": "alpha", "root": "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/work/alpha", "created_at": 1} + ], + "sessions": [ + { + "id": "mixed", "title": "Mixed", "provider": "codex", "cwd": "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/work/alpha", + "project_id": "p1", "created_at": 1, "updated_at": 30, + "forked_from": "source", + "checkpoints": [{"turn": 2, "commit": "deadbeef", "event_offset": 7}] + }, + { + "id": "empty", "title": "Empty", "provider": "claude_code", + "cwd": "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/work/beta", "created_at": 2, "updated_at": 20 + } + ] + }) +} + +fn write_legacy_fixture(root: &Path) { + fs::write(root.join("sessions.json"), legacy_index().to_string()).unwrap(); + fs::write(root.join("mixed.jsonl"), MIXED_LOG).unwrap(); + fs::write(root.join("empty.jsonl"), b"").unwrap(); + fs::write(root.join("orphan.jsonl"), ORPHAN_LOG).unwrap(); +} + +const LEGACY_FILES: [&str; 4] = [ + "sessions.json", + "mixed.jsonl", + "empty.jsonl", + "orphan.jsonl", +]; + +fn turn_ids(events: &[StoredEvent]) -> Vec<(Option, String)> { + events + .iter() + .map(|stored| { + let id = match &stored.event { + AgentEvent::TurnStarted { turn_id } => format!("started {turn_id}"), + AgentEvent::TurnCompleted { + turn_id, + status: TurnStatus::Completed, + .. + } => format!("completed {turn_id}"), + other => format!("{other:?}"), + }; + (stored.ts, id) + }) + .collect() +} + +/// What the fixture must read back as, wherever it was migrated. +fn assert_fixture_migrated(store: &SessionStore, root: &Path) { + let file = store.read_file().unwrap(); + assert_eq!(file.projects.len(), 2); + assert_eq!(file.projects[0].id, "p1"); + let derived = &file.projects[1]; + assert_eq!(derived.root, Path::new("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/work/beta")); + assert_eq!(derived.name, "beta"); + assert_eq!( + file.sessions + .iter() + .map(|meta| (meta.id.as_str(), meta.project_id.as_deref())) + .collect::>(), + [("mixed", Some("p1")), ("empty", Some(derived.id.as_str()))] + ); + let mixed = &file.sessions[0]; + assert_eq!(mixed.title, "Mixed"); + assert_eq!(mixed.approval_mode, ApprovalMode::FullAccess); + assert_eq!(mixed.archived_at, None); + + assert_eq!(store.read_event_log("mixed").unwrap(), MIXED_LOG); + assert_eq!(store.read_event_log("orphan").unwrap(), ORPHAN_LOG); + assert!(store.read_event_log("empty").unwrap().is_empty()); + assert_eq!( + turn_ids(&store.read_events("mixed").unwrap()), + [ + (None, "started legacy".to_owned()), + (Some(2000), "completed legacy".to_owned()), + (Some(2500), "started second".to_owned()), + ] + ); + assert_eq!( + turn_ids(&store.read_events("orphan").unwrap()), + [(Some(1), "started orphan".to_owned())] + ); + + for name in LEGACY_FILES { + assert!(!root.join(name).exists(), "{name} left in the data dir"); + } + assert_eq!( + fs::read(root.join("legacy/sessions.json")).unwrap(), + legacy_index().to_string().as_bytes() + ); + assert_eq!( + fs::read(root.join("legacy/mixed.jsonl")).unwrap(), + MIXED_LOG + ); + assert_eq!( + fs::read(root.join("legacy/orphan.jsonl")).unwrap(), + ORPHAN_LOG + ); + assert!( + fs::read(root.join("legacy/empty.jsonl")) + .unwrap() + .is_empty() + ); + for leftover in [ + "tcode.db.migrating", + "tcode.db.migrating-wal", + "tcode.db.archive", + ] { + assert!(!root.join(leftover).exists(), "{leftover} left behind"); + } +} + +#[test] +fn first_start_migrates_the_json_index_and_every_log_byte_for_byte() { + let dir = DataDir::new(); + write_legacy_fixture(dir.path()); + let store = dir.store(); + store.open().unwrap(); + assert_fixture_migrated(&store, dir.path()); + + // Appending to a log that lost its final newline terminates that line + // first, as appending to the file did. + store + .append_event( + "mixed", + 3000, + &AgentEvent::TurnStarted { + turn_id: "next".into(), + }, + ) + .unwrap(); + let mut expected = MIXED_LOG.to_vec(); + expected.extend_from_slice( + b"\n{\"ts\":3000,\"event\":{\"type\":\"turn_started\",\"turn_id\":\"next\"}}\n", + ); + assert_eq!(store.read_event_log("mixed").unwrap(), expected); + store.close().unwrap(); + assert!(store.read_file().is_err(), "a closed store serves nothing"); + + // A later start opens the database as it is: the data dir now holds only + // what an older build would write, and that is ignored. + fs::write(dir.path().join("sessions.json"), b"[]").unwrap(); + fs::write(dir.path().join("mixed.jsonl"), b"").unwrap(); + let reopened = dir.store(); + assert_eq!(reopened.read_event_log("mixed").unwrap(), expected); + assert_eq!(reopened.read_file().unwrap().sessions.len(), 2); + assert_eq!(fs::read(dir.path().join("sessions.json")).unwrap(), b"[]"); + assert_eq!( + fs::read(dir.path().join("legacy/mixed.jsonl")).unwrap(), + MIXED_LOG + ); +} + +#[test] +fn legacy_bare_array_index_migrates_with_one_derived_project_per_root() { + let dir = DataDir::new(); + // Old-format file: a bare JSON array with no project_id fields. + let legacy = serde_json::json!([ + { + "id": "s1", "title": "One", "provider": "claude_code", + "cwd": "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/work/alpha", "created_at": 1, "updated_at": 10 + }, + { + "id": "s2", "title": "Two", "provider": "codex", + "cwd": "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/work/alpha", "created_at": 2, "updated_at": 20 + }, + { + "id": "s3", "title": "Three", "provider": "codex", + "cwd": "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/work/beta", "created_at": 3, "updated_at": 30 + } + ]); + fs::write(dir.path().join("sessions.json"), legacy.to_string()).unwrap(); + + let file = dir.store().read_file().unwrap(); + // Two distinct roots -> two derived projects, deduped by root. + assert_eq!(file.projects.len(), 2); + let alpha = file + .projects + .iter() + .find(|p| p.root == Path::new("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/work/alpha")) + .unwrap(); + assert_eq!(alpha.name, "alpha"); + let project_of = |id: &str| { + file.sessions + .iter() + .find(|session| session.id == id) + .unwrap() + .project_id + .clone() + }; + assert_eq!(project_of("s1"), Some(alpha.id.clone())); + assert_eq!(project_of("s2"), project_of("s1")); + assert_ne!(project_of("s3"), project_of("s1")); + // Newest first. + assert_eq!( + dir.store() + .load_index() + .unwrap() + .iter() + .map(|meta| meta.id.as_str()) + .collect::>(), + ["s3", "s2", "s1"] + ); +} + +#[test] +fn an_unparseable_index_is_preserved_and_the_logs_still_migrate() { + let dir = DataDir::new(); + let corrupt = b"not valid session json"; + fs::write(dir.path().join("sessions.json"), corrupt).unwrap(); + fs::write(dir.path().join("orphan.jsonl"), ORPHAN_LOG).unwrap(); + let store = dir.store(); + + let file = store.read_file().unwrap(); + assert!(file.projects.is_empty() && file.sessions.is_empty()); + assert_eq!(store.read_event_log("orphan").unwrap(), ORPHAN_LOG); + let backups: Vec<_> = fs::read_dir(dir.path()) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with("sessions.json.corrupt-") + }) + .collect(); + assert_eq!(backups.len(), 1); + assert_eq!(fs::read(backups[0].path()).unwrap(), corrupt); +} + +#[test] +fn duplicate_index_ids_fail_the_migration_and_leave_the_sources_alone() { + let dir = DataDir::new(); + let session = serde_json::json!({ + "id": "twice", "title": "One", "provider": "codex", + "cwd": "/work", "created_at": 1, "updated_at": 1 + }); + let index = serde_json::json!({"projects": [], "sessions": [session, session]}).to_string(); + fs::write(dir.path().join("sessions.json"), &index).unwrap(); + fs::write(dir.path().join("twice.jsonl"), ORPHAN_LOG).unwrap(); + + let error = dir.store().open().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("twice"), "{error}"); + assert!(!dir.path().join(DB_FILE).exists()); + assert!(!dir.path().join(LEGACY_DIR_NAME).exists()); + assert_eq!( + fs::read(dir.path().join("sessions.json")).unwrap(), + index.as_bytes() + ); + assert_eq!( + fs::read(dir.path().join("twice.jsonl")).unwrap(), + ORPHAN_LOG + ); +} + +const LEGACY_DIR_NAME: &str = "legacy"; + +/// Put the migrated sources back where an interruption would have left them. +fn restore_sources(root: &Path) { + for name in LEGACY_FILES { + fs::rename(root.join("legacy").join(name), root.join(name)).unwrap(); + } +} + +fn archive_list(names: &[&str]) -> Vec { + serde_json::to_vec(names).unwrap() +} + +#[test] +fn a_start_after_an_interrupted_migration_discards_the_staging_files_and_starts_over() { + // Killed while building: a torn staging file and WAL, no final database. + let dir = DataDir::new(); + write_legacy_fixture(dir.path()); + fs::write(dir.path().join("tcode.db.migrating"), b"partial").unwrap(); + fs::write(dir.path().join("tcode.db.migrating-wal"), b"torn frames").unwrap(); + let store = dir.store(); + store.open().unwrap(); + assert_fixture_migrated(&store, dir.path()); + store.close().unwrap(); + + // Killed after the staged database was complete and the archive list was + // written, but before the rename was durable: the sources are all still + // in the data dir. + fs::rename( + dir.path().join(DB_FILE), + dir.path().join("tcode.db.migrating"), + ) + .unwrap(); + restore_sources(dir.path()); + fs::write( + dir.path().join("tcode.db.archive"), + archive_list(&LEGACY_FILES), + ) + .unwrap(); + let store = dir.store(); + store.open().unwrap(); + assert_fixture_migrated(&store, dir.path()); +} + +#[test] +fn a_start_after_an_interrupted_archival_finishes_it_without_overwriting() { + let dir = DataDir::new(); + write_legacy_fixture(dir.path()); + let store = dir.store(); + store.open().unwrap(); + store.close().unwrap(); + + // Killed after the rename while moving the sources: two are still in the + // data dir. One of them also has a copy in legacy/ already, which must + // survive untouched while the source stays where it is. + fs::rename( + dir.path().join("legacy/mixed.jsonl"), + dir.path().join("mixed.jsonl"), + ) + .unwrap(); + fs::copy( + dir.path().join("legacy/orphan.jsonl"), + dir.path().join("orphan.jsonl"), + ) + .unwrap(); + fs::write(dir.path().join("legacy/orphan.jsonl"), b"older copy").unwrap(); + fs::write( + dir.path().join("tcode.db.archive"), + archive_list(&LEGACY_FILES), + ) + .unwrap(); + + let store = dir.store(); + store.open().unwrap(); + assert!(!dir.path().join("tcode.db.archive").exists()); + assert!(!dir.path().join("mixed.jsonl").exists()); + assert_eq!( + fs::read(dir.path().join("legacy/mixed.jsonl")).unwrap(), + MIXED_LOG + ); + assert_eq!( + fs::read(dir.path().join("orphan.jsonl")).unwrap(), + ORPHAN_LOG + ); + assert_eq!( + fs::read(dir.path().join("legacy/orphan.jsonl")).unwrap(), + b"older copy" + ); + assert_eq!(store.read_event_log("mixed").unwrap(), MIXED_LOG); +} + +#[test] +fn a_fresh_data_dir_gets_a_completed_database_through_the_staged_path() { + let dir = DataDir::new(); + let store = dir.store(); + assert!(store.load_index().unwrap().is_empty()); + store.close().unwrap(); + let db = db::Db::open(&dir.path().join(DB_FILE), false).unwrap(); + assert_eq!(db.user_version().unwrap(), SCHEMA_VERSION); + drop(db); + assert!(!dir.path().join("legacy").exists()); + assert!(!dir.path().join("tcode.db.migrating").exists()); +} + +#[test] +fn an_unfinished_newer_or_corrupt_database_is_refused_and_left_untouched() { + let set_version = |root: &Path, version: i64| { + let db = db::Db::open(&root.join(DB_FILE), false).unwrap(); + db.write(|db, connection| { + db.execute(connection, &format!("PRAGMA user_version = {version}"), ()) + .map(drop) + }) + .unwrap(); + db.checkpoint().unwrap(); + }; + for (version, expected) in [(0, "never completed"), (2, "newer Tcode")] { + let dir = DataDir::new(); + dir.store() + .upsert_meta(&SessionMeta::new(ProviderKind::Codex, "/w".into(), None)) + .unwrap(); + dir.store().close().unwrap(); + set_version(dir.path(), version); + let before = fs::read(dir.path().join(DB_FILE)).unwrap(); + let error = dir.store().open().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains(expected), "{error}"); + assert_eq!(fs::read(dir.path().join(DB_FILE)).unwrap(), before); + } + + let dir = DataDir::new(); + let garbage = vec![0x5a_u8; 8192]; + fs::write(dir.path().join(DB_FILE), &garbage).unwrap(); + let error = dir.store().open().unwrap_err(); + assert!(error.to_string().contains("sqlite3"), "{error}"); + assert!(error.to_string().contains(DB_FILE), "{error}"); + assert_eq!(fs::read(dir.path().join(DB_FILE)).unwrap(), garbage); +} + +#[test] +fn session_index_upserts_orders_and_removes_only_the_selected_session() { + let dir = DataDir::new(); + let store = dir.store(); + let mut a = SessionMeta::new(ProviderKind::Codex, PathBuf::from("/a"), None); + a.updated_at = 100; + let mut b = SessionMeta::new(ProviderKind::ClaudeCode, PathBuf::from("/b"), None); + b.updated_at = 200; + store.upsert_meta(&a).unwrap(); + store.upsert_meta(&b).unwrap(); + + let index = store.load_index().unwrap(); + assert_eq!(index.len(), 2); + // newest first + assert_eq!(index[0].id, b.id); + assert_eq!(index[1].id, a.id); + + // upsert replaces + let mut a2 = a.clone(); + a2.title = "renamed".into(); + store.upsert_meta(&a2).unwrap(); + let index = store.load_index().unwrap(); + assert_eq!(index.len(), 2); + assert_eq!( + index.iter().find(|m| m.id == a.id).unwrap().title, + "renamed" + ); + for (meta, ts) in [(&a, 1), (&b, 2)] { + store + .append_event( + &meta.id, + ts, + &AgentEvent::TurnStarted { + turn_id: format!("turn-{ts}"), + }, + ) + .unwrap(); + } + store.remove_session(&a.id).unwrap(); + store.remove_session(&a.id).unwrap(); + let remaining = store.load_index().unwrap(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].id, b.id); + assert!(store.read_event_log(&a.id).unwrap().is_empty()); + assert_eq!( + store.read_event_log(&b.id).unwrap(), + b"{\"ts\":2,\"event\":{\"type\":\"turn_started\",\"turn_id\":\"turn-2\"}}\n" + ); +} + +#[test] +fn a_cloned_or_imported_log_keeps_its_exact_bytes() { + let dir = DataDir::new(); + let store = dir.store(); + let fork = SessionMeta::new(ProviderKind::Codex, PathBuf::from("/w"), None); + store + .apply(&[ + Mutation::replace_event_log("source", MIXED_LOG.to_vec()), + Mutation::clone_events("source", &fork.id), + Mutation::upsert_meta(fork.clone()), + Mutation::clone_events("missing", "empty-fork"), + ]) + .unwrap(); + assert_eq!(store.read_event_log("source").unwrap(), MIXED_LOG); + assert_eq!(store.read_event_log(&fork.id).unwrap(), MIXED_LOG); + assert!(store.read_event_log("empty-fork").unwrap().is_empty()); + assert_eq!(store.load_index().unwrap(), [fork]); +} + +#[test] +fn removing_project_cleans_only_managed_icons() { + let dir = DataDir::new(); + let root = dir.path(); + let store = dir.store(); + fs::create_dir(root.join("project-icons")).unwrap(); + let mut project = Project::from_root(root.join("project")); + for (path, managed) in [ + (root.join("project-icons/icon.png"), true), + (root.join("original.png"), false), + ] { + fs::write(&path, b"stored image").unwrap(); + project.icon_path = Some(path.clone()); + store.upsert_project(&project).unwrap(); + assert_eq!( + store.read_file().unwrap().projects[0].icon_path.as_ref(), + Some(&path) + ); + store.remove_project(&project.id).unwrap(); + assert!(store.read_file().unwrap().projects.is_empty()); + assert_eq!(path.exists(), !managed); + } +} + +#[test] +fn command_cache_roundtrips_per_provider_and_acp_agent() { + let dir = DataDir::new(); + let root = dir.path().to_path_buf(); + let store = dir.store(); + let native = vec![ProviderCommand { + name: "review".into(), + description: Some("Review the current changes".into()), + kind: ProviderCommandKind::Command, + }]; + let acp = vec![ProviderCommand { + name: "browser".into(), + description: None, + kind: ProviderCommandKind::Skill, + }]; + store + .save_commands(ProviderKind::ClaudeCode, None, &native) + .unwrap(); + store + .save_commands(ProviderKind::Acp, Some("vendor/agent"), &acp) + .unwrap(); + + // Reopen the store to prove the values come from disk, not memory. + let reopened = SessionStore::open_at(root.clone()).unwrap(); + assert_eq!( + reopened.load_commands(ProviderKind::ClaudeCode, None), + native + ); + assert_eq!( + reopened.load_commands(ProviderKind::Acp, Some("vendor/agent")), + acp + ); + assert!( + reopened + .load_commands(ProviderKind::Acp, Some("different-agent")) + .is_empty() + ); + assert!(root.join("commands-claude.json").is_file()); + assert!( + root.join("commands-acp-76656e646f722f6167656e74.json") + .is_file() + ); +} + +/// This test binary, re-run as another process that opens `root` and holds +/// it until its stdin closes. It reports `OPEN` or `ERR `. +struct Child { + process: std::process::Child, + stdout: BufReader, + report: String, +} + +impl Child { + fn spawn(root: &Path) -> std::process::Child { + crate::process::command(std::env::current_exe().unwrap()) + .args([ + "--exact", + "store::tests::store_owner_process", + "--ignored", + "--nocapture", + "--test-threads=1", + ]) + .env("TCODE_STORE_OWNER_DIR", root) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap() + } + + /// Wait for the child to say whether it owns the data dir. + fn report(mut process: std::process::Child) -> Self { + let mut stdout = BufReader::new(process.stdout.take().unwrap()); + // libtest prints the test's name on the same line first. + let mut line = String::new(); + let report = loop { + line.clear(); + assert!( + stdout.read_line(&mut line).unwrap() > 0, + "the child exited without reporting" + ); + if let Some((_, report)) = line.trim_end().split_once(OWNER_REPORT) { + break report.to_owned(); + } + }; + Self { + process, + stdout, + report, + } + } + + /// Close the child's stdin, which makes it close the store and exit. + fn release(mut self) { + drop(self.process.stdin.take()); + std::io::copy(&mut self.stdout, &mut std::io::sink()).unwrap(); + assert!(self.process.wait().unwrap().success()); + } +} + +const OWNER_REPORT: &str = "store owner: "; + +#[test] +#[ignore = "the other process of the ownership tests below, which run it"] +fn store_owner_process() { + let root = PathBuf::from(std::env::var_os("TCODE_STORE_OWNER_DIR").unwrap()); + let store = SessionStore::open_at(root).unwrap(); + match store.open() { + Ok(()) => println!("{OWNER_REPORT}OPEN"), + Err(error) => println!("{OWNER_REPORT}ERR {:?} {error}", error.kind()), + } + std::io::stdout().flush().unwrap(); + let _ = std::io::stdin().read(&mut [0_u8; 1]); + store.close().unwrap(); +} + +#[test] +fn a_second_process_is_refused_and_a_relaunch_waits_for_the_owner_to_exit() { + let dir = DataDir::new(); + let owner = Child::report(Child::spawn(dir.path())); + assert_eq!(owner.report, "OPEN"); + + let error = dir.store().open().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::ResourceBusy); + assert!( + error + .to_string() + .contains(&dir.path().display().to_string()), + "{error}" + ); + + // A permission relaunch starts while the old instance is still closing. + crate::relaunch::write( + dir.path(), + &crate::relaunch::RelaunchMarker { + reopen_settings: "computer_use".into(), + active_session: None, + }, + ) + .unwrap(); + let waiting = std::thread::spawn({ + let store = dir.store(); + move || store.open() + }); + std::thread::sleep(Duration::from_millis(300)); + assert!(!waiting.is_finished(), "the relaunch waits for the owner"); + owner.release(); + waiting.join().unwrap().unwrap(); + // Only peeked: the host consumes the marker once it owns the data dir. + assert!(crate::relaunch::pending(dir.path())); +} + +#[test] +fn two_simultaneous_first_launches_migrate_once() { + let dir = DataDir::new(); + write_legacy_fixture(dir.path()); + let (first, second) = (Child::spawn(dir.path()), Child::spawn(dir.path())); + let (first, second) = (Child::report(first), Child::report(second)); + let mut reports = [first.report.clone(), second.report.clone()]; + reports.sort(); + assert_eq!(reports[1], "OPEN", "{reports:?}"); + assert!(reports[0].starts_with("ERR ResourceBusy"), "{reports:?}"); + first.release(); + second.release(); + + let store = dir.store(); + assert_fixture_migrated(&store, dir.path()); +} + +/// Kill the migration at random points — while logs are copied, around the +/// checkpoint, the rename, the directory syncs and the archival — and restart +/// it: there is either no `tcode.db` and every source is where it was, or a +/// complete one, and the finished migration always holds every source byte +/// for byte. `TCODE_MIGRATION_KILLS` sets the number of kills. +#[test] +#[ignore = "spawns and SIGKILLs child processes over a large fixture; run deliberately"] +fn migration_survives_sigkill() { + let runs: usize = std::env::var("TCODE_MIGRATION_KILLS") + .ok() + .and_then(|runs| runs.parse().ok()) + .unwrap_or(20); + let originals = DataDir::new(); + write_legacy_fixture(originals.path()); + let line = format!( + "{{\"ts\":1,\"event\":{{\"type\":\"turn_started\",\"turn_id\":\"{}\"}}}}\n", + "x".repeat(1000) + ); + for index in 0..24 { + fs::write( + originals.path().join(format!("bulk-{index}.jsonl")), + line.repeat(300 + index * 20), + ) + .unwrap(); + } + let names: Vec = fs::read_dir(originals.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .collect(); + let mut seed = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64 + | 1; + // Kills spread over the time an uninterrupted migration takes. + let calibration = DataDir::new(); + for name in &names { + fs::copy(originals.path().join(name), calibration.path().join(name)).unwrap(); + } + let started = Instant::now(); + let full = Child::report(Child::spawn(calibration.path())); + let window = started.elapsed().as_millis() as u64; + assert_eq!(full.report, "OPEN"); + full.release(); + let mut outcomes = HashMap::<&str, usize>::new(); + for run in 0..runs { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + let dir = DataDir::new(); + for name in &names { + fs::copy(originals.path().join(name), dir.path().join(name)).unwrap(); + } + let mut owner = Child::spawn(dir.path()); + std::thread::sleep(Duration::from_millis(seed % (window + window / 10))); + owner.kill().unwrap(); + owner.wait().unwrap(); + + // What the kill left: no database and untouched sources, or a + // completed database and every source in the data dir or legacy/. + let source = |name: &str| { + [dir.path().join(name), dir.path().join("legacy").join(name)] + .into_iter() + .find(|path| path.exists()) + }; + let outcome = if dir.path().join(DB_FILE).exists() { + let db = db::Db::open(&dir.path().join(DB_FILE), false).unwrap(); + assert_eq!(db.user_version().unwrap(), SCHEMA_VERSION, "run {run}"); + drop(db); + if dir.path().join("tcode.db.archive").exists() { + "published, archival pending" + } else { + "complete" + } + } else { + for name in &names { + assert!( + dir.path().join(name).exists(), + "run {run}: {name} moved early" + ); + } + if dir.path().join("tcode.db.migrating").exists() { + "staging" + } else { + "not started" + } + }; + *outcomes.entry(outcome).or_default() += 1; + for name in &names { + assert_eq!( + fs::read(source(name).unwrap_or_else(|| panic!("run {run}: {name} lost"))).unwrap(), + fs::read(originals.path().join(name)).unwrap(), + "run {run}: {name}" + ); + } + + let store = dir.store(); + store.open().unwrap(); + for name in names.iter().filter_map(|name| name.strip_suffix(".jsonl")) { + assert_eq!( + store.read_event_log(name).unwrap(), + fs::read(originals.path().join(format!("{name}.jsonl"))).unwrap(), + "run {run}: {name}" + ); + } + assert_eq!(store.read_file().unwrap().sessions.len(), 2, "run {run}"); + store.close().unwrap(); + } + println!("{runs} kills over a {window} ms migration: {outcomes:?}"); +} diff --git a/locales/en.yml b/locales/en.yml index 788ee4b12..446b39767 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -218,6 +218,7 @@ quit: title: "Quit Tcode?" description: "%{count} session(s) are still working. Quitting will stop them." confirm: "Quit" + startup_failed: "Tcode could not start" settings: title: "Settings" sections: "All settings" diff --git a/locales/zh-CN.yml b/locales/zh-CN.yml index 35cda3304..e8b3b9bfa 100644 --- a/locales/zh-CN.yml +++ b/locales/zh-CN.yml @@ -218,6 +218,7 @@ quit: title: "退出 Tcode？" description: "仍有 %{count} 个会话正在工作。退出将停止这些会话。" confirm: "退出" + startup_failed: "Tcode 无法启动" settings: title: "设置" sections: "全部设置" From 3d9c62b354839a9edbfe7a6a3b2d25d6acd24e35 Mon Sep 17 00:00:00 2001 From: Tryanks Date: Tue, 6 Oct 2026 14:40:53 +0800 Subject: [PATCH 2/4] feat: run the storage migration behind a progress dialog that can be quit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The migration is its own cancellable step with progress, run under the data directory's ownership lock before the host starts. The desktop opens a modal dialog with a progress bar and a Quit button first and starts the kernel and shell in the same process once it completes; Quit, ⌘Q or closing the window stops the migration at its next chunk, discards the staging files and exits. Headless prints progress and treats SIGINT and SIGTERM as cancel. No source file is written, renamed or removed before the staged database is published; an unparseable sessions.json is no longer renamed during the build but archived unchanged afterwards. --- crates/app/src/main.rs | 393 +++++++++++++++------------ crates/app/src/migration.rs | 325 ++++++++++++++++++++++ crates/headless/src/main.rs | 92 ++++++- crates/headless/tests/migration.rs | 97 +++++++ crates/runtime/src/pipe.rs | 11 +- crates/services/src/store/migrate.rs | 286 +++++++++++++++---- crates/services/src/store/mod.rs | 152 +++++++++-- crates/services/src/store/tests.rs | 183 +++++++++++-- crates/ui/src/lib.rs | 6 +- crates/ui/src/run.rs | 110 +++++--- crates/ui/src/thread_export.rs | 6 +- locales/en.yml | 10 + locales/zh-CN.yml | 10 + 13 files changed, 1351 insertions(+), 330 deletions(-) create mode 100644 crates/app/src/migration.rs create mode 100644 crates/headless/tests/migration.rs diff --git a/crates/app/src/main.rs b/crates/app/src/main.rs index f3648b510..f9782394f 100644 --- a/crates/app/src/main.rs +++ b/crates/app/src/main.rs @@ -2,7 +2,7 @@ // Debug builds keep the console so `RUST_LOG` output stays visible. #![cfg_attr(all(windows, not(debug_assertions)), windows_subsystem = "windows")] -use std::{borrow::Cow, rc::Rc, time::Duration}; +use std::{borrow::Cow, cell::RefCell, rc::Rc, time::Duration}; use gpui::{ App, BorrowAppContext as _, Entity, ParentElement as _, Styled as _, TitlebarOptions, @@ -20,6 +20,7 @@ use tcode_ui::{assets, settings}; use tcode_ui::overlay::{DialogActions, OverlayExt as _}; use tcode_ui::widgets::button::{Button, ButtonVariants as _}; +mod migration; #[cfg(not(target_os = "linux"))] mod preview_smoke; @@ -45,6 +46,10 @@ fn main_window_background() -> WindowBackgroundAppearance { } } +/// Background redraws of an inactive window are capped to this interval +/// unless Settings turns the throttle off. +const INACTIVE_FRAME_INTERVAL: Duration = Duration::from_millis(500); + const QUIT_PROMPT_TIMEOUT: Duration = Duration::from_secs(15); fn finish_quit_prompt(window_state: &Entity, epoch: u64, cx: &mut App) -> bool { @@ -260,6 +265,53 @@ fn exit_with_startup_failure(error: std::io::Error) -> ! { std::process::exit(1) } +/// Every desktop window: the main one and the startup migration's. +fn window_options(cx: &App, inactive_frame_interval: Option) -> WindowOptions { + WindowOptions { + window_bounds: Some(WindowBounds::centered(size(px(1200.), px(800.)), cx)), + // Low enough that the window can actually be dragged into the + // compact layout; the launch geometry above is unchanged. + window_min_size: Some(size(px(360.), px(480.))), + // macOS: seamless titlebar — transparent, with the traffic lights + // nudged down to sit vertically centered in the 52px top strip. + // + // Windows: also client-decorated — a transparent titlebar hides + // the system one — and `tcode_ui`'s window caption cluster draws + // the minimize/maximize/close controls into whichever top strip + // is rightmost (`crates/ui/src/window_caption.rs`). + // + // Linux: we draw no controls of our own there, so a transparent + // titlebar would leave the window with no way to be closed from + // the chrome. Keep the native system titlebar; our top strip + // simply sits below it. + titlebar: Some(TitlebarOptions { + title: None, + appears_transparent: cfg!(any(target_os = "macos", target_os = "windows")), + traffic_light_position: Some(point(px(12.), px(19.))), + }), + // macOS: the app owns titlebar dragging. AppKit's native + // titlebar-region drag sits on top of whatever gpui draws in the + // top strip — dragging the Preview URL bar moved the window + // instead of selecting text. Every draggable strip already goes + // through `window_drag_area` (`start_window_move`), so hand the + // whole content view to the app and let controls own their input. + app_owns_titlebar_drag: true, + // Spell out that Windows is client-decorated. (This field is + // advisory off Wayland; leaving it `None` elsewhere keeps Linux + // on whatever its compositor/backend already chose.) + window_decorations: cfg!(target_os = "windows").then_some(WindowDecorations::Client), + // Persistent windows use the platform's system material: + // macOS sidebar vibrancy, Windows Acrylic, or an opaque fallback. + window_background: main_window_background(), + // Throttle background redraws (spinners, streaming output) to + // ~2 FPS while the window is inactive; gpui lifts the cap the + // moment the window is active or receiving high-rate input. + // This setting is captured at window creation and requires a restart. + inactive_frame_interval, + ..Default::default() + } +} + struct LocalKernel { host: SpawnedHost, mux: HostMux, @@ -406,13 +458,19 @@ fn main() { } None => AttachmentTarget::Local, }; + // An older build's threads move into tcode.db before the kernel starts, + // behind a window of their own. With nothing to migrate, the kernel + // starts before the event loop. + let migration_needed = match store.needs_migration() { + Ok(needed) => needed, + Err(error) => exit_with_startup_failure(error), + }; // Kernel ownership is process composition, not a property of whichever // host the window currently views. - let kernel = match LocalKernel::start(store) { + let kernel = (!migration_needed).then(|| match LocalKernel::start(store.clone()) { Ok(kernel) => Rc::new(kernel), Err(error) => exit_with_startup_failure(error), - }; - let local_settings = kernel.settings(); + }); gpui_platform::application() .with_assets(assets::Assets) @@ -436,191 +494,182 @@ fn main() { Cow::Owned(tcode_ui::flattened_theme_json()) }; - // Hosting belongs to the process-owned local kernel; it carries no - // current-attachment mode. - cx.set_global(RemoteController::new( - kernel.mux.clone(), - data_dir.clone(), - kernel.control_link.clone(), - local_settings.clone(), - )); - if local_settings.remote_hosting_enabled { - let name = local_settings - .remote_host_name - .clone() - .unwrap_or_else(machine_name); - cx.update_global::(|controller, _| { - if let Err(error) = controller.start_hosting(&local_settings.traverse, name) { - log::error!("remote hosting could not start: {error}"); - } - }); - } + let shell_options = ShellOptions { + title: tcode_ui::tr!("app.name").into(), + fonts: application_fonts, + theme_json, + activate: true, + system_locale: None, + // A desktop window is never suspended by the OS. + lifecycle: None, + ..ShellOptions::default() + }; + tcode_ui::init_client(cx, native_client.as_ref(), &shell_options); #[cfg(target_os = "macos")] cx.set_menus([gpui::Menu::new(tcode_ui::tr!("app.name")).items([ gpui::MenuItem::action(tcode_ui::tr!("quit.menu_item"), Quit), ])]); - // Process ownership, not the window's current attachment, grants - // authority to stop the local kernel on application quit. - let quit_subscription = cx.on_app_quit({ - let link = kernel.control_link.clone(); - let to_host = kernel.host.to_host.clone(); - let stopped = kernel.host.stopped.clone(); - move |_cx| { - let link = link.clone(); - let to_host = to_host.clone(); - let stopped = stopped.clone(); - async move { - let _ = link.shutdown().await; - // `shutdown` only closes this client's mux connection; - // the host loop ends when its own inbox closes. - to_host.close(); - let _ = stopped.recv().await; + let appearance = native_client.load_preferences().appearance; + let shell_slot: Rc>>> = Rc::default(); + cx.on_action::({ + let shell_slot = shell_slot.clone(); + move |action, cx| { + if migration::quit(cx) { + return; + } + let shell = shell_slot.borrow().clone(); + match shell { + Some(shell) => handle_quit(action, &shell, cx), + None => cx.quit(), } } }); - quit_subscription.detach(); - - let window_options = WindowOptions { - window_bounds: Some(WindowBounds::centered(size(px(1200.), px(800.)), cx)), - // Low enough that the window can actually be dragged into the - // compact layout; the launch geometry above is unchanged. - window_min_size: Some(size(px(360.), px(480.))), - // macOS: seamless titlebar — transparent, with the traffic lights - // nudged down to sit vertically centered in the 52px top strip. - // - // Windows: also client-decorated — a transparent titlebar hides - // the system one — and `tcode_ui`'s window caption cluster draws - // the minimize/maximize/close controls into whichever top strip - // is rightmost (`crates/ui/src/window_caption.rs`). - // - // Linux: we draw no controls of our own there, so a transparent - // titlebar would leave the window with no way to be closed from - // the chrome. Keep the native system titlebar; our top strip - // simply sits below it. - titlebar: Some(TitlebarOptions { - title: None, - appears_transparent: cfg!(any(target_os = "macos", target_os = "windows")), - traffic_light_position: Some(point(px(12.), px(19.))), - }), - // macOS: the app owns titlebar dragging. AppKit's native - // titlebar-region drag sits on top of whatever gpui draws in the - // top strip — dragging the Preview URL bar moved the window - // instead of selecting text. Every draggable strip already goes - // through `window_drag_area` (`start_window_move`), so hand the - // whole content view to the app and let controls own their input. - app_owns_titlebar_drag: true, - // Spell out that Windows is client-decorated. (This field is - // advisory off Wayland; leaving it `None` elsewhere keeps Linux - // on whatever its compositor/backend already chose.) - window_decorations: cfg!(target_os = "windows") - .then_some(WindowDecorations::Client), - // Persistent windows use the platform's system material: - // macOS sidebar vibrancy, Windows Acrylic, or an opaque fallback. - window_background: main_window_background(), - // Throttle background redraws (spinners, streaming output) to - // ~2 FPS while the window is inactive; gpui lifts the cap the - // moment the window is active or receiving high-rate input. - // This setting is captured at window creation and requires a restart. - inactive_frame_interval: (!local_settings.inactive_frame_throttle_disabled) - .then(|| Duration::from_millis(500)), - ..Default::default() - }; - let local_kernel = kernel.clone(); - let (window, shell) = tcode_ui::run_shell( - cx, - native_client.clone(), - ShellOptions { - window: window_options, - title: tcode_ui::tr!("app.name").into(), - fonts: application_fonts, - theme_json, - activate: true, - system_locale: None, - // A desktop window is never suspended by the OS. - lifecycle: None, - setup: ShellSetup { - client_host: Some(native_client.clone()), - local: Some(Rc::new(move || local_kernel.transport())), - initial: Some(initial_target.clone()), - initial_pairing_error: None, - // Only here: bootstrap applies locale and theme from the - // host's own settings before the first frame. - seed_blocking: true, - restore_navigation: false, - }, - }, - ); - - cx.on_action::({ - let shell = shell.clone(); - move |action, cx| handle_quit(action, &shell, cx) - }); - // Restart continuity: if this launch follows a permission-grant - // relaunch, reopen the recorded session and Settings page. Only - // meaningful for a host in this process: the marker lives in this - // machine's data dir, and a remote host's marker is its own. - if shell - .read(cx) - .store() - .is_some_and(|store| !store.read(cx).is_remote()) - && let Ok(CommandResponse::PendingRelaunchSection { - section: Some(section), - session_id, - }) = kernel - .control_link - .command_blocking(Command::ApplyPendingRelaunch) - { - if let Some(id) = session_id - && let Some(store) = shell.read(cx).store() - { - store.update(cx, |store, _| store.select_session(id)); + let launch = move |cx: &mut App, kernel: Rc| { + let local_settings = kernel.settings(); + // Hosting belongs to the process-owned local kernel; it carries no + // current-attachment mode. + cx.set_global(RemoteController::new( + kernel.mux.clone(), + data_dir.clone(), + kernel.control_link.clone(), + local_settings.clone(), + )); + if local_settings.remote_hosting_enabled { + let name = local_settings + .remote_host_name + .clone() + .unwrap_or_else(machine_name); + cx.update_global::(|controller, _| { + if let Err(error) = controller.start_hosting(&local_settings.traverse, name) + { + log::error!("remote hosting could not start: {error}"); + } + }); } - let window_state = shell.read(cx).window_state(); - window_state.update(cx, |state, cx| { - state.pending_settings_section = Some(section); - state.open_settings(cx); + // Process ownership, not the window's current attachment, grants + // authority to stop the local kernel on application quit. + let quit_subscription = cx.on_app_quit({ + let link = kernel.control_link.clone(); + let to_host = kernel.host.to_host.clone(); + let stopped = kernel.host.stopped.clone(); + move |_cx| { + let link = link.clone(); + let to_host = to_host.clone(); + let stopped = stopped.clone(); + async move { + let _ = link.shutdown().await; + // `shutdown` only closes this client's mux connection; + // the host loop ends when its own inbox closes. + to_host.close(); + let _ = stopped.recv().await; + } + } }); - } + quit_subscription.detach(); - cx.spawn(async move |cx| { - #[cfg(not(target_os = "linux"))] - if let Some(watchdog) = preview_smoke_watchdog { - preview_smoke::run(watchdog, shell, window, cx).await; - return; + let local_kernel = kernel.clone(); + let options = window_options( + cx, + (!local_settings.inactive_frame_throttle_disabled) + .then_some(INACTIVE_FRAME_INTERVAL), + ); + let (window, shell) = tcode_ui::run_shell( + cx, + native_client.clone(), + ShellOptions { + window: options, + setup: ShellSetup { + client_host: Some(native_client.clone()), + local: Some(Rc::new(move || local_kernel.transport())), + initial: Some(initial_target.clone()), + initial_pairing_error: None, + // Only here: bootstrap applies locale and theme from the + // host's own settings before the first frame. + seed_blocking: true, + restore_navigation: false, + }, + ..shell_options + }, + ); + *shell_slot.borrow_mut() = Some(shell.clone()); + // Restart continuity: if this launch follows a permission-grant + // relaunch, reopen the recorded session and Settings page. Only + // meaningful for a host in this process: the marker lives in this + // machine's data dir, and a remote host's marker is its own. + if shell + .read(cx) + .store() + .is_some_and(|store| !store.read(cx).is_remote()) + && let Ok(CommandResponse::PendingRelaunchSection { + section: Some(section), + session_id, + }) = kernel + .control_link + .command_blocking(Command::ApplyPendingRelaunch) + { + if let Some(id) = session_id + && let Some(store) = shell.read(cx).store() + { + store.update(cx, |store, _| store.select_session(id)); + } + let window_state = shell.read(cx).window_state(); + window_state.update(cx, |state, cx| { + state.pending_settings_section = Some(section); + state.open_settings(cx); + }); } - let _ = window; - if open_latest { - let Some(link) = cx.update(|cx| shell.read(cx).link()) else { + cx.spawn(async move |cx| { + #[cfg(not(target_os = "linux"))] + if let Some(watchdog) = preview_smoke_watchdog { + preview_smoke::run(watchdog, shell, window, cx).await; return; - }; - if let Ok(CommandResponse::SessionId(Some(id))) = - link.command(Command::OpenLatestSession).await - && let Some(store) = cx.update(|cx| shell.read(cx).store()) - { - store.update(cx, |store, _| store.select_session(id)); } - for _ in 0..100 { - if cx.update(|cx| { - shell - .read(cx) - .store() - .is_some_and(|store| store.read(cx).active_session_id().is_some()) - }) { - break; + let _ = window; + + if open_latest { + let Some(link) = cx.update(|cx| shell.read(cx).link()) else { + return; + }; + if let Ok(CommandResponse::SessionId(Some(id))) = + link.command(Command::OpenLatestSession).await + && let Some(store) = cx.update(|cx| shell.read(cx).store()) + { + store.update(cx, |store, _| store.select_session(id)); + } + for _ in 0..100 { + if cx.update(|cx| { + shell.read(cx).store().is_some_and(|store| { + store.read(cx).active_session_id().is_some() + }) + }) { + break; + } + cx.background_executor() + .timer(std::time::Duration::from_millis(10)) + .await; + } + if let Some(store) = cx.update(|cx| shell.read(cx).store()) { + store.update(cx, |store, _cx| { + store.sync_active_conversation_ui(); + }); } - cx.background_executor() - .timer(std::time::Duration::from_millis(10)) - .await; - } - if let Some(store) = cx.update(|cx| shell.read(cx).store()) { - store.update(cx, |store, _cx| { - store.sync_active_conversation_ui(); - }); } + }) + .detach(); + }; + + match kernel { + Some(kernel) => launch(cx, kernel), + None => { + let options = window_options(cx, Some(INACTIVE_FRAME_INTERVAL)); + let kernel_store = store.clone(); + migration::run(cx, store, options, appearance, move |cx| { + launch(cx, Rc::new(LocalKernel::start(kernel_store)?)); + Ok(()) + }); } - }) - .detach(); + } }); } diff --git a/crates/app/src/migration.rs b/crates/app/src/migration.rs new file mode 100644 index 000000000..66a0377b8 --- /dev/null +++ b/crates/app/src/migration.rs @@ -0,0 +1,325 @@ +//! The one-time migration into `tcode.db`, run before the local kernel starts +//! behind a window that offers nothing but its progress and Quit. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use gpui::{ + AnyWindowHandle, App, AppContext as _, Context, Entity, Global, IntoElement, + ParentElement as _, Render, Styled as _, Window, WindowOptions, div, + prelude::FluentBuilder as _, px, +}; +use tcode_services::store::{Migration, MigrationPhase, MigrationProgress, SessionStore}; +use tcode_ui::overlay::{DialogActions, OverlayExt as _}; +use tcode_ui::theme::{self, ActiveTheme as _, ThemeMode}; +use tcode_ui::widgets::{Button, Progress}; + +/// What the dialog shows. +enum Status { + Running(Option), + /// Quit was asked for: the migration stops at its next chunk, and the + /// process exits once it has. + Quitting, + Failed(String), +} + +struct MigrationView { + status: Status, + cancel: Arc, +} + +/// The view a Quit reaches while the migration window is up. +struct Running(Entity); +impl Global for Running {} + +/// Behind the dialog; the window shows nothing else. +struct Backdrop; + +impl Render for Backdrop { + fn render(&mut self, _: &mut Window, _: &mut Context) -> impl IntoElement { + div().size_full() + } +} + +enum Update { + Progress(MigrationProgress), + Done(std::io::Result), +} + +/// Migrate `store` behind a modal progress dialog, then hand over to `start` +/// and close the window. Quit, ⌘Q or closing the window cancels the +/// migration and exits once it has stopped; a failure, of the migration or of +/// `start`, stays in the dialog until Quit, which then exits non-zero. +pub(crate) fn run( + cx: &mut App, + store: SessionStore, + window_options: WindowOptions, + appearance: Option, + start: impl FnOnce(&mut App) -> std::io::Result<()> + 'static, +) { + let cancel = Arc::new(AtomicBool::new(false)); + let view = cx.new(|_| MigrationView { + status: Status::Running(None), + cancel: cancel.clone(), + }); + cx.set_global(Running(view.clone())); + + let (updates, received) = async_channel::unbounded(); + let (finished, stopped) = async_channel::bounded::<()>(1); + let spawned = std::thread::Builder::new() + .name("tcode-migration".into()) + .spawn({ + let cancel = cancel.clone(); + move || { + let progress = updates.clone(); + let result = store.migrate( + |value| { + let _ = progress.try_send(Update::Progress(value)); + }, + &cancel, + ); + let _ = updates.try_send(Update::Done(result)); + drop(finished); + } + }); + // A quit the dialog does not see first (the Dock menu, logging out) + // still stops the migration; gpui waits for it only briefly, and a + // migration it outlasts is discarded by the next start. + cx.on_app_quit({ + let cancel = cancel.clone(); + move |_| { + cancel.store(true, Ordering::Relaxed); + let stopped = stopped.clone(); + async move { + let _ = stopped.recv().await; + } + } + }) + .detach(); + + let window = tcode_ui::open_client_window( + cx, + window_options, + tcode_ui::tr!("app.name").into(), + |_, cx| cx.new(|_| Backdrop), + ); + cx.activate(true); + let _ = window.update(cx, |_, window, cx| { + // The host's theme setting is unknown until its kernel starts; this + // client's own choice is not. + match appearance.as_deref() { + Some("light") => theme::change_mode(ThemeMode::Light, Some(window), cx), + Some("dark") => theme::change_mode(ThemeMode::Dark, Some(window), cx), + _ => {} + } + window.on_window_should_close(cx, |_, cx| { + quit(cx); + false + }); + let content = view.clone(); + window.open_dialog(cx, move |dialog, _, _| { + let content = content.clone(); + dialog + .title(tcode_ui::tr!("migration.title").into_owned()) + .close_button(false) + .overlay_closable(false) + .keyboard(false) + .content(move |body, _, _| body.child(content.clone())) + .footer( + DialogActions::new().child( + Button::new("migration-quit") + .label(tcode_ui::tr!("quit.confirm")) + .on_click(|_, _, cx| { + quit(cx); + }), + ), + ) + }); + window.activate_window(); + }); + + if let Err(error) = spawned { + fail(&view, error.to_string(), cx); + return; + } + cx.spawn(async move |cx| { + while let Ok(mut update) = received.recv().await { + // Only the latest progress is drawn; `Done` is always the last. + while let Ok(next) = received.try_recv() { + update = next; + } + match update { + Update::Progress(progress) => view.update(cx, |view, cx| { + if let Status::Running(shown) = &mut view.status { + *shown = Some(progress); + cx.notify(); + } + }), + Update::Done(result) => { + cx.update(|cx| finish(result, &view, window, start, cx)); + return; + } + } + } + }) + .detach(); +} + +fn finish( + result: std::io::Result, + view: &Entity, + window: AnyWindowHandle, + start: impl FnOnce(&mut App) -> std::io::Result<()>, + cx: &mut App, +) { + let quitting = matches!(view.read(cx).status, Status::Quitting); + match result { + Err(error) => { + log::error!("migration failed: {error}"); + fail(view, error.to_string(), cx); + if quitting { + cx.quit(); + } + } + Ok(Migration::Cancelled) => cx.quit(), + Ok(Migration::Completed) if quitting => cx.quit(), + Ok(Migration::Completed) => { + cx.remove_global::(); + match start(cx) { + Ok(()) => { + let _ = window.update(cx, |_, window, _| window.remove_window()); + } + Err(error) => { + log::error!("the local host did not start after the migration: {error}"); + cx.set_global(Running(view.clone())); + fail(view, error.to_string(), cx); + } + } + } + } +} + +/// Show `reason` in the dialog; the process exits non-zero once it quits. +fn fail(view: &Entity, reason: String, cx: &mut App) { + // gpui ends a quit with status 0; this observer is the last code that + // runs before it. + cx.on_app_quit(|_| async { std::process::exit(1) }).detach(); + view.update(cx, |view, cx| { + view.status = Status::Failed(reason); + cx.notify(); + }); +} + +/// Quit while the migration window is up. Returns whether it was. +pub(crate) fn quit(cx: &mut App) -> bool { + let Some(Running(view)) = cx.try_global::() else { + return false; + }; + let view = view.clone(); + view.update(cx, |view, cx| match view.status { + Status::Running(_) => { + view.cancel.store(true, Ordering::Relaxed); + view.status = Status::Quitting; + cx.notify(); + } + Status::Quitting => {} + Status::Failed(_) => cx.quit(), + }); + true +} + +impl Render for MigrationView { + fn render(&mut self, _: &mut Window, cx: &mut Context) -> impl IntoElement { + let theme = cx.theme(); + let (percent, line, failed) = match &self.status { + Status::Running(progress) => ( + progress.as_ref().map_or(0., overall_percent), + progress.as_ref().map_or_else( + || phase_line(MigrationPhase::Scanning, None), + |progress| phase_line(progress.phase, Some(progress)), + ), + false, + ), + Status::Quitting => (0., tcode_ui::tr!("migration.quitting").into_owned(), false), + Status::Failed(reason) => ( + 0., + tcode_ui::tr!("migration.failed", reason = reason.clone()).into_owned(), + true, + ), + }; + let quitting = matches!(self.status, Status::Quitting); + div() + .flex() + .flex_col() + .gap_3() + .text_size(px(13.)) + .child( + div() + .text_color(theme.muted_foreground) + .child(tcode_ui::tr!("migration.description")), + ) + .when(!failed, |column| { + column.child( + Progress::new("migration-progress") + .value(percent) + .loading(quitting), + ) + }) + .child( + div() + .text_color(if failed { + theme.danger + } else { + theme.muted_foreground + }) + .child(line), + ) + } +} + +/// Import and verification each go through every log once, but import +/// writes and syncs each chunk while verification only reads it back, which +/// takes a fraction of the time. The phases around them take a moment. +fn overall_percent(progress: &MigrationProgress) -> f32 { + let fraction = |done: u64, total: u64| { + if total == 0 { + 1. + } else { + (done as f32 / total as f32).min(1.) + } + }; + let phase = fraction(progress.bytes_done, progress.bytes_total); + match progress.phase { + MigrationPhase::Scanning => 0., + MigrationPhase::Importing => phase * 85., + MigrationPhase::Verifying => 85. + phase * 15., + MigrationPhase::Publishing | MigrationPhase::Archiving => 100., + } +} + +fn phase_line(phase: MigrationPhase, progress: Option<&MigrationProgress>) -> String { + let counts = |key: &str| { + let progress = progress.copied().unwrap_or(MigrationProgress { + phase, + threads_done: 0, + threads_total: 0, + bytes_done: 0, + bytes_total: 0, + }); + tcode_ui::tr!( + key, + done = progress.threads_done, + total = progress.threads_total, + bytes = tcode_ui::format_size(progress.bytes_done as usize), + total_bytes = tcode_ui::format_size(progress.bytes_total as usize) + ) + .into_owned() + }; + match phase { + MigrationPhase::Scanning => tcode_ui::tr!("migration.scanning").into_owned(), + MigrationPhase::Importing => counts("migration.importing"), + MigrationPhase::Verifying => counts("migration.verifying"), + MigrationPhase::Publishing => tcode_ui::tr!("migration.publishing").into_owned(), + MigrationPhase::Archiving => tcode_ui::tr!("migration.archiving").into_owned(), + } +} diff --git a/crates/headless/src/main.rs b/crates/headless/src/main.rs index ed2492e32..f6e401cd3 100644 --- a/crates/headless/src/main.rs +++ b/crates/headless/src/main.rs @@ -1,13 +1,14 @@ use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::Duration; +use std::sync::atomic::AtomicBool; +use std::time::{Duration, Instant}; use qrcode::QrCode; use qrcode::render::unicode::Dense1x2; use tcode_client::pairing::{PairInvite, pair_url, parse_pair_url}; use tcode_runtime::pipe::{HostServices, spawn_host}; -use tcode_services::store::SessionStore; +use tcode_services::store::{Migration, MigrationPhase, MigrationProgress, SessionStore}; use tcode_traverse::browser::{BrowserConfig, StaticBundle, check_bind, serve, set_password}; use tcode_traverse::identity::write_private; use tcode_traverse::lan::DEFAULT_PORT; @@ -35,6 +36,11 @@ const STATIC_BUNDLE: Option = None; const DEFAULT_BROWSER_LISTEN: &str = "127.0.0.1:47420"; /// The current invitation, for `pair` to print; absent while none is valid. const INVITATION_FILE: &str = "invitation.json"; +/// How often a running migration prints its progress. +const MIGRATION_REPORT_INTERVAL: Duration = Duration::from_secs(1); + +/// Set by SIGINT or SIGTERM: cancels a startup migration, then stops `serve`. +static INTERRUPTED: AtomicBool = AtomicBool::new(false); fn main() { env_logger::init(); @@ -115,6 +121,10 @@ fn serve_command(args: &[String]) -> Result<(), String> { } // Nothing else starts for a bind the listener would refuse anyway. check_bind(browser_listen, &remote_data_dir).map_err(|error| error.to_string())?; + install_interrupt_handler(); + if migrate_store(&store)? == Migration::Cancelled { + return Ok(()); + } let mut services = HostServices { background_startup_probes: true, ai_title_generation: true, @@ -230,6 +240,59 @@ fn serve_command(args: &[String]) -> Result<(), String> { Ok(()) } +/// Move an older build's threads into `tcode.db` before the host starts, +/// printing progress; an interrupt cancels it and leaves them as they were. +fn migrate_store(store: &SessionStore) -> Result { + let needed = store + .needs_migration() + .map_err(|error| format!("could not open session store: {error}"))?; + if !needed { + return Ok(Migration::Completed); + } + println!( + "Migrating threads into {}; the original files are kept in legacy/. Ctrl-C cancels.", + store.root().join("tcode.db").display() + ); + let mut last: Option<(MigrationPhase, Instant)> = None; + let outcome = store + .migrate( + |progress| { + let due = last.is_none_or(|(phase, at)| { + phase != progress.phase || at.elapsed() >= MIGRATION_REPORT_INTERVAL + }); + if due { + println!("{}", migration_line(&progress)); + last = Some((progress.phase, Instant::now())); + } + }, + &INTERRUPTED, + ) + .map_err(|error| format!("migration failed: {error}"))?; + match outcome { + Migration::Completed => println!("Migration complete"), + Migration::Cancelled => println!("Migration cancelled; no file was changed"), + } + Ok(outcome) +} + +fn migration_line(progress: &MigrationProgress) -> String { + let phase = match progress.phase { + MigrationPhase::Scanning => "scanning", + MigrationPhase::Importing => "importing", + MigrationPhase::Verifying => "verifying", + MigrationPhase::Publishing => "publishing", + MigrationPhase::Archiving => "archiving", + }; + let mib = |bytes: u64| bytes as f64 / (1024. * 1024.); + format!( + "{phase}: {}/{} threads, {:.1}/{:.1} MiB", + progress.threads_done, + progress.threads_total, + mib(progress.bytes_done), + mib(progress.bytes_total) + ) +} + fn set_password_command(args: &[String]) -> Result<(), String> { let revoke = args.iter().any(|arg| arg == "--revoke-tokens"); let values: Vec<_> = args @@ -351,26 +414,33 @@ fn reject_unknown_options(args: &[String], options_with_values: &[&str]) -> Resu } #[cfg(unix)] -fn wait_for_interrupt() { - use std::sync::atomic::{AtomicBool, Ordering}; - use std::time::Duration; - - static INTERRUPTED: AtomicBool = AtomicBool::new(false); - +fn install_interrupt_handler() { type SignalHandler = extern "C" fn(i32); unsafe extern "C" { fn signal(signal: i32, handler: SignalHandler) -> SignalHandler; } extern "C" fn handle_interrupt(_: i32) { - INTERRUPTED.store(true, Ordering::Relaxed); + INTERRUPTED.store(true, std::sync::atomic::Ordering::Relaxed); } const SIGINT: i32 = 2; - // SAFETY: installs a process-global handler with the C ABI expected by + const SIGTERM: i32 = 15; + // SAFETY: installs process-global handlers with the C ABI expected by // signal(3); the handler performs only a lock-free atomic store. unsafe { signal(SIGINT, handle_interrupt); + signal(SIGTERM, handle_interrupt); } - while !INTERRUPTED.load(Ordering::Relaxed) { +} + +/// Console interrupts keep their default effect: a migration they end is +/// discarded by the next start, and the sources are untouched until it +/// publishes. +#[cfg(not(unix))] +fn install_interrupt_handler() {} + +#[cfg(unix)] +fn wait_for_interrupt() { + while !INTERRUPTED.load(std::sync::atomic::Ordering::Relaxed) { std::thread::sleep(Duration::from_millis(100)); } } diff --git a/crates/headless/tests/migration.rs b/crates/headless/tests/migration.rs new file mode 100644 index 000000000..3a8c47fc7 --- /dev/null +++ b/crates/headless/tests/migration.rs @@ -0,0 +1,97 @@ +//! `tcode-headless serve` runs the one-time migration into `tcode.db` before +//! anything else starts, and an interrupt cancels it. + +#![cfg(unix)] + +use std::collections::BTreeMap; +use std::fs; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::process::Stdio; + +/// Every file in `root` with its bytes. +fn snapshot(root: &Path) -> BTreeMap> { + fs::read_dir(root) + .unwrap() + .map(|entry| { + let entry = entry.unwrap(); + let bytes = if entry.file_type().unwrap().is_dir() { + b"".to_vec() + } else { + fs::read(entry.path()).unwrap() + }; + (entry.file_name().into_string().unwrap(), bytes) + }) + .collect() +} + +#[test] +fn sigint_cancels_the_startup_migration_and_leaves_every_source_as_it_was() { + let root: PathBuf = std::env::temp_dir().join(format!( + "tcode-headless-migration-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&root).unwrap(); + fs::write( + root.join("sessions.json"), + r#"{"projects":[],"sessions":[{"id":"log-0","title":"Zero","provider":"codex","cwd":"/work","created_at":1,"updated_at":1}]}"#, + ) + .unwrap(); + // Enough that the migration is still importing well after it starts. + let line = format!( + "{{\"ts\":1,\"event\":{{\"type\":\"turn_started\",\"turn_id\":\"{}\"}}}}\n", + "x".repeat(2000) + ); + for index in 0..24 { + fs::write(root.join(format!("log-{index}.jsonl")), line.repeat(1000)).unwrap(); + } + let sources = snapshot(&root); + + let mut serve = tcode_services::process::command(env!("CARGO_BIN_EXE_tcode-headless")) + .args(["serve", "--traverse", "off", "--data-dir"]) + .arg(&root) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut output = BufReader::new(serve.stdout.take().unwrap()).lines(); + let mut printed = Vec::new(); + loop { + let line = output + .next() + .expect("serve exited before the migration imported anything") + .unwrap(); + let importing = line.starts_with("importing:"); + printed.push(line); + if importing { + break; + } + } + let interrupted = tcode_services::process::command("kill") + .args(["-INT", &serve.id().to_string()]) + .status() + .unwrap(); + assert!(interrupted.success()); + printed.extend(output.map(Result::unwrap)); + let status = serve.wait().unwrap(); + + assert!(status.success(), "{status}: {printed:#?}"); + assert!( + printed + .iter() + .any(|line| line.starts_with("Migration cancelled")), + "{printed:#?}" + ); + assert!( + !printed.iter().any(|line| line.starts_with("Machine id")), + "the host started after the cancel: {printed:#?}" + ); + let mut after = snapshot(&root); + // The data dir's ownership lock, which every start creates. + assert!(after.remove("tcode.lock").is_some()); + assert_eq!(after, sources); + fs::remove_dir_all(&root).unwrap(); +} diff --git a/crates/runtime/src/pipe.rs b/crates/runtime/src/pipe.rs index a469e7923..c146968cd 100644 --- a/crates/runtime/src/pipe.rs +++ b/crates/runtime/src/pipe.rs @@ -1272,6 +1272,9 @@ mod tests { .to_string(), ) .unwrap(); + store + .migrate(|_| {}, &std::sync::atomic::AtomicBool::new(false)) + .unwrap(); let host = spawn_host(store.clone(), HostServices::default()).unwrap(); let link = host.link(); link.subscribe(Subscription { @@ -1390,7 +1393,7 @@ mod tests { /// and the JSONL logs into its database once, and serves every thread's /// timeline exactly as the logs fold, before and after a restart. #[test] - fn a_host_on_legacy_files_migrates_once_and_serves_the_same_timelines() { + fn legacy_files_migrated_at_startup_serve_the_same_timelines() { use agent::{AgentEvent, ItemContent, ThreadItem, TurnStatus}; use tcode_core::session::{StoredEvent, Timeline}; @@ -1484,6 +1487,12 @@ mod tests { for start in ["migrating", "restarted"] { let store = SessionStore::open_at(data_root.clone()).unwrap(); + assert_eq!(store.needs_migration().unwrap(), start == "migrating"); + if start == "migrating" { + store + .migrate(|_| {}, &std::sync::atomic::AtomicBool::new(false)) + .unwrap(); + } let host = spawn_host(store, HostServices::default()).unwrap(); let link = host.link(); let stream = link.events(); diff --git a/crates/services/src/store/migrate.rs b/crates/services/src/store/migrate.rs index e35c6f060..aaa99107a 100644 --- a/crates/services/src/store/migrate.rs +++ b/crates/services/src/store/migrate.rs @@ -4,25 +4,31 @@ //! The database is built and verified under a staging name and published by //! a rename, so a file at the final name is always complete: //! -//! 1. `tcode.db.migrating` is created with the schema and `user_version` 0. -//! 2. The index is imported, then every `*.jsonl` in the data dir (orphans -//! included) in bounded transactions; each log is read back and compared -//! byte for byte with its file, then the metadata field for field. -//! 3. `user_version` is set to 1, the WAL is checkpointed and truncated, -//! every handle is dropped, and a fresh open re-counts what was committed. +//! 1. `tcode.db.migrating` is created with the schema and `user_version` 0, +//! and the index is imported. +//! 2. Every `*.jsonl` in the data dir (orphans included) is imported in +//! bounded transactions. +//! 3. Each log is read back and compared byte for byte with its file, then +//! the metadata field for field; `user_version` is set to 1, the WAL is +//! checkpointed and truncated, every handle is dropped, and a fresh open +//! re-counts what was committed. //! 4. The staged file is synced, the list of sources to archive is written to //! `tcode.db.archive`, and the staged file is renamed to `tcode.db`; the //! directory is synced around both. //! 5. The sources move into `legacy/`, and the archive list is removed. //! -//! An interrupted run before step 4's rename leaves no `tcode.db`, and the -//! next start discards the staging files and begins again from the untouched -//! sources. After the rename, the next start finishes step 5 from the list. +//! Nothing before step 4's rename writes, renames or removes a source: the +//! steps before it only read them. A migration cancelled or failed before +//! the rename, or a start after one interrupted there, discards the staging +//! files and leaves no `tcode.db`, so the next run begins again from the +//! untouched sources. Cancellation is honoured only up to the rename; after +//! it the next start finishes step 5 from the list. use std::collections::HashSet; use std::fs::{self, File}; use std::io::{self, BufRead, BufReader}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; use tcode_core::project::{IndexFile, SessionMeta, migrate_index}; @@ -37,21 +43,111 @@ const LEGACY_DIR: &str = "legacy"; /// A thread's log is committed in transactions of about this size; one /// transaction for a multi-hundred-megabyte log holds every dirty page in /// memory and doubles the WAL. -const CHUNK_BYTES: usize = 8 << 20; +pub(super) const CHUNK_BYTES: usize = 8 << 20; /// At most this many rows, and about [`CHUNK_BYTES`], are compared per read /// when verifying, so a reader never pins the WAL or memory for a whole log. const VERIFY_ROWS: i64 = 4096; -/// Make sure a complete `tcode.db` exists, migrating or creating it, and -/// finish an archival a previous start left half done. Called with the data -/// dir's ownership lock held. +/// Where a running migration is. Each of the importing and verifying phases +/// goes through every log once, so their counts start again from zero; +/// publishing and archiving report everything done. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MigrationProgress { + pub phase: MigrationPhase, + pub threads_done: usize, + pub threads_total: usize, + pub bytes_done: u64, + pub bytes_total: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MigrationPhase { + /// Listing the logs and importing the index; the totals are not known + /// until it ends. + Scanning, + Importing, + Verifying, + /// Past the last point a cancellation is honoured. + Publishing, + Archiving, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Migration { + /// `tcode.db` is complete and the sources are in `legacy/`. + Completed, + /// The staging files are gone, no `tcode.db` exists and every source is + /// where it was. + Cancelled, +} + +/// Whether the data dir holds an older build's files and no `tcode.db`. +pub(super) fn needed(root: &Path) -> io::Result { + Ok(!root.join(DB_FILE).exists() && !legacy_sources(root)?.is_empty()) +} + +/// Make sure a complete `tcode.db` exists for opening, creating an empty one +/// on a fresh install, and finish an archival a previous start left half +/// done. A data dir that still needs migrating is refused: that runs only +/// through [`run`], which reports progress and can be cancelled. Called with +/// the data dir's ownership lock held. pub(super) fn prepare(root: &Path) -> io::Result<()> { + if needed(root)? { + return Err(io::Error::other(format!( + "{} holds sessions.json or *.jsonl files from an older Tcode that have not been \ + migrated into {DB_FILE} yet", + root.display() + ))); + } + run(root, &mut |_| {}, &AtomicBool::new(false)).map(drop) +} + +/// Migrate the older build's files into `tcode.db`, or create an empty one, +/// reporting progress after every chunk and every log. `cancel` is checked +/// at the same points until the staged database is complete; once it is set +/// there, the staging files are removed and nothing else is changed. Called +/// with the data dir's ownership lock held. +pub(super) fn run( + root: &Path, + report: &mut dyn FnMut(MigrationProgress), + cancel: &AtomicBool, +) -> io::Result { if root.join(DB_FILE).exists() { - return archive_sources(root); + archive_sources(root)?; + return Ok(Migration::Completed); } remove_staging(root)?; let started = Instant::now(); - let summary = build_staging(root)?; + let mut progress = Progress { + report, + cancel, + current: MigrationProgress { + phase: MigrationPhase::Scanning, + threads_done: 0, + threads_total: 0, + bytes_done: 0, + bytes_total: 0, + }, + }; + (progress.report)(progress.current); + let summary = match build_staging(root, &mut progress) { + Ok(summary) => summary, + Err(error) => { + // `build_staging` has dropped every handle on the staging files. + if let Err(cleanup) = remove_staging(root) { + log::warn!("could not discard the staged database: {cleanup}"); + } + if error.get_ref().is_some_and(|inner| inner.is::()) { + log::info!( + "migration into {} cancelled; the sources are unchanged", + root.join(DB_FILE).display() + ); + return Ok(Migration::Cancelled); + } + return Err(error); + } + }; + progress.finish(MigrationPhase::Publishing); publish(root, &summary.sources)?; log::info!( "migrated {} thread(s) ({} line(s), {} blank, {} bytes) and {} session(s) in {} \ @@ -65,7 +161,64 @@ pub(super) fn prepare(root: &Path) -> io::Result<()> { root.join(DB_FILE).display(), started.elapsed().as_secs_f64() ); - archive_sources(root) + progress.finish(MigrationPhase::Archiving); + archive_sources(root)?; + Ok(Migration::Completed) +} + +#[derive(Debug)] +struct Cancelled; + +impl std::fmt::Display for Cancelled { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("the migration was cancelled") + } +} + +impl std::error::Error for Cancelled {} + +struct Progress<'a> { + report: &'a mut dyn FnMut(MigrationProgress), + cancel: &'a AtomicBool, + current: MigrationProgress, +} + +impl Progress<'_> { + fn check(&self) -> io::Result<()> { + if self.cancel.load(Ordering::Relaxed) { + return Err(io::Error::other(Cancelled)); + } + Ok(()) + } + + /// Begin a cancellable phase over `threads` logs of `bytes` in all. + fn start(&mut self, phase: MigrationPhase, threads: usize, bytes: u64) -> io::Result<()> { + self.check()?; + self.current = MigrationProgress { + phase, + threads_done: 0, + threads_total: threads, + bytes_done: 0, + bytes_total: bytes, + }; + (self.report)(self.current); + Ok(()) + } + + fn advance(&mut self, threads: usize, bytes: u64) -> io::Result<()> { + self.current.threads_done += threads; + self.current.bytes_done += bytes; + (self.report)(self.current); + self.check() + } + + /// Enter a phase that runs to the end whatever `cancel` says. + fn finish(&mut self, phase: MigrationPhase) { + self.current.phase = phase; + self.current.threads_done = self.current.threads_total; + self.current.bytes_done = self.current.bytes_total; + (self.report)(self.current); + } } /// An older build that ran after the migration writes JSON again; this build @@ -105,7 +258,7 @@ fn remove_staging(root: &Path) -> io::Result<()> { root.join(ARCHIVE_LIST), ] { match fs::remove_file(&path) { - Ok(()) => log::warn!("discarded {} from an interrupted migration", path.display()), + Ok(()) => log::warn!("discarded {} from an unfinished migration", path.display()), Err(error) if error.kind() == io::ErrorKind::NotFound => {} Err(error) => return Err(error), } @@ -142,11 +295,20 @@ fn legacy_sources(root: &Path) -> io::Result> { Ok(names) } -fn build_staging(root: &Path) -> io::Result { - let staging = root.join(STAGING_FILE); - let db = Db::open(&staging, true)?; +fn build_staging(root: &Path, progress: &mut Progress) -> io::Result { + let sources = legacy_sources(root)?; + let mut logs = Vec::new(); + let mut bytes_total = 0; + for name in &sources { + if let Some(id) = name.strip_suffix(".jsonl") { + bytes_total += fs::metadata(root.join(name))?.len(); + logs.push((id, root.join(name))); + } + } let index = migrate_index(read_legacy_index(root)?); reject_duplicates(&index)?; + let staging = root.join(STAGING_FILE); + let db = Db::open(&staging, true)?; db.write(|db, connection| { for project in &index.projects { let body = serde_json::to_vec(project).map_err(io::Error::other)?; @@ -167,7 +329,6 @@ fn build_staging(root: &Path) -> io::Result { Ok(()) })?; - let sources = legacy_sources(root)?; let mut summary = Summary { sources: sources.clone(), projects: index.projects.len(), @@ -177,29 +338,31 @@ fn build_staging(root: &Path) -> io::Result { blank_lines: 0, bytes: 0, }; + progress.start(MigrationPhase::Importing, logs.len(), bytes_total)?; let mut tallies = Vec::new(); - for name in &sources { - let Some(id) = name.strip_suffix(".jsonl") else { - continue; - }; - let path = root.join(name); - let tally = import_log(&db, id, &path)?; - verify_log(&db, id, &path, &tally)?; - log::info!( - "migrated {name}: {} line(s), {} blank, {} bytes, verified byte-identical", - tally.lines, - tally.blank_lines, - tally.bytes - ); + for (id, path) in &logs { + let tally = import_log(&db, id, path, progress)?; summary.logs += 1; summary.lines += tally.lines; summary.blank_lines += tally.blank_lines; summary.bytes += tally.bytes; - tallies.push((id.to_owned(), tally)); + tallies.push(((*id).to_owned(), tally)); } + progress.start(MigrationPhase::Verifying, logs.len(), bytes_total)?; + for ((id, path), (_, tally)) in logs.iter().zip(&tallies) { + verify_log(&db, id, path, tally, progress)?; + log::info!( + "migrated {}: {} line(s), {} blank, {} bytes, verified byte-identical", + path.display(), + tally.lines, + tally.blank_lines, + tally.bytes + ); + } let stored = db.read(read_index)?; verify_index(&index, &stored)?; + progress.check()?; db.write(|db, connection| { db.execute( connection, @@ -211,6 +374,8 @@ fn build_staging(root: &Path) -> io::Result { db.checkpoint()?; drop(db); + progress.check()?; + // A fresh open sees only what reached the main file. let reopened = Db::open(&staging, false)?; if reopened.user_version()? != SCHEMA_VERSION { @@ -276,12 +441,15 @@ fn build_staging(root: &Path) -> io::Result { Err(error) if error.kind() == io::ErrorKind::NotFound => {} Err(error) => return Err(error), } + // The last point a cancellation is honoured: what follows publishes. + progress.check()?; sync_file(&staging)?; Ok(summary) } -/// Today's tolerance: the object schema or the legacy bare array; an -/// unparseable file is preserved beside itself and the logs still migrate. +/// Today's tolerance: the object schema or the legacy bare array. An +/// unparseable file is left as it is, to be archived into `legacy/` with the +/// logs, which still migrate. fn read_legacy_index(root: &Path) -> io::Result { let path = root.join(LEGACY_INDEX); let bytes = match fs::read(&path) { @@ -295,23 +463,13 @@ fn read_legacy_index(root: &Path) -> io::Result { sessions, }) }); - match parsed { - Ok(file) => Ok(file), - Err(error) => { - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|duration| duration.as_nanos()) - .unwrap_or(0); - let corrupt = root.join(format!("{LEGACY_INDEX}.corrupt-{timestamp}")); - fs::rename(&path, &corrupt)?; - log::warn!( - "failed to parse {LEGACY_INDEX}: {error}; preserved it as {} and migrated the \ - event logs without it", - corrupt.display() - ); - Ok(IndexFile::default()) - } - } + Ok(parsed.unwrap_or_else(|error| { + log::warn!( + "failed to parse {LEGACY_INDEX}: {error}; migrating the event logs without it, and \ + keeping it in {LEGACY_DIR}/" + ); + IndexFile::default() + })) } /// Upserting by id would silently keep only the last of two entries. @@ -381,7 +539,7 @@ fn read_segments(path: &Path, mut each: impl FnMut(Vec) -> io::Result<()>) - } } -fn import_log(db: &Db, id: &str, path: &Path) -> io::Result { +fn import_log(db: &Db, id: &str, path: &Path, progress: &mut Progress) -> io::Result { let mut tally = Tally::default(); let mut pending: Vec> = Vec::new(); let mut pending_bytes = 0; @@ -405,16 +563,24 @@ fn import_log(db: &Db, id: &str, path: &Path) -> io::Result { pending.push(segment); if pending_bytes >= CHUNK_BYTES { flush(&mut pending, &mut position)?; + progress.advance(0, pending_bytes as u64)?; pending_bytes = 0; } Ok(()) })?; flush(&mut pending, &mut position)?; + progress.advance(1, pending_bytes as u64)?; Ok(tally) } /// Compare the committed rows with the file again, segment for segment. -fn verify_log(db: &Db, id: &str, path: &Path, tally: &Tally) -> io::Result<()> { +fn verify_log( + db: &Db, + id: &str, + path: &Path, + tally: &Tally, + progress: &mut Progress, +) -> io::Result<()> { let mismatch = |position: i64| { io::Error::other(format!( "{}: line {} differs from the migrated copy", @@ -425,8 +591,13 @@ fn verify_log(db: &Db, id: &str, path: &Path, tally: &Tally) -> io::Result<()> { let mut page: Vec> = Vec::new(); let mut page_start = 0_i64; let mut position = 0_i64; + let mut compared = 0; read_segments(path, |segment| { if position - page_start == page.len() as i64 { + if compared > 0 { + progress.advance(0, compared)?; + compared = 0; + } page_start = position; page = db.read(|db, connection| { // Sizes first, so a page holds about CHUNK_BYTES however long @@ -470,6 +641,7 @@ fn verify_log(db: &Db, id: &str, path: &Path, tally: &Tally) -> io::Result<()> { if *stored != segment { return Err(mismatch(position)); } + compared += segment.len() as u64; position += 1; Ok(()) })?; @@ -488,7 +660,7 @@ fn verify_log(db: &Db, id: &str, path: &Path, tally: &Tally) -> io::Result<()> { if extra != 0 || position as u64 != tally.lines { return Err(mismatch(position)); } - Ok(()) + progress.advance(1, compared) } fn publish(root: &Path, sources: &[String]) -> io::Result<()> { diff --git a/crates/services/src/store/mod.rs b/crates/services/src/store/mod.rs index 31aad88ff..e0f7b9553 100644 --- a/crates/services/src/store/mod.rs +++ b/crates/services/src/store/mod.rs @@ -19,12 +19,14 @@ mod migrate; #[cfg(test)] mod tests; +pub use migrate::{Migration, MigrationPhase, MigrationProgress}; + use std::collections::HashMap; use std::fs::{self, File}; use std::io; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Condvar, LazyLock, Mutex, MutexGuard, Weak}; use std::time::{Duration, Instant}; @@ -94,6 +96,11 @@ enum State { /// Opened on first use, so a handle that only needs the directory (a /// subcommand, a settings file) never takes the data dir. Unopened, + /// The data dir is owned, for a migration and then the database, but the + /// database is not open yet. + Owned(File), + /// [`SessionStore::migrate`] holds the ownership lock while it runs. + Migrating, Open(Live), /// [`SessionStore::close`] is waiting for in-flight operations. Closing(Live), @@ -261,13 +268,90 @@ impl SessionStore { &self.root } - /// Take ownership of the data dir and open its database now, migrating the - /// JSON index and JSONL logs on first start. Fails with + /// Take ownership of the data dir and open its database now. A data dir + /// that still needs migrating ([`SessionStore::needs_migration`]) is + /// refused. Fails with /// [`io::ErrorKind::ResourceBusy`] while another host owns the directory. pub fn open(&self) -> io::Result<()> { self.run("open", |_| Ok(())) } + /// Take ownership of the data dir, keeping it for the database opened + /// later, and say whether it holds the JSON index or JSONL logs of an + /// older build that [`SessionStore::migrate`] must move into `tcode.db` + /// before the store opens. Fails with [`io::ErrorKind::ResourceBusy`] + /// while another host owns the directory. + pub fn needs_migration(&self) -> io::Result { + let mut state = self.shared.lock_state()?; + match &*state { + State::Unopened => *state = State::Owned(acquire_ownership(&self.root)?), + State::Owned(_) => {} + State::Open(_) => return Ok(false), + other => return Err(unavailable(other, "check for a migration")), + } + migrate::needed(&self.root) + } + + /// Migrate an older build's files into `tcode.db` under the data dir's + /// ownership lock, which the store keeps for the database afterwards. + /// `progress` is called after every chunk of about 8 MiB and every log; + /// `cancel` is checked at the same points and between phases until the + /// migrated database is complete. A cancelled migration removes its + /// staging files and changes nothing else; a later one starts over. + pub fn migrate( + &self, + mut progress: impl FnMut(MigrationProgress), + cancel: &AtomicBool, + ) -> io::Result { + let ownership = { + let mut state = self.shared.lock_state()?; + match std::mem::replace(&mut *state, State::Migrating) { + State::Unopened => match acquire_ownership(&self.root) { + Ok(ownership) => ownership, + Err(error) => { + *state = State::Unopened; + return Err(error); + } + }, + State::Owned(ownership) => ownership, + State::Open(live) => { + *state = State::Open(live); + return Ok(Migration::Completed); + } + other => { + let error = unavailable(&other, "migrate"); + *state = other; + return Err(error); + } + } + }; + let outcome = catch_unwind(AssertUnwindSafe(|| { + migrate::run(&self.root, &mut progress, cancel) + })); + let mut state = self.shared.lock_state()?; + let result = match outcome { + Ok(result) => { + *state = State::Owned(ownership); + result + } + Err(panic) => { + let reason = format!( + "session store panicked while migrating {}: {}", + self.root.display(), + panic_message(panic.as_ref()) + ); + log::error!("{reason}"); + *state = State::Failed { + reason: reason.clone(), + _ownership: Some(ownership), + }; + Err(io::Error::other(reason)) + } + }; + self.shared.idle.notify_all(); + result + } + /// Whether the store stopped serving after a panic or a broken connection. pub fn is_failed(&self) -> bool { matches!( @@ -280,15 +364,23 @@ impl SessionStore { ) } - /// Wait for every in-flight operation, checkpoint the WAL into the main - /// file and release the database and the data dir. Every handle of this - /// store fails afterwards; a store that already failed is not - /// checkpointed. + /// Wait for every in-flight operation and a running migration, checkpoint + /// the WAL into the main file and release the database and the data dir. + /// Every handle of this store fails afterwards; a store that already + /// failed is not checkpointed. pub fn close(&self) -> io::Result<()> { let mut state = self.shared.lock_state()?; loop { match std::mem::replace(&mut *state, State::Closed) { - State::Unopened | State::Closed => return Ok(()), + State::Unopened | State::Closed | State::Owned(_) => return Ok(()), + State::Migrating => { + *state = State::Migrating; + state = self + .shared + .idle + .wait(state) + .unwrap_or_else(|poisoned| poisoned.into_inner()); + } State::Open(live) => *state = State::Closing(live), State::Closing(live) if live.in_flight > 0 => { *state = State::Closing(live); @@ -354,8 +446,17 @@ impl SessionStore { fn acquire(&self, operation: &str) -> io::Result> { let mut state = self.shared.lock_state()?; - if matches!(*state, State::Unopened) { - match catch_unwind(AssertUnwindSafe(|| open_live(&self.root))) { + let ownership = match std::mem::replace(&mut *state, State::Unopened) { + State::Unopened => Some(acquire_ownership(&self.root)?), + State::Owned(ownership) => Some(ownership), + other => { + *state = other; + None + } + }; + if let Some(ownership) = ownership { + // Dropping the ownership on failure lets the next attempt start over. + match catch_unwind(AssertUnwindSafe(|| open_live(&self.root, ownership))) { Ok(Ok(live)) => *state = State::Open(live), Ok(Err(error)) => return Err(error), Err(panic) => { @@ -378,12 +479,7 @@ impl SessionStore { live.in_flight += 1; Ok(live.db.clone()) } - State::Closing(_) | State::Closed => Err(io::Error::new( - io::ErrorKind::BrokenPipe, - format!("cannot {operation}: the session store is closed"), - )), - failed @ State::Failed { .. } => Err(failed_error(failed)), - State::Unopened => unreachable!("opened above"), + other => Err(unavailable(other, operation)), } } @@ -687,8 +783,9 @@ impl Shared { .unwrap_or_else(|poisoned| poisoned.into_inner()); let ownership = match std::mem::replace(&mut *state, State::Closed) { State::Open(live) | State::Closing(live) => Some(live.ownership), + State::Owned(ownership) => Some(ownership), State::Failed { _ownership, .. } => _ownership, - State::Unopened | State::Closed => None, + State::Unopened | State::Migrating | State::Closed => None, }; *state = State::Failed { reason, @@ -707,6 +804,21 @@ fn failed_error(state: &State) -> io::Error { ))) } +/// Why a store in `state` cannot `operation` now. +fn unavailable(state: &State, operation: &str) -> io::Error { + match state { + State::Migrating => io::Error::new( + io::ErrorKind::ResourceBusy, + format!("cannot {operation}: the data directory is being migrated"), + ), + failed @ State::Failed { .. } => failed_error(failed), + _ => io::Error::new( + io::ErrorKind::BrokenPipe, + format!("cannot {operation}: the session store is closed"), + ), + } +} + fn panic_message(panic: &(dyn std::any::Any + Send)) -> String { panic .downcast_ref::<&str>() @@ -715,10 +827,8 @@ fn panic_message(panic: &(dyn std::any::Any + Send)) -> String { .unwrap_or_else(|| "unknown panic".into()) } -/// Take the data dir, bring `tcode.db` up to date (migrating on first start) -/// and open it. -fn open_live(root: &Path) -> io::Result { - let ownership = acquire_ownership(root)?; +/// Bring `tcode.db` up to date in the data dir `ownership` holds, and open it. +fn open_live(root: &Path, ownership: File) -> io::Result { migrate::prepare(root)?; let path = root.join(DB_FILE); let db = Db::open(&path, false)?; diff --git a/crates/services/src/store/tests.rs b/crates/services/src/store/tests.rs index e7943a97c..fe5005225 100644 --- a/crates/services/src/store/tests.rs +++ b/crates/services/src/store/tests.rs @@ -3,6 +3,8 @@ use std::path::Path; use agent::{ApprovalMode, ProviderCommand, ProviderCommandKind, TurnStatus}; +use std::sync::atomic::AtomicBool; + use super::*; /// A data dir that is removed when the test ends. @@ -22,6 +24,17 @@ impl DataDir { fn store(&self) -> SessionStore { SessionStore::open_at(self.0.clone()).unwrap() } + + /// A store over this data dir after the migration a start runs first. + fn migrated(&self) -> SessionStore { + let store = self.store(); + assert_eq!(migrate(&store).unwrap(), Migration::Completed); + store + } +} + +fn migrate(store: &SessionStore) -> io::Result { + store.migrate(|_| {}, &AtomicBool::new(false)) } impl Drop for DataDir { @@ -169,6 +182,11 @@ fn first_start_migrates_the_json_index_and_every_log_byte_for_byte() { let dir = DataDir::new(); write_legacy_fixture(dir.path()); let store = dir.store(); + assert!(store.needs_migration().unwrap()); + let error = store.open().unwrap_err(); + assert!(error.to_string().contains("not been migrated"), "{error}"); + assert_eq!(migrate(&store).unwrap(), Migration::Completed); + assert!(!store.needs_migration().unwrap()); store.open().unwrap(); assert_fixture_migrated(&store, dir.path()); @@ -196,6 +214,7 @@ fn first_start_migrates_the_json_index_and_every_log_byte_for_byte() { fs::write(dir.path().join("sessions.json"), b"[]").unwrap(); fs::write(dir.path().join("mixed.jsonl"), b"").unwrap(); let reopened = dir.store(); + assert!(!reopened.needs_migration().unwrap()); assert_eq!(reopened.read_event_log("mixed").unwrap(), expected); assert_eq!(reopened.read_file().unwrap().sessions.len(), 2); assert_eq!(fs::read(dir.path().join("sessions.json")).unwrap(), b"[]"); @@ -225,7 +244,7 @@ fn legacy_bare_array_index_migrates_with_one_derived_project_per_root() { ]); fs::write(dir.path().join("sessions.json"), legacy.to_string()).unwrap(); - let file = dir.store().read_file().unwrap(); + let file = dir.migrated().read_file().unwrap(); // Two distinct roots -> two derived projects, deduped by root. assert_eq!(file.projects.len(), 2); let alpha = file @@ -258,28 +277,21 @@ fn legacy_bare_array_index_migrates_with_one_derived_project_per_root() { } #[test] -fn an_unparseable_index_is_preserved_and_the_logs_still_migrate() { +fn an_unparseable_index_is_archived_unchanged_and_the_logs_still_migrate() { let dir = DataDir::new(); let corrupt = b"not valid session json"; fs::write(dir.path().join("sessions.json"), corrupt).unwrap(); fs::write(dir.path().join("orphan.jsonl"), ORPHAN_LOG).unwrap(); - let store = dir.store(); + let store = dir.migrated(); let file = store.read_file().unwrap(); assert!(file.projects.is_empty() && file.sessions.is_empty()); assert_eq!(store.read_event_log("orphan").unwrap(), ORPHAN_LOG); - let backups: Vec<_> = fs::read_dir(dir.path()) - .unwrap() - .filter_map(Result::ok) - .filter(|entry| { - entry - .file_name() - .to_string_lossy() - .starts_with("sessions.json.corrupt-") - }) - .collect(); - assert_eq!(backups.len(), 1); - assert_eq!(fs::read(backups[0].path()).unwrap(), corrupt); + assert!(!dir.path().join("sessions.json").exists()); + assert_eq!( + fs::read(dir.path().join("legacy/sessions.json")).unwrap(), + corrupt + ); } #[test] @@ -293,7 +305,7 @@ fn duplicate_index_ids_fail_the_migration_and_leave_the_sources_alone() { fs::write(dir.path().join("sessions.json"), &index).unwrap(); fs::write(dir.path().join("twice.jsonl"), ORPHAN_LOG).unwrap(); - let error = dir.store().open().unwrap_err(); + let error = migrate(&dir.store()).unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::InvalidData); assert!(error.to_string().contains("twice"), "{error}"); assert!(!dir.path().join(DB_FILE).exists()); @@ -328,8 +340,7 @@ fn a_start_after_an_interrupted_migration_discards_the_staging_files_and_starts_ write_legacy_fixture(dir.path()); fs::write(dir.path().join("tcode.db.migrating"), b"partial").unwrap(); fs::write(dir.path().join("tcode.db.migrating-wal"), b"torn frames").unwrap(); - let store = dir.store(); - store.open().unwrap(); + let store = dir.migrated(); assert_fixture_migrated(&store, dir.path()); store.close().unwrap(); @@ -347,8 +358,127 @@ fn a_start_after_an_interrupted_migration_discards_the_staging_files_and_starts_ archive_list(&LEGACY_FILES), ) .unwrap(); + let store = dir.migrated(); + assert_fixture_migrated(&store, dir.path()); +} + +/// Every file in `root`, by name, with its bytes; `legacy/` as a marker. +fn snapshot(root: &Path) -> std::collections::BTreeMap> { + fs::read_dir(root) + .unwrap() + .map(|entry| { + let entry = entry.unwrap(); + let name = entry.file_name().into_string().unwrap(); + let bytes = if entry.file_type().unwrap().is_dir() { + b"".to_vec() + } else { + fs::read(entry.path()).unwrap() + }; + (name, bytes) + }) + .collect() +} + +#[test] +fn a_cancelled_migration_changes_no_source_and_the_next_one_completes() { + let dir = DataDir::new(); + write_legacy_fixture(dir.path()); + // Longer than one chunk, so a cancel can land inside a log. + let bulk_line = format!( + "{{\"ts\":1,\"event\":{{\"type\":\"turn_started\",\"turn_id\":\"{}\"}}}}\n", + "x".repeat(4000) + ); + let bulk = bulk_line.repeat(2600); + assert!(bulk.len() > migrate::CHUNK_BYTES); + fs::write(dir.path().join("bulk.jsonl"), &bulk).unwrap(); let store = dir.store(); - store.open().unwrap(); + assert!(store.needs_migration().unwrap()); + let mut sources = snapshot(dir.path()); + // Owned for the whole start, and kept from the first check. + sources.remove(LOCK_FILE); + + type Stop = fn(&MigrationProgress) -> bool; + let stops: [(&str, Stop); 4] = [ + ("scanning", |progress| { + progress.phase == MigrationPhase::Scanning + }), + ("inside a log", |progress| { + progress.phase == MigrationPhase::Importing + && progress.bytes_done > 0 + && progress.threads_done == 0 + }), + ("between logs", |progress| { + progress.phase == MigrationPhase::Importing && progress.threads_done == 2 + }), + ("verifying", |progress| { + progress.phase == MigrationPhase::Verifying && progress.threads_done == 1 + }), + ]; + for (point, stop) in stops { + let cancel = AtomicBool::new(false); + let mut seen = None; + let outcome = store + .migrate( + |progress| { + if !cancel.load(Ordering::Relaxed) && stop(&progress) { + seen = Some(progress); + cancel.store(true, Ordering::Relaxed); + } + }, + &cancel, + ) + .unwrap(); + assert_eq!(outcome, Migration::Cancelled, "{point}"); + assert!(seen.is_some(), "{point}: never reached"); + let mut after = snapshot(dir.path()); + assert!(after.remove(LOCK_FILE).is_some()); + assert_eq!(after, sources, "{point}"); + assert!(store.needs_migration().unwrap(), "{point}"); + } + + let mut reports = Vec::new(); + let outcome = store + .migrate(|progress| reports.push(progress), &AtomicBool::new(false)) + .unwrap(); + assert_eq!(outcome, Migration::Completed); + let mut phases = reports + .iter() + .map(|progress| progress.phase) + .collect::>(); + phases.dedup(); + assert_eq!( + phases, + [ + MigrationPhase::Scanning, + MigrationPhase::Importing, + MigrationPhase::Verifying, + MigrationPhase::Publishing, + MigrationPhase::Archiving + ] + ); + let log_bytes = (MIXED_LOG.len() + ORPHAN_LOG.len() + bulk.len()) as u64; + for phase in [MigrationPhase::Importing, MigrationPhase::Verifying] { + let last = reports + .iter() + .rfind(|progress| progress.phase == phase) + .unwrap(); + assert_eq!( + ( + last.threads_done, + last.threads_total, + last.bytes_done, + last.bytes_total + ), + (4, 4, log_bytes, log_bytes), + "{phase:?}" + ); + } + let last = reports.last().unwrap(); + assert_eq!( + (last.threads_done, last.bytes_done), + (last.threads_total, last.bytes_total) + ); + assert_eq!(store.read_event_log("bulk").unwrap(), bulk.as_bytes()); assert_fixture_migrated(&store, dir.path()); } @@ -356,9 +486,7 @@ fn a_start_after_an_interrupted_migration_discards_the_staging_files_and_starts_ fn a_start_after_an_interrupted_archival_finishes_it_without_overwriting() { let dir = DataDir::new(); write_legacy_fixture(dir.path()); - let store = dir.store(); - store.open().unwrap(); - store.close().unwrap(); + dir.migrated().close().unwrap(); // Killed after the rename while moving the sources: two are still in the // data dir. One of them also has a copy in legacy/ already, which must @@ -645,7 +773,13 @@ const OWNER_REPORT: &str = "store owner: "; fn store_owner_process() { let root = PathBuf::from(std::env::var_os("TCODE_STORE_OWNER_DIR").unwrap()); let store = SessionStore::open_at(root).unwrap(); - match store.open() { + let opened = store.needs_migration().and_then(|needed| { + if needed { + migrate(&store)?; + } + store.open() + }); + match opened { Ok(()) => println!("{OWNER_REPORT}OPEN"), Err(error) => println!("{OWNER_REPORT}ERR {:?} {error}", error.kind()), } @@ -803,8 +937,7 @@ fn migration_survives_sigkill() { ); } - let store = dir.store(); - store.open().unwrap(); + let store = dir.migrated(); for name in names.iter().filter_map(|name| name.strip_suffix(".jsonl")) { assert_eq!( store.read_event_log(name).unwrap(), diff --git a/crates/ui/src/lib.rs b/crates/ui/src/lib.rs index a3255293b..12f45e376 100644 --- a/crates/ui/src/lib.rs +++ b/crates/ui/src/lib.rs @@ -73,9 +73,13 @@ pub use i18n::{ LANGUAGE_ENGLISH, LANGUAGE_SIMPLIFIED_CHINESE, apply_locale, resolve_locale, set_locale, translate, translate_with_args, }; -pub use run::{ShellOptions, THEME_JSON, flattened_theme_json, last_host_target, run_shell}; +pub use run::{ + ShellOptions, THEME_JSON, flattened_theme_json, init_client, last_host_target, + open_client_window, run_shell, +}; pub(crate) use shell::window_drag_area; pub use shell::{AppShell, Quit, ShellSetup, TogglePalette, handle_back}; +pub use thread_export::format_size; pub use window_seam::force_mobile_layout; pub use window_state::{OpenThread, WindowState}; diff --git a/crates/ui/src/run.rs b/crates/ui/src/run.rs index 15e6bfc7c..3dd836c3d 100644 --- a/crates/ui/src/run.rs +++ b/crates/ui/src/run.rs @@ -11,7 +11,8 @@ use std::cell::RefCell; use std::rc::Rc; use gpui::{ - AnyWindowHandle, App, AppContext as _, Entity, KeyBinding, SharedString, WindowOptions, + AnyWindowHandle, App, AppContext as _, Entity, KeyBinding, Render, SharedString, Window, + WindowOptions, }; use tcode_client::host::ClientHost; @@ -99,15 +100,17 @@ impl ShellOptions { } } -/// Open this client's window on the shared shell. -/// -/// Returns the window and its shell, so bootstrap can keep doing whatever is -/// genuinely its own (a smoke run, a launch flag, a platform Back callback). -pub fn run_shell( - cx: &mut App, - host: Rc, - options: ShellOptions, -) -> (AnyWindowHandle, Entity) { +/// Process-wide setup every window of this client needs: locale, fonts, +/// theme, markdown and keybindings. [`run_shell`] runs it; a bootstrap that +/// opens a window of its own before the shell runs it first, and the second +/// call does nothing. +pub fn init_client(cx: &mut App, host: &dyn ClientHost, options: &ShellOptions) { + struct Initialized; + impl gpui::Global for Initialized {} + if cx.has_global::() { + return; + } + cx.set_global(Initialized); // Browser bootstrap supplies its Fetch client; native image URLs need an HTTP client too. #[cfg(not(target_family = "wasm"))] { @@ -127,7 +130,7 @@ pub fn run_shell( override_locale => override_locale, }); cx.text_system() - .add_fonts(options.fonts) + .add_fonts(options.fonts.clone()) .expect("failed to register bundled application fonts"); theme::init_with_json(&options.theme_json, cx); crate::markdown::init(cx); @@ -139,6 +142,50 @@ pub fn run_shell( cx.bind_keys([KeyBinding::new("secondary-k", TogglePalette, None)]); #[cfg(target_os = "macos")] cx.bind_keys([KeyBinding::new("cmd-q", crate::shell::Quit, None)]); +} + +/// Open a window of this client around the view `build` returns: titled, in +/// the system appearance, on the platform material its options ask for, and +/// with the overlay host every dialog and toast needs. +pub fn open_client_window( + cx: &mut App, + options: WindowOptions, + title: SharedString, + build: impl FnOnce(&mut Window, &mut App) -> Entity + 'static, +) -> AnyWindowHandle { + #[cfg(target_os = "macos")] + let window_background = options.window_background; + let window = cx + .open_window(options, move |window, cx| { + window.set_window_title(&title); + theme::sync_system_appearance(Some(window), cx); + let view = build(window, cx); + cx.new(|cx| gpui_base::Root::new(view, window, cx)) + }) + .expect("failed to open the tcode window"); + // A transparent macOS window gets its blur from a stock semantic material + // rather than GPUI's `Blurred` path; see `macos_backdrop`. + #[cfg(target_os = "macos")] + if window_background == gpui::WindowBackgroundAppearance::Transparent { + use crate::theme::ActiveTheme as _; + let _ = window.update(cx, |_, window, cx| { + crate::macos_backdrop::install(window, cx); + theme::change_mode(cx.theme().mode, Some(window), cx); + }); + } + window.into() +} + +/// Open this client's window on the shared shell. +/// +/// Returns the window and its shell, so bootstrap can keep doing whatever is +/// genuinely its own (a smoke run, a launch flag, a platform Back callback). +pub fn run_shell( + cx: &mut App, + host: Rc, + options: ShellOptions, +) -> (AnyWindowHandle, Entity) { + init_client(cx, host.as_ref(), &options); if options.activate { cx.activate(true); } @@ -163,36 +210,19 @@ pub fn run_shell( }, )); let captured = mounted.clone(); - #[cfg(target_os = "macos")] - let window_background = options.window.window_background; - let window: AnyWindowHandle = cx - .open_window(options.window, move |window, cx| { - window.set_window_title(&title); - theme::sync_system_appearance(Some(window), cx); - let window_state = cx.new(|_| WindowState::new(false)); - let shell = cx.new(|cx| { - AppShell::new( - window_state, - setup.borrow_mut().take().unwrap_or_default(), - window, - cx, - ) - }); - *captured.borrow_mut() = Some(shell.clone()); - cx.new(|cx| gpui_base::Root::new(shell, window, cx)) - }) - .expect("failed to open the tcode window") - .into(); - // A transparent macOS window gets its blur from a stock semantic material - // rather than GPUI's `Blurred` path; see `macos_backdrop`. - #[cfg(target_os = "macos")] - if window_background == gpui::WindowBackgroundAppearance::Transparent { - use crate::theme::ActiveTheme as _; - let _ = window.update(cx, |_, window, cx| { - crate::macos_backdrop::install(window, cx); - theme::change_mode(cx.theme().mode, Some(window), cx); + let window = open_client_window(cx, options.window, title, move |window, cx| { + let window_state = cx.new(|_| WindowState::new(false)); + let shell = cx.new(|cx| { + AppShell::new( + window_state, + setup.borrow_mut().take().unwrap_or_default(), + window, + cx, + ) }); - } + *captured.borrow_mut() = Some(shell.clone()); + shell + }); let shell = mounted .borrow_mut() .take() diff --git a/crates/ui/src/thread_export.rs b/crates/ui/src/thread_export.rs index 7b8bb09aa..04585f564 100644 --- a/crates/ui/src/thread_export.rs +++ b/crates/ui/src/thread_export.rs @@ -228,12 +228,14 @@ fn save_to_disk( ) { } -pub(crate) fn format_size(bytes: usize) -> String { +pub fn format_size(bytes: usize) -> String { if bytes < 1024 { format!("{bytes} B") } else if bytes < 1024 * 1024 { format!("{:.1} KB", bytes as f64 / 1024.) - } else { + } else if bytes < 1024 * 1024 * 1024 { format!("{:.1} MB", bytes as f64 / (1024. * 1024.)) + } else { + format!("{:.1} GB", bytes as f64 / (1024. * 1024. * 1024.)) } } diff --git a/locales/en.yml b/locales/en.yml index 446b39767..263cdb092 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -219,6 +219,16 @@ quit: description: "%{count} session(s) are still working. Quitting will stop them." confirm: "Quit" startup_failed: "Tcode could not start" +migration: + title: "Upgrading thread storage" + description: "Tcode is copying your threads into a new database, tcode.db. The original files are kept in the data folder's legacy/ directory." + scanning: "Looking for threads…" + importing: "Copying threads: %{done} of %{total} · %{bytes} of %{total_bytes}" + verifying: "Checking the copy: %{done} of %{total} · %{bytes} of %{total_bytes}" + publishing: "Finishing…" + archiving: "Moving the original files into legacy/…" + quitting: "Stopping. The original files are unchanged." + failed: "The upgrade could not finish: %{reason}" settings: title: "Settings" sections: "All settings" diff --git a/locales/zh-CN.yml b/locales/zh-CN.yml index e8b3b9bfa..b565a9ee6 100644 --- a/locales/zh-CN.yml +++ b/locales/zh-CN.yml @@ -219,6 +219,16 @@ quit: description: "仍有 %{count} 个会话正在工作。退出将停止这些会话。" confirm: "退出" startup_failed: "Tcode 无法启动" +migration: + title: "正在升级对话存储" + description: "Tcode 正在把你的对话复制到新的数据库 tcode.db，原始文件会保留在数据目录的 legacy/ 文件夹中。" + scanning: "正在查找对话…" + importing: "正在复制对话：%{done}/%{total} · %{bytes}/%{total_bytes}" + verifying: "正在校验副本：%{done}/%{total} · %{bytes}/%{total_bytes}" + publishing: "即将完成…" + archiving: "正在把原始文件移入 legacy/…" + quitting: "正在停止，原始文件未被改动。" + failed: "升级未能完成：%{reason}" settings: title: "设置" sections: "全部设置" From cfd256bea1c3d37371398c60af9ec3dd615bd12d Mon Sep 17 00:00:00 2001 From: Tryanks Date: Tue, 6 Oct 2026 14:53:49 +0800 Subject: [PATCH 3/4] test: check the store's own files through the store, not by reading them The export test snapshotted every file in the data dir to prove an export writes nothing; the running host now holds tcode.db and tcode.lock with locks that Windows enforces, so the read failed there. The snapshot skips those files and the test compares the event log and the index through the store instead. --- crates/runtime/src/pipe.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/runtime/src/pipe.rs b/crates/runtime/src/pipe.rs index c146968cd..2b3081b12 100644 --- a/crates/runtime/src/pipe.rs +++ b/crates/runtime/src/pipe.rs @@ -1029,6 +1029,8 @@ mod tests { .expect("append event"); store.upsert_meta(&meta).expect("write meta"); let event_log = store.read_event_log(&meta.id).expect("read event log"); + let index = store.load_index().expect("read index"); + let reader = store.clone(); let host = spawn_host(store, HostServices::default()).expect("spawn host"); let before = tree_snapshot(&data_root); @@ -1071,6 +1073,8 @@ mod tests { before, "rendering an export must not write anything on the host" ); + assert_eq!(reader.read_event_log(&meta.id).unwrap(), event_log); + assert_eq!(reader.load_index().unwrap(), index); let missing = smol::block_on(link.query(Query::RenderThreadExport { session_id: "nope".into(), @@ -1232,6 +1236,15 @@ mod tests { }; for entry in entries.flatten() { let path = entry.path(); + // The live store's own files are checked through the store + // above; Windows locks them while the host runs, so they + // cannot be read here. + if matches!( + entry.file_name().to_str(), + Some("tcode.db" | "tcode.db-wal" | "tcode.db-shm" | "tcode.lock") + ) { + continue; + } match entry.metadata() { Ok(metadata) if metadata.is_dir() => walk(&path, out), Ok(_) => { From 783af828bcb0792a6565ba2c897bcbf9e3c7d518 Mon Sep 17 00:00:00 2001 From: Tryanks Date: Tue, 6 Oct 2026 15:46:49 +0800 Subject: [PATCH 4/4] test: start each legacy-migration host in its own process; snapshot store files by name The store's lock file cannot be read while it is held on Windows, so the migration snapshot records store files by name instead of bytes. An in-process restart right after a host stops can find the data directory still locked: a spawn the stopped host started (a git status refresh) keeps a copy of the lock descriptor until the child execs. The app restarts as a new process, and the test now does the same. --- crates/runtime/src/pipe.rs | 143 +++++++++++++++++++---------- crates/services/src/store/tests.rs | 4 + 2 files changed, 97 insertions(+), 50 deletions(-) diff --git a/crates/runtime/src/pipe.rs b/crates/runtime/src/pipe.rs index 2b3081b12..49a37a1aa 100644 --- a/crates/runtime/src/pipe.rs +++ b/crates/runtime/src/pipe.rs @@ -1402,17 +1402,12 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } - /// A host started on a data dir an older build wrote moves the JSON index - /// and the JSONL logs into its database once, and serves every thread's - /// timeline exactly as the logs fold, before and after a restart. - #[test] - fn legacy_files_migrated_at_startup_serve_the_same_timelines() { + const LEGACY_START_DIR: &str = "TCODE_LEGACY_START_DIR"; + const LEGACY_START_REPORT: &str = "legacy start: "; + + fn legacy_events() -> [(Option, agent::AgentEvent); 5] { use agent::{AgentEvent, ItemContent, ThreadItem, TurnStatus}; - use tcode_core::session::{StoredEvent, Timeline}; - let data_root = - std::env::temp_dir().join(format!("tcode-host-migration-{}", uuid::Uuid::new_v4())); - std::fs::create_dir_all(&data_root).unwrap(); let item = |id: &str, content| { AgentEvent::ItemCompleted(ThreadItem { id: id.into(), @@ -1420,7 +1415,7 @@ mod tests { content, }) }; - let events = [ + [ ( None, AgentEvent::TurnStarted { @@ -1461,8 +1456,24 @@ mod tests { turn_id: "t2".into(), }, ), - ]; - let line = |(ts, event): &(Option, AgentEvent)| match ts { + ] + } + + /// A host started on a data dir an older build wrote moves the JSON index + /// and the JSONL logs into its database once, and serves every thread's + /// timeline exactly as the logs fold, before and after a restart. + /// + /// Each start is its own process, as the app's are. A restart inside one + /// process can find the data dir still locked: a stopped host's detached + /// work (here a git status refresh) may be inside a spawn, and the child's + /// copy of the process's descriptors holds the lock until it execs. + #[test] + fn legacy_files_migrated_at_startup_serve_the_same_timelines() { + let data_root = + std::env::temp_dir().join(format!("tcode-host-migration-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&data_root).unwrap(); + let events = legacy_events(); + let line = |(ts, event): &(Option, agent::AgentEvent)| match ts { None => serde_json::to_string(event).unwrap(), Some(ts) => serde_json::json!({"ts": ts, "event": event}).to_string(), }; @@ -1489,47 +1500,28 @@ mod tests { .to_string(), ) .unwrap(); - let expected = format!( - "{:?}", - Timeline::fold_events(events.iter().map(|(ts, event)| StoredEvent { - ts: *ts, - event: event.clone(), - elided: None, - })) - ); for start in ["migrating", "restarted"] { - let store = SessionStore::open_at(data_root.clone()).unwrap(); - assert_eq!(store.needs_migration().unwrap(), start == "migrating"); - if start == "migrating" { - store - .migrate(|_| {}, &std::sync::atomic::AtomicBool::new(false)) - .unwrap(); - } - let host = spawn_host(store, HostServices::default()).unwrap(); - let link = host.link(); - let stream = link.events(); - let topic = Topic::SessionEvents { - session_id: "legacy-thread".into(), - }; - link.subscribe(Subscription { - after: None, - topic: topic.clone(), - }) - .unwrap(); - let snapshot = next_event(&stream, |event| { - event.topic == topic && matches!(event.event, ServerEvent::SessionSnapshot { .. }) - }); - let ServerEvent::SessionSnapshot { records, total, .. } = snapshot.event else { - unreachable!("filtered to snapshots") - }; - assert_eq!(total, 5, "{start}"); - assert_eq!( - format!("{:?}", Timeline::fold_events(records)), - expected, - "{start}" + let output = tcode_services::process::command(std::env::current_exe().unwrap()) + .args([ + "--exact", + "pipe::tests::legacy_files_start_process", + "--ignored", + "--nocapture", + ]) + .env(LEGACY_START_DIR, &data_root) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "{start}: {stdout}{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + stdout.contains(&format!("{LEGACY_START_REPORT}{start}")), + "{start}: {stdout}" ); - host.shutdown_blocking().unwrap(); assert!(!data_root.join("sessions.json").exists(), "{start}"); assert!(!data_root.join("legacy-thread.jsonl").exists(), "{start}"); assert_eq!( @@ -1541,6 +1533,57 @@ mod tests { std::fs::remove_dir_all(data_root).unwrap(); } + /// One start of the app: migrate if an older build's files are there, + /// serve the legacy thread's timeline, and stop. + #[test] + #[ignore = "a start of legacy_files_migrated_at_startup_serve_the_same_timelines, which runs it"] + fn legacy_files_start_process() { + use tcode_core::session::{StoredEvent, Timeline}; + + let data_root = std::path::PathBuf::from(std::env::var_os(LEGACY_START_DIR).unwrap()); + let store = SessionStore::open_at(data_root).unwrap(); + let migrating = store.needs_migration().unwrap(); + if migrating { + store + .migrate(|_| {}, &std::sync::atomic::AtomicBool::new(false)) + .unwrap(); + } + let host = spawn_host(store, HostServices::default()).unwrap(); + let link = host.link(); + let stream = link.events(); + let topic = Topic::SessionEvents { + session_id: "legacy-thread".into(), + }; + link.subscribe(Subscription { + after: None, + topic: topic.clone(), + }) + .unwrap(); + let snapshot = next_event(&stream, |event| { + event.topic == topic && matches!(event.event, ServerEvent::SessionSnapshot { .. }) + }); + let ServerEvent::SessionSnapshot { records, total, .. } = snapshot.event else { + unreachable!("filtered to snapshots") + }; + assert_eq!(total, 5); + assert_eq!( + format!("{:?}", Timeline::fold_events(records)), + format!( + "{:?}", + Timeline::fold_events(legacy_events().into_iter().map(|(ts, event)| { + StoredEvent { + ts, + event, + elided: None, + } + })) + ) + ); + host.shutdown_blocking().unwrap(); + let start = if migrating { "migrating" } else { "restarted" }; + println!("{LEGACY_START_REPORT}{start}"); + } + #[test] fn dedicated_host_round_trips_commands_queries_events_and_quit_barrier() { let data_root = diff --git a/crates/services/src/store/tests.rs b/crates/services/src/store/tests.rs index fe5005225..19db868a1 100644 --- a/crates/services/src/store/tests.rs +++ b/crates/services/src/store/tests.rs @@ -363,6 +363,8 @@ fn a_start_after_an_interrupted_migration_discards_the_staging_files_and_starts_ } /// Every file in `root`, by name, with its bytes; `legacy/` as a marker. +/// The store's own files are recorded by name only: Windows locks the whole +/// of a file the store holds, so even the empty `tcode.lock` cannot be read. fn snapshot(root: &Path) -> std::collections::BTreeMap> { fs::read_dir(root) .unwrap() @@ -371,6 +373,8 @@ fn snapshot(root: &Path) -> std::collections::BTreeMap> { let name = entry.file_name().into_string().unwrap(); let bytes = if entry.file_type().unwrap().is_dir() { b"".to_vec() + } else if name == LOCK_FILE || name.starts_with(DB_FILE) { + b"".to_vec() } else { fs::read(entry.path()).unwrap() };