+20

.config/nextest.toml

Reviewed

··· 106 106 filter = "binary(compaction_restart) | binary(mst_refcount_integrity) | binary(gc_compaction_restart)" 107 107 slow-timeout = { period = "120s", terminate-after = 4 } 108 108 109 109 + [[profile.default.overrides]] 110 110 + filter = "test(/retention_time_travel_survives_many_seeds/)" 111 111 + slow-timeout = { period = "300s", terminate-after = 24 } 112 112 + test-group = "io-heavy-sim" 113 113 + 114 114 + [[profile.default.overrides]] 115 115 + filter = "binary(fd_lifecycle)" 116 116 + slow-timeout = { period = "300s", terminate-after = 4 } 117 117 + test-group = "io-heavy-sim" 118 118 + 109 119 [[profile.ci.overrides]] 110 120 filter = "test(/import_with_verification/) | test(/plc_migration/)" 111 121 test-group = "serial-env-tests" ··· 137 147 [[profile.ci.overrides]] 138 148 filter = "binary(repo_lifecycle)" 139 149 test-group = "heavy-load-tests" 150 150 + 151 151 + [[profile.ci.overrides]] 152 152 + filter = "test(/retention_time_travel_survives_many_seeds/)" 153 153 + slow-timeout = { period = "300s", terminate-after = 24 } 154 154 + test-group = "io-heavy-sim" 155 155 + 156 156 + [[profile.ci.overrides]] 157 157 + filter = "binary(fd_lifecycle)" 158 158 + slow-timeout = { period = "300s", terminate-after = 4 } 159 159 + test-group = "io-heavy-sim"

+12

.containerignore

Reviewed

··· 1 1 + target/ 2 2 + .git/ 3 3 + .jj/ 4 4 + **/node_modules/ 5 5 + frontend/dist/ 6 6 + frontend/coverage/ 7 7 + frontend/.pnpm-store/ 8 8 + .direnv/ 9 9 + result 10 10 + .env 11 11 + *.output 12 12 + reference-pds-bsky/

+1

.gitignore

Reviewed

··· 4 4 result 5 5 frontend/node_modules/ 6 6 frontend/dist/ 7 7 + frontend/coverage/ 7 8 frontend/.pnpm-store 8 9 frontend/.npmrc

+3 -2

crates/tranquil-pds/src/repo/mod.rs

Reviewed

··· 8 8 use jacquard_repo::repo::CommitData; 9 9 use jacquard_repo::storage::BlockStore; 10 10 use tranquil_store::blockstore::TranquilBlockStore; 11 11 + use tranquil_store::{RealIO, SystemClock}; 11 12 12 13 #[derive(Clone)] 13 14 pub enum AnyBlockStore { 14 15 Postgres(PostgresBlockStore), 15 15 - TranquilStore(TranquilBlockStore), 16 16 + TranquilStore(TranquilBlockStore<RealIO, SystemClock>), 16 17 } 17 18 18 19 impl AnyBlockStore { ··· 23 24 } 24 25 } 25 26 26 26 - pub fn as_tranquil_store(&self) -> Option<&TranquilBlockStore> { 27 27 + pub fn as_tranquil_store(&self) -> Option<&TranquilBlockStore<RealIO, SystemClock>> { 27 28 match self { 28 29 Self::TranquilStore(s) => Some(s), 29 30 Self::Postgres(_) => None,

+12 -3

crates/tranquil-pds/src/scheduled.rs

Reviewed

··· 574 574 } 575 575 576 576 fn run_compaction_pass( 577 577 - store: &tranquil_store::blockstore::TranquilBlockStore, 577 577 + store: &tranquil_store::blockstore::TranquilBlockStore< 578 578 + tranquil_store::RealIO, 579 579 + tranquil_store::SystemClock, 580 580 + >, 578 581 liveness_threshold: f64, 579 582 grace_period_ms: u64, 580 583 blocklist: &parking_lot::Mutex<CompactionBlocklist>, ··· 836 839 } 837 840 838 841 fn walk_repo_dag_sync( 839 839 - store: &tranquil_store::blockstore::TranquilBlockStore, 842 842 + store: &tranquil_store::blockstore::TranquilBlockStore< 843 843 + tranquil_store::RealIO, 844 844 + tranquil_store::SystemClock, 845 845 + >, 840 846 head_cid: &Cid, 841 847 reachable: &mut std::collections::HashSet<CidBytes>, 842 848 phantom_files: &mut std::collections::HashSet<tranquil_store::blockstore::DataFileId>, ··· 960 966 } 961 967 962 968 pub fn run_reachability_walk( 963 963 - store: &tranquil_store::blockstore::TranquilBlockStore, 969 969 + store: &tranquil_store::blockstore::TranquilBlockStore< 970 970 + tranquil_store::RealIO, 971 971 + tranquil_store::SystemClock, 972 972 + >, 964 973 repo_repo: &dyn RepoRepository, 965 974 ) -> anyhow::Result<ReachabilityResult> { 966 975 let rt = tokio::runtime::Handle::current();

+4 -1

crates/tranquil-pds/src/state.rs

Reviewed

··· 471 471 } 472 472 473 473 struct TranquilStoreWiring { 474 474 - blockstore: tranquil_store::blockstore::TranquilBlockStore, 474 474 + blockstore: tranquil_store::blockstore::TranquilBlockStore< 475 475 + tranquil_store::RealIO, 476 476 + tranquil_store::SystemClock, 477 477 + >, 475 478 signal_provider: Arc<dyn tranquil_signal::SignalStoreProvider>, 476 479 repos: PostgresRepositories, 477 480 segments_dir: PathBuf,

+6 -1

crates/tranquil-pds/tests/gc_compaction_restart.rs

Reviewed

··· 5 5 use reqwest::StatusCode; 6 6 use serde_json::{Value, json}; 7 7 8 8 - fn run_compaction(store: &tranquil_store::blockstore::TranquilBlockStore) { 8 8 + fn run_compaction( 9 9 + store: &tranquil_store::blockstore::TranquilBlockStore< 10 10 + tranquil_store::RealIO, 11 11 + tranquil_store::SystemClock, 12 12 + >, 13 13 + ) { 9 14 let liveness = store.compaction_liveness(0).unwrap(); 10 15 liveness 11 16 .iter()

+6 -3

crates/tranquil-store/benches/blockstore.rs

Reviewed

··· 13 13 use tranquil_store::blockstore::{ 14 14 BlockStoreConfig, DEFAULT_MAX_FILE_SIZE, GroupCommitConfig, TranquilBlockStore, 15 15 }; 16 16 + use tranquil_store::{RealIO, SystemClock}; 16 17 17 18 const DAG_CBOR_CODEC: u64 = 0x71; 18 19 const SHA2_256_CODE: u64 = 0x12; ··· 60 61 }) 61 62 } 62 63 63 63 - fn open_store(dir: &Path) -> TranquilBlockStore { 64 64 + fn open_store(dir: &Path) -> TranquilBlockStore<RealIO, SystemClock> { 64 65 open_store_sharded(dir, 1) 65 66 } 66 67 67 67 - fn open_store_sharded(dir: &Path, shard_count: u8) -> TranquilBlockStore { 68 68 + fn open_store_sharded(dir: &Path, shard_count: u8) -> TranquilBlockStore<RealIO, SystemClock> { 68 69 TranquilBlockStore::open(BlockStoreConfig { 69 70 data_dir: dir.join("data"), 70 71 index_dir: dir.join("index"), ··· 183 184 cids 184 185 }; 185 186 186 186 - let run_reads = |label: &'static str, store: TranquilBlockStore, cids: Vec<Cid>| async move { 187 187 + let run_reads = |label: &'static str, 188 188 + store: TranquilBlockStore<RealIO, SystemClock>, 189 189 + cids: Vec<Cid>| async move { 187 190 let start = Instant::now(); 188 191 189 192 let handles: Vec<_> = (0..concurrency)

+4 -3

crates/tranquil-store/src/backup.rs

Reviewed

··· 8 8 BlockOffset, BlockstoreSnapshot, CommitEpoch, CommitError, DataFileId, QuiesceGuard, 9 9 RebuildError, TranquilBlockStore, 10 10 }; 11 11 + use crate::clock::SystemClock; 11 12 use crate::eventlog::{ 12 13 EventLog, EventLogConfig, EventLogFreezeGuard, EventLogSnapshotState, EventSequence, 13 14 EventWithMutations, SegmentId, SegmentOffset, ··· 157 158 } 158 159 159 160 pub struct BackupCoordinator<'a, S: StorageIO> { 160 160 - blockstore: &'a TranquilBlockStore, 161 161 + blockstore: &'a TranquilBlockStore<RealIO, SystemClock>, 161 162 eventlog: &'a EventLog<S>, 162 163 metastore: &'a Metastore, 163 164 } 164 165 165 166 impl<'a, S: StorageIO + Send + Sync + 'static> BackupCoordinator<'a, S> { 166 167 pub fn new( 167 167 - blockstore: &'a TranquilBlockStore, 168 168 + blockstore: &'a TranquilBlockStore<RealIO, SystemClock>, 168 169 eventlog: &'a EventLog<S>, 169 170 metastore: &'a Metastore, 170 171 ) -> Self { ··· 312 313 313 314 BackupManifest { 314 315 version: BACKUP_FORMAT_VERSION, 315 315 - created_at_ms: crate::wall_clock_ms().raw(), 316 316 + created_at_ms: crate::blockstore::WallClockMs::now().raw(), 316 317 blockstore: { 317 318 let max_cursor = bs 318 319 .shard_cursors

+5 -1

crates/tranquil-store/src/blockstore/compaction.rs

Reviewed

··· 9 9 use super::manager::DataFileManager; 10 10 use super::types::{ 11 11 BlockLocation, CidBytes, CommitEpoch, CompactionResult, CompactionStats, DataFileId, 12 12 + WallClockMs, 12 13 }; 13 14 14 15 #[derive(Debug)] ··· 65 66 active_files: &ActiveFileSet, 66 67 hint_positions: &super::group_commit::ShardHintPositions, 67 68 epoch: &super::types::EpochCounter, 69 69 + now: WallClockMs, 68 70 ) -> Result<CompactionResult, CompactionError> { 69 71 if active_files.contains(source_file_id) { 70 72 return Err(CompactionError::ActiveFileCannotBeCompacted); ··· 89 91 new_file_id, 90 92 current_epoch, 91 93 grace_period_ms, 94 94 + now, 92 95 ); 93 96 94 97 match result { ··· 183 186 }) 184 187 } 185 188 189 189 + #[allow(clippy::too_many_arguments)] 186 190 fn stream_compact<S: StorageIO>( 187 191 manager: &DataFileManager<S>, 188 192 index: &BlockIndex, ··· 191 195 new_file_id: DataFileId, 192 196 current_epoch: CommitEpoch, 193 197 grace_period_ms: u64, 198 198 + now: WallClockMs, 194 199 ) -> Result<(u64, u64, u64, super::types::HintOffset), CompactionError> { 195 200 let mut reader = DataFileReader::open(manager.io(), source_fd)?; 196 196 - let now = crate::wall_clock_ms(); 197 201 198 202 let new_handle = manager.open_for_append(new_file_id)?; 199 203 let mut writer = DataFileWriter::new(manager.io(), new_handle.fd(), new_file_id)?;

+37 -23

crates/tranquil-store/src/blockstore/group_commit.rs

Reviewed

··· 6 6 7 7 use parking_lot::RwLock; 8 8 9 9 + use crate::clock::{Clock, LogicalNanos}; 9 10 use crate::fsync_order::PostBlockstoreHook; 10 11 11 12 use super::BlocksSynced; ··· 224 225 } 225 226 } 226 227 227 227 - struct ShardContext { 228 228 + struct ShardContext<C: Clock> { 228 229 shard_id: ShardId, 229 230 epoch: EpochCounter, 230 231 file_ids: Arc<FileIdAllocator>, 231 232 active_files: Arc<ActiveFileSet>, 232 233 hint_positions: Arc<ShardHintPositions>, 233 234 verify_persisted_blocks: bool, 235 235 + clock: C, 234 236 } 235 237 236 238 struct ActiveState { ··· 256 258 } 257 259 258 260 impl SingleShardWriter { 259 259 - fn spawn<S: StorageIO + 'static>( 260 260 - ctx: ShardContext, 261 261 + fn spawn<S: StorageIO + 'static, C: Clock>( 262 262 + ctx: ShardContext<C>, 261 263 manager: DataFileManager<S>, 262 264 index: Arc<BlockIndex>, 263 265 config: GroupCommitConfig, ··· 359 361 } 360 362 361 363 impl GroupCommitWriter { 362 362 - pub fn spawn<S: StorageIO + 'static>( 364 364 + pub fn spawn<S: StorageIO + 'static, C: Clock>( 363 365 make_manager: impl Fn() -> DataFileManager<S>, 364 366 index: Arc<BlockIndex>, 365 367 config: GroupCommitConfig, 368 368 + clock: C, 366 369 ) -> Result<Self, CommitError> { 367 367 - Self::spawn_sharded(make_manager, index, config, None, None, 1, None) 370 370 + Self::spawn_sharded(make_manager, index, config, None, None, 1, None, clock) 368 371 } 369 372 370 370 - pub fn spawn_with_hook<S: StorageIO + 'static>( 373 373 + pub fn spawn_with_hook<S: StorageIO + 'static, C: Clock>( 371 374 make_manager: impl Fn() -> DataFileManager<S>, 372 375 index: Arc<BlockIndex>, 373 376 config: GroupCommitConfig, 374 377 post_sync_hook: Option<Arc<dyn PostBlockstoreHook>>, 375 378 initial_epoch: Option<CommitEpoch>, 379 379 + clock: C, 376 380 ) -> Result<Self, CommitError> { 377 381 Self::spawn_sharded( 378 382 make_manager, ··· 382 386 initial_epoch, 383 387 1, 384 388 None, 389 389 + clock, 385 390 ) 386 391 } 387 392 388 388 - pub fn spawn_sharded<F, S>( 393 393 + #[allow(clippy::too_many_arguments)] 394 394 + pub fn spawn_sharded<F, S, C>( 389 395 make_manager: F, 390 396 index: Arc<BlockIndex>, 391 397 config: GroupCommitConfig, ··· 393 399 initial_epoch: Option<CommitEpoch>, 394 400 shard_count: u8, 395 401 checkpoint_positions: Option<&CheckpointPositions>, 402 402 + clock: C, 396 403 ) -> Result<Self, CommitError> 397 404 where 398 405 S: StorageIO + 'static, 406 406 + C: Clock, 399 407 F: Fn() -> DataFileManager<S>, 400 408 { 401 409 let shard_count = shard_count.max(1); ··· 435 443 active_files: Arc::clone(&active_files), 436 444 hint_positions: Arc::clone(&hint_positions), 437 445 verify_persisted_blocks: config.verify_persisted_blocks, 446 446 + clock: clock.clone(), 438 447 }; 439 448 SingleShardWriter::spawn( 440 449 ctx, ··· 843 852 let _ = resume.blocking_recv(); 844 853 } 845 854 846 846 - fn maybe_checkpoint( 855 855 + fn maybe_checkpoint<C: Clock>( 847 856 index: &BlockIndex, 848 857 epoch: &EpochCounter, 849 858 config: &GroupCommitConfig, 850 850 - last_checkpoint: &mut std::time::Instant, 859 859 + clock: &C, 860 860 + last_checkpoint: &mut LogicalNanos, 851 861 writes_since_checkpoint: &mut u64, 852 862 hint_positions: &ShardHintPositions, 853 863 ) { 854 854 - let interval = std::time::Duration::from_millis(config.checkpoint_interval_ms); 855 855 - let elapsed = last_checkpoint.elapsed() >= interval; 864 864 + let interval = LogicalNanos::from_millis(config.checkpoint_interval_ms); 865 865 + let elapsed = clock.monotonic().saturating_sub(*last_checkpoint) >= interval; 856 866 let threshold = *writes_since_checkpoint >= config.checkpoint_write_threshold; 857 867 if !elapsed && !threshold { 858 868 return; 859 869 } 860 870 match index.write_checkpoint(epoch.current(), hint_positions) { 861 871 Ok(()) => { 862 862 - *last_checkpoint = std::time::Instant::now(); 872 872 + *last_checkpoint = clock.monotonic(); 863 873 *writes_since_checkpoint = 0; 864 874 tracing::debug!("periodic checkpoint written"); 865 875 } ··· 880 890 } 881 891 } 882 892 883 883 - fn commit_loop<S: StorageIO>( 893 893 + fn commit_loop<S: StorageIO, C: Clock>( 884 894 manager: &DataFileManager<S>, 885 895 index: &BlockIndex, 886 896 receiver: &flume::Receiver<CommitRequest>, 887 897 config: &GroupCommitConfig, 888 898 state: &mut ActiveState, 889 899 post_sync_hook: Option<&dyn PostBlockstoreHook>, 890 890 - ctx: &ShardContext, 900 900 + ctx: &ShardContext<C>, 891 901 ) { 892 902 let epoch = &ctx.epoch; 893 893 - let mut last_checkpoint = std::time::Instant::now(); 903 903 + let mut last_checkpoint = ctx.clock.monotonic(); 894 904 let mut writes_since_checkpoint: u64 = 0; 895 905 896 906 loop { ··· 914 924 &ctx.active_files, 915 925 &ctx.hint_positions, 916 926 epoch, 927 927 + ctx.clock.wall_millis(), 917 928 ); 918 929 let _ = response.send(result); 919 930 continue; ··· 925 936 let repaired = index.repair_leaked_refcounts( 926 937 &leaked_cids, 927 938 epoch.current(), 928 928 - crate::wall_clock_ms(), 939 939 + ctx.clock.wall_millis(), 929 940 ); 930 941 let _ = response.send(Ok(repaired)); 931 942 continue; ··· 985 996 &ctx.active_files, 986 997 &ctx.hint_positions, 987 998 epoch, 999 999 + ctx.clock.wall_millis(), 988 1000 ); 989 1001 let _ = response.send(result); 990 1002 }); ··· 996 1008 let repaired = index.repair_leaked_refcounts( 997 1009 &leaked_cids, 998 1010 epoch.current(), 999 999 - crate::wall_clock_ms(), 1011 1011 + ctx.clock.wall_millis(), 1000 1012 ); 1001 1013 let _ = response.send(Ok(repaired)); 1002 1014 }); ··· 1005 1017 index, 1006 1018 epoch, 1007 1019 config, 1020 1020 + &ctx.clock, 1008 1021 &mut last_checkpoint, 1009 1022 &mut writes_since_checkpoint, 1010 1023 &ctx.hint_positions, ··· 1032 1045 } 1033 1046 } 1034 1047 1035 1035 - fn drain_and_process_remaining<S: StorageIO>( 1048 1048 + fn drain_and_process_remaining<S: StorageIO, C: Clock>( 1036 1049 manager: &DataFileManager<S>, 1037 1050 index: &BlockIndex, 1038 1051 receiver: &flume::Receiver<CommitRequest>, 1039 1052 state: &mut ActiveState, 1040 1053 post_sync_hook: Option<&dyn PostBlockstoreHook>, 1041 1041 - ctx: &ShardContext, 1054 1054 + ctx: &ShardContext<C>, 1042 1055 ) { 1043 1056 let epoch = &ctx.epoch; 1044 1057 let mut entries: Vec<BatchEntry> = Vec::new(); ··· 1082 1095 &ctx.active_files, 1083 1096 &ctx.hint_positions, 1084 1097 epoch, 1098 1098 + ctx.clock.wall_millis(), 1085 1099 ); 1086 1100 let _ = response.send(result); 1087 1101 }); 1088 1102 1089 1103 repairs.into_iter().for_each(|(leaked_cids, response)| { 1090 1104 let repaired = 1091 1091 - index.repair_leaked_refcounts(&leaked_cids, epoch.current(), crate::wall_clock_ms()); 1105 1105 + index.repair_leaked_refcounts(&leaked_cids, epoch.current(), ctx.clock.wall_millis()); 1092 1106 let _ = response.send(Ok(repaired)); 1093 1107 }); 1094 1108 ··· 1215 1229 }); 1216 1230 } 1217 1231 1218 1218 - fn process_batch<S: StorageIO>( 1232 1232 + fn process_batch<S: StorageIO, C: Clock>( 1219 1233 manager: &DataFileManager<S>, 1220 1234 index: &BlockIndex, 1221 1235 batch: &[BatchEntry], 1222 1236 state: &mut ActiveState, 1223 1223 - ctx: &ShardContext, 1237 1237 + ctx: &ShardContext<C>, 1224 1238 ) -> Result<(HashMap<[u8; CID_SIZE], BlockLocation>, BlocksSynced), CommitError> { 1225 1239 let epoch = &ctx.epoch; 1226 1240 let batch_start = std::time::Instant::now(); ··· 1330 1344 let write_nanos = batch_start.elapsed().as_nanos() as u64; 1331 1345 1332 1346 let current_epoch = epoch.current(); 1333 1333 - let now = crate::wall_clock_ms(); 1347 1347 + let now = ctx.clock.wall_millis(); 1334 1348 1335 1349 let rollback_on_err = |e: CommitError| -> CommitError { 1336 1350 rollback_batch(manager, state, &rotations);

+27 -13

crates/tranquil-store/src/blockstore/store.rs

Reviewed

··· 11 11 use jacquard_repo::repo::CommitData; 12 12 use jacquard_repo::storage::BlockStore; 13 13 14 14 + use crate::clock::{Clock, SystemClock}; 14 15 use crate::fsync_order::PostBlockstoreHook; 15 16 use crate::io::{OpenOptions, RealIO, StorageIO}; 16 17 ··· 23 24 use super::reader::{BlockStoreReader, ReadError}; 24 25 use super::types::{ 25 26 BlockLocation, BlockOffset, CollectionResult, CompactionResult, DataFileId, EpochCounter, 26 26 - LivenessInfo, WallClockMs, 27 27 + LivenessInfo, 27 28 }; 28 29 29 30 fn cid_to_bytes(cid: &Cid) -> Result<[u8; CID_SIZE], RepoError> { ··· 111 112 } 112 113 } 113 114 114 114 - pub struct TranquilBlockStore<S: StorageIO + Send + Sync + 'static = RealIO> { 115 115 + pub struct TranquilBlockStore<S: StorageIO + Send + Sync + 'static, C: Clock> { 115 116 writer: Arc<WriterHandle>, 116 117 reader: Arc<BlockStoreReader<S>>, 117 118 index: Arc<BlockIndex>, 118 119 epoch: EpochCounter, 119 120 data_dir: PathBuf, 121 121 + clock: C, 120 122 } 121 123 122 122 - impl<S: StorageIO + Send + Sync + 'static> Clone for TranquilBlockStore<S> { 124 124 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Clone for TranquilBlockStore<S, C> { 123 125 fn clone(&self) -> Self { 124 126 Self { 125 127 writer: Arc::clone(&self.writer), ··· 127 129 index: Arc::clone(&self.index), 128 130 epoch: self.epoch.clone(), 129 131 data_dir: self.data_dir.clone(), 132 132 + clock: self.clock.clone(), 130 133 } 131 134 } 132 135 } ··· 170 173 } 171 174 } 172 175 173 173 - impl TranquilBlockStore<RealIO> { 176 176 + impl TranquilBlockStore<RealIO, SystemClock> { 174 177 pub fn open(config: BlockStoreConfig) -> Result<Self, RepoError> { 175 178 Self::open_with_hook(config, None) 176 179 } ··· 179 182 config: BlockStoreConfig, 180 183 post_sync_hook: Option<Arc<dyn PostBlockstoreHook>>, 181 184 ) -> Result<Self, RepoError> { 182 182 - Self::open_with_io_hook(config, RealIO::new, post_sync_hook) 185 185 + Self::open_with_io_hook(config, RealIO::new, post_sync_hook, SystemClock) 183 186 } 184 187 185 188 pub fn open_with_retry( ··· 227 230 } 228 231 } 229 232 230 230 - impl<S: StorageIO + Send + Sync + 'static> TranquilBlockStore<S> { 231 231 - pub fn open_with_io<F>(config: BlockStoreConfig, make_io: F) -> Result<Self, RepoError> 233 233 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> TranquilBlockStore<S, C> { 234 234 + pub fn open_with_io<F>( 235 235 + config: BlockStoreConfig, 236 236 + make_io: F, 237 237 + clock: C, 238 238 + ) -> Result<Self, RepoError> 232 239 where 233 240 F: Fn() -> S + Send + Sync + Clone + 'static, 234 241 { 235 235 - Self::open_with_io_hook(config, make_io, None) 242 242 + Self::open_with_io_hook(config, make_io, None, clock) 236 243 } 237 244 238 245 pub fn open_with_io_hook<F>( 239 246 config: BlockStoreConfig, 240 247 make_io: F, 241 248 post_sync_hook: Option<Arc<dyn PostBlockstoreHook>>, 249 249 + clock: C, 242 250 ) -> Result<Self, RepoError> 243 251 where 244 252 F: Fn() -> S + Send + Sync + Clone + 'static, ··· 295 303 checkpoint_epoch, 296 304 shard_count, 297 305 checkpoint_positions, 306 306 + clock.clone(), 298 307 ) 299 308 .map_err(commit_error_to_repo)?; 300 309 let epoch = writer.epoch().clone(); ··· 317 326 index, 318 327 epoch, 319 328 data_dir, 329 329 + clock, 320 330 }) 321 331 } 322 332 ··· 495 505 resume_txs: resumes, 496 506 }, 497 507 )) 508 508 + } 509 509 + 510 510 + pub fn clock(&self) -> &C { 511 511 + &self.clock 498 512 } 499 513 500 514 pub fn collect_dead_blocks(&self, grace_period_ms: u64) -> Result<CollectionResult, RepoError> { 501 515 let current_epoch = self.epoch.current(); 502 502 - let now = WallClockMs::now(); 516 516 + let now = self.clock.wall_millis(); 503 517 Ok(self 504 518 .index 505 519 .collect_dead_blocks(current_epoch, now, grace_period_ms)) ··· 536 550 grace_period_ms: u64, 537 551 ) -> Result<HashMap<DataFileId, LivenessInfo>, RepoError> { 538 552 let current_epoch = self.epoch.current(); 539 539 - let now = WallClockMs::now(); 553 553 + let now = self.clock.wall_millis(); 540 554 Ok(self 541 555 .index 542 556 .liveness_by_file(current_epoch, now, grace_period_ms)) ··· 700 714 } 701 715 } 702 716 703 703 - impl<S: StorageIO + Send + Sync + 'static> BlockStore for TranquilBlockStore<S> { 717 717 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> BlockStore for TranquilBlockStore<S, C> { 704 718 async fn get(&self, cid: &Cid) -> Result<Option<Bytes>, RepoError> { 705 719 let cid_bytes = cid_to_bytes(cid)?; 706 720 let reader = Arc::clone(&self.reader); ··· 772 786 } 773 787 } 774 788 775 775 - impl<S: StorageIO + Send + Sync + 'static> TranquilBlockStore<S> { 789 789 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> TranquilBlockStore<S, C> { 776 790 pub async fn decrement_refs(&self, cids: &[Cid]) -> Result<(), RepoError> { 777 791 if cids.is_empty() { 778 792 return Ok(()); ··· 920 934 921 935 let index = BlockIndex::open(&index_dir).unwrap(); 922 936 923 923 - let result = TranquilBlockStore::<EioOnReadAtRange>::replay_single_file( 937 937 + let result = TranquilBlockStore::<EioOnReadAtRange, SystemClock>::replay_single_file( 924 938 &wrapper, 925 939 &data_dir, 926 940 &index,

+124

crates/tranquil-store/src/clock.rs

Reviewed

··· 1 1 + use std::sync::OnceLock; 2 2 + use std::time::{Duration, Instant}; 3 3 + 4 4 + use crate::blockstore::WallClockMs; 5 5 + use crate::eventlog::TimestampMicros; 6 6 + 7 7 + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 8 8 + #[repr(transparent)] 9 9 + pub struct LogicalNanos(u64); 10 10 + 11 11 + impl LogicalNanos { 12 12 + pub const fn new(nanos: u64) -> Self { 13 13 + Self(nanos) 14 14 + } 15 15 + 16 16 + pub fn raw(self) -> u64 { 17 17 + self.0 18 18 + } 19 19 + 20 20 + pub fn from_millis(ms: u64) -> Self { 21 21 + Self(ms.saturating_mul(1_000_000)) 22 22 + } 23 23 + 24 24 + pub fn saturating_sub(self, other: Self) -> Self { 25 25 + Self(self.0.saturating_sub(other.0)) 26 26 + } 27 27 + } 28 28 + 29 29 + pub trait Clock: Clone + Send + Sync + 'static { 30 30 + fn unix_micros(&self) -> TimestampMicros; 31 31 + fn wall_millis(&self) -> WallClockMs; 32 32 + fn monotonic(&self) -> LogicalNanos; 33 33 + fn advance(&self, by: Duration); 34 34 + } 35 35 + 36 36 + #[derive(Debug, Clone, Copy, Default)] 37 37 + pub struct SystemClock; 38 38 + 39 39 + static PROCESS_ORIGIN: OnceLock<Instant> = OnceLock::new(); 40 40 + 41 41 + impl Clock for SystemClock { 42 42 + fn unix_micros(&self) -> TimestampMicros { 43 43 + TimestampMicros::now() 44 44 + } 45 45 + 46 46 + fn wall_millis(&self) -> WallClockMs { 47 47 + WallClockMs::now() 48 48 + } 49 49 + 50 50 + fn monotonic(&self) -> LogicalNanos { 51 51 + let origin = PROCESS_ORIGIN.get_or_init(Instant::now); 52 52 + LogicalNanos::new(u64::try_from(origin.elapsed().as_nanos()).unwrap_or(u64::MAX)) 53 53 + } 54 54 + 55 55 + fn advance(&self, _by: Duration) {} 56 56 + } 57 57 + 58 58 + #[cfg(any(test, feature = "test-harness"))] 59 59 + pub use sim_clock::SimClock; 60 60 + 61 61 + #[cfg(any(test, feature = "test-harness"))] 62 62 + mod sim_clock { 63 63 + use std::sync::Arc; 64 64 + use std::sync::atomic::{AtomicU64, Ordering}; 65 65 + use std::time::Duration; 66 66 + 67 67 + use super::{Clock, LogicalNanos}; 68 68 + use crate::blockstore::WallClockMs; 69 69 + use crate::eventlog::TimestampMicros; 70 70 + use crate::sim::splitmix64; 71 71 + 72 72 + const EPOCH_BASE_MICROS: u64 = 1_700_000_000_000_000; 73 73 + const ORIGIN_SPREAD_MICROS: u64 = 86_400 * 1_000_000; 74 74 + 75 75 + #[derive(Debug)] 76 76 + struct SimClockState { 77 77 + logical_nanos: AtomicU64, 78 78 + unix_origin_micros: u64, 79 79 + } 80 80 + 81 81 + #[derive(Debug, Clone)] 82 82 + pub struct SimClock { 83 83 + state: Arc<SimClockState>, 84 84 + } 85 85 + 86 86 + impl SimClock { 87 87 + pub fn new(seed: u64) -> Self { 88 88 + Self { 89 89 + state: Arc::new(SimClockState { 90 90 + logical_nanos: AtomicU64::new(0), 91 91 + unix_origin_micros: EPOCH_BASE_MICROS 92 92 + + (splitmix64(seed) % ORIGIN_SPREAD_MICROS), 93 93 + }), 94 94 + } 95 95 + } 96 96 + } 97 97 + 98 98 + impl Clock for SimClock { 99 99 + fn unix_micros(&self) -> TimestampMicros { 100 100 + let logical = self.state.logical_nanos.load(Ordering::Acquire); 101 101 + TimestampMicros::new( 102 102 + self.state 103 103 + .unix_origin_micros 104 104 + .saturating_add(logical / 1_000), 105 105 + ) 106 106 + } 107 107 + 108 108 + fn wall_millis(&self) -> WallClockMs { 109 109 + let logical = self.state.logical_nanos.load(Ordering::Acquire); 110 110 + WallClockMs::new( 111 111 + (self.state.unix_origin_micros / 1_000).saturating_add(logical / 1_000_000), 112 112 + ) 113 113 + } 114 114 + 115 115 + fn monotonic(&self) -> LogicalNanos { 116 116 + LogicalNanos::new(self.state.logical_nanos.load(Ordering::Acquire)) 117 117 + } 118 118 + 119 119 + fn advance(&self, by: Duration) { 120 120 + let nanos = u64::try_from(by.as_nanos()).unwrap_or(u64::MAX); 121 121 + self.state.logical_nanos.fetch_add(nanos, Ordering::AcqRel); 122 122 + } 123 123 + } 124 124 + }

+13 -9

crates/tranquil-store/src/consistency.rs

Reviewed

··· 5 5 use crate::blockstore::CID_SIZE; 6 6 use crate::blockstore::hash_index::BlockIndex; 7 7 use crate::blockstore::{DataFileId, TranquilBlockStore}; 8 8 + use crate::clock::{Clock, SystemClock}; 8 9 use crate::eventlog::{EventLog, EventSequence, SequenceContiguityResult}; 9 9 - use crate::io::StorageIO; 10 10 + use crate::io::{RealIO, StorageIO}; 10 11 use crate::metastore::Metastore; 11 12 use crate::metastore::encoding::KeyBuilder; 12 13 use crate::metastore::event_keys::metastore_cursor_key; ··· 245 246 } 246 247 247 248 pub fn verify_store_consistency<S: StorageIO + 'static>( 248 248 - blockstore: &TranquilBlockStore, 249 249 + blockstore: &TranquilBlockStore<RealIO, SystemClock>, 249 250 metastore: &Metastore, 250 251 eventlog: &EventLog<S>, 251 252 ) -> ConsistencyReport { ··· 258 259 } 259 260 260 261 pub fn verify_store_consistency_with_options<S: StorageIO + 'static>( 261 261 - blockstore: &TranquilBlockStore, 262 262 + blockstore: &TranquilBlockStore<RealIO, SystemClock>, 262 263 metastore: &Metastore, 263 264 eventlog: &EventLog<S>, 264 265 options: ConsistencyCheckOptions, ··· 567 568 } 568 569 569 570 fn check_orphan_data_files( 570 570 - blockstore: &TranquilBlockStore, 571 571 + blockstore: &TranquilBlockStore<RealIO, SystemClock>, 571 572 block_index: &BlockIndex, 572 573 report: &mut ConsistencyReport, 573 574 ) { ··· 580 581 }; 581 582 582 583 let epoch = blockstore.epoch().current(); 583 583 - let now = crate::wall_clock_ms(); 584 584 + let now = blockstore.clock().wall_millis(); 584 585 let indexed_files = block_index.liveness_by_file(epoch, now, 0); 585 586 586 587 let indexed_file_ids: HashSet<DataFileId> = indexed_files.keys().copied().collect(); ··· 600 601 } 601 602 602 603 fn check_missing_indexed_files( 603 603 - blockstore: &TranquilBlockStore, 604 604 + blockstore: &TranquilBlockStore<RealIO, SystemClock>, 604 605 block_index: &BlockIndex, 605 606 report: &mut ConsistencyReport, 606 607 ) { ··· 613 614 }; 614 615 615 616 let epoch = blockstore.epoch().current(); 616 616 - let now = crate::wall_clock_ms(); 617 617 + let now = blockstore.clock().wall_millis(); 617 618 let indexed_files = block_index.liveness_by_file(epoch, now, 0); 618 619 619 620 indexed_files ··· 622 623 .for_each(|(fid, _)| report.missing_indexed_files.push(*fid)); 623 624 } 624 625 625 625 - fn check_orphan_hint_files(blockstore: &TranquilBlockStore, report: &mut ConsistencyReport) { 626 626 + fn check_orphan_hint_files( 627 627 + blockstore: &TranquilBlockStore<RealIO, SystemClock>, 628 628 + report: &mut ConsistencyReport, 629 629 + ) { 626 630 let data_files: HashSet<DataFileId> = match blockstore.list_data_files() { 627 631 Ok(files) => files.into_iter().collect(), 628 632 Err(e) => { ··· 682 686 } 683 687 684 688 pub fn repair_known_issues( 685 685 - blockstore: &TranquilBlockStore, 689 689 + blockstore: &TranquilBlockStore<RealIO, SystemClock>, 686 690 report: &ConsistencyReport, 687 691 ) -> RepairResult { 688 692 let mut result = RepairResult::default();

+13 -4

crates/tranquil-store/src/eventlog/writer.rs

Reviewed

··· 3 3 4 4 use tracing::warn; 5 5 6 6 + use crate::clock::{Clock, SystemClock}; 6 7 use crate::io::{FileId, StorageIO}; 7 8 8 9 use super::manager::SegmentManager; ··· 12 13 }; 13 14 use super::segment_index::{DEFAULT_INDEX_INTERVAL, SegmentIndex, rebuild_from_segment}; 14 15 use super::sidecar::build_sidecar_from_segment; 15 15 - use super::types::{ 16 16 - DidHash, EventSequence, EventTypeTag, SegmentId, SegmentOffset, TimestampMicros, 17 17 - }; 16 16 + use super::types::{DidHash, EventSequence, EventTypeTag, SegmentId, SegmentOffset}; 18 17 19 18 const VALIDATE_RETRY_ATTEMPTS: u32 = 32; 20 19 ··· 241 240 event_type: EventTypeTag, 242 241 payload: Vec<u8>, 243 242 ) -> io::Result<EventSequence> { 243 243 + self.append_with_clock(did_hash, event_type, payload, &SystemClock) 244 244 + } 245 245 + 246 246 + pub fn append_with_clock<C: Clock>( 247 247 + &mut self, 248 248 + did_hash: DidHash, 249 249 + event_type: EventTypeTag, 250 250 + payload: Vec<u8>, 251 251 + clock: &C, 252 252 + ) -> io::Result<EventSequence> { 244 253 let payload_len = u32::try_from(payload.len()) 245 254 .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "payload exceeds u32::MAX"))?; 246 255 if payload_len > self.max_payload { ··· 252 261 } 253 262 254 263 let seq = self.next_seq; 255 255 - let timestamp = TimestampMicros::now(); 264 264 + let timestamp = clock.unix_micros(); 256 265 257 266 let event = ValidEvent { 258 267 seq,

+7 -6

crates/tranquil-store/src/gauntlet/chaos_walker.rs

Reviewed

··· 6 6 use super::oracle::{hex_short, try_cid_to_fixed}; 7 7 use crate::StorageIO; 8 8 use crate::blockstore::{CidBytes, TranquilBlockStore}; 9 9 + use crate::clock::Clock; 9 10 10 11 pub enum LookupResult { 11 12 Found(Cid), ··· 13 14 LostPath, 14 15 } 15 16 16 16 - pub fn walk_mst_node_cids_tolerant<S: StorageIO + Send + Sync + 'static>( 17 17 - store: &TranquilBlockStore<S>, 17 17 + pub fn walk_mst_node_cids_tolerant<S: StorageIO + Send + Sync + 'static, C: Clock>( 18 18 + store: &TranquilBlockStore<S, C>, 18 19 root: Cid, 19 20 lost: &HashSet<CidBytes>, 20 21 ) -> Result<Vec<CidBytes>, String> { ··· 44 45 Ok(result) 45 46 } 46 47 47 47 - pub fn mst_get_tolerant<S: StorageIO + Send + Sync + 'static>( 48 48 - store: &TranquilBlockStore<S>, 48 48 + pub fn mst_get_tolerant<S: StorageIO + Send + Sync + 'static, C: Clock>( 49 49 + store: &TranquilBlockStore<S, C>, 49 50 root: Cid, 50 51 target: &str, 51 52 lost: &HashSet<CidBytes>, ··· 76 77 } 77 78 } 78 79 79 79 - fn read_node<S: StorageIO + Send + Sync + 'static>( 80 80 - store: &TranquilBlockStore<S>, 80 80 + fn read_node<S: StorageIO + Send + Sync + 'static, C: Clock>( 81 81 + store: &TranquilBlockStore<S, C>, 81 82 cid_bytes: &CidBytes, 82 83 ) -> Result<NodeData, String> { 83 84 let bytes = match store.get_block_sync(cid_bytes) {

+43 -42

crates/tranquil-store/src/gauntlet/invariants.rs

Reviewed

··· 10 10 use crate::blockstore::{ 11 11 BLOCK_HEADER_SIZE, CidBytes, CompactionError, TranquilBlockStore, hash_to_cid_bytes, 12 12 }; 13 13 + use crate::clock::Clock; 13 14 use crate::eventlog::{EventSequence, SegmentId}; 14 14 - use crate::io::{RealIO, StorageIO}; 15 15 + use crate::io::StorageIO; 15 16 16 17 #[derive(Debug, Clone, Copy, PartialEq, Eq)] 17 18 pub struct InvariantSet(u32); ··· 100 101 pub segment_last_ts: Vec<(SegmentId, u64)>, 101 102 } 102 103 103 103 - pub struct InvariantCtx<'a, S: StorageIO + Send + Sync + 'static = RealIO> { 104 104 - pub store: &'a Arc<TranquilBlockStore<S>>, 104 104 + pub struct InvariantCtx<'a, S: StorageIO + Send + Sync + 'static, C: Clock> { 105 105 + pub store: &'a Arc<TranquilBlockStore<S, C>>, 105 106 pub oracle: &'a Oracle, 106 107 pub root: Option<Cid>, 107 108 pub eventlog: Option<&'a EventLogSnapshot>, 108 109 } 109 110 110 111 #[async_trait] 111 111 - pub trait Invariant<S: StorageIO + Send + Sync + 'static>: Send + Sync { 112 112 + pub trait Invariant<S: StorageIO + Send + Sync + 'static, C: Clock>: Send + Sync { 112 113 fn name(&self) -> &'static str; 113 113 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation>; 114 114 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation>; 114 115 } 115 116 116 117 pub struct RefcountConservation; 117 118 118 119 #[async_trait] 119 119 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for RefcountConservation { 120 120 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for RefcountConservation { 120 121 fn name(&self) -> &'static str { 121 122 "RefcountConservation" 122 123 } 123 124 124 124 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 125 125 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 125 126 let live: Vec<(String, CidBytes)> = ctx.oracle.live_cids_labeled(); 126 127 let live_set: HashSet<CidBytes> = live.iter().map(|(_, c)| *c).collect(); 127 128 let index: HashMap<CidBytes, u32> = ctx ··· 162 163 pub struct Reachability; 163 164 164 165 #[async_trait] 165 165 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for Reachability { 166 166 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for Reachability { 166 167 fn name(&self) -> &'static str { 167 168 "Reachability" 168 169 } 169 170 170 170 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 171 171 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 171 172 let violations: Vec<String> = ctx 172 173 .oracle 173 174 .live_cids_labeled() ··· 193 194 pub struct AckedWritePersistence; 194 195 195 196 #[async_trait] 196 196 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for AckedWritePersistence { 197 197 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for AckedWritePersistence { 197 198 fn name(&self) -> &'static str { 198 199 "AckedWritePersistence" 199 200 } 200 201 201 201 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 202 202 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 202 203 let Some(root) = ctx.root else { 203 204 if ctx.oracle.live_count() == 0 { 204 205 return Ok(()); ··· 241 242 pub struct ReadAfterWrite; 242 243 243 244 #[async_trait] 244 244 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ReadAfterWrite { 245 245 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for ReadAfterWrite { 245 246 fn name(&self) -> &'static str { 246 247 "ReadAfterWrite" 247 248 } 248 249 249 249 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 250 250 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 250 251 let Some(root) = ctx.root else { 251 252 return Ok(()); 252 253 }; ··· 295 296 pub struct CompactionIdempotent; 296 297 297 298 #[async_trait] 298 298 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for CompactionIdempotent { 299 299 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for CompactionIdempotent { 299 300 fn name(&self) -> &'static str { 300 301 "CompactionIdempotent" 301 302 } 302 303 303 303 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 304 304 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 304 305 let store_a = ctx.store.clone(); 305 306 let first = tokio::task::spawn_blocking(move || compact_by_liveness(&store_a)) 306 307 .await ··· 348 349 } 349 350 } 350 351 351 351 - fn snapshot<S: StorageIO + Send + Sync + 'static>( 352 352 - store: &Arc<TranquilBlockStore<S>>, 352 352 + fn snapshot<S: StorageIO + Send + Sync + 'static, C: Clock>( 353 353 + store: &Arc<TranquilBlockStore<S, C>>, 353 354 ) -> Vec<(CidBytes, u32)> { 354 355 let mut v: Vec<(CidBytes, u32)> = store 355 356 .block_index() ··· 363 364 364 365 const COMPACT_LIVENESS_CEILING: f64 = 0.99; 365 366 366 366 - fn compact_by_liveness<S: StorageIO + Send + Sync + 'static>( 367 367 - store: &TranquilBlockStore<S>, 367 367 + fn compact_by_liveness<S: StorageIO + Send + Sync + 'static, C: Clock>( 368 368 + store: &TranquilBlockStore<S, C>, 368 369 ) -> Result<(), String> { 369 370 let liveness = store 370 371 .compaction_liveness(0) ··· 386 387 pub struct HintBackedByData; 387 388 388 389 #[async_trait] 389 389 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for HintBackedByData { 390 390 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for HintBackedByData { 390 391 fn name(&self) -> &'static str { 391 392 "HintBackedByData" 392 393 } 393 394 394 394 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 395 395 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 395 396 let store_c = ctx.store.clone(); 396 397 let result = tokio::task::spawn_blocking(move || { 397 398 let data: std::collections::HashSet<_> = store_c ··· 435 436 pub struct IndexBlocksReadable; 436 437 437 438 #[async_trait] 438 438 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for IndexBlocksReadable { 439 439 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for IndexBlocksReadable { 439 440 fn name(&self) -> &'static str { 440 441 "IndexBlocksReadable" 441 442 } 442 443 443 443 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 444 444 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 444 445 let store_c = ctx.store.clone(); 445 446 let result = tokio::task::spawn_blocking(move || { 446 447 let entries = store_c.block_index().live_entries_snapshot(); ··· 490 491 pub struct IndexBackedByDisk; 491 492 492 493 #[async_trait] 493 493 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for IndexBackedByDisk { 494 494 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for IndexBackedByDisk { 494 495 fn name(&self) -> &'static str { 495 496 "IndexBackedByDisk" 496 497 } 497 498 498 498 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 499 499 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 499 500 let store_c = ctx.store.clone(); 500 501 let result = tokio::task::spawn_blocking(move || { 501 502 let disk: std::collections::HashSet<_> = store_c ··· 544 545 pub struct NoOrphanFiles; 545 546 546 547 #[async_trait] 547 547 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for NoOrphanFiles { 548 548 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for NoOrphanFiles { 548 549 fn name(&self) -> &'static str { 549 550 "NoOrphanFiles" 550 551 } 551 552 552 552 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 553 553 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 553 554 let store_c = ctx.store.clone(); 554 555 let result = tokio::task::spawn_blocking(move || { 555 556 let disk = store_c.list_data_files().map_err(|e| e.to_string())?; ··· 606 607 } 607 608 608 609 #[async_trait] 609 609 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ByteBudget { 610 610 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for ByteBudget { 610 611 fn name(&self) -> &'static str { 611 612 "ByteBudget" 612 613 } 613 614 614 614 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 615 615 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 615 616 let store = ctx.store.clone(); 616 617 let factor = self.overhead_factor; 617 618 let floor = self.floor_bytes; ··· 643 644 pub struct ManifestEqualsReality; 644 645 645 646 #[async_trait] 646 646 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ManifestEqualsReality { 647 647 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for ManifestEqualsReality { 647 648 fn name(&self) -> &'static str { 648 649 "ManifestEqualsReality" 649 650 } 650 651 651 651 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 652 652 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 652 653 let store = ctx.store.clone(); 653 654 tokio::task::spawn_blocking(move || { 654 655 let listed = store.list_data_files().map_err(|e| e.to_string())?; ··· 715 716 pub struct ChecksumCoverage; 716 717 717 718 #[async_trait] 718 718 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ChecksumCoverage { 719 719 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for ChecksumCoverage { 719 720 fn name(&self) -> &'static str { 720 721 "ChecksumCoverage" 721 722 } 722 723 723 723 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 724 724 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 724 725 let violations: Vec<String> = ctx 725 726 .oracle 726 727 .live_cids_labeled() ··· 761 762 pub struct MonotonicSeq; 762 763 763 764 #[async_trait] 764 764 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for MonotonicSeq { 765 765 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for MonotonicSeq { 765 766 fn name(&self) -> &'static str { 766 767 "MonotonicSeq" 767 768 } 768 769 769 769 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 770 770 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 770 771 let Some(el) = ctx.eventlog else { 771 772 return Ok(()); 772 773 }; ··· 819 820 pub struct FsyncOrdering; 820 821 821 822 #[async_trait] 822 822 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for FsyncOrdering { 823 823 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for FsyncOrdering { 823 824 fn name(&self) -> &'static str { 824 825 "FsyncOrdering" 825 826 } 826 827 827 827 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 828 828 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 828 829 let Some(el) = ctx.eventlog else { 829 830 return Ok(()); 830 831 }; ··· 873 874 pub struct TombstoneBound; 874 875 875 876 #[async_trait] 876 876 - impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for TombstoneBound { 877 877 + impl<S: StorageIO + Send + Sync + 'static, C: Clock> Invariant<S, C> for TombstoneBound { 877 878 fn name(&self) -> &'static str { 878 879 "TombstoneBound" 879 880 } 880 881 881 881 - async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> { 882 882 + async fn check(&self, ctx: &InvariantCtx<'_, S, C>) -> Result<(), InvariantViolation> { 882 883 let Some(el) = ctx.eventlog else { 883 884 return Ok(()); 884 885 }; ··· 906 907 } 907 908 } 908 909 909 909 - pub fn invariants_for<S: StorageIO + Send + Sync + 'static>( 910 910 + pub fn invariants_for<S: StorageIO + Send + Sync + 'static, C: Clock>( 910 911 set: InvariantSet, 911 911 - ) -> Vec<Box<dyn Invariant<S>>> { 912 912 + ) -> Vec<Box<dyn Invariant<S, C>>> { 912 913 let unknown = set.unknown_bits(); 913 914 assert!( 914 915 unknown == 0, 915 916 "invariants_for: unknown InvariantSet bits 0x{unknown:x}; all bits must map to an impl" 916 917 ); 917 917 - let candidates: Vec<(InvariantSet, Box<dyn Invariant<S>>)> = vec![ 918 918 + let candidates: Vec<(InvariantSet, Box<dyn Invariant<S, C>>)> = vec![ 918 919 ( 919 920 InvariantSet::REFCOUNT_CONSERVATION, 920 921 Box::new(RefcountConservation),

+5 -4

crates/tranquil-store/src/gauntlet/metrics.rs

Reviewed

··· 7 7 8 8 use super::runner::{EventLogState, Harness}; 9 9 use crate::blockstore::TranquilBlockStore; 10 10 + use crate::clock::Clock; 10 11 use crate::io::StorageIO; 11 12 12 13 #[derive(Debug, Clone, Copy, Serialize, Deserialize)] ··· 94 95 } 95 96 } 96 97 97 97 - pub fn sample_harness<S: StorageIO + Send + Sync + 'static>( 98 98 - harness: &Harness<S>, 98 98 + pub fn sample_harness<S: StorageIO + Send + Sync + 'static, C: Clock>( 99 99 + harness: &Harness<S, C>, 99 100 elapsed: Duration, 100 101 ) -> MetricsSample { 101 102 MetricsSample { ··· 116 117 } 117 118 } 118 119 119 119 - fn data_file_count<S: StorageIO + Send + Sync + 'static>( 120 120 - store: &Arc<TranquilBlockStore<S>>, 120 120 + fn data_file_count<S: StorageIO + Send + Sync + 'static, C: Clock>( 121 121 + store: &Arc<TranquilBlockStore<S, C>>, 121 122 ) -> Option<u64> { 122 123 match store.list_data_files() { 123 124 Ok(v) => Some(v.len() as u64),

+4 -4

crates/tranquil-store/src/gauntlet/mod.rs

Reviewed

··· 24 24 pub use leak::{LeakGateBuildError, LeakGateConfig, LeakViolation, evaluate as evaluate_leak_gate}; 25 25 pub use metrics::{MetricName, MetricsSample, sample_harness}; 26 26 pub use op::{ 27 27 - CollectionName, DidSeed, EventKind, FileChoice, Op, OpStream, PayloadSeed, RecordKey, 28 28 - RetentionSecs, Seed, ValueSeed, 27 27 + AdvanceNanos, CollectionName, DidSeed, EventKind, FileChoice, Op, OpStream, PayloadSeed, 28 28 + RecordKey, RetentionSecs, Seed, ValueSeed, 29 29 }; 30 30 pub use oracle::{EventExpectation, Oracle}; 31 31 pub use overrides::{ConfigOverrides, GroupCommitOverrides, StoreOverrides}; ··· 42 42 SoakEvent, SoakReport, run_soak, 43 43 }; 44 44 pub use workload::{ 45 45 - ByteRange, DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, SizeDistribution, 46 46 - ValueBytes, WorkloadModel, 45 45 + AdvanceMaxSecs, ByteRange, DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, 46 46 + SizeDistribution, ValueBytes, WorkloadModel, 47 47 };

+6

crates/tranquil-store/src/gauntlet/op.rs

Reviewed

··· 22 22 pub struct RetentionSecs(pub u32); 23 23 24 24 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] 25 25 + pub struct AdvanceNanos(pub u64); 26 26 + 27 27 + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] 25 28 pub enum EventKind { 26 29 Commit, 27 30 Identity, ··· 53 56 SyncEventLog, 54 57 RunRetention { 55 58 max_age_secs: RetentionSecs, 59 59 + }, 60 60 + AdvanceTime { 61 61 + by: AdvanceNanos, 56 62 }, 57 63 ReadRecord { 58 64 collection: CollectionName,

+250 -75

crates/tranquil-store/src/gauntlet/runner.rs

Reviewed

··· 18 18 BlockStoreConfig, CidBytes, CompactionError, GroupCommitConfig, TranquilBlockStore, 19 19 hash_to_cid, hash_to_cid_bytes, 20 20 }; 21 21 + use crate::clock::{Clock, SimClock, SystemClock}; 21 22 use crate::eventlog::{ 22 23 DEFAULT_INDEX_INTERVAL, DidHash, EventLogWriter, EventTypeTag, MAX_EVENT_PAYLOAD, SegmentId, 23 23 - SegmentManager, SegmentReader, TimestampMicros, ValidEvent, 24 24 + SegmentManager, SegmentReader, ValidEvent, 24 25 }; 25 26 use crate::io::{RealIO, StorageIO}; 26 27 use crate::sim::{FaultConfig, PristineGuard, SimulatedIO}; ··· 161 162 pub max_segment_size: u64, 162 163 } 163 164 164 164 - pub struct Harness<S: StorageIO + Send + Sync + 'static> { 165 165 - pub store: Arc<TranquilBlockStore<S>>, 165 165 + pub struct Harness<S: StorageIO + Send + Sync + 'static, C: Clock> { 166 166 + pub store: Arc<TranquilBlockStore<S, C>>, 166 167 pub eventlog: Option<EventLogState<S>>, 167 168 } 168 169 ··· 172 173 pub eventlog: Option<EventLogState<S>>, 173 174 } 174 175 175 175 - pub struct SharedState<S: StorageIO + Send + Sync + 'static> { 176 176 - pub store: Arc<TranquilBlockStore<S>>, 176 176 + pub struct SharedState<S: StorageIO + Send + Sync + 'static, C: Clock> { 177 177 + pub store: Arc<TranquilBlockStore<S, C>>, 177 178 pub write: tokio::sync::Mutex<WriteState<S>>, 178 179 } 179 180 ··· 381 382 let segments_dir = segments_subdir(&root); 382 383 let open = { 383 384 let segments_dir = segments_dir.clone(); 384 384 - move |_attempt: usize| -> Result<Harness<RealIO>, String> { 385 385 + move |_attempt: usize| -> Result<Harness<RealIO, SystemClock>, String> { 385 386 let store = TranquilBlockStore::open(cfg.clone()) 386 387 .map(Arc::new) 387 388 .map_err(|e| e.to_string())?; ··· 396 397 } 397 398 }; 398 399 if config.writer_concurrency.0 > 1 { 399 399 - run_inner_generic_concurrent::<RealIO, _, _>( 400 400 + run_inner_generic_concurrent::<RealIO, SystemClock, _, _>( 400 401 config, 401 402 ops, 402 403 ops_counter, ··· 406 407 || {}, 407 408 tolerate_op_errors, 408 409 reopen_backoff, 410 410 + SystemClock, 409 411 ) 410 412 .await 411 413 } else { 412 412 - run_inner_generic::<RealIO, _, _>( 414 414 + run_inner_generic::<RealIO, SystemClock, _, _>( 413 415 config, 414 416 ops, 415 417 ops_counter, ··· 419 421 || {}, 420 422 tolerate_op_errors, 421 423 reopen_backoff, 424 424 + SystemClock, 422 425 ) 423 426 .await 424 427 } ··· 438 441 let eventlog_cfg = config.eventlog; 439 442 let segments_dir = segments_subdir(dir.path()); 440 443 let sim: Arc<SimulatedIO> = Arc::new(SimulatedIO::new(config.seed.0, fault)); 444 444 + let clock = sim.clock(); 441 445 let sim_for_open = Arc::clone(&sim); 446 446 + let clock_for_open = clock.clone(); 442 447 let open = { 443 448 let segments_dir = segments_dir.clone(); 444 444 - move |attempt: usize| -> Result<Harness<Arc<SimulatedIO>>, String> { 449 449 + move |attempt: usize| -> Result<Harness<Arc<SimulatedIO>, SimClock>, String> { 445 450 let _pristine = PristineGuard::new(Arc::clone(&sim_for_open), attempt > 0); 446 451 let factory_sim = Arc::clone(&sim_for_open); 447 452 let make_io = move || Arc::clone(&factory_sim); 448 448 - let store = TranquilBlockStore::<Arc<SimulatedIO>>::open_with_io(cfg.clone(), make_io) 449 449 - .map(Arc::new) 450 450 - .map_err(|e| e.to_string())?; 453 453 + let store = TranquilBlockStore::<Arc<SimulatedIO>, SimClock>::open_with_io( 454 454 + cfg.clone(), 455 455 + make_io, 456 456 + clock_for_open.clone(), 457 457 + ) 458 458 + .map(Arc::new) 459 459 + .map_err(|e| e.to_string())?; 451 460 let eventlog = match eventlog_cfg { 452 461 None => None, 453 462 Some(elc) => Some( ··· 465 474 let sim_for_crash = Arc::clone(&sim); 466 475 let crash = move || sim_for_crash.crash(); 467 476 if config.writer_concurrency.0 > 1 { 468 468 - run_inner_generic_concurrent::<Arc<SimulatedIO>, _, _>( 477 477 + run_inner_generic_concurrent::<Arc<SimulatedIO>, SimClock, _, _>( 469 478 config, 470 479 ops, 471 480 ops_counter, ··· 475 484 crash, 476 485 tolerate_errors, 477 486 Duration::ZERO, 487 487 + clock, 478 488 ) 479 489 .await 480 490 } else { 481 481 - run_inner_generic::<Arc<SimulatedIO>, _, _>( 491 491 + run_inner_generic::<Arc<SimulatedIO>, SimClock, _, _>( 482 492 config, 483 493 ops, 484 494 ops_counter, ··· 488 498 crash, 489 499 tolerate_errors, 490 500 Duration::ZERO, 501 501 + clock, 491 502 ) 492 503 .await 493 504 } ··· 517 528 } 518 529 519 530 #[allow(clippy::too_many_arguments)] 520 520 - async fn run_inner_generic<S, Open, Crash>( 531 531 + async fn run_inner_generic<S, C, Open, Crash>( 521 532 config: GauntletConfig, 522 533 op_stream: OpStream, 523 534 ops_counter: Arc<AtomicUsize>, ··· 527 538 mut crash: Crash, 528 539 tolerate_op_errors: bool, 529 540 reopen_backoff: Duration, 541 541 + clock: C, 530 542 ) -> GauntletReport 531 543 where 532 544 S: StorageIO + Send + Sync + 'static, 533 533 - Open: FnMut(usize) -> Result<Harness<S>, String>, 545 545 + C: Clock, 546 546 + Open: FnMut(usize) -> Result<Harness<S, C>, String>, 534 547 Crash: FnMut(), 535 548 { 536 549 let mut oracle = Oracle::new(); 537 550 let mut violations: Vec<InvariantViolation> = Vec::new(); 538 551 539 539 - let mut harness: Option<Harness<S>> = 552 552 + let mut harness: Option<Harness<S, C>> = 540 553 match reopen_with_recovery(&mut open, &mut crash, tolerate_op_errors, reopen_backoff).await 541 554 { 542 555 Ok(h) => Some(h), ··· 569 582 .as_mut() 570 583 .expect("harness invariant: present when halt_ops is false"); 571 584 let root_before = root; 572 572 - match apply_op(live, &mut root, &mut oracle, op, &config.workload).await { 585 585 + match apply_op(live, &mut root, &mut oracle, op, &config.workload, &clock).await { 573 586 Ok(()) => {} 574 587 Err(e) => { 575 588 if tolerate_op_errors { ··· 750 763 }) 751 764 } 752 765 753 753 - fn shutdown_harness<S: StorageIO + Send + Sync + 'static>(harness: &mut Option<Harness<S>>) { 766 766 + fn shutdown_harness<S: StorageIO + Send + Sync + 'static, C: Clock>( 767 767 + harness: &mut Option<Harness<S, C>>, 768 768 + ) { 754 769 if let Some(h) = harness.as_mut() 755 770 && let Some(el) = h.eventlog.as_mut() 756 771 { ··· 762 777 763 778 const MAX_REOPEN_ATTEMPTS: usize = 5; 764 779 765 765 - async fn reopen_with_recovery<S, Open, Crash>( 780 780 + async fn reopen_with_recovery<S, C, Open, Crash>( 766 781 open: &mut Open, 767 782 crash: &mut Crash, 768 783 tolerate: bool, 769 784 backoff: Duration, 770 770 - ) -> Result<Harness<S>, String> 785 785 + ) -> Result<Harness<S, C>, String> 771 786 where 772 787 S: StorageIO + Send + Sync + 'static, 773 773 - Open: FnMut(usize) -> Result<Harness<S>, String>, 788 788 + C: Clock, 789 789 + Open: FnMut(usize) -> Result<Harness<S, C>, String>, 774 790 Crash: FnMut(), 775 791 { 776 792 let mut errors: Vec<String> = Vec::new(); ··· 808 824 .collect() 809 825 } 810 826 811 811 - async fn run_quick_check<S: StorageIO + Send + Sync + 'static>( 812 812 - store: &Arc<TranquilBlockStore<S>>, 827 827 + async fn run_quick_check<S: StorageIO + Send + Sync + 'static, C: Clock>( 828 828 + store: &Arc<TranquilBlockStore<S, C>>, 813 829 oracle: &Oracle, 814 830 root: Option<Cid>, 815 831 rng: &mut Lcg, ··· 890 906 } 891 907 } 892 908 893 893 - pub(super) async fn run_invariants<S: StorageIO + Send + Sync + 'static>( 894 894 - store: &Arc<TranquilBlockStore<S>>, 909 909 + pub(super) async fn run_invariants<S: StorageIO + Send + Sync + 'static, C: Clock>( 910 910 + store: &Arc<TranquilBlockStore<S, C>>, 895 911 oracle: &Oracle, 896 912 root: Option<Cid>, 897 913 eventlog: Option<EventLogSnapshot>, 898 914 set: InvariantSet, 899 915 ) -> Vec<InvariantViolation> { 900 900 - let ctx = InvariantCtx::<S> { 916 916 + let ctx = InvariantCtx::<S, C> { 901 917 store, 902 918 oracle, 903 919 root, 904 920 eventlog: eventlog.as_ref(), 905 921 }; 906 922 let mut out = Vec::new(); 907 907 - for inv in invariants_for::<S>(set) { 923 923 + for inv in invariants_for::<S, C>(set) { 908 924 if let Err(v) = inv.check(&ctx).await { 909 925 out.push(v); 910 926 } ··· 912 928 out 913 929 } 914 930 915 915 - fn snapshot_block_index<S: StorageIO + Send + Sync + 'static>( 916 916 - store: &TranquilBlockStore<S>, 931 931 + fn snapshot_block_index<S: StorageIO + Send + Sync + 'static, C: Clock>( 932 932 + store: &TranquilBlockStore<S, C>, 917 933 ) -> Vec<(CidBytes, u32)> { 918 934 let mut v: Vec<(CidBytes, u32)> = store 919 935 .block_index() ··· 972 988 )) 973 989 } 974 990 975 975 - pub(super) async fn refresh_oracle_graph<S: StorageIO + Send + Sync + 'static>( 976 976 - store: &Arc<TranquilBlockStore<S>>, 991 991 + pub(super) async fn refresh_oracle_graph<S: StorageIO + Send + Sync + 'static, C: Clock>( 992 992 + store: &Arc<TranquilBlockStore<S, C>>, 977 993 oracle: &mut Oracle, 978 994 root: Option<Cid>, 979 995 ) -> Result<(), String> { ··· 1095 1111 DidHash::from_did(&format!("did:plc:gauntlet{:08x}", seed.0)) 1096 1112 } 1097 1113 1098 1098 - pub(super) async fn apply_op<S: StorageIO + Send + Sync + 'static>( 1099 1099 - harness: &mut Harness<S>, 1114 1114 + pub(super) async fn apply_op<S: StorageIO + Send + Sync + 'static, C: Clock>( 1115 1115 + harness: &mut Harness<S, C>, 1100 1116 root: &mut Option<Cid>, 1101 1117 oracle: &mut Oracle, 1102 1118 op: &Op, 1103 1119 workload: &WorkloadModel, 1120 1120 + clock: &C, 1104 1121 ) -> Result<(), OpError> { 1105 1122 match op { 1106 1123 Op::AddRecord { ··· 1165 1182 let did_hash = did_hash_for_seed(*did_seed); 1166 1183 let tag = event_kind_to_tag(*event_kind); 1167 1184 let payload = event_payload_bytes(*payload_seed); 1168 1168 - let ts_before = TimestampMicros::now().raw(); 1169 1169 - match el.writer.append(did_hash, tag, payload) { 1185 1185 + let ts_before = clock.unix_micros().raw(); 1186 1186 + match el.writer.append_with_clock(did_hash, tag, payload, clock) { 1170 1187 Ok(seq) => { 1171 1188 oracle.record_event_append(EventExpectation { 1172 1189 seq, ··· 1198 1215 let Some(el) = harness.eventlog.as_mut() else { 1199 1216 return Ok(()); 1200 1217 }; 1201 1201 - run_retention(el, oracle, *max_age_secs).map_err(OpError::EventLogRetention) 1218 1218 + run_retention(el, oracle, *max_age_secs, clock).map_err(OpError::EventLogRetention) 1219 1219 + } 1220 1220 + Op::AdvanceTime { by } => { 1221 1221 + clock.advance(Duration::from_nanos(by.0)); 1222 1222 + Ok(()) 1202 1223 } 1203 1224 Op::ReadRecord { collection, rkey } => { 1204 1225 let Some(r) = *root else { return Ok(()) }; ··· 1228 1249 } 1229 1250 } 1230 1251 1231 1231 - fn externally_delete_data_file( 1232 1232 - store: &std::sync::Arc<TranquilBlockStore<impl StorageIO + 'static>>, 1252 1252 + fn externally_delete_data_file<S: StorageIO + Send + Sync + 'static, C: Clock>( 1253 1253 + store: &std::sync::Arc<TranquilBlockStore<S, C>>, 1233 1254 pick: u32, 1234 1255 ) -> Result<Vec<CidBytes>, OpError> { 1235 1256 let active = store.block_index().read_write_cursor().map(|c| c.file_id); ··· 1250 1271 } 1251 1272 } 1252 1273 1253 1253 - fn run_retention<S: StorageIO + Send + Sync + 'static>( 1274 1274 + fn run_retention<S: StorageIO + Send + Sync + 'static, C: Clock>( 1254 1275 el: &mut EventLogState<S>, 1255 1276 oracle: &mut Oracle, 1256 1277 max_age: RetentionSecs, 1278 1278 + clock: &C, 1257 1279 ) -> Result<(), String> { 1258 1280 let sync_result = el.writer.sync().map_err(|e| e.to_string())?; 1259 1281 el.manager 1260 1282 .io() 1261 1283 .sync_dir(el.segments_dir.as_path()) 1262 1284 .map_err(|e| e.to_string())?; 1263 1263 - let _ = el.writer.rotate_if_needed(); 1264 1285 oracle.record_event_sync(sync_result.synced_through); 1265 1265 - let active_id = sync_result.segment_id; 1266 1266 - let now_us = TimestampMicros::now().raw(); 1286 1286 + let now_us = clock.unix_micros().raw(); 1267 1287 let max_age_us = u64::from(max_age.0).saturating_mul(1_000_000); 1268 1288 let cutoff_us = now_us.saturating_sub(max_age_us); 1269 1289 let segments = el.manager.list_segments().map_err(|e| e.to_string())?; 1290 1290 + let active_id = segments.last().copied(); 1270 1291 segments 1271 1292 .iter() 1272 1272 - .take_while(|&&id| id != active_id) 1293 1293 + .filter(|&&id| Some(id) != active_id) 1273 1294 .try_for_each(|&id| -> Result<(), String> { 1274 1295 let last_ts = segment_last_timestamp(&el.manager, id).map_err(|e| e.to_string())?; 1275 1296 match last_ts { ··· 1293 1314 Ok(events.last().map(|e: &ValidEvent| e.timestamp.raw())) 1294 1315 } 1295 1316 1296 1296 - async fn add_record_atomic<S: StorageIO + Send + Sync + 'static>( 1297 1297 - store: &Arc<TranquilBlockStore<S>>, 1317 1317 + async fn add_record_atomic<S: StorageIO + Send + Sync + 'static, C: Clock>( 1318 1318 + store: &Arc<TranquilBlockStore<S, C>>, 1298 1319 root: Option<Cid>, 1299 1320 collection: &super::op::CollectionName, 1300 1321 rkey: &super::op::RecordKey, ··· 1330 1351 Ok((new_root, true)) 1331 1352 } 1332 1353 1333 1333 - async fn delete_record_atomic<S: StorageIO + Send + Sync + 'static>( 1334 1334 - store: &Arc<TranquilBlockStore<S>>, 1354 1354 + async fn delete_record_atomic<S: StorageIO + Send + Sync + 'static, C: Clock>( 1355 1355 + store: &Arc<TranquilBlockStore<S, C>>, 1335 1356 old_root: Cid, 1336 1357 collection: &super::op::CollectionName, 1337 1358 rkey: &super::op::RecordKey, ··· 1387 1408 .map_err(OpError::from) 1388 1409 } 1389 1410 1390 1390 - async fn commit_atomic<S: StorageIO + Send + Sync + 'static>( 1391 1391 - store: &Arc<TranquilBlockStore<S>>, 1411 1411 + async fn commit_atomic<S: StorageIO + Send + Sync + 'static, C: Clock>( 1412 1412 + store: &Arc<TranquilBlockStore<S, C>>, 1392 1413 blocks: Vec<(CidBytes, Vec<u8>)>, 1393 1414 obsolete: Vec<CidBytes>, 1394 1415 ) -> Result<(), OpError> { ··· 1404 1425 1405 1426 const COMPACT_LIVENESS_CEILING: f64 = 0.99; 1406 1427 1407 1407 - fn compact_by_liveness<S: StorageIO + Send + Sync + 'static>( 1408 1408 - store: &TranquilBlockStore<S>, 1428 1428 + fn compact_by_liveness<S: StorageIO + Send + Sync + 'static, C: Clock>( 1429 1429 + store: &TranquilBlockStore<S, C>, 1409 1430 ) -> Result<(), OpError> { 1410 1431 let liveness = store 1411 1432 .compaction_liveness(0) ··· 1425 1446 }) 1426 1447 } 1427 1448 1428 1428 - async fn apply_op_concurrent<S: StorageIO + Send + Sync + 'static>( 1429 1429 - shared: &Arc<SharedState<S>>, 1449 1449 + async fn apply_op_concurrent<S: StorageIO + Send + Sync + 'static, C: Clock>( 1450 1450 + shared: &Arc<SharedState<S, C>>, 1430 1451 op: &Op, 1431 1452 workload: &WorkloadModel, 1453 1453 + clock: &C, 1432 1454 ) -> Result<(), OpError> { 1433 1455 match op { 1434 1456 Op::AddRecord { ··· 1497 1519 let did_hash = did_hash_for_seed(*did_seed); 1498 1520 let tag = event_kind_to_tag(*event_kind); 1499 1521 let payload = event_payload_bytes(*payload_seed); 1500 1500 - let ts_before = TimestampMicros::now().raw(); 1522 1522 + let ts_before = clock.unix_micros().raw(); 1501 1523 let mut state = shared.write.lock().await; 1502 1524 let Some(el) = state.eventlog.as_mut() else { 1503 1525 return Ok(()); 1504 1526 }; 1505 1505 - match el.writer.append(did_hash, tag, payload) { 1527 1527 + match el.writer.append_with_clock(did_hash, tag, payload, clock) { 1506 1528 Ok(seq) => { 1507 1529 let _ = el.writer.rotate_if_needed(); 1508 1530 state.oracle.record_event_append(EventExpectation { ··· 1540 1562 let Some(el) = eventlog.as_mut() else { 1541 1563 return Ok(()); 1542 1564 }; 1543 1543 - run_retention(el, oracle, *max_age_secs).map_err(OpError::EventLogRetention) 1565 1565 + run_retention(el, oracle, *max_age_secs, clock).map_err(OpError::EventLogRetention) 1566 1566 + } 1567 1567 + Op::AdvanceTime { by } => { 1568 1568 + clock.advance(Duration::from_nanos(by.0)); 1569 1569 + Ok(()) 1544 1570 } 1545 1571 Op::ReadRecord { collection, rkey } => { 1546 1572 let r = { shared.write.lock().await.root }; ··· 1575 1601 } 1576 1602 1577 1603 #[allow(clippy::too_many_arguments)] 1578 1578 - async fn writer_task<S: StorageIO + Send + Sync + 'static>( 1579 1579 - shared: Arc<SharedState<S>>, 1604 1604 + async fn writer_task<S: StorageIO + Send + Sync + 'static, C: Clock>( 1605 1605 + shared: Arc<SharedState<S, C>>, 1580 1606 ops: Arc<Vec<Op>>, 1581 1607 index: Arc<AtomicUsize>, 1582 1608 end: usize, ··· 1584 1610 ops_counter: Arc<AtomicUsize>, 1585 1611 op_errors_counter: Arc<AtomicUsize>, 1586 1612 tolerate_op_errors: bool, 1613 1613 + clock: C, 1587 1614 ) -> Option<InvariantViolation> { 1588 1615 loop { 1589 1616 let idx = index.fetch_add(1, Ordering::Relaxed); ··· 1591 1618 return None; 1592 1619 } 1593 1620 let op = &ops[idx]; 1594 1594 - match apply_op_concurrent(&shared, op, &workload).await { 1621 1621 + match apply_op_concurrent(&shared, op, &workload, &clock).await { 1595 1622 Ok(()) => { 1596 1623 ops_counter.fetch_max(idx + 1, Ordering::Relaxed); 1597 1624 } ··· 1633 1660 } 1634 1661 1635 1662 #[allow(clippy::too_many_arguments)] 1636 1636 - async fn run_inner_generic_concurrent<S, Open, Crash>( 1663 1663 + async fn run_inner_generic_concurrent<S, C, Open, Crash>( 1637 1664 config: GauntletConfig, 1638 1665 op_stream: OpStream, 1639 1666 ops_counter: Arc<AtomicUsize>, ··· 1643 1670 mut crash: Crash, 1644 1671 tolerate_op_errors: bool, 1645 1672 reopen_backoff: Duration, 1673 1673 + clock: C, 1646 1674 ) -> GauntletReport 1647 1675 where 1648 1676 S: StorageIO + Send + Sync + 'static, 1649 1649 - Open: FnMut(usize) -> Result<Harness<S>, String>, 1677 1677 + C: Clock, 1678 1678 + Open: FnMut(usize) -> Result<Harness<S, C>, String>, 1650 1679 Crash: FnMut(), 1651 1680 { 1652 1681 let ops: Vec<Op> = op_stream.into_vec(); ··· 1659 1688 let mut sample_rng = Lcg::new(Seed(config.seed.0 ^ 0x5A5A_5A5A_5A5A_5A5A)); 1660 1689 let chunks = compute_chunks(config.restart_policy, total_ops, &mut restart_rng); 1661 1690 1662 1662 - let mut harness: Option<Harness<S>> = 1691 1691 + let mut harness: Option<Harness<S, C>> = 1663 1692 match reopen_with_recovery(&mut open, &mut crash, tolerate_op_errors, reopen_backoff).await 1664 1693 { 1665 1694 Ok(h) => Some(h), ··· 1711 1740 Arc::clone(&ops_counter), 1712 1741 Arc::clone(&op_errors_counter), 1713 1742 tolerate_op_errors, 1743 1743 + clock.clone(), 1714 1744 ))); 1715 1745 } 1716 1746 for h in handles.drain(..) { ··· 1916 1946 attempts: Arc<AtomicUsize>, 1917 1947 sim: Arc<SimulatedIO>, 1918 1948 store_cfg: BlockStoreConfig, 1919 1919 - ) -> impl FnMut(usize) -> Result<Harness<Arc<SimulatedIO>>, String> + Send + 'static { 1920 1920 - move |_attempt: usize| -> Result<Harness<Arc<SimulatedIO>>, String> { 1949 1949 + clock: SimClock, 1950 1950 + ) -> impl FnMut(usize) -> Result<Harness<Arc<SimulatedIO>, SimClock>, String> + Send + 'static 1951 1951 + { 1952 1952 + move |_attempt: usize| -> Result<Harness<Arc<SimulatedIO>, SimClock>, String> { 1921 1953 let n = attempts.fetch_add(1, Ordering::Relaxed); 1922 1954 if n == 0 { 1923 1955 return Err("simulated EIO on initial open".to_string()); 1924 1956 } 1925 1957 let factory_sim = Arc::clone(&sim); 1926 1958 let make_io = move || Arc::clone(&factory_sim); 1927 1927 - TranquilBlockStore::<Arc<SimulatedIO>>::open_with_io(store_cfg.clone(), make_io) 1928 1928 - .map(|s| Harness { 1929 1929 - store: Arc::new(s), 1930 1930 - eventlog: None, 1931 1931 - }) 1932 1932 - .map_err(|e| e.to_string()) 1959 1959 + TranquilBlockStore::<Arc<SimulatedIO>, SimClock>::open_with_io( 1960 1960 + store_cfg.clone(), 1961 1961 + make_io, 1962 1962 + clock.clone(), 1963 1963 + ) 1964 1964 + .map(|s| Harness { 1965 1965 + store: Arc::new(s), 1966 1966 + eventlog: None, 1967 1967 + }) 1968 1968 + .map_err(|e| e.to_string()) 1933 1969 } 1934 1970 } 1935 1971 ··· 1939 1975 let cfg = minimal_config(); 1940 1976 let store_cfg = blockstore_config(dir.path(), &cfg.store); 1941 1977 let sim: Arc<SimulatedIO> = Arc::new(SimulatedIO::pristine(0)); 1978 1978 + let clock = sim.clock(); 1942 1979 let attempts = Arc::new(AtomicUsize::new(0)); 1943 1980 1944 1944 - let report = run_inner_generic::<Arc<SimulatedIO>, _, _>( 1981 1981 + let report = run_inner_generic::<Arc<SimulatedIO>, SimClock, _, _>( 1945 1982 cfg, 1946 1983 OpStream::empty(), 1947 1984 Arc::new(AtomicUsize::new(0)), 1948 1985 Arc::new(AtomicUsize::new(0)), 1949 1986 Arc::new(AtomicUsize::new(0)), 1950 1950 - flaky_open(Arc::clone(&attempts), Arc::clone(&sim), store_cfg), 1987 1987 + flaky_open( 1988 1988 + Arc::clone(&attempts), 1989 1989 + Arc::clone(&sim), 1990 1990 + store_cfg, 1991 1991 + clock.clone(), 1992 1992 + ), 1951 1993 || {}, 1952 1994 true, 1953 1995 Duration::ZERO, 1996 1996 + clock, 1954 1997 ) 1955 1998 .await; 1956 1999 ··· 1977 2020 cfg.writer_concurrency = WriterConcurrency(2); 1978 2021 let store_cfg = blockstore_config(dir.path(), &cfg.store); 1979 2022 let sim: Arc<SimulatedIO> = Arc::new(SimulatedIO::pristine(0)); 2023 2023 + let clock = sim.clock(); 1980 2024 let attempts = Arc::new(AtomicUsize::new(0)); 1981 2025 1982 1982 - let report = run_inner_generic_concurrent::<Arc<SimulatedIO>, _, _>( 2026 2026 + let report = run_inner_generic_concurrent::<Arc<SimulatedIO>, SimClock, _, _>( 1983 2027 cfg, 1984 2028 OpStream::empty(), 1985 2029 Arc::new(AtomicUsize::new(0)), 1986 2030 Arc::new(AtomicUsize::new(0)), 1987 2031 Arc::new(AtomicUsize::new(0)), 1988 1988 - flaky_open(Arc::clone(&attempts), Arc::clone(&sim), store_cfg), 2032 2032 + flaky_open( 2033 2033 + Arc::clone(&attempts), 2034 2034 + Arc::clone(&sim), 2035 2035 + store_cfg, 2036 2036 + clock.clone(), 2037 2037 + ), 1989 2038 || {}, 1990 2039 true, 1991 2040 Duration::ZERO, 2041 2041 + clock, 1992 2042 ) 1993 2043 .await; 1994 2044 ··· 2005 2055 assert!( 2006 2056 total >= 2, 2007 2057 "expected at least one retry after first failure, attempts={total}" 2058 2058 + ); 2059 2059 + } 2060 2060 + 2061 2061 + #[tokio::test] 2062 2062 + async fn dst_retention_advances_logical_hours_in_real_milliseconds() { 2063 2063 + use crate::gauntlet::op::AdvanceNanos; 2064 2064 + use futures::stream::StreamExt; 2065 2065 + use std::time::Instant; 2066 2066 + 2067 2067 + const NANOS_PER_HOUR: u64 = 3600 * 1_000_000_000; 2068 2068 + 2069 2069 + let dir = tempfile::TempDir::new().expect("tempdir"); 2070 2070 + let store_cfg = blockstore_config(dir.path(), &minimal_config().store); 2071 2071 + let sim: Arc<SimulatedIO> = Arc::new(SimulatedIO::pristine(0xDADA)); 2072 2072 + let clock = sim.clock(); 2073 2073 + 2074 2074 + let store = { 2075 2075 + let s = Arc::clone(&sim); 2076 2076 + TranquilBlockStore::<Arc<SimulatedIO>, SimClock>::open_with_io( 2077 2077 + store_cfg, 2078 2078 + move || Arc::clone(&s), 2079 2079 + clock.clone(), 2080 2080 + ) 2081 2081 + .map(Arc::new) 2082 2082 + .expect("open store") 2083 2083 + }; 2084 2084 + let eventlog = open_eventlog(Arc::clone(&sim), segments_subdir(dir.path()), 200) 2085 2085 + .expect("open eventlog"); 2086 2086 + let harness = Harness { 2087 2087 + store, 2088 2088 + eventlog: Some(eventlog), 2089 2089 + }; 2090 2090 + 2091 2091 + let workload = WorkloadModel::default(); 2092 2092 + let appends: Vec<Op> = (0..80u32) 2093 2093 + .map(|i| Op::AppendEvent { 2094 2094 + did_seed: DidSeed(i), 2095 2095 + event_kind: EventKind::Commit, 2096 2096 + payload_seed: PayloadSeed(i), 2097 2097 + }) 2098 2098 + .collect(); 2099 2099 + 2100 2100 + let start = Instant::now(); 2101 2101 + let (mut harness, mut root, mut oracle) = futures::stream::iter(appends) 2102 2102 + .fold( 2103 2103 + (harness, None::<Cid>, Oracle::new()), 2104 2104 + |(mut h, mut r, mut o), op| { 2105 2105 + let clk = clock.clone(); 2106 2106 + let wl = workload.clone(); 2107 2107 + async move { 2108 2108 + apply_op(&mut h, &mut r, &mut o, &op, &wl, &clk) 2109 2109 + .await 2110 2110 + .expect("append op"); 2111 2111 + (h, r, o) 2112 2112 + } 2113 2113 + }, 2114 2114 + ) 2115 2115 + .await; 2116 2116 + 2117 2117 + apply_op( 2118 2118 + &mut harness, 2119 2119 + &mut root, 2120 2120 + &mut oracle, 2121 2121 + &Op::SyncEventLog, 2122 2122 + &workload, 2123 2123 + &clock, 2124 2124 + ) 2125 2125 + .await 2126 2126 + .expect("sync"); 2127 2127 + let segments_before = harness 2128 2128 + .eventlog 2129 2129 + .as_ref() 2130 2130 + .unwrap() 2131 2131 + .manager 2132 2132 + .list_segments() 2133 2133 + .expect("list before"); 2134 2134 + 2135 2135 + apply_op( 2136 2136 + &mut harness, 2137 2137 + &mut root, 2138 2138 + &mut oracle, 2139 2139 + &Op::AdvanceTime { 2140 2140 + by: AdvanceNanos(2 * NANOS_PER_HOUR), 2141 2141 + }, 2142 2142 + &workload, 2143 2143 + &clock, 2144 2144 + ) 2145 2145 + .await 2146 2146 + .expect("advance"); 2147 2147 + apply_op( 2148 2148 + &mut harness, 2149 2149 + &mut root, 2150 2150 + &mut oracle, 2151 2151 + &Op::RunRetention { 2152 2152 + max_age_secs: RetentionSecs(3600), 2153 2153 + }, 2154 2154 + &workload, 2155 2155 + &clock, 2156 2156 + ) 2157 2157 + .await 2158 2158 + .expect("retention"); 2159 2159 + let elapsed = start.elapsed(); 2160 2160 + 2161 2161 + let segments_after = harness 2162 2162 + .eventlog 2163 2163 + .as_ref() 2164 2164 + .unwrap() 2165 2165 + .manager 2166 2166 + .list_segments() 2167 2167 + .expect("list after"); 2168 2168 + 2169 2169 + assert!( 2170 2170 + segments_before.len() > 1, 2171 2171 + "expected appends to seal old segments, got {} segment(s)", 2172 2172 + segments_before.len() 2173 2173 + ); 2174 2174 + assert!( 2175 2175 + segments_after.len() < segments_before.len(), 2176 2176 + "retention after advancing 2 logical hours must delete sealed segments older than the 1h cutoff: before={} after={}", 2177 2177 + segments_before.len(), 2178 2178 + segments_after.len() 2179 2179 + ); 2180 2180 + assert!( 2181 2181 + elapsed < Duration::from_secs(10), 2182 2182 + "advancing 2 logical hours took {elapsed:?} of wall time" 2008 2183 ); 2009 2184 } 2010 2185 }

+53 -2

crates/tranquil-store/src/gauntlet/scenarios.rs

Reviewed

··· 6 6 RestartPolicy, RunLimits, ShardCount, StoreConfig, WallMs, WriterConcurrency, 7 7 }; 8 8 use super::workload::{ 9 9 - ByteRange, DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, SizeDistribution, 10 10 - ValueBytes, WorkloadModel, 9 9 + AdvanceMaxSecs, ByteRange, DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, 10 10 + SizeDistribution, ValueBytes, WorkloadModel, 11 11 }; 12 12 use crate::blockstore::{GroupCommitConfig, MAX_BLOCK_SIZE}; 13 13 use crate::sim::FaultConfig; ··· 32 32 ContendedWriters, 33 33 FlakyDevice, 34 34 ExternalCorruption, 35 35 + RetentionTimeTravel, 35 36 } 36 37 37 38 impl Scenario { ··· 55 56 Self::ContendedWriters => "ContendedWriters", 56 57 Self::FlakyDevice => "FlakyDevice", 57 58 Self::ExternalCorruption => "ExternalCorruption", 59 59 + Self::RetentionTimeTravel => "RetentionTimeTravel", 58 60 } 59 61 } 60 62 ··· 78 80 Self::ContendedWriters => "contended-writers", 79 81 Self::FlakyDevice => "flaky-device", 80 82 Self::ExternalCorruption => "external-corruption", 83 83 + Self::RetentionTimeTravel => "retention-time-travel", 81 84 } 82 85 } 83 86 ··· 115 118 Self::ExternalCorruption => { 116 119 "Rare external data-file deletion mid-workload. Validates phantom-purge self-heal under chaos." 117 120 } 121 121 + Self::RetentionTimeTravel => { 122 122 + "Eventlog retention under logical time travel: random multi-day AdvanceTime jumps interleaved with append/sync/retention. TOMBSTONE_BOUND across fake weeks." 123 123 + } 118 124 } 119 125 } 120 126 ··· 145 151 Self::ContendedWriters, 146 152 Self::FlakyDevice, 147 153 Self::ExternalCorruption, 154 154 + Self::RetentionTimeTravel, 148 155 ]; 149 156 } 150 157 ··· 219 226 Scenario::ContendedWriters => contended_writers(seed), 220 227 Scenario::FlakyDevice => flaky_device(seed), 221 228 Scenario::ExternalCorruption => external_corruption(seed), 229 229 + Scenario::RetentionTimeTravel => retention_time_travel(seed), 222 230 } 223 231 } 224 232 ··· 251 259 key_space, 252 260 did_space: DidSpaceSize(32), 253 261 retention_max_secs: RetentionMaxSecs(3600), 262 262 + advance_max_secs: AdvanceMaxSecs(7200), 254 263 } 255 264 } 256 265 ··· 666 675 key_space: KeySpaceSize(500), 667 676 did_space: DidSpaceSize(64), 668 677 retention_max_secs: RetentionMaxSecs(60), 678 678 + advance_max_secs: AdvanceMaxSecs(7200), 669 679 }, 670 680 op_count: OpCount(20_000), 671 681 invariants: sim_invariants() ··· 706 716 key_space: KeySpaceSize(400), 707 717 did_space: DidSpaceSize(32), 708 718 retention_max_secs: RetentionMaxSecs(3600), 719 719 + advance_max_secs: AdvanceMaxSecs(7200), 709 720 }, 710 721 op_count: OpCount(20_000), 711 722 invariants: sim_invariants(), ··· 773 784 key_space: KeySpaceSize(1_000), 774 785 did_space: DidSpaceSize(32), 775 786 retention_max_secs: RetentionMaxSecs(3600), 787 787 + advance_max_secs: AdvanceMaxSecs(7200), 776 788 }, 777 789 op_count: OpCount(20_000), 778 790 invariants: sim_invariants(), ··· 815 827 tolerate_op_errors: true, 816 828 } 817 829 } 830 830 + 831 831 + fn retention_time_travel(seed: Seed) -> GauntletConfig { 832 832 + GauntletConfig { 833 833 + seed, 834 834 + io: IoBackend::Simulated { 835 835 + fault: FaultConfig::none(), 836 836 + }, 837 837 + workload: WorkloadModel { 838 838 + weights: OpWeights { 839 839 + add: 20, 840 840 + compact: 3, 841 841 + checkpoint: 2, 842 842 + append_event: 30, 843 843 + sync_event_log: 15, 844 844 + run_retention: 10, 845 845 + advance_time: 20, 846 846 + ..OpWeights::default() 847 847 + }, 848 848 + size_distribution: SizeDistribution::Fixed(ValueBytes(96)), 849 849 + collections: default_collections(), 850 850 + key_space: KeySpaceSize(400), 851 851 + did_space: DidSpaceSize(64), 852 852 + retention_max_secs: RetentionMaxSecs(1_209_600), 853 853 + advance_max_secs: AdvanceMaxSecs(1_209_600), 854 854 + }, 855 855 + op_count: OpCount(1_200), 856 856 + invariants: sim_invariants() | InvariantSet::MONOTONIC_SEQ | InvariantSet::TOMBSTONE_BOUND, 857 857 + limits: RunLimits { 858 858 + max_wall_ms: Some(WallMs(5 * 60_000)), 859 859 + }, 860 860 + restart_policy: RestartPolicy::EveryNOps(OpInterval(400)), 861 861 + store: sim_store(), 862 862 + eventlog: Some(EventLogConfig { 863 863 + max_segment_size: MaxSegmentSize(8 * 1024), 864 864 + }), 865 865 + writer_concurrency: WriterConcurrency(1), 866 866 + tolerate_op_errors: false, 867 867 + } 868 868 + }

+3 -2

crates/tranquil-store/src/gauntlet/shrink.rs

Reviewed

··· 112 112 RestartPolicy, RunLimits, ShardCount, StoreConfig, WriterConcurrency, 113 113 }; 114 114 use crate::gauntlet::workload::{ 115 115 - DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, SizeDistribution, 116 116 - ValueBytes, WorkloadModel, 115 115 + AdvanceMaxSecs, DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, 116 116 + SizeDistribution, ValueBytes, WorkloadModel, 117 117 }; 118 118 use crate::sim::FaultConfig; 119 119 ··· 130 130 key_space: KeySpaceSize(4), 131 131 did_space: DidSpaceSize(1), 132 132 retention_max_secs: RetentionMaxSecs(60), 133 133 + advance_max_secs: AdvanceMaxSecs(7200), 133 134 }, 134 135 op_count: OpCount(4), 135 136 invariants: InvariantSet::EMPTY,

+16 -4

crates/tranquil-store/src/gauntlet/soak.rs

Reviewed

··· 18 18 }; 19 19 use super::workload::OpCount; 20 20 use crate::blockstore::TranquilBlockStore; 21 21 + use crate::clock::Clock; 21 22 use crate::io::{RealIO, StorageIO}; 22 23 23 24 const OP_ERROR_LOG_THROTTLE: u64 = 1024; ··· 137 138 outcome 138 139 } 139 140 140 140 - fn shutdown_harness<S: StorageIO + Send + Sync + 'static>(harness: &mut Harness<S>) { 141 141 + fn shutdown_harness<S: StorageIO + Send + Sync + 'static, C: Clock>(harness: &mut Harness<S, C>) { 141 142 if let Some(el) = harness.eventlog.as_mut() { 142 143 if let Err(e) = el.writer.shutdown() { 143 144 warn!(error = %e, "soak: event log writer shutdown failed"); ··· 146 147 } 147 148 } 148 149 149 149 - async fn drive_soak<S, W>( 150 150 - harness: &mut Harness<S>, 150 150 + async fn drive_soak<S, C, W>( 151 151 + harness: &mut Harness<S, C>, 151 152 cfg: &SoakConfig, 152 153 emitter: &mut W, 153 154 ) -> Result<SoakReport, SoakError> 154 155 where 155 156 S: StorageIO + Send + Sync + 'static, 157 157 + C: Clock, 156 158 W: Write + Send, 157 159 { 160 160 + let clock = harness.store.clock().clone(); 158 161 let mut oracle = Oracle::new(); 159 162 let mut root: Option<Cid> = None; 160 163 ··· 194 197 if start.elapsed() >= cfg.total_duration { 195 198 break; 196 199 } 197 197 - match apply_op(harness, &mut root, &mut oracle, op, &cfg.gauntlet.workload).await { 200 200 + match apply_op( 201 201 + harness, 202 202 + &mut root, 203 203 + &mut oracle, 204 204 + op, 205 205 + &cfg.gauntlet.workload, 206 206 + &clock, 207 207 + ) 208 208 + .await 209 209 + { 198 210 Ok(()) => { 199 211 ops_executed = ops_executed.saturating_add(1); 200 212 }

+20 -3

crates/tranquil-store/src/gauntlet/workload.rs

Reviewed

··· 1 1 use super::op::{ 2 2 - CollectionName, DidSeed, EventKind, FileChoice, Op, OpStream, PayloadSeed, RecordKey, 3 3 - RetentionSecs, Seed, ValueSeed, 2 2 + AdvanceNanos, CollectionName, DidSeed, EventKind, FileChoice, Op, OpStream, PayloadSeed, 3 3 + RecordKey, RetentionSecs, Seed, ValueSeed, 4 4 }; 5 5 + 6 6 + const NANOS_PER_SEC: u64 = 1_000_000_000; 5 7 6 8 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] 7 9 pub struct ValueBytes(pub u32); ··· 24 26 pub read_record: u32, 25 27 pub read_block: u32, 26 28 pub external_delete_data_file: u32, 29 29 + pub advance_time: u32, 27 30 } 28 31 29 32 impl OpWeights { ··· 38 41 + self.read_record 39 42 + self.read_block 40 43 + self.external_delete_data_file 44 44 + + self.advance_time 41 45 } 42 46 43 47 pub const fn touches_eventlog(&self) -> bool { ··· 82 86 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] 83 87 pub struct RetentionMaxSecs(pub u32); 84 88 89 89 + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] 90 90 + pub struct AdvanceMaxSecs(pub u32); 91 91 + 85 92 #[derive(Debug, Clone)] 86 93 pub struct WorkloadModel { 87 94 pub weights: OpWeights, ··· 90 97 pub key_space: KeySpaceSize, 91 98 pub did_space: DidSpaceSize, 92 99 pub retention_max_secs: RetentionMaxSecs, 100 100 + pub advance_max_secs: AdvanceMaxSecs, 93 101 } 94 102 95 103 impl Default for WorkloadModel { ··· 106 114 read_record: 0, 107 115 read_block: 0, 108 116 external_delete_data_file: 0, 117 117 + advance_time: 0, 109 118 }, 110 119 size_distribution: SizeDistribution::Fixed(ValueBytes(64)), 111 120 collections: vec![CollectionName("app.bsky.feed.post".to_string())], 112 121 key_space: KeySpaceSize(200), 113 122 did_space: DidSpaceSize(32), 114 123 retention_max_secs: RetentionMaxSecs(3600), 124 124 + advance_max_secs: AdvanceMaxSecs(7200), 115 125 } 116 126 } 117 127 } ··· 142 152 let t7 = t6 + w.run_retention; 143 153 let t8 = t7 + w.read_record; 144 154 let t9 = t8 + w.read_block; 155 155 + let t10 = t9 + w.external_delete_data_file; 145 156 146 157 match bucket { 147 158 b if b < t1 => Op::AddRecord { ··· 173 184 b if b < t9 => Op::ReadBlock { 174 185 value_seed: ValueSeed(rng.next_u32()), 175 186 }, 176 176 - _ => Op::ExternalDeleteDataFile { 187 187 + b if b < t10 => Op::ExternalDeleteDataFile { 177 188 choice: FileChoice(rng.next_u32()), 189 189 + }, 190 190 + _ => Op::AdvanceTime { 191 191 + by: AdvanceNanos( 192 192 + u64::from(rng.next_u32() % self.advance_max_secs.0.max(1)) 193 193 + * NANOS_PER_SEC, 194 194 + ), 178 195 }, 179 196 } 180 197 })

+4 -4

crates/tranquil-store/src/lib.rs

Reviewed

··· 2 2 pub mod backup; 3 3 pub mod blockstore; 4 4 pub mod bloom; 5 5 + pub mod clock; 5 6 pub mod consistency; 6 7 pub mod eventlog; 7 8 pub mod fsync_order; ··· 16 17 mod sim; 17 18 18 19 pub use blockstore::BlocksSynced; 20 20 + #[cfg(any(test, feature = "test-harness"))] 21 21 + pub use clock::SimClock; 22 22 + pub use clock::{Clock, LogicalNanos, SystemClock}; 19 23 pub use fsync_order::PostBlockstoreHook; 20 24 #[cfg(any(test, feature = "test-harness"))] 21 25 pub use harness::{ ··· 31 35 FaultConfig, LatencyNs, OpRecord, PristineGuard, Probability, SimulatedIO, SyncReorderWindow, 32 36 sim_proptest_cases, sim_seed_count, sim_seed_range, sim_single_seed, 33 37 }; 34 34 - 35 35 - pub(crate) fn wall_clock_ms() -> blockstore::WallClockMs { 36 36 - blockstore::WallClockMs::now() 37 37 - }

+4 -4

crates/tranquil-store/src/metastore/commit_ops.rs

Reviewed

··· 20 20 use super::user_blocks::user_block_user_prefix; 21 21 use super::user_hash::UserHashMap; 22 22 use crate::blockstore::TranquilBlockStore; 23 23 + use crate::clock::SystemClock; 23 24 use crate::eventlog::EventLogBridge; 24 24 - use crate::io::StorageIO; 25 25 + use crate::io::{RealIO, StorageIO}; 25 26 26 27 use tranquil_db_traits::{ 27 28 ApplyCommitError, ApplyCommitInput, ApplyCommitResult, ImportBlock, ImportRecord, ··· 81 82 user_block_ops: UserBlockOps, 82 83 backlink_ops: BacklinkOps, 83 84 event_ops: EventOps<S>, 84 84 - blockstore: Option<TranquilBlockStore>, 85 85 + blockstore: Option<TranquilBlockStore<RealIO, SystemClock>>, 85 86 } 86 87 87 88 impl<S: StorageIO + 'static> CommitOps<S> { ··· 110 111 } 111 112 } 112 113 113 113 - pub fn with_blockstore(mut self, blockstore: TranquilBlockStore) -> Self { 114 114 + pub fn with_blockstore(mut self, blockstore: TranquilBlockStore<RealIO, SystemClock>) -> Self { 114 115 self.blockstore = Some(blockstore); 115 116 self 116 117 } ··· 474 475 mod tests { 475 476 use super::*; 476 477 use crate::eventlog::{EventLog, EventLogConfig}; 477 477 - use crate::io::RealIO; 478 478 use crate::metastore::{Metastore, MetastoreConfig}; 479 479 use tranquil_db_traits::{CommitEventData, RepoEventType}; 480 480 use tranquil_types::{Handle, Nsid, Rkey};

+4 -4

crates/tranquil-store/src/metastore/handler.rs

Reviewed

··· 42 42 use super::record_ops::ListRecordsQuery; 43 43 use super::user_hash::UserHashMap; 44 44 use crate::blockstore::TranquilBlockStore; 45 45 + use crate::clock::SystemClock; 45 46 use crate::eventlog::EventLogBridge; 46 46 - use crate::io::StorageIO; 47 47 + use crate::io::{RealIO, StorageIO}; 47 48 use crate::metastore::Metastore; 48 49 49 50 type Tx<T> = oneshot::Sender<Result<T, DbError>>; ··· 5956 5957 fn handler_loop<S: StorageIO + 'static>( 5957 5958 metastore: Metastore, 5958 5959 bridge: Arc<EventLogBridge<S>>, 5959 5959 - blockstore: Option<TranquilBlockStore>, 5960 5960 + blockstore: Option<TranquilBlockStore<RealIO, SystemClock>>, 5960 5961 rx: flume::Receiver<MetastoreRequest>, 5961 5962 thread_index: usize, 5962 5963 ) { ··· 6004 6005 pub fn spawn<S: StorageIO + 'static>( 6005 6006 metastore: Metastore, 6006 6007 bridge: Arc<EventLogBridge<S>>, 6007 6007 - blockstore: Option<TranquilBlockStore>, 6008 6008 + blockstore: Option<TranquilBlockStore<RealIO, SystemClock>>, 6008 6009 thread_count: Option<usize>, 6009 6010 ) -> Self { 6010 6011 let count = thread_count ··· 6106 6107 mod tests { 6107 6108 use super::*; 6108 6109 use crate::eventlog::{EventLog, EventLogConfig}; 6109 6109 - use crate::io::RealIO; 6110 6110 use crate::metastore::MetastoreConfig; 6111 6111 use tranquil_types::{Did, Handle}; 6112 6112

+19 -8

crates/tranquil-store/src/sim.rs

Reviewed

··· 6 6 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; 7 7 use std::time::Duration; 8 8 9 9 + use crate::clock::{Clock, SimClock}; 9 10 use crate::io::{FileId, OpenOptions, StorageIO}; 10 11 11 12 pub const TORN_PAGE_BYTES: usize = 4096; 12 13 pub const SECTOR_BYTES: usize = 512; 14 14 + 15 15 + const BASE_IO_SERVICE_NS: u64 = 1_000; 13 16 14 17 #[derive(Debug, Clone, Copy, PartialEq)] 15 18 pub struct Probability(f64); ··· 331 334 pristine_mode: AtomicBool, 332 335 rng_seed: u64, 333 336 latency_counter: AtomicU64, 337 337 + clock: SimClock, 334 338 } 335 339 336 340 impl SimulatedIO { ··· 352 356 pristine_mode: AtomicBool::new(false), 353 357 rng_seed: seed, 354 358 latency_counter: AtomicU64::new(0), 359 359 + clock: SimClock::new(seed), 355 360 } 361 361 + } 362 362 + 363 363 + pub fn clock(&self) -> SimClock { 364 364 + self.clock.clone() 356 365 } 357 366 358 367 fn effective_fault_config(&self) -> FaultConfig { ··· 369 378 370 379 fn jitter(&self) { 371 380 let max_ns = self.effective_fault_config().latency_distribution_ns.0; 372 372 - if max_ns == 0 { 373 373 - return; 374 374 - } 375 375 - let c = self.latency_counter.fetch_add(1, Ordering::Relaxed); 376 376 - let r = splitmix64(self.rng_seed.wrapping_add(c)); 377 377 - let ns = r % max_ns; 378 378 - std::thread::sleep(Duration::from_nanos(ns)); 381 381 + let extra_ns = match max_ns { 382 382 + 0 => 0, 383 383 + max => { 384 384 + let c = self.latency_counter.fetch_add(1, Ordering::Relaxed); 385 385 + splitmix64(self.rng_seed.wrapping_add(c)) % max 386 386 + } 387 387 + }; 388 388 + self.clock 389 389 + .advance(Duration::from_nanos(BASE_IO_SERVICE_NS + extra_ns)); 379 390 } 380 391 381 392 pub fn pristine(seed: u64) -> Self { ··· 907 918 u32::try_from(sim_seed_count()).unwrap_or(u32::MAX) 908 919 } 909 920 910 910 - fn splitmix64(mut x: u64) -> u64 { 921 921 + pub(crate) fn splitmix64(mut x: u64) -> u64 { 911 922 x = x.wrapping_add(0x9e3779b97f4a7c15); 912 923 x = (x ^ (x >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); 913 924 x = (x ^ (x >> 27)).wrapping_mul(0x94d049bb133111eb);

+6 -3

crates/tranquil-store/tests/backup.rs

Reviewed

··· 1 1 use std::path::Path; 2 2 use std::sync::Arc; 3 3 4 4 - use tranquil_store::RealIO; 5 4 use tranquil_store::backup::{ 6 5 BackupCoordinator, BackupKind, read_manifest, recover_to_sequence, restore_from_backup, 7 6 restore_from_incremental, verify_backup, ··· 11 10 }; 12 11 use tranquil_store::eventlog::{EventLog, EventLogConfig}; 13 12 use tranquil_store::metastore::{Metastore, MetastoreConfig}; 13 13 + use tranquil_store::{RealIO, SystemClock}; 14 14 15 15 fn test_cid(seed: u16) -> CidBytes { 16 16 let mut cid = [0u8; 36]; ··· 32 32 33 33 struct TestStore { 34 34 _dir: tempfile::TempDir, 35 35 - blockstore: TranquilBlockStore, 35 35 + blockstore: TranquilBlockStore<RealIO, SystemClock>, 36 36 eventlog: Arc<EventLog<RealIO>>, 37 37 metastore: Metastore, 38 38 } ··· 163 163 f(); 164 164 } 165 165 166 166 - fn verify_blocks_readable(store: &TranquilBlockStore, range: std::ops::Range<u16>) { 166 166 + fn verify_blocks_readable( 167 167 + store: &TranquilBlockStore<RealIO, SystemClock>, 168 168 + range: std::ops::Range<u16>, 169 169 + ) { 167 170 range.for_each(|i| { 168 171 let cid = test_cid(i); 169 172 let data = store.get_block_sync(&cid).unwrap();

+2 -2

crates/tranquil-store/tests/checkpoint_race.rs

Reviewed

··· 4 4 use std::sync::Arc; 5 5 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; 6 6 7 7 - use tranquil_store::PostBlockstoreHook; 8 7 use tranquil_store::blockstore::{ 9 8 BlockStoreConfig, BlocksSynced, CidBytes, GroupCommitConfig, TranquilBlockStore, 10 9 }; 10 10 + use tranquil_store::{PostBlockstoreHook, RealIO, SystemClock}; 11 11 12 12 struct SlowHook; 13 13 ··· 18 18 } 19 19 } 20 20 21 21 - fn refcount(store: &TranquilBlockStore, cid: &CidBytes) -> Option<u32> { 21 21 + fn refcount(store: &TranquilBlockStore<RealIO, SystemClock>, cid: &CidBytes) -> Option<u32> { 22 22 store.block_index().get(cid).map(|e| e.refcount.raw()) 23 23 } 24 24

+11 -8

crates/tranquil-store/tests/common/mod.rs

Reviewed

··· 3 3 use std::collections::HashSet; 4 4 use std::sync::Arc; 5 5 6 6 - use tranquil_store::RealIO; 7 6 use tranquil_store::blockstore::{ 8 7 BlockStoreConfig, CidBytes, DEFAULT_MAX_FILE_SIZE, GroupCommitConfig, TranquilBlockStore, 9 8 }; 10 9 use tranquil_store::eventlog::{EventLog, EventLogConfig}; 11 10 use tranquil_store::metastore::{Metastore, MetastoreConfig}; 11 11 + use tranquil_store::{RealIO, SystemClock}; 12 12 use tranquil_types::{CidLink, Did, Handle}; 13 13 use uuid::Uuid; 14 14 ··· 78 78 f(); 79 79 } 80 80 81 81 - pub fn advance_epoch(store: &TranquilBlockStore) { 81 81 + pub fn advance_epoch(store: &TranquilBlockStore<RealIO, SystemClock>) { 82 82 store.apply_commit_blocking(vec![], vec![]).unwrap(); 83 83 } 84 84 85 85 - pub fn collect_all_dead(store: &TranquilBlockStore) -> HashSet<CidBytes> { 85 85 + pub fn collect_all_dead(store: &TranquilBlockStore<RealIO, SystemClock>) -> HashSet<CidBytes> { 86 86 let result = store.collect_dead_blocks(0).unwrap(); 87 87 result 88 88 .candidates ··· 91 91 .collect() 92 92 } 93 93 94 94 - pub fn compact_all_sealed(store: &TranquilBlockStore) { 94 94 + pub fn compact_all_sealed(store: &TranquilBlockStore<RealIO, SystemClock>) { 95 95 let Ok(files) = store.list_data_files() else { 96 96 return; 97 97 }; ··· 118 118 } 119 119 } 120 120 121 121 - pub fn compact_by_liveness(store: &TranquilBlockStore) { 121 121 + pub fn compact_by_liveness(store: &TranquilBlockStore<RealIO, SystemClock>) { 122 122 let liveness = store.compaction_liveness(0).unwrap(); 123 123 liveness 124 124 .iter() ··· 133 133 }); 134 134 } 135 135 136 136 - pub fn compact_lowest_liveness(store: &TranquilBlockStore) { 136 136 + pub fn compact_lowest_liveness(store: &TranquilBlockStore<RealIO, SystemClock>) { 137 137 let liveness = store.compaction_liveness(0).unwrap(); 138 138 let candidate = liveness 139 139 .iter() ··· 154 154 } 155 155 } 156 156 157 157 - pub fn collect_refcounts(store: &TranquilBlockStore, cids: &[CidBytes]) -> Vec<(u32, u32)> { 157 157 + pub fn collect_refcounts( 158 158 + store: &TranquilBlockStore<RealIO, SystemClock>, 159 159 + cids: &[CidBytes], 160 160 + ) -> Vec<(u32, u32)> { 158 161 cids.iter() 159 162 .map(|cid| { 160 163 let seed = u32::from_le_bytes([cid[4], cid[5], cid[6], cid[7]]); ··· 169 172 } 170 173 171 174 pub struct TestStores { 172 172 - pub blockstore: TranquilBlockStore, 175 175 + pub blockstore: TranquilBlockStore<RealIO, SystemClock>, 173 176 pub eventlog: Arc<EventLog<RealIO>>, 174 177 pub metastore: Metastore, 175 178 }

+7 -2

crates/tranquil-store/tests/compaction_liveness.rs

Reviewed

··· 5 5 use tranquil_store::blockstore::{ 6 6 BlockStoreConfig, CidBytes, GroupCommitConfig, TranquilBlockStore, 7 7 }; 8 8 + use tranquil_store::{RealIO, SystemClock}; 8 9 9 10 fn tiny_store_config(dir: &std::path::Path) -> BlockStoreConfig { 10 11 BlockStoreConfig { ··· 31 32 ) 32 33 } 33 34 34 34 - fn verify_live_blocks(store: &TranquilBlockStore, live: &HashSet<u32>, context: &str) { 35 35 + fn verify_live_blocks( 36 36 + store: &TranquilBlockStore<RealIO, SystemClock>, 37 37 + live: &HashSet<u32>, 38 38 + context: &str, 39 39 + ) { 35 40 let missing: Vec<u32> = live 36 41 .iter() 37 42 .copied() ··· 50 55 ); 51 56 } 52 57 53 53 - fn compact_sealed(store: &TranquilBlockStore) { 58 58 + fn compact_sealed(store: &TranquilBlockStore<RealIO, SystemClock>) { 54 59 let files = store.list_data_files().unwrap(); 55 60 files 56 61 .iter()

+3 -3

crates/tranquil-store/tests/compaction_restart.rs

Reviewed

··· 7 7 collect_refcounts, compact_by_liveness, compact_lowest_liveness, test_cid, 8 8 tiny_blockstore_config, with_runtime, 9 9 }; 10 10 - use tranquil_store::RealIO; 11 10 use tranquil_store::blockstore::{CidBytes, TranquilBlockStore}; 12 11 use tranquil_store::eventlog::{EventLog, EventLogBridge, EventLogConfig}; 13 12 use tranquil_store::metastore::handler::HandlerPool; 14 13 use tranquil_store::metastore::partitions::Partition; 15 14 use tranquil_store::metastore::{Metastore, MetastoreConfig}; 15 15 + use tranquil_store::{RealIO, SystemClock}; 16 16 17 17 struct FullStack { 18 18 - blockstore: TranquilBlockStore, 18 18 + blockstore: TranquilBlockStore<RealIO, SystemClock>, 19 19 _pool: Arc<HandlerPool>, 20 20 _event_log: Arc<EventLog<RealIO>>, 21 21 } ··· 109 109 } 110 110 111 111 fn verify_blocks_and_refcounts( 112 112 - store: &TranquilBlockStore, 112 112 + store: &TranquilBlockStore<RealIO, SystemClock>, 113 113 live_cids: &[CidBytes], 114 114 expected_refcounts: Option<&[(u32, u32)]>, 115 115 label: &str,

+5 -1

crates/tranquil-store/tests/external_corruption_recovery.rs

Reviewed

··· 6 6 use tranquil_store::blockstore::{ 7 7 CompactionResult, DataFileId, TranquilBlockStore, hint_file_path, 8 8 }; 9 9 + use tranquil_store::{RealIO, SystemClock}; 9 10 10 11 fn data_file_path(dir: &std::path::Path, file_id: DataFileId) -> std::path::PathBuf { 11 12 dir.join(format!("{file_id}.tqb")) 12 13 } 13 14 14 14 - fn populate_with_compaction_history(store: &TranquilBlockStore, live_cids: &[u32]) { 15 15 + fn populate_with_compaction_history( 16 16 + store: &TranquilBlockStore<RealIO, SystemClock>, 17 17 + live_cids: &[u32], 18 18 + ) { 15 19 live_cids.iter().for_each(|&seed| { 16 20 store 17 21 .put_blocks_blocking(vec![(test_cid(seed), block_data(seed))])

+39 -2

crates/tranquil-store/tests/gauntlet_index_backed.rs

Reviewed

··· 5 5 use common::with_runtime; 6 6 use tranquil_store::blockstore::{BlockStoreConfig, GroupCommitConfig, TranquilBlockStore}; 7 7 use tranquil_store::gauntlet::{ 8 8 - Gauntlet, IndexBackedByDisk, Invariant, InvariantCtx, InvariantSet, Oracle, Scenario, Seed, 9 9 - config_for, 8 8 + Gauntlet, IndexBackedByDisk, Invariant, InvariantCtx, InvariantSet, Op, OpStream, Oracle, 9 9 + RetentionSecs, Scenario, Seed, config_for, 10 10 }; 11 11 12 12 #[test] ··· 82 82 let report = Gauntlet::new(cfg).expect("build gauntlet").run().await; 83 83 (seed, report) 84 84 })) 85 85 + .await 86 86 + .into_iter() 87 87 + .filter(|(_, r)| !r.is_clean()) 88 88 + .map(|(seed, r)| { 89 89 + format!( 90 90 + "seed {}: {} violations\n {}", 91 91 + seed.0, 92 92 + r.violations.len(), 93 93 + r.violations 94 94 + .iter() 95 95 + .map(|v| format!("{}: {}", v.invariant, v.detail)) 96 96 + .collect::<Vec<_>>() 97 97 + .join("\n ") 98 98 + ) 99 99 + }) 100 100 + .collect(); 101 101 + assert!(failures.is_empty(), "{}", failures.join("\n---\n")); 102 102 + } 103 103 + 104 104 + #[tokio::test] 105 105 + async fn retention_time_travel_survives_many_seeds() { 106 106 + use futures::stream::StreamExt; 107 107 + 108 108 + let failures: Vec<String> = futures::stream::iter(tranquil_store::sim_seed_range().map(Seed)) 109 109 + .map(|seed| async move { 110 110 + let g = Gauntlet::new(config_for(Scenario::RetentionTimeTravel, seed)) 111 111 + .expect("build gauntlet"); 112 112 + let mut ops = g.generate_ops().into_vec(); 113 113 + ops.push(Op::SyncEventLog); 114 114 + ops.push(Op::RunRetention { 115 115 + max_age_secs: RetentionSecs(604_800), 116 116 + }); 117 117 + let report = g.run_with_ops(OpStream::from_vec(ops)).await; 118 118 + (seed, report) 119 119 + }) 120 120 + .buffer_unordered(16) 121 121 + .collect::<Vec<_>>() 85 122 .await 86 123 .into_iter() 87 124 .filter(|(_, r)| !r.is_clean())

+12 -4

crates/tranquil-store/tests/gauntlet_smoke.rs

Reviewed

··· 1 1 use tranquil_store::FaultConfig; 2 2 use tranquil_store::blockstore::GroupCommitConfig; 3 3 use tranquil_store::gauntlet::{ 4 4 - CollectionName, ConfigOverrides, DidSpaceSize, Gauntlet, GauntletConfig, GauntletReport, 5 5 - InvariantSet, IoBackend, KeySpaceSize, MaxFileSize, OpCount, OpInterval, OpWeights, 6 6 - RegressionRecord, RestartPolicy, RetentionMaxSecs, RunLimits, Scenario, Seed, ShardCount, 7 7 - SizeDistribution, StoreConfig, StoreOverrides, ValueBytes, WallMs, WorkloadModel, 4 4 + AdvanceMaxSecs, CollectionName, ConfigOverrides, DidSpaceSize, Gauntlet, GauntletConfig, 5 5 + GauntletReport, InvariantSet, IoBackend, KeySpaceSize, MaxFileSize, OpCount, OpInterval, 6 6 + OpWeights, RegressionRecord, RestartPolicy, RetentionMaxSecs, RunLimits, Scenario, Seed, 7 7 + ShardCount, SizeDistribution, StoreConfig, StoreOverrides, ValueBytes, WallMs, WorkloadModel, 8 8 WriterConcurrency, config_for, farm, 9 9 }; 10 10 ··· 60 60 key_space: KeySpaceSize(100), 61 61 did_space: DidSpaceSize(32), 62 62 retention_max_secs: RetentionMaxSecs(3600), 63 63 + advance_max_secs: AdvanceMaxSecs(7200), 63 64 }, 64 65 op_count: OpCount(200), 65 66 invariants: InvariantSet::REFCOUNT_CONSERVATION ··· 126 127 key_space: KeySpaceSize(50), 127 128 did_space: DidSpaceSize(32), 128 129 retention_max_secs: RetentionMaxSecs(3600), 130 130 + advance_max_secs: AdvanceMaxSecs(7200), 129 131 }, 130 132 op_count: OpCount(300), 131 133 invariants: InvariantSet::REFCOUNT_CONSERVATION ··· 165 167 key_space: KeySpaceSize(80), 166 168 did_space: DidSpaceSize(32), 167 169 retention_max_secs: RetentionMaxSecs(3600), 170 170 + advance_max_secs: AdvanceMaxSecs(7200), 168 171 }, 169 172 op_count: OpCount(200), 170 173 invariants: InvariantSet::REFCOUNT_CONSERVATION ··· 209 212 key_space: KeySpaceSize(80), 210 213 did_space: DidSpaceSize(32), 211 214 retention_max_secs: RetentionMaxSecs(3600), 215 215 + advance_max_secs: AdvanceMaxSecs(7200), 212 216 }, 213 217 op_count: OpCount(300), 214 218 invariants: InvariantSet::REFCOUNT_CONSERVATION ··· 260 264 key_space: KeySpaceSize(100), 261 265 did_space: DidSpaceSize(32), 262 266 retention_max_secs: RetentionMaxSecs(60), 267 267 + advance_max_secs: AdvanceMaxSecs(7200), 263 268 }, 264 269 op_count: OpCount(2_000), 265 270 invariants: InvariantSet::REFCOUNT_CONSERVATION ··· 312 317 key_space: KeySpaceSize(200), 313 318 did_space: DidSpaceSize(32), 314 319 retention_max_secs: RetentionMaxSecs(3600), 320 320 + advance_max_secs: AdvanceMaxSecs(7200), 315 321 }, 316 322 op_count: OpCount(1_000), 317 323 invariants: InvariantSet::REFCOUNT_CONSERVATION ··· 360 366 key_space: KeySpaceSize(500), 361 367 did_space: DidSpaceSize(32), 362 368 retention_max_secs: RetentionMaxSecs(3600), 369 369 + advance_max_secs: AdvanceMaxSecs(7200), 363 370 }, 364 371 op_count: OpCount(1_000), 365 372 invariants: InvariantSet::REFCOUNT_CONSERVATION ··· 491 498 key_space: KeySpaceSize(500), 492 499 did_space: DidSpaceSize(32), 493 500 retention_max_secs: RetentionMaxSecs(3600), 501 501 + advance_max_secs: AdvanceMaxSecs(7200), 494 502 }, 495 503 op_count: OpCount(2_000), 496 504 invariants: InvariantSet::REFCOUNT_CONSERVATION

+6 -5

crates/tranquil-store/tests/gc_stress.rs

Reviewed

··· 5 5 BlockStoreConfig, CidBytes, DEFAULT_MAX_FILE_SIZE, DataFileId, GroupCommitConfig, 6 6 TranquilBlockStore, 7 7 }; 8 8 + use tranquil_store::{RealIO, SystemClock}; 8 9 9 10 fn test_cid_u32(seed: u32) -> [u8; 36] { 10 11 let mut cid = [0u8; 36]; ··· 50 51 f(); 51 52 } 52 53 53 53 - fn collect_all_dead(store: &TranquilBlockStore) -> HashSet<CidBytes> { 54 54 + fn collect_all_dead(store: &TranquilBlockStore<RealIO, SystemClock>) -> HashSet<CidBytes> { 54 55 let result = store.collect_dead_blocks(0).unwrap(); 55 56 result 56 57 .candidates ··· 59 60 .collect() 60 61 } 61 62 62 62 - fn compact_all_sealed(store: &TranquilBlockStore) { 63 63 + fn compact_all_sealed(store: &TranquilBlockStore<RealIO, SystemClock>) { 63 64 let files = store.list_data_files().unwrap(); 64 65 let sealed: Vec<DataFileId> = files 65 66 .iter() ··· 72 73 }); 73 74 } 74 75 75 75 - fn verify_live_readable(store: &TranquilBlockStore, oracle: &GcOracle) { 76 76 + fn verify_live_readable(store: &TranquilBlockStore<RealIO, SystemClock>, oracle: &GcOracle) { 76 77 oracle.live_seeds().iter().for_each(|&seed| { 77 78 let cid = test_cid_u32(seed); 78 79 let data = store ··· 92 93 }); 93 94 } 94 95 95 95 - fn verify_no_live_in_dead(store: &TranquilBlockStore, oracle: &GcOracle) { 96 96 + fn verify_no_live_in_dead(store: &TranquilBlockStore<RealIO, SystemClock>, oracle: &GcOracle) { 96 97 let dead = collect_all_dead(store); 97 98 oracle.live_seeds().iter().for_each(|&seed| { 98 99 let cid = test_cid_u32(seed); ··· 150 151 } 151 152 } 152 153 153 153 - fn advance_epoch(store: &TranquilBlockStore) { 154 154 + fn advance_epoch(store: &TranquilBlockStore<RealIO, SystemClock>) { 154 155 store.apply_commit_blocking(vec![], vec![]).unwrap(); 155 156 } 156 157

+8 -4

crates/tranquil-store/tests/rotation_robustness.rs

Reviewed

··· 10 10 BlockStoreConfig, DataFileId, DataFileManager, DataFileWriter, GroupCommitConfig, 11 11 TranquilBlockStore, 12 12 }; 13 13 - use tranquil_store::{FileId, MappedFile, OpenOptions, RealIO, SimulatedIO, StorageIO}; 13 13 + use tranquil_store::{ 14 14 + FileId, MappedFile, OpenOptions, RealIO, SimulatedIO, StorageIO, SystemClock, 15 15 + }; 14 16 15 17 use common::{test_cid, with_runtime}; 16 18 ··· 149 151 shard_count: 1, 150 152 }; 151 153 152 152 - let store = TranquilBlockStore::<FailingIO>::open_with_io(config, move || { 153 153 - FailingIO::new(Arc::clone(&spec_for_factory)) 154 154 - }) 154 154 + let store = TranquilBlockStore::<FailingIO, SystemClock>::open_with_io( 155 155 + config, 156 156 + move || FailingIO::new(Arc::clone(&spec_for_factory)), 157 157 + SystemClock, 158 158 + ) 155 159 .unwrap(); 156 160 157 161 store

+13 -9

crates/tranquil-store/tests/sim_blockstore.rs

Reviewed

··· 13 13 WallClockMs, WriteCursor, hint_file_path, 14 14 }; 15 15 use tranquil_store::{ 16 16 - FaultConfig, OpenOptions, SimulatedIO, StorageIO, SyncReorderWindow, sim_seed_range, 16 16 + FaultConfig, OpenOptions, SimClock, SimulatedIO, StorageIO, SyncReorderWindow, sim_seed_range, 17 17 }; 18 18 19 19 use common::{Rng, advance_epoch, block_data, test_cid, with_runtime}; ··· 717 717 718 718 { 719 719 let s = Arc::clone(&sim); 720 720 - let store = 721 721 - TranquilBlockStore::<Arc<SimulatedIO>>::open_with_io(config.clone(), move || { 722 722 - Arc::clone(&s) 723 723 - }) 724 724 - .unwrap(); 720 720 + let store = TranquilBlockStore::<Arc<SimulatedIO>, SimClock>::open_with_io( 721 721 + config.clone(), 722 722 + move || Arc::clone(&s), 723 723 + sim.clock(), 724 724 + ) 725 725 + .unwrap(); 725 726 store 726 727 .put_blocks_blocking(vec![(cid, data.clone())]) 727 728 .unwrap(); ··· 730 731 sim.crash(); 731 732 732 733 let s = Arc::clone(&sim); 733 733 - let store = 734 734 - TranquilBlockStore::<Arc<SimulatedIO>>::open_with_io(config, move || Arc::clone(&s)) 735 735 - .unwrap(); 734 734 + let store = TranquilBlockStore::<Arc<SimulatedIO>, SimClock>::open_with_io( 735 735 + config, 736 736 + move || Arc::clone(&s), 737 737 + sim.clock(), 738 738 + ) 739 739 + .unwrap(); 736 740 737 741 match store.get_block_sync(&cid) { 738 742 Ok(Some(d)) => assert_eq!(&d[..], &data[..], "block content mismatch after crash"),

+5

frontend/package.json

Reviewed

··· 7 7 "dev": "vite", 8 8 "build": "vite build", 9 9 "preview": "vite preview", 10 10 + "check": "svelte-check --tsconfig ./tsconfig.json", 10 11 "test": "vitest", 11 12 "test:run": "vitest run", 13 13 + "test:watch": "vitest", 14 14 + "test:ui": "vitest --ui", 12 15 "test:coverage": "vitest run --coverage" 13 16 }, 14 17 "dependencies": { ··· 26 29 "@testing-library/jest-dom": "^6.9.1", 27 30 "@testing-library/svelte": "^5.3.1", 28 31 "@testing-library/user-event": "^14.6.1", 32 32 + "@vitest/coverage-v8": "4.1.4", 33 33 + "@vitest/ui": "4.1.4", 29 34 "jsdom": "^25.0.1", 30 35 "svelte": "^5.46.1", 31 36 "svelte-check": "^4.3.5",

+208 -5

frontend/pnpm-lock.yaml

Reviewed

··· 41 41 version: 6.9.1 42 42 '@testing-library/svelte': 43 43 specifier: ^5.3.1 44 44 - version: 5.3.1(svelte@5.55.3)(vite@7.3.2)(vitest@4.1.4(jsdom@25.0.1)(vite@7.3.2)) 44 44 + version: 5.3.1(svelte@5.55.3)(vite@7.3.2)(vitest@4.1.4) 45 45 '@testing-library/user-event': 46 46 specifier: ^14.6.1 47 47 version: 14.6.1(@testing-library/dom@10.4.1) 48 48 + '@vitest/coverage-v8': 49 49 + specifier: 4.1.4 50 50 + version: 4.1.4(vitest@4.1.4) 51 51 + '@vitest/ui': 52 52 + specifier: 4.1.4 53 53 + version: 4.1.4(vitest@4.1.4) 48 54 jsdom: 49 55 specifier: ^25.0.1 50 56 version: 25.0.1 ··· 62 68 version: 7.3.2 63 69 vitest: 64 70 specifier: ^4.0.16 65 65 - version: 4.1.4(jsdom@25.0.1)(vite@7.3.2) 71 71 + version: 4.1.4(@vitest/coverage-v8@4.1.4)(@vitest/ui@4.1.4)(jsdom@25.0.1)(vite@7.3.2) 66 72 67 73 packages: 68 74 ··· 104 110 105 111 '@babel/code-frame@7.29.0': 106 112 resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==} 113 113 + engines: {node: '>=6.9.0'} 114 114 + 115 115 + '@babel/helper-string-parser@7.29.7': 116 116 + resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} 107 117 engines: {node: '>=6.9.0'} 108 118 109 119 '@babel/helper-validator-identifier@7.28.5': 110 120 resolution: {integrity: sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==} 111 121 engines: {node: '>=6.9.0'} 112 122 123 123 + '@babel/helper-validator-identifier@7.29.7': 124 124 + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} 125 125 + engines: {node: '>=6.9.0'} 126 126 + 127 127 + '@babel/parser@7.29.7': 128 128 + resolution: {integrity: sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==} 129 129 + engines: {node: '>=6.0.0'} 130 130 + hasBin: true 131 131 + 113 132 '@babel/runtime@7.29.2': 114 133 resolution: {integrity: sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==} 134 134 + engines: {node: '>=6.9.0'} 135 135 + 136 136 + '@babel/types@7.29.7': 137 137 + resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==} 115 138 engines: {node: '>=6.9.0'} 116 139 117 140 '@badrap/valita@0.4.6': 118 141 resolution: {integrity: sha512-4kdqcjyxo/8RQ8ayjms47HCWZIF5981oE5nIenbfThKDxWXtEHKipAOWlflpPJzZx9y/JWYQkp18Awr7VuepFg==} 119 142 engines: {node: '>= 18'} 120 143 144 144 + '@bcoe/v8-coverage@1.0.2': 145 145 + resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} 146 146 + engines: {node: '>=18'} 147 147 + 121 148 '@csstools/color-helpers@5.1.0': 122 149 resolution: {integrity: sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==} 123 150 engines: {node: '>=18'} ··· 474 501 '@noble/secp256k1@3.1.0': 475 502 resolution: {integrity: sha512-+F7iS7tUMaNGXcc9X3PjmjvuQnXEuSjCRNzVVA2xAcKXgCaP0dHYz4SFyt4FKNHef7sOP//xihowcySSS7PK9g==} 476 503 504 504 + '@polka/url@1.0.0-next.29': 505 505 + resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==} 506 506 + 477 507 '@rollup/rollup-android-arm-eabi@4.60.1': 478 508 resolution: {integrity: sha512-d6FinEBLdIiK+1uACUttJKfgZREXrF0Qc2SmLII7W2AD8FfiZ9Wjd+rD/iRuf5s5dWrr1GgwXCvPqOuDquOowA==} 479 509 cpu: [arm] ··· 683 713 '@types/trusted-types@2.0.7': 684 714 resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} 685 715 716 716 + '@vitest/coverage-v8@4.1.4': 717 717 + resolution: {integrity: sha512-x7FptB5oDruxNPDNY2+S8tCh0pcq7ymCe1gTHcsp733jYjrJl8V1gMUlVysuCD9Kz46Xz9t1akkv08dPcYDs1w==} 718 718 + peerDependencies: 719 719 + '@vitest/browser': 4.1.4 720 720 + vitest: 4.1.4 721 721 + peerDependenciesMeta: 722 722 + '@vitest/browser': 723 723 + optional: true 724 724 + 686 725 '@vitest/expect@4.1.4': 687 726 resolution: {integrity: sha512-iPBpra+VDuXmBFI3FMKHSFXp3Gx5HfmSCE8X67Dn+bwephCnQCaB7qWK2ldHa+8ncN8hJU8VTMcxjPpyMkUjww==} 688 727 ··· 708 747 709 748 '@vitest/spy@4.1.4': 710 749 resolution: {integrity: sha512-XxNdAsKW7C+FLydqFJLb5KhJtl3PGCMmYwFRfhvIgxJvLSXhhVI1zM8f1qD3Zg7RCjTSzDVyct6sghs9UEgBEQ==} 750 750 + 751 751 + '@vitest/ui@4.1.4': 752 752 + resolution: {integrity: sha512-EgFR7nlj5iTDYZYCvavjFokNYwr3c3ry0sFiCg+N7B233Nwp+NNx7eoF/XvMWDCKY71xXAG3kFkt97ZHBJVL8A==} 753 753 + peerDependencies: 754 754 + vitest: 4.1.4 711 755 712 756 '@vitest/utils@4.1.4': 713 757 resolution: {integrity: sha512-13QMT+eysM5uVGa1rG4kegGYNp6cnQcsTc67ELFbhNLQO+vgsygtYJx2khvdt4gVQqSSpC/KT5FZZxUpP3Oatw==} ··· 743 787 assertion-error@2.0.1: 744 788 resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} 745 789 engines: {node: '>=12'} 790 790 + 791 791 + ast-v8-to-istanbul@1.0.2: 792 792 + resolution: {integrity: sha512-dKmJxJsGItLmc5CYZKuEjuG6GnBs6PG4gohMhyFOWKaNQoYCuRZJDECaBlHmcG0lv2wc2E0uU8lESmBEumC3DQ==} 746 793 747 794 asynckit@0.4.0: 748 795 resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} ··· 917 964 picomatch: 918 965 optional: true 919 966 967 967 + fflate@0.8.3: 968 968 + resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} 969 969 + 970 970 + flatted@3.4.2: 971 971 + resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==} 972 972 + 920 973 form-data@4.0.5: 921 974 resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} 922 975 engines: {node: '>= 6'} ··· 946 999 gopd@1.2.0: 947 1000 resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} 948 1001 engines: {node: '>= 0.4'} 1002 1002 + 1003 1003 + has-flag@4.0.0: 1004 1004 + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} 1005 1005 + engines: {node: '>=8'} 949 1006 950 1007 has-symbols@1.1.0: 951 1008 resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} ··· 963 1020 resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==} 964 1021 engines: {node: '>=18'} 965 1022 1023 1023 + html-escaper@2.0.2: 1024 1024 + resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} 1025 1025 + 966 1026 http-proxy-agent@7.0.2: 967 1027 resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} 968 1028 engines: {node: '>= 14'} ··· 991 1051 is-reference@3.0.3: 992 1052 resolution: {integrity: sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==} 993 1053 1054 1054 + istanbul-lib-coverage@3.2.2: 1055 1055 + resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==} 1056 1056 + engines: {node: '>=8'} 1057 1057 + 1058 1058 + istanbul-lib-report@3.0.1: 1059 1059 + resolution: {integrity: sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==} 1060 1060 + engines: {node: '>=10'} 1061 1061 + 1062 1062 + istanbul-reports@3.2.0: 1063 1063 + resolution: {integrity: sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==} 1064 1064 + engines: {node: '>=8'} 1065 1065 + 1066 1066 + js-tokens@10.0.0: 1067 1067 + resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==} 1068 1068 + 994 1069 js-tokens@4.0.0: 995 1070 resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} 996 1071 ··· 1019 1094 magic-string@0.30.21: 1020 1095 resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} 1021 1096 1097 1097 + magicast@0.5.3: 1098 1098 + resolution: {integrity: sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==} 1099 1099 + 1100 1100 + make-dir@4.0.0: 1101 1101 + resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} 1102 1102 + engines: {node: '>=10'} 1103 1103 + 1022 1104 math-intrinsics@1.1.0: 1023 1105 resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} 1024 1106 engines: {node: '>= 0.4'} ··· 1042 1124 mri@1.2.0: 1043 1125 resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} 1044 1126 engines: {node: '>=4'} 1127 1127 + 1128 1128 + mrmime@2.0.1: 1129 1129 + resolution: {integrity: sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==} 1130 1130 + engines: {node: '>=10'} 1045 1131 1046 1132 ms@2.1.3: 1047 1133 resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} ··· 1121 1207 resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} 1122 1208 engines: {node: '>=v12.22.7'} 1123 1209 1210 1210 + semver@7.8.1: 1211 1211 + resolution: {integrity: sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==} 1212 1212 + engines: {node: '>=10'} 1213 1213 + hasBin: true 1214 1214 + 1124 1215 siginfo@2.0.0: 1125 1216 resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} 1217 1217 + 1218 1218 + sirv@3.0.2: 1219 1219 + resolution: {integrity: sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==} 1220 1220 + engines: {node: '>=18'} 1126 1221 1127 1222 source-map-js@1.2.1: 1128 1223 resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} ··· 1136 1231 1137 1232 strip-indent@3.0.0: 1138 1233 resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==} 1234 1234 + engines: {node: '>=8'} 1235 1235 + 1236 1236 + supports-color@7.2.0: 1237 1237 + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} 1139 1238 engines: {node: '>=8'} 1140 1239 1141 1240 svelte-check@4.4.6: ··· 1188 1287 tldts@6.1.86: 1189 1288 resolution: {integrity: sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==} 1190 1289 hasBin: true 1290 1290 + 1291 1291 + totalist@3.0.1: 1292 1292 + resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} 1293 1293 + engines: {node: '>=6'} 1191 1294 1192 1295 tough-cookie@5.1.2: 1193 1296 resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==} ··· 1311 1414 whatwg-encoding@3.1.1: 1312 1415 resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} 1313 1416 engines: {node: '>=18'} 1417 1417 + deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation 1314 1418 1315 1419 whatwg-mimetype@4.0.0: 1316 1420 resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} ··· 1423 1527 js-tokens: 4.0.0 1424 1528 picocolors: 1.1.1 1425 1529 1530 1530 + '@babel/helper-string-parser@7.29.7': {} 1531 1531 + 1426 1532 '@babel/helper-validator-identifier@7.28.5': {} 1427 1533 1534 1534 + '@babel/helper-validator-identifier@7.29.7': {} 1535 1535 + 1536 1536 + '@babel/parser@7.29.7': 1537 1537 + dependencies: 1538 1538 + '@babel/types': 7.29.7 1539 1539 + 1428 1540 '@babel/runtime@7.29.2': {} 1541 1541 + 1542 1542 + '@babel/types@7.29.7': 1543 1543 + dependencies: 1544 1544 + '@babel/helper-string-parser': 7.29.7 1545 1545 + '@babel/helper-validator-identifier': 7.29.7 1429 1546 1430 1547 '@badrap/valita@0.4.6': {} 1431 1548 1549 1549 + '@bcoe/v8-coverage@1.0.2': {} 1550 1550 + 1432 1551 '@csstools/color-helpers@5.1.0': {} 1433 1552 1434 1553 '@csstools/css-calc@2.1.4(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4)': ··· 1643 1762 1644 1763 '@noble/secp256k1@3.1.0': {} 1645 1764 1765 1765 + '@polka/url@1.0.0-next.29': {} 1766 1766 + 1646 1767 '@rollup/rollup-android-arm-eabi@4.60.1': 1647 1768 optional: true 1648 1769 ··· 1765 1886 dependencies: 1766 1887 svelte: 5.55.3 1767 1888 1768 1768 - '@testing-library/svelte@5.3.1(svelte@5.55.3)(vite@7.3.2)(vitest@4.1.4(jsdom@25.0.1)(vite@7.3.2))': 1889 1889 + '@testing-library/svelte@5.3.1(svelte@5.55.3)(vite@7.3.2)(vitest@4.1.4)': 1769 1890 dependencies: 1770 1891 '@testing-library/dom': 10.4.1 1771 1892 '@testing-library/svelte-core': 1.0.0(svelte@5.55.3) 1772 1893 svelte: 5.55.3 1773 1894 optionalDependencies: 1774 1895 vite: 7.3.2 1775 1775 - vitest: 4.1.4(jsdom@25.0.1)(vite@7.3.2) 1896 1896 + vitest: 4.1.4(@vitest/coverage-v8@4.1.4)(@vitest/ui@4.1.4)(jsdom@25.0.1)(vite@7.3.2) 1776 1897 1777 1898 '@testing-library/user-event@14.6.1(@testing-library/dom@10.4.1)': 1778 1899 dependencies: ··· 1791 1912 1792 1913 '@types/trusted-types@2.0.7': {} 1793 1914 1915 1915 + '@vitest/coverage-v8@4.1.4(vitest@4.1.4)': 1916 1916 + dependencies: 1917 1917 + '@bcoe/v8-coverage': 1.0.2 1918 1918 + '@vitest/utils': 4.1.4 1919 1919 + ast-v8-to-istanbul: 1.0.2 1920 1920 + istanbul-lib-coverage: 3.2.2 1921 1921 + istanbul-lib-report: 3.0.1 1922 1922 + istanbul-reports: 3.2.0 1923 1923 + magicast: 0.5.3 1924 1924 + obug: 2.1.1 1925 1925 + std-env: 4.0.0 1926 1926 + tinyrainbow: 3.1.0 1927 1927 + vitest: 4.1.4(@vitest/coverage-v8@4.1.4)(@vitest/ui@4.1.4)(jsdom@25.0.1)(vite@7.3.2) 1928 1928 + 1794 1929 '@vitest/expect@4.1.4': 1795 1930 dependencies: 1796 1931 '@standard-schema/spec': 1.1.0 ··· 1826 1961 1827 1962 '@vitest/spy@4.1.4': {} 1828 1963 1964 1964 + '@vitest/ui@4.1.4(vitest@4.1.4)': 1965 1965 + dependencies: 1966 1966 + '@vitest/utils': 4.1.4 1967 1967 + fflate: 0.8.3 1968 1968 + flatted: 3.4.2 1969 1969 + pathe: 2.0.3 1970 1970 + sirv: 3.0.2 1971 1971 + tinyglobby: 0.2.16 1972 1972 + tinyrainbow: 3.1.0 1973 1973 + vitest: 4.1.4(@vitest/coverage-v8@4.1.4)(@vitest/ui@4.1.4)(jsdom@25.0.1)(vite@7.3.2) 1974 1974 + 1829 1975 '@vitest/utils@4.1.4': 1830 1976 dependencies: 1831 1977 '@vitest/pretty-format': 4.1.4 ··· 1849 1995 aria-query@5.3.2: {} 1850 1996 1851 1997 assertion-error@2.0.1: {} 1998 1998 + 1999 1999 + ast-v8-to-istanbul@1.0.2: 2000 2000 + dependencies: 2001 2001 + '@jridgewell/trace-mapping': 0.3.31 2002 2002 + estree-walker: 3.0.3 2003 2003 + js-tokens: 10.0.0 1852 2004 1853 2005 asynckit@0.4.0: {} 1854 2006 ··· 2055 2207 optionalDependencies: 2056 2208 picomatch: 4.0.4 2057 2209 2210 2210 + fflate@0.8.3: {} 2211 2211 + 2212 2212 + flatted@3.4.2: {} 2213 2213 + 2058 2214 form-data@4.0.5: 2059 2215 dependencies: 2060 2216 asynckit: 0.4.0 ··· 2092 2248 2093 2249 gopd@1.2.0: {} 2094 2250 2251 2251 + has-flag@4.0.0: {} 2252 2252 + 2095 2253 has-symbols@1.1.0: {} 2096 2254 2097 2255 has-tostringtag@1.0.2: ··· 2106 2264 dependencies: 2107 2265 whatwg-encoding: 3.1.1 2108 2266 2267 2267 + html-escaper@2.0.2: {} 2268 2268 + 2109 2269 http-proxy-agent@7.0.2: 2110 2270 dependencies: 2111 2271 agent-base: 7.1.4 ··· 2141 2301 dependencies: 2142 2302 '@types/estree': 1.0.8 2143 2303 2304 2304 + istanbul-lib-coverage@3.2.2: {} 2305 2305 + 2306 2306 + istanbul-lib-report@3.0.1: 2307 2307 + dependencies: 2308 2308 + istanbul-lib-coverage: 3.2.2 2309 2309 + make-dir: 4.0.0 2310 2310 + supports-color: 7.2.0 2311 2311 + 2312 2312 + istanbul-reports@3.2.0: 2313 2313 + dependencies: 2314 2314 + html-escaper: 2.0.2 2315 2315 + istanbul-lib-report: 3.0.1 2316 2316 + 2317 2317 + js-tokens@10.0.0: {} 2318 2318 + 2144 2319 js-tokens@4.0.0: {} 2145 2320 2146 2321 jsdom@25.0.1: ··· 2185 2360 dependencies: 2186 2361 '@jridgewell/sourcemap-codec': 1.5.5 2187 2362 2363 2363 + magicast@0.5.3: 2364 2364 + dependencies: 2365 2365 + '@babel/parser': 7.29.7 2366 2366 + '@babel/types': 7.29.7 2367 2367 + source-map-js: 1.2.1 2368 2368 + 2369 2369 + make-dir@4.0.0: 2370 2370 + dependencies: 2371 2371 + semver: 7.8.1 2372 2372 + 2188 2373 math-intrinsics@1.1.0: {} 2189 2374 2190 2375 memoizee@0.4.17: ··· 2207 2392 min-indent@1.0.1: {} 2208 2393 2209 2394 mri@1.2.0: {} 2395 2395 + 2396 2396 + mrmime@2.0.1: {} 2210 2397 2211 2398 ms@2.1.3: {} 2212 2399 ··· 2298 2485 dependencies: 2299 2486 xmlchars: 2.2.0 2300 2487 2488 2488 + semver@7.8.1: {} 2489 2489 + 2301 2490 siginfo@2.0.0: {} 2302 2491 2492 2492 + sirv@3.0.2: 2493 2493 + dependencies: 2494 2494 + '@polka/url': 1.0.0-next.29 2495 2495 + mrmime: 2.0.1 2496 2496 + totalist: 3.0.1 2497 2497 + 2303 2498 source-map-js@1.2.1: {} 2304 2499 2305 2500 stackback@0.0.2: {} ··· 2309 2504 strip-indent@3.0.0: 2310 2505 dependencies: 2311 2506 min-indent: 1.0.1 2507 2507 + 2508 2508 + supports-color@7.2.0: 2509 2509 + dependencies: 2510 2510 + has-flag: 4.0.0 2312 2511 2313 2512 svelte-check@4.4.6(picomatch@4.0.4)(svelte@5.55.3)(typescript@5.9.3): 2314 2513 dependencies: ··· 2383 2582 dependencies: 2384 2583 tldts-core: 6.1.86 2385 2584 2585 2585 + totalist@3.0.1: {} 2586 2586 + 2386 2587 tough-cookie@5.1.2: 2387 2588 dependencies: 2388 2589 tldts: 6.1.86 ··· 2414 2615 optionalDependencies: 2415 2616 vite: 7.3.2 2416 2617 2417 2417 - vitest@4.1.4(jsdom@25.0.1)(vite@7.3.2): 2618 2618 + vitest@4.1.4(@vitest/coverage-v8@4.1.4)(@vitest/ui@4.1.4)(jsdom@25.0.1)(vite@7.3.2): 2418 2619 dependencies: 2419 2620 '@vitest/expect': 4.1.4 2420 2621 '@vitest/mocker': 4.1.4(vite@7.3.2) ··· 2437 2638 vite: 7.3.2 2438 2639 why-is-node-running: 2.3.0 2439 2640 optionalDependencies: 2641 2641 + '@vitest/coverage-v8': 4.1.4(vitest@4.1.4) 2642 2642 + '@vitest/ui': 4.1.4(vitest@4.1.4) 2440 2643 jsdom: 25.0.1 2441 2644 transitivePeerDependencies: 2442 2645 - msw

+7 -7

justfile

Reviewed

··· 164 164 podman pull atcr.io/tranquil.farm/tranquil-pds:latest 165 165 166 166 frontend-dev: 167 167 - . ~/.deno/env && cd frontend && deno task dev 167 167 + cd frontend && pnpm run dev 168 168 frontend-build: 169 169 - . ~/.deno/env && cd frontend && deno task build 169 169 + cd frontend && pnpm run build 170 170 frontend-check: 171 171 - . ~/.deno/env && cd frontend && deno task check 171 171 + cd frontend && pnpm run check 172 172 frontend-clean: 173 173 rm -rf frontend/dist frontend/node_modules 174 174 175 175 frontend-test *args: 176 176 - . ~/.deno/env && cd frontend && VITEST=true deno task test:run {{args}} 176 176 + cd frontend && VITEST=true pnpm run test:run {{args}} 177 177 frontend-test-watch: 178 178 - . ~/.deno/env && cd frontend && VITEST=true deno task test:watch 178 178 + cd frontend && VITEST=true pnpm run test:watch 179 179 frontend-test-ui: 180 180 - . ~/.deno/env && cd frontend && VITEST=true deno task test:ui 180 180 + cd frontend && VITEST=true pnpm run test:ui 181 181 frontend-test-coverage: 182 182 - . ~/.deno/env && cd frontend && VITEST=true deno task test:run --coverage 182 182 + cd frontend && VITEST=true pnpm run test:run --coverage 183 183 184 184 build-all: frontend-build build