diff --git a/lib/services/stale_history_sweep.dart b/lib/services/stale_history_sweep.dart index 629f4154..f4652cc4 100644 --- a/lib/services/stale_history_sweep.dart +++ b/lib/services/stale_history_sweep.dart @@ -103,9 +103,19 @@ class StaleHistorySweep { if (migratedAt == null) continue; // unstamped: nothing to age against if (at.difference(migratedAt) < window) continue; - if (await _blobs.read(key) == null) continue; // already reclaimed + // Verify and delete under the key's own lock, as one step. Deciding on + // one call and deleting on another leaves a window, and a delete made on + // a stale answer cannot be taken back. + final result = await _blobs.reclaimVerifiedHistoryBlob( + key, + converterFor(key), + ); - final missing = await _blobs.missingFromRows(key, converterFor(key)); + if (result.deleted) { + cleared.add(key); + continue; + } + final missing = result.missing; if (missing == null) { // Unverifiable, NOT verified-empty. Keep it. refused.add(key); @@ -119,16 +129,31 @@ class StaleHistorySweep { tag: 'Storage', ); refused.add(key); - continue; } - - await _blobs.delete(key); - cleared.add(key); + // Otherwise there was no blob left to reclaim: not cleared, not refused. } if (cleared.isNotEmpty) { + // VACUUM takes an exclusive lock and rebuilds the file, so database + // access queues behind it. It does NOT block the UI thread: drift_flutter + // runs SQLite in a spawned isolate (`_isolateEntrypoint` -> + // `DriftIsolate.inCurrent`), so this is a stall on history-backed reads + // rather than an ANR, which is the distinction that matters for an app + // whose storage work started as an ANR investigation. + // + // It is also close to once per install: a key is cleared once and then + // has no blob to consider, so later launches delete nothing and never + // reach here. Measured at 0.6 s on the owner's 70.9 MiB database. The + // duration is logged because that is a number worth watching in the + // field rather than assuming. + final sw = Stopwatch()..start(); try { await _blobs.vacuum(); + appLogger.info( + 'VACUUM took ${sw.elapsedMilliseconds} ms after clearing ' + '${cleared.length} stale blob(s).', + tag: 'Storage', + ); } catch (e) { // The deletes already happened and were each verified; VACUUM only // affects file size, so a failure here costs space and not data. The diff --git a/lib/storage/drift/blob_store.dart b/lib/storage/drift/blob_store.dart index fc5b33b1..9d891e78 100644 --- a/lib/storage/drift/blob_store.dart +++ b/lib/storage/drift/blob_store.dart @@ -241,8 +241,16 @@ class BlobStore { final blob = await readWithPrefsFallback(storeKey); if (blob == null || blob.isEmpty) return const []; + final int sourceCount; final List rows; try { + // The converter's own output cannot be the measure of the converter. + // `countJsonArrayElements` decodes the blob and does nothing but count, + // in a separate `compute`, exactly as the migration gate does. Without + // it a converter that silently dropped a record would report every + // record it produced as present and this would delete the blob holding + // the only copy of the one it dropped (Gemini review). + sourceCount = await compute(countJsonArrayElements, blob); rows = await compute(toRows, blob); } catch (e) { appLogger.error( @@ -254,12 +262,56 @@ class BlobStore { return null; } + if (rows.length != sourceCount) { + appLogger.error( + 'Cannot verify $storeKey against its rows: its blob holds ' + '$sourceCount records but the converter produced ${rows.length}. ' + 'Treating this as unverifiable; the blob is kept.', + tag: 'Storage', + ); + return null; + } + return _absentFrom( rows.map((r) => r.mergeKey), await messageMergeKeys(storeKey), ); } + /// Verifies [storeKey]'s stale blob against its rows and deletes it **only** + /// if every record is accounted for, as one atomic step (#962). + /// + /// The verify and the delete have to hold the same per-key lock. Checking + /// and then deleting through two separate calls leaves a window, and a + /// delete decided on a stale answer is not recoverable (Gemini review). This + /// also serialises the reclaim against [ensureMessageRowsMigrated] and + /// [mergeOffThread], which take the same lock, so the sweep cannot run + /// against a key mid-migration. + /// + /// `deleted` says whether a blob actually went, so the caller does not have + /// to infer it. `missing` is null when the blob could not be verified at all + /// and the key must be refused, empty when nothing was missing, and + /// otherwise the records that have no row. + /// + /// Nothing inside the locked section may take the same lock again: the chain + /// is not re-entrant and would deadlock. Everything called here reads. + Future<({bool deleted, List? missing})> reclaimVerifiedHistoryBlob( + String storeKey, + BlobToRowsCallback toRows, + ) { + return synchronized(storeKey, () async { + if (await read(storeKey) == null) { + return (deleted: false, missing: const []); + } + final missing = await missingFromRows(storeKey, toRows); + if (missing == null || missing.isNotEmpty) { + return (deleted: false, missing: missing); + } + await delete(storeKey); + return (deleted: true, missing: const []); + }); + } + /// The one place the "no blob record is missing from rows" rule lives. /// /// Both the migration gate and the sweep decide by this. The gate cannot diff --git a/test/storage/stale_history_sweep_test.dart b/test/storage/stale_history_sweep_test.dart index d69a0bee..1551716b 100644 --- a/test/storage/stale_history_sweep_test.dart +++ b/test/storage/stale_history_sweep_test.dart @@ -197,6 +197,28 @@ void main() { expect(await blobs.read(key), isNull); }); + test('a converter that silently drops a record is refused', () async { + // The hole Gemini found, closed. A converter can fail without throwing: + // it returns fewer rows than the blob holds. Comparing only the records + // it DID produce would find them all present and delete the blob that + // holds the only copy of the dropped one. The verify path therefore + // cross-checks against an independent count, as the migration gate does. + await migratedDaysAgo(key, 31); + + final result = await blobs.reclaimVerifiedHistoryBlob(key, (json) { + final all = toRows(json); + return all.take(all.length - 1).toList(); + }); + + expect(result.deleted, isFalse); + expect( + result.missing, + isNull, + reason: 'a lossy converter is UNVERIFIABLE, not verified-empty', + ); + expect(await blobs.read(key), isNotNull); + }); + test('an undecodable blob is refused, not treated as empty', () async { await migratedDaysAgo(key, 31); await blobs.write(key, 'this is not json'); @@ -246,6 +268,46 @@ void main() { ); }); + test('a migrated key with no blob is neither cleared nor refused', () async { + // The empty-blob migration path marks a key without there ever being a + // blob, and a second sweep sees a key it already reclaimed. Both must be + // silent: counting them as cleared would overstate what was reclaimed, + // and as refused would raise a false alarm. On the owner's database 7 of + // 47 aged keys are in exactly this state. + await migratedDaysAgo(key, 31); + expect((await StaleHistorySweep(blobs).run()).cleared, [key]); + + final second = await StaleHistorySweep(blobs).run(); + + expect(second.cleared, isEmpty); + expect(second.refused, isEmpty); + expect(second.bytesReclaimed, 0); + }); + + test( + 'reclaim is atomic: verify and delete hold the same key lock', + () async { + // Checking through one call and deleting through another leaves a window + // where the answer can go stale, and a delete decided on a stale answer + // cannot be taken back. BlobStore.reclaimVerifiedHistoryBlob exists to + // close that, so assert the sweep goes through it rather than doing its + // own read-then-delete. + await migratedDaysAgo(key, 31); + + final result = await blobs.reclaimVerifiedHistoryBlob(key, toRows); + + expect(result.deleted, isTrue); + expect(result.missing, isEmpty); + expect(await blobs.read(key), isNull); + + // Called again on the now-blobless key it reports no delete and no + // problem, which is what makes a second sweep silent. + final again = await blobs.reclaimVerifiedHistoryBlob(key, toRows); + expect(again.deleted, isFalse); + expect(again.missing, isEmpty); + }, + ); + test('an unmigrated history key keeps its blob', () async { await blobs.write(key, bigBlobWith(20)); // Never migrated: the blob is authoritative. @@ -274,12 +336,33 @@ void main() { isFalse, reason: '$k must never be considered a history key', ); + // Asserting the helper alone would pass even if `run` stopped calling + // it, because the loop only visits keys that HAVE rows and these + // normally have none (Gemini review). So give them rows and a stale + // timestamp: now the only thing standing between them and a delete is + // the prefix guard inside `run`. + await blobs.upsertMessageRows(k, const [ + MessageRowInput( + mergeKey: 'id:planted', + timestampMs: 1, + payload: '{}', + ), + ]); + await blobs.setMessageRowsMigratedAtForTest( + k, + DateTime.now().subtract(const Duration(days: 99)), + ); } await migratedDaysAgo(key, 31); final outcome = await StaleHistorySweep(blobs).run(); expect(outcome.cleared, [key]); + expect( + outcome.refused, + isEmpty, + reason: 'they must be skipped outright, not refused', + ); for (final k in others) { expect(await blobs.read(k), '[{"name":"keep me"}]', reason: k); }