From d780b7bbb9cfb8b802758fb338d2e86b8e443958 Mon Sep 17 00:00:00 2001 From: diffwireauto <330573775+diffwireauto@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:36:11 -0400 Subject: [PATCH] task(#936): A1 scoping inventory, with a gate that already caught four things Feature #853's acceptance bar is that every store has "a verified scoping verdict and no hole remains". Until now, verifying that meant a human re-reading an audit in an issue body, which is how three separate bleeds (#754, #854, #856) survived earlier rounds of bleed fixes. `scoping_inventory.dart` is that audit in code: every persistence surface with perRadio / globalOnPurpose / knownHole, a mandatory reason, and the file that owns it. A reason is required because an unexplained verdict is how a settled surface gets re-opened as a hole later. The test makes it a gate. It scans lib/ for the shape every key in this repo uses, a `const String` with a lower snake_case value, and fails when one is neither a declared surface nor an explicitly justified non-key. It also checks the reverse: a surface declared here but gone from the source. That one will fire when Epic B deletes `pending_messages`, which is intended. **It caught four things on its first two runs**, which is the argument for building it before Epic B rather than after: - `channel_cyr2lat_` and `contact_cyr2lat_` were never audited. #853 listed channel_settings_store as one surface; it owns two keys. - `los_error_elevation_unavailable`, a third error sentinel I had not found by hand. - `__msgrows_migrated`, declared by me and then reported missing, because my pattern required a letter first and internal markers start with underscores. The gate caught my own declaration bug. Scope of the scan, stated so nobody mistakes it for total: it would miss a key built inline with no constant, and a key that is not snake_case. Scanning every string literal in lib/ was tried and rejected, because it fires on asset paths and log tags, and a check that cries wolf gets worked around. That is the #927 lesson, which is also why there are three tests that prove this gate can fail rather than assuming it. Citadel: meshcore-open-z36 Refs #936, #853 Co-Authored-By: Claude Opus 5 --- lib/storage/scoping_inventory.dart | 334 +++++++++++++++++++++++ test/storage/scoping_inventory_test.dart | 211 ++++++++++++++ 2 files changed, 545 insertions(+) create mode 100644 lib/storage/scoping_inventory.dart create mode 100644 test/storage/scoping_inventory_test.dart diff --git a/lib/storage/scoping_inventory.dart b/lib/storage/scoping_inventory.dart new file mode 100644 index 00000000..b1cff4ae --- /dev/null +++ b/lib/storage/scoping_inventory.dart @@ -0,0 +1,334 @@ +/// The declared scoping verdict for every persistence surface (#853, #936). +/// +/// Feature #853's acceptance bar is that "every store has a verified scoping +/// verdict (per-radio or deliberately-global) and no hole remains". Until this +/// file existed, the only way to check that was for a human to re-read an audit +/// in an issue body, which is how #754 and #856 survived three earlier bleed +/// fixes. This file is that audit, in code, with a test that fails when a new +/// candidate key appears undeclared. +/// +/// **Adding a storage key means adding a row here.** The test in +/// `test/storage/scoping_inventory_test.dart` scans `lib/` for the shape every +/// key in this repo uses, a `const String` whose value is lower snake_case, and +/// fails if one is not classified below. A key that is not a storage key goes +/// in [nonStorageConstants] with a reason, so "it isn't a key" is a written +/// claim rather than a silence. +library; + +enum ScopingVerdict { + /// Scoped by the first 10 hex of the connected radio's public key, the + /// scheme every per-radio store in this repo uses. Radio A's data is never + /// visible while radio B is connected. + perRadio, + + /// Global, and that is correct. The reason is mandatory: without it a later + /// audit re-opens the surface as a hole, which is exactly the churn #853 + /// exists to end. + globalOnPurpose, + + /// Global, and that is a defect. Each of these has an open issue. The test + /// asserts the set matches the known holes, so a NEW hole cannot hide among + /// them and a FIXED one cannot be left mislabelled. + knownHole, +} + +class StorageSurface { + const StorageSurface( + this.key, { + required this.verdict, + required this.reason, + required this.owner, + this.issue, + }); + + /// The key or key prefix exactly as the source declares it. + final String key; + + final ScopingVerdict verdict; + + /// Why this verdict. For [ScopingVerdict.perRadio], where the scoping comes + /// from. For [ScopingVerdict.globalOnPurpose], why radio scoping would be + /// wrong. For [ScopingVerdict.knownHole], what bleeds. + final String reason; + + /// The file that owns the key, so a reader can go check. + final String owner; + + /// Required for [ScopingVerdict.knownHole], so every hole is tracked. + final int? issue; +} + +/// Every persistence surface in the client, with its verdict. +const List storageSurfaces = [ + // --------------------------------------------------------------------------- + // Per-radio. Each is fed the device-key prefix on connect. + // --------------------------------------------------------------------------- + StorageSurface( + 'channel_messages_', + verdict: ScopingVerdict.perRadio, + reason: + 'prefix + device key, then PSK identity within the radio (#194), ' + 'so history survives a channel rename and does not follow a reused ' + 'slot. The cross-radio bleed was #472', + owner: 'lib/storage/channel_message_store.dart', + ), + StorageSurface( + 'messages_', + verdict: ScopingVerdict.perRadio, + reason: 'prefix + device key + contact key', + owner: 'lib/storage/message_store.dart', + ), + StorageSurface( + 'channels', + verdict: ScopingVerdict.perRadio, + reason: 'prefix + device key', + owner: 'lib/storage/channel_store.dart', + ), + StorageSurface( + 'channel_order_', + verdict: ScopingVerdict.perRadio, + reason: + 'prefix + device key; display order is per radio because the ' + 'channel list is', + owner: 'lib/storage/channel_order_store.dart', + ), + StorageSurface( + 'channel_region_scope_', + verdict: ScopingVerdict.perRadio, + reason: 'prefix + device key', + owner: 'lib/storage/channel_region_scope_store.dart', + ), + StorageSurface( + 'contacts', + verdict: ScopingVerdict.perRadio, + reason: + 'prefix + device key; a contact list belongs to the radio that ' + 'holds it', + owner: 'lib/storage/contact_store.dart', + ), + StorageSurface( + 'contact_groups', + verdict: ScopingVerdict.perRadio, + reason: 'prefix + device key; groups reference that radio\'s contacts', + owner: 'lib/storage/contact_group_store.dart', + ), + StorageSurface( + 'contact_unread_count', + verdict: ScopingVerdict.perRadio, + reason: 'prefix + device key; unread counts follow the message history', + owner: 'lib/storage/unread_store.dart', + ), + StorageSurface( + 'discovered_contacts', + verdict: ScopingVerdict.perRadio, + reason: + 'prefix + device key since #754. A discovered node is what THIS ' + 'radio heard. An unset key loads nothing and never writes, so it ' + 'cannot touch the orphaned legacy blob', + owner: 'lib/storage/contact_discovery_store.dart', + ), + StorageSurface( + 'communities_v1', + verdict: ScopingVerdict.perRadio, + reason: 'prefix + device key', + owner: 'lib/storage/community_store.dart', + ), + StorageSurface( + 'channel_cyr2lat_', + verdict: ScopingVerdict.perRadio, + reason: + 'prefix + device key + channel index, via keyForCyr2Lat. NOT ' + 'previously audited: #853\'s audit listed channel_settings_store as ' + 'one surface, but it owns two keys and this inventory found the ' + 'second', + owner: 'lib/storage/channel_settings_store.dart', + ), + StorageSurface( + 'contact_cyr2lat_', + verdict: ScopingVerdict.perRadio, + reason: + 'prefix + device key + contact key. NOT previously audited, same ' + 'reason as channel_cyr2lat_', + owner: 'lib/storage/contact_settings_store.dart', + ), + StorageSurface( + 'block_keys_v1', + verdict: ScopingVerdict.perRadio, + reason: + 'prefix + device key. A blank radio is no longer offloaded the ' + 'client list (#471)', + owner: 'lib/storage/block_store.dart', + ), + StorageSurface( + 'block_names_v1', + verdict: ScopingVerdict.perRadio, + reason: 'prefix + device key, alongside block_keys_v1', + owner: 'lib/storage/block_store.dart', + ), + + // --------------------------------------------------------------------------- + // Known holes. Each has an issue and each is fixed in Epic B (#937). + // --------------------------------------------------------------------------- + StorageSurface( + 'path_history_', + verdict: ScopingVerdict.knownHole, + issue: 854, + reason: + 'keyed by contact public key only, with no radio in the key, so a ' + 'learned mesh route bleeds between any two radios that share a ' + 'contact. Also unbounded at 2,041 keys and 86% of the prefs file ' + '(#924), while PathHistoryService caches only 50', + owner: 'lib/services/storage_service.dart', + ), + StorageSurface( + 'delivery_observations', + verdict: ScopingVerdict.knownHole, + issue: 856, + reason: + 'one global list feeding the ACK-timeout ML model, so one radio\'s ' + 'RF timing trains another radio\'s predictions. Silent accuracy loss, ' + 'which is why it was never reported', + owner: 'lib/services/storage_service.dart', + ), + StorageSurface( + 'pending_messages', + verdict: ScopingVerdict.knownHole, + issue: 855, + reason: + 'global, but DEAD: loadPendingMessages, savePendingMessages and ' + 'clearPendingMessages have no callers anywhere in lib/ or test/, and ' + 'the key is absent from the owner\'s live prefs. The fix is deletion, ' + 'not scoping', + owner: 'lib/services/storage_service.dart', + ), + + // --------------------------------------------------------------------------- + // Global on purpose. The reason is the point: it stops a later audit + // re-opening these as holes. + // --------------------------------------------------------------------------- + StorageSurface( + 'client_install_id', + verdict: ScopingVerdict.globalOnPurpose, + reason: + 'the identity of this app INSTALL, not of any radio. Scoping it ' + 'per radio would mint a new install id per radio, which is the ' + 'opposite of what it is for', + owner: 'lib/storage/client_id_store.dart', + ), + StorageSurface( + 'app_settings', + verdict: ScopingVerdict.globalOnPurpose, + reason: + 'app-wide UX: theme, language, units, map and notification ' + 'preferences. The per-radio data it does carry is scoped INSIDE it, by ' + 'channelNotifyModesByRadio: device key -> channel PSK -> mode (#839). ' + 'So the blob is global and the radio-specific part within it is not', + owner: 'lib/models/app_settings.dart', + ), + StorageSurface( + 'repeater_passwords', + verdict: ScopingVerdict.globalOnPurpose, + reason: + 'keyed by the repeater NODE, not by the radio connecting to it. ' + 'The same repeater has the same password whichever radio reaches it, ' + 'so per-radio scoping would make the user re-enter it per radio', + owner: 'lib/services/storage_service.dart', + ), + StorageSurface( + 'repeater_auto_clock_sync_after_login', + verdict: ScopingVerdict.globalOnPurpose, + reason: 'keyed by repeater node, same reasoning as repeater_passwords', + owner: 'lib/services/storage_service.dart', + ), + StorageSurface( + 'map_removed_marker_ids', + verdict: ScopingVerdict.globalOnPurpose, + reason: + 'a set of node ids the user dismissed from the map. "Do not show ' + 'me this node" is a statement about the NODE, like repeater_passwords, ' + 'not about the radio that happened to hear it. NOT previously audited: ' + 'surfaced by this inventory, see #936', + owner: 'lib/services/map_marker_service.dart', + ), + StorageSurface( + 'map_tile_cache', + verdict: ScopingVerdict.globalOnPurpose, + reason: + 'cached OSM tiles. Geography does not depend on which radio is ' + 'connected, and per-radio caching would multiply the download', + owner: 'lib/services/map_tile_cache_service.dart', + ), + StorageSurface( + 'radio_presets_v1_', + verdict: ScopingVerdict.globalOnPurpose, + reason: + 'prefix + preset FILE name. A cached catalogue of region presets ' + 'fetched from assets or upstream, not state belonging to a radio. NOT ' + 'previously audited: surfaced by this inventory', + owner: 'lib/services/radio_preset_service.dart', + ), + StorageSurface( + 'anr_last_exit_ts_ms', + verdict: ScopingVerdict.globalOnPurpose, + reason: + 'watermark for which ApplicationExitInfo records have been ' + 'processed (#906). A property of the app process, not of a radio. NOT ' + 'previously audited: surfaced by this inventory', + owner: 'lib/services/app_exit_info_service.dart', + ), + StorageSurface( + '__msgrows_migrated', + verdict: ScopingVerdict.globalOnPurpose, + reason: + 'the set of store keys whose blob has been moved into message rows ' + '(#920). Global because the keys it lists are themselves already ' + 'radio-scoped, so scoping the marker would scope it twice', + owner: 'lib/storage/drift/blob_store.dart', + ), +]; + +/// Constants that match the key shape but are not persistence keys. +/// +/// This list exists so "that is not a storage key" has to be written down and +/// justified, instead of a new key slipping through because nobody looked. +const Map nonStorageConstants = { + 'offband_store': 'the drift database FILE name, not a key within it', + '__all__': + 'the sentinel value for "all groups" in the contacts filter ' + '(UiViewStateService). A filter VALUE, not a key, and not persisted ' + 'under this name', + '__reaction_send__': + 'a prefix for an in-memory send-queue id in the ' + 'connector, distinguishing a reaction send from a message send. Never ' + 'reaches storage', + 'psk_': + 'a marker inside a channel-history key, distinguishing a PSK ' + 'identity from a legacy slot index (#194). Not a key by itself', + 'anr': + 'a log tag / exit-reason discriminator in the ANR capture path, not ' + 'persisted under this name', + 'los_error_select_start_end': + 'an in-memory error sentinel in the line-of-sight screen, assigned to ' + '_error and compared against. Never written to storage', + 'los_error_invalid_input': 'the same, for invalid input', + 'los_error_elevation_unavailable': + 'the same, raised by the line-of-sight service when elevation data is ' + 'missing. Never written to storage', + // MQTT broker config field names. These are keys within a config map that is + // itself persisted under another surface, not top-level storage keys. + 'auth_type': 'MQTT broker config field name', + 'ca_cert': 'MQTT broker config field name', + 'enabled': 'MQTT broker config field name', + 'password': 'MQTT broker config field name', + 'port': 'MQTT broker config field name', + 'topic_prefix': 'MQTT broker config field name', + 'transport': 'MQTT broker config field name', + 'url': 'MQTT broker config field name', + 'username': 'MQTT broker config field name', + 'jwt_aud': 'JWT claim name in a broker auth payload', + 'jwt_email': 'JWT claim name', + 'jwt_owner': 'JWT claim name', + 'jwt_refresh': 'JWT claim name', + 'jwt_token': 'JWT claim name', + 'iata_override': 'a field name within a config payload, not a storage key', +}; diff --git a/test/storage/scoping_inventory_test.dart b/test/storage/scoping_inventory_test.dart new file mode 100644 index 00000000..0d923eee --- /dev/null +++ b/test/storage/scoping_inventory_test.dart @@ -0,0 +1,211 @@ +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:meshcore_open/storage/scoping_inventory.dart'; + +/// #936: the scoping verdict for every persistence surface, enforced. +/// +/// Feature #853's acceptance bar is that every store has a verified scoping +/// verdict and no hole remains. Before this test, "verified" meant a human +/// re-reading an audit in an issue body, which is how three separate bleeds +/// (#754, #854, #856) survived earlier rounds of bleed fixes. +/// +/// The scan deliberately covers one shape: a `const String` whose value is +/// lower snake_case, which is the shape every storage key in this repo uses. +/// What it would miss, stated plainly so nobody mistakes it for total: a key +/// built inline at the call site without a constant, and a key that is not +/// snake_case. Covering every string literal in `lib/` was tried and rejected: +/// it fires on asset paths and log tags, and a check that cries wolf gets +/// worked around, which is the #927 lesson. +void main() { + final declaredKeys = {for (final s in storageSurfaces) s.key}; + + group('inventory is internally coherent', () { + test('no surface is declared twice', () { + expect( + declaredKeys.length, + storageSurfaces.length, + reason: 'a duplicated key means two verdicts for one surface', + ); + }); + + test('every verdict carries a reason', () { + for (final s in storageSurfaces) { + expect( + s.reason.trim(), + isNotEmpty, + reason: + '${s.key} has no reason. An unexplained verdict is how a ' + 'surface gets re-opened as a hole later', + ); + expect(s.owner.trim(), isNotEmpty, reason: '${s.key} has no owner'); + } + }); + + test('every known hole names its issue', () { + final holes = storageSurfaces.where( + (s) => s.verdict == ScopingVerdict.knownHole, + ); + expect(holes, isNotEmpty, reason: 'sanity: the holes are the point'); + for (final s in holes) { + expect( + s.issue, + isNotNull, + reason: '${s.key} is a hole with no tracking issue', + ); + } + }); + + test('nothing is both a surface and a non-storage constant', () { + final overlap = declaredKeys.intersection( + nonStorageConstants.keys.toSet(), + ); + expect( + overlap, + isEmpty, + reason: 'a key cannot be both a storage surface and not one', + ); + }); + + test('the known holes are exactly the ones Epic B fixes', () { + // Pins the set. A NEW hole cannot hide among the known ones, and a hole + // fixed in Epic B cannot be left mislabelled here. + final holes = { + for (final s in storageSurfaces) + if (s.verdict == ScopingVerdict.knownHole) s.issue!, + }; + expect(holes, {854, 855, 856}); + }); + }); + + group('source scan', () { + test('every candidate key in lib/ is classified', () { + final found = _scanLibForCandidateKeys(); + + expect( + found, + isNotEmpty, + reason: 'the scan found nothing, so it is broken, not clean', + ); + + final unclassified = + found.keys + .where((k) => !declaredKeys.contains(k)) + .where((k) => !nonStorageConstants.containsKey(k)) + .toList() + ..sort(); + + expect( + unclassified, + isEmpty, + reason: + 'undeclared candidate keys:\n' + '${unclassified.map((k) => ' $k (${found[k]})').join('\n')}\n\n' + 'Add each to storageSurfaces with a verdict and a reason, or to ' + 'nonStorageConstants with why it is not a key. This is the gate ' + 'Feature #853 asked for: a new global key cannot land unnoticed.', + ); + }); + + test('nothing declared has vanished from the source', () { + // The other direction: a surface removed from the code but left in the + // inventory is a lie in the opposite direction. pending_messages will + // trip this the moment Epic B deletes it, which is intended. + final found = _scanLibForCandidateKeys().keys.toSet(); + final stale = declaredKeys.difference(found).toList()..sort(); + + expect( + stale, + isEmpty, + reason: + 'declared but not found in lib/:\n${stale.join('\n')}\n\n' + 'If the surface was deleted, remove its row. If it was renamed, ' + 'update it.', + ); + }); + }); + + group('the gate can fail', () { + // A check nobody has watched fail is not evidence. These run the scan + // logic over synthetic source instead of lib/, so the failure is provable + // without breaking the real tree. + + test('an undeclared snake_case key is caught', () { + final found = _extractCandidateKeys( + "class Foo { static const String _k = 'a_brand_new_global_key'; }", + 'synthetic.dart', + ); + + expect(found.keys, contains('a_brand_new_global_key')); + expect( + declaredKeys.contains('a_brand_new_global_key') || + nonStorageConstants.containsKey('a_brand_new_global_key'), + isFalse, + reason: 'so the real test would fail on it, which is the point', + ); + }); + + test('a declared key is not flagged', () { + final found = _extractCandidateKeys( + "const String _p = 'channel_messages_';", + 'synthetic.dart', + ); + + expect(found.keys, contains('channel_messages_')); + expect(declaredKeys, contains('channel_messages_')); + }); + + test('a non-key shape is ignored rather than demanding a verdict', () { + final found = _extractCandidateKeys( + "const String a = 'assets/radio_presets/';\n" + "const String b = 'com.meshcore.open';\n" + "const String c = 'linux connect stage failure';", + 'synthetic.dart', + ); + + expect( + found, + isEmpty, + reason: + 'paths, package ids and prose must not demand a verdict, or ' + 'the check gets worked around', + ); + }); + }); +} + +/// `const String = ''`, the shape every storage key in +/// this repo is declared with. +/// +/// Leading underscores are allowed in the VALUE because internal markers use +/// them (`__msgrows_migrated`). The first run of this test declared that key +/// and then reported it missing, because the pattern required a letter first. +final RegExp _candidate = RegExp( + r"const\s+String\s+[A-Za-z_][A-Za-z0-9_]*\s*=\s*'(_{0,2}[a-z][a-z0-9_]*)'", +); + +Map _extractCandidateKeys(String source, String path) { + final out = {}; + for (final m in _candidate.allMatches(source)) { + out[m.group(1)!] = path; + } + return out; +} + +/// Candidate key -> the file it was found in. +Map _scanLibForCandidateKeys() { + final lib = Directory('lib'); + expect( + lib.existsSync(), + isTrue, + reason: 'this test must run from the package root so lib/ is visible', + ); + + final out = {}; + for (final f in lib.listSync(recursive: true).whereType()) { + if (!f.path.endsWith('.dart')) continue; + if (f.path.endsWith('.g.dart')) continue; // generated, not authored + out.addAll(_extractCandidateKeys(f.readAsStringSync(), f.path)); + } + return out; +}