From 636f36b0f6b89ba0a287dbe324a3e13bee962a3d Mon Sep 17 00:00:00 2001 From: "shoney.arickathil" Date: Sat, 29 Aug 2026 20:47:52 +0200 Subject: [PATCH] feat(db2-keys): the query paths read through the iterator and borrow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit databasev2 2, task 5d. Every reader in db.c now works for both backings. The measured problem: a keys-resident table's bitmap is EMPTY by construction (its payloads live in the log), so all four bitmap walks would have silently returned no rows — a query over such a table would find nothing, with no error. - wo_row_next_id: one iterator, two backings. Keys tables walk the id map; resident tables keep walking the BITMAP deliberately, because the id map holds the same set in hash order and switching would reorder the results of every unordered query in the repo. No behaviour change where none was needed - the three id-collecting scans move onto it. They only ever collected ids (the 9b cursor-stability rule materialises the list up front), so they needed no row access at all — which is why this was far smaller than the plan feared - the two filtered scans borrow, compare, and RELEASE BEFORE any exit. The scratch is per-table, so a borrow leaked past a `return` or `break` would make the next borrow on that table fail as a nested one. That is a real hazard, not a hypothetical: the request-path GET_FIELD borrowed and then `break`ed without releasing until this commit - point reads decode or clone BEFORE releasing, because a keys-resident row's slots point into the scratch that release frees Verified: just wovm-test — 36 suites 0 fail. Still to do in 5d: table.c's unique shadow and its three remaining wo_row_ptr sites, wal.c's append encode, and compaction's own walk — which is where the recorded `resident: keys` offset obligation has to be honoured. The loader refusal stays until all of it lands. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 0c97fa48d3e31ea9eea3287d9c23ed29f0260488) --- database/src/db.c | 84 +++++++++++++++++++++++++++----------------- database/src/table.c | 30 ++++++++++++++++ database/src/table.h | 14 ++++++++ 3 files changed, 95 insertions(+), 33 deletions(-) diff --git a/database/src/db.c b/database/src/db.c index f35ed6b..f46aaff 100644 --- a/database/src/db.c +++ b/database/src/db.c @@ -136,15 +136,13 @@ int wo_builtin_db(wo_vm *vm, uint64_t *R, uint32_t ins, const char **msg) { /* materialize the id list up front — the 9b cursor-stability rule: * the loop body then point-reads each id, so a row updated mid-loop * (even an indexed column) cannot disturb the iteration */ - db_table *t = &db->tables[cid]; - if (t->row_size) { - uint32_t total = t->slab_cnt * DB_SLAB_ROWS; - for (uint32_t g = 0; g < total; g++) { - if (!(t->bitmap[g >> 6] & (1ull << (g & 63)))) continue; - db_row *row = - (db_row *)(t->slabs[g / DB_SLAB_ROWS] + (size_t)(g % DB_SLAB_ROWS) * t->row_size); - if (wo_multi_push(ids, row->id) != 0) return WO_T_OOM; - } + { /* databasev2 2 (5d): through the shared iterator, because a + * keys-resident table's bitmap is empty by construction — this + * walk would otherwise see no rows at all */ + size_t cur = 0; + uint64_t rid; + while (wo_row_next_id(db, cid, &cur, &rid)) + if (wo_multi_push(ids, rid) != 0) return WO_T_OOM; } R[A] = (uint64_t)(uintptr_t)ids; return 0; @@ -157,14 +155,17 @@ int wo_builtin_db(wo_vm *vm, uint64_t *R, uint32_t ins, const char **msg) { *msg = "no such field"; return WO_T_DB; } - db_row *row = wo_row_ptr(db, cid, id); + db_row *row = wo_row_borrow(db, cid, id, msg); if (!row) { *msg = "no such row"; return WO_T_DB; } int ok = 1; + /* decode BEFORE releasing: for a keys-resident row the slots point at + * the borrow's scratch, which release frees */ uint64_t v = wo_val_decode_vm(db, &vm->rt, db->classes[cid].kinds[field], row->slots[field], &ok, msg); + wo_row_release(db, cid, row); if (!ok) return WO_T_OOM; R[A] = v; return 0; @@ -210,21 +211,29 @@ int wo_builtin_db(wo_vm *vm, uint64_t *R, uint32_t ins, const char **msg) { return 0; } } - uint32_t total = t->slab_cnt * DB_SLAB_ROWS; - for (uint32_t g = 0; g < total; g++) { - if (!(t->bitmap[g >> 6] & (1ull << (g & 63)))) continue; - db_row *row = - (db_row *)(t->slabs[g / DB_SLAB_ROWS] + (size_t)(g % DB_SLAB_ROWS) * t->row_size); - int eq; - if (kind == WO_K_TEXT) { - const wo_str *want = (const wo_str *)(uintptr_t)key; - const db_text *have = (const db_text *)(uintptr_t)row->slots[col]; - eq = (!want && !have) || - (want && have && want->len == have->len && - memcmp(want->data, have->bytes, have->len) == 0); - } else - eq = row->slots[col] == key; - if (eq && wo_multi_push(ids, row->id) != 0) return WO_T_OOM; + { /* databasev2 2 (5d): the filtered scan, through the shared + * iterator and a borrow. The borrow is released BEFORE any + * exit from the loop body: the scratch is per-table, so a + * borrow leaked past a `return` would make the next borrow on + * that table fail as a nested one. */ + size_t cur = 0; + uint64_t rid; + const char *bmsg = NULL; + while (wo_row_next_id(db, cid, &cur, &rid)) { + db_row *row = wo_row_borrow(db, cid, rid, &bmsg); + if (!row) continue; + int eq; + if (kind == WO_K_TEXT) { + const wo_str *want = (const wo_str *)(uintptr_t)key; + const db_text *have = (const db_text *)(uintptr_t)row->slots[col]; + eq = (!want && !have) || + (want && have && want->len == have->len && + memcmp(want->data, have->bytes, have->len) == 0); + } else + eq = row->slots[col] == key; + wo_row_release(db, cid, row); + if (eq && wo_multi_push(ids, rid) != 0) return WO_T_OOM; + } } } R[A] = (uint64_t)(uintptr_t)ids; @@ -340,11 +349,15 @@ void wo_db_exec_req(wo_vm *vm, wo_db_req *q) { } if (prc == 1) break; /* probed; reply fields already set */ } - uint32_t total = t->slab_cnt * DB_SLAB_ROWS; - for (uint32_t g = 0; g < total; g++) { - if (!(t->bitmap[g >> 6] & (1ull << (g & 63)))) continue; - db_row *row = - (db_row *)(t->slabs[g / DB_SLAB_ROWS] + (size_t)(g % DB_SLAB_ROWS) * t->row_size); + { /* databasev2 2 (5d): shared iterator + borrow, with the borrow + * released before the realloc that can `break` — a borrow held + * past an exit would poison the table's scratch. */ + size_t cur = 0; + uint64_t rid; + const char *bmsg = NULL; + while (wo_row_next_id(db, q->cid, &cur, &rid)) { + db_row *row = wo_row_borrow(db, q->cid, rid, &bmsg); + if (!row) continue; if (q->op == WO_B_DB_PROBE) { int eq; if (kind == WO_K_TEXT || kind == WO_K_BYTES) { @@ -357,8 +370,9 @@ void wo_db_exec_req(wo_vm *vm, wo_db_req *q) { memcmp(want->bytes, have->bytes, have->len) == 0); } else eq = row->slots[col] == q->slots[0]; - if (!eq) continue; + if (!eq) { wo_row_release(db, q->cid, row); continue; } } + wo_row_release(db, q->cid, row); if (n == cap) { uint32_t ncap = cap ? cap * 2 : 16; uint64_t *no = realloc(out, (size_t)ncap * 8u); @@ -372,7 +386,8 @@ void wo_db_exec_req(wo_vm *vm, wo_db_req *q) { out = no; cap = ncap; } - out[n++] = row->id; + out[n++] = rid; + } } } if (!q->status) { @@ -387,7 +402,7 @@ void wo_db_exec_req(wo_vm *vm, wo_db_req *q) { q->msg = "no such field"; break; } - db_row *row = wo_row_ptr(db, q->cid, q->id); + db_row *row = wo_row_borrow(db, q->cid, q->id, &m); if (!row) { q->status = WO_T_DB; q->msg = "no such row"; @@ -395,7 +410,10 @@ void wo_db_exec_req(wo_vm *vm, wo_db_req *q) { } int ok = 1; q->val_kind = db->classes[q->cid].kinds[q->field]; + /* clone BEFORE releasing: a keys-resident row's slots point into the + * borrow's scratch, which release frees */ q->val = wo_db_val_clone(db->classes, q->val_kind, row->slots[q->field], &ok); + wo_row_release(db, q->cid, row); if (!ok) { q->status = WO_T_OOM; q->msg = "out of memory"; diff --git a/database/src/table.c b/database/src/table.c index 46b5215..b5efd07 100644 --- a/database/src/table.c +++ b/database/src/table.c @@ -1086,6 +1086,36 @@ int wo_row_has_referrers(wo_db *db, uint32_t class_id, uint64_t id) { return 0; } +int wo_row_next_id(const wo_db *db, uint32_t class_id, size_t *cursor, uint64_t *id_out) { + if (class_id >= db->class_cnt) return 0; + const db_table *t = &db->tables[class_id]; + if (!t->row_size) return 0; + if (wo_table_is_keys_resident(db, class_id)) { + /* the id map IS the live set here: hkeys non-zero, hvals holding an + * offset + 1 */ + for (size_t j = *cursor; j < t->hcap; j++) { + if (t->hkeys[j] && t->hvals[j]) { + *id_out = t->hkeys[j]; + *cursor = j + 1; + return 1; + } + } + *cursor = t->hcap; + return 0; + } + { /* resident: the bitmap, in slab order, exactly as before */ + uint32_t total = t->slab_cnt * DB_SLAB_ROWS; + for (size_t g = *cursor; g < total; g++) { + if (!(t->bitmap[g >> 6] & (1ull << (g & 63)))) continue; + *id_out = slot_row((db_table *)t, (uint32_t)g)->id; + *cursor = g + 1; + return 1; + } + *cursor = total; + return 0; + } +} + int wo_table_is_keys_resident(const wo_db *db, uint32_t class_id) { if (class_id >= db->class_cnt) return 0; return (db->classes[class_id].flags & WO_CLASSF_RESIDENT_KEYS) != 0u; diff --git a/database/src/table.h b/database/src/table.h index e8b3cc9..df59243 100644 --- a/database/src/table.h +++ b/database/src/table.h @@ -194,6 +194,20 @@ int wo_row_remove(wo_db *db, uint32_t class_id, uint64_t id); * 0 ok, -1 unknown class/row. */ int wo_row_drop_payload(wo_db *db, uint32_t class_id, uint64_t id, uint64_t wal_off); +/* databasev2 2 (5d): iterate the live row IDS of a table, whichever backing it + * has. [*cursor] starts at 0 and is opaque; returns 1 with *id_out set, or 0 + * when exhausted. + * + * A keys-resident table has an EMPTY bitmap by construction — its payloads live + * in the log — so every bitmap walk in the engine would silently see no rows. + * This is the one primitive those walks move onto. + * + * Resident tables keep walking the bitmap, deliberately: the id map holds the + * same set, but in hash order, and switching would reorder the results of every + * unordered query in the repo. Two backings, one interface, no behaviour change + * where nothing needed to change. */ +int wo_row_next_id(const wo_db *db, uint32_t class_id, size_t *cursor, uint64_t *id_out); + /* databasev2 2 (5c): is this table's row data in the log rather than in slabs? */ int wo_table_is_keys_resident(const wo_db *db, uint32_t class_id);