From df09158b7f5d23e08163f94a7c6b52f7283c357c Mon Sep 17 00:00:00 2001 From: "shoney.arickathil" Date: Mon, 31 Aug 2026 21:25:11 +0200 Subject: [PATCH] =?UTF-8?q?feat(db2-migrate):=20the=20boot=20diff=20?= =?UTF-8?q?=E2=80=94=20name-keyed,=20poisons=20instead=20of=20errors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - wo_schema_diff matches classes and fields by NAME, so declaration reordering is identity apart from the cid map — the silent cid-renumbering hole closes as a side effect - owned-field references (fclass) compare by the NAME the number resolves to, never the number: a raw compare would false-poison retype on every pure reorder - refusals are per-class POISONS carried in the plan, not diff errors: a poison bites only when a record of the class is met, so a retyped class with no stored rows never blocks a boot - poison set: retype, same-shape delete+add (a disguised rename, one reading destroys a column), vanished class, storage-flag change, and the embed closure — any class whose old records carry values of a class whose shape changed, iterated to a fixpoint - identity plans skip the rewrite entirely; a NEW class in the binary does not break identity (no records; the head refreshes at the next compaction) - ten verdict tests; test_wal 5778 pass, 0 fail Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 63a063b822af381a10c2e599c58cf3455d9c5bf7) --- database/src/wal.c | 197 ++++++++++++++++++++++++++++++++++++++++ database/src/wal.h | 31 +++++++ runtime/test/test_wal.c | 136 +++++++++++++++++++++++++++ 3 files changed, 364 insertions(+) diff --git a/database/src/wal.c b/database/src/wal.c index 55cd3d4..c6f994f 100644 --- a/database/src/wal.c +++ b/database/src/wal.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -547,6 +548,202 @@ void wo_schema_free(wo_schema *sc) { free(sc); } +/* ---- databasev2 12: the boot diff ------------------------------------- */ + +static int sc_name_eq(const uint8_t *a, uint32_t al, const uint8_t *b, uint32_t bl) { + return al == bl && (al == 0 || memcmp(a, b, al) == 0); +} +static uint32_t sc_find_class(const wo_schema *sc, const uint8_t *name, uint32_t len) { + for (uint32_t c = 0; c < sc->class_cnt; c++) + if (sc_name_eq(sc->classes[c].name, sc->classes[c].name_len, name, len)) return c; + return WO_SCHEMA_NONE; +} +static uint32_t sc_find_field(const wo_schema_class *k, const uint8_t *name, uint32_t len) { + for (uint32_t f = 0; f < k->field_cnt; f++) + if (sc_name_eq(k->fields[f].name, k->fields[f].name_len, name, len)) return f; + return WO_SCHEMA_NONE; +} +/* fclass words number classes in their OWN schema, so under reordering the + * same referenced class carries different numbers — equality is by the NAME + * the number resolves to, never the number itself */ +static int sc_ref_eq(const wo_schema *osc, uint32_t ofc, const wo_schema *nsc, uint32_t nfc) { + if (ofc == WO_SCHEMA_NONE || nfc == WO_SCHEMA_NONE) return ofc == nfc; + if (ofc >= osc->class_cnt || nfc >= nsc->class_cnt) return 0; + return sc_name_eq(osc->classes[ofc].name, osc->classes[ofc].name_len, + nsc->classes[nfc].name, nsc->classes[nfc].name_len); +} +static int sc_field_shape_eq(const wo_schema *osc, const wo_schema_field *of, + const wo_schema *nsc, const wo_schema_field *nf) { + return of->kind == nf->kind && of->felem == nf->felem && + sc_ref_eq(osc, of->fclass, nsc, nf->fclass); +} +#if defined(__GNUC__) +__attribute__((format(printf, 1, 2))) +#endif +static char *sc_poisonf(const char *fmt, ...) { + char buf[512]; + va_list ap; + va_start(ap, fmt); + vsnprintf(buf, sizeof buf, fmt, ap); + va_end(ap); + return strdup(buf); +} + +void wo_mig_plan_free(wo_mig_plan *plan) { + if (!plan->classes) return; + for (uint32_t c = 0; c < plan->old_class_cnt; c++) { + free(plan->classes[c].poison); + free(plan->classes[c].fmap); + } + free(plan->classes); + plan->classes = NULL; +} + +int wo_schema_diff(const wo_schema *osc, const wo_schema *nsc, wo_mig_plan *plan) { + memset(plan, 0, sizeof *plan); + plan->old_class_cnt = osc->class_cnt; + plan->classes = calloc(osc->class_cnt ? osc->class_cnt : 1, sizeof *plan->classes); + if (!plan->classes) return -1; + + for (uint32_t c = 0; c < osc->class_cnt; c++) { + const wo_schema_class *ok = &osc->classes[c]; + wo_mig_class *mc = &plan->classes[c]; + mc->old_field_cnt = ok->field_cnt; + mc->new_cid = sc_find_class(nsc, ok->name, ok->name_len); + if (mc->new_cid == WO_SCHEMA_NONE) { + mc->poison = sc_poisonf("class `%.*s` has stored rows this binary no " + "longer declares — restore the class, or delete " + "WO_DATA if the rows are expendable", + (int)ok->name_len, (const char *)ok->name); + continue; + } + const wo_schema_class *nk = &nsc->classes[mc->new_cid]; + if (ok->flags != nk->flags) { + mc->new_cid = WO_SCHEMA_NONE; + mc->poison = sc_poisonf("class `%.*s` changed its storage declaration " + "(durable/resident) with rows in the log — v1 " + "migrates fields, not storage modes", + (int)ok->name_len, (const char *)ok->name); + continue; + } + mc->fmap = malloc((ok->field_cnt ? ok->field_cnt : 1) * sizeof *mc->fmap); + if (!mc->fmap) return -1; + uint32_t deleted = 0; + for (uint32_t f = 0; f < ok->field_cnt; f++) { + const wo_schema_field *of = &ok->fields[f]; + uint32_t nf = sc_find_field(nk, of->name, of->name_len); + if (nf == WO_SCHEMA_NONE) { + mc->fmap[f] = -1; + deleted++; + continue; + } + if (!sc_field_shape_eq(osc, of, nsc, &nk->fields[nf])) { + free(mc->fmap); + mc->fmap = NULL; + mc->new_cid = WO_SCHEMA_NONE; + mc->poison = sc_poisonf("class `%.*s`: field `%.*s` changed its type " + "— v1 has no conversions; add a new field " + "and backfill instead", + (int)ok->name_len, (const char *)ok->name, + (int)of->name_len, (const char *)of->name); + break; + } + mc->fmap[f] = (int32_t)nf; + } + if (!mc->fmap) continue; /* poisoned above */ + uint32_t added = 0; + for (uint32_t f = 0; f < nk->field_cnt; f++) + if (sc_find_field(ok, nk->fields[f].name, nk->fields[f].name_len) == + WO_SCHEMA_NONE) + added++; + mc->changed = (deleted > 0 || added > 0); + /* a deleted and an added field of the SAME shape in one step is + * byte-for-byte indistinguishable from a rename, and the two + * readings differ by exactly one column of destroyed data */ + if (deleted && added) { + for (uint32_t f = 0; f < ok->field_cnt && mc->fmap; f++) { + if (mc->fmap[f] != -1) continue; + for (uint32_t g = 0; g < nk->field_cnt; g++) { + if (sc_find_field(ok, nk->fields[g].name, nk->fields[g].name_len) != + WO_SCHEMA_NONE) + continue; + if (sc_field_shape_eq(osc, &ok->fields[f], nsc, &nk->fields[g])) { + free(mc->fmap); + mc->fmap = NULL; + mc->new_cid = WO_SCHEMA_NONE; + mc->poison = sc_poisonf( + "class `%.*s`: `%.*s` was deleted and a field of the " + "same type added — a rename and a delete+add are " + "indistinguishable here and one of them destroys data. " + "Deploy the delete and the add as two separate steps", + (int)ok->name_len, (const char *)ok->name, + (int)ok->fields[f].name_len, + (const char *)ok->fields[f].name); + break; + } + } + if (!mc->fmap) break; + } + } + } + + /* the embed closure: a class whose old records EMBED (owned or container + * values of) a class whose shape changed cannot be transcoded — the + * nested bytes are in the OLD sub-shape and v1 does not rewrite value + * trees recursively. Iterate to a fixpoint so chains of embedding + * poison through. */ + for (int again = 1; again;) { + again = 0; + for (uint32_t c = 0; c < osc->class_cnt; c++) { + wo_mig_class *mc = &plan->classes[c]; + if (mc->poison) continue; + for (uint32_t f = 0; f < osc->classes[c].field_cnt; f++) { + const wo_schema_field *of = &osc->classes[c].fields[f]; + if (of->kind != WO_K_OWNED && of->kind != WO_K_MULTI && + of->kind != WO_K_MAP) + continue; + int tainted = 0; + if (of->fclass != WO_SCHEMA_NONE && of->fclass < osc->class_cnt) { + const wo_mig_class *ref = &plan->classes[of->fclass]; + tainted = ref->poison != NULL || ref->changed; + } else if (of->kind == WO_K_OWNED) { + /* an owned field with no recorded target class: assume the + * worst whenever anything at all changed */ + for (uint32_t x = 0; x < osc->class_cnt && !tainted; x++) + tainted = plan->classes[x].poison != NULL || + plan->classes[x].changed; + } + if (tainted) { + free(mc->fmap); + mc->fmap = NULL; + mc->new_cid = WO_SCHEMA_NONE; + mc->poison = sc_poisonf( + "class `%.*s`: field `%.*s` embeds a class whose shape " + "changed — its stored values carry the old sub-shape, " + "which v1 does not rewrite. Migrate the embedded class " + "on its own first", + (int)osc->classes[c].name_len, + (const char *)osc->classes[c].name, (int)of->name_len, + (const char *)of->name); + again = 1; + break; + } + } + } + } + + plan->identity = 1; + for (uint32_t c = 0; c < osc->class_cnt; c++) { + const wo_mig_class *mc = &plan->classes[c]; + if (mc->poison) continue; /* a poison alone never forces a rewrite */ + if (mc->new_cid != c || mc->changed) { + plan->identity = 0; + break; + } + } + return 0; +} + int wo_wal_set_schema(wo_wal *w, const wo_schema *sc) { uint8_t *p; uint32_t len; diff --git a/database/src/wal.h b/database/src/wal.h index 0675234..00b74fc 100644 --- a/database/src/wal.h +++ b/database/src/wal.h @@ -161,6 +161,37 @@ int wo_schema_encode(const wo_schema *sc, uint8_t **payload_out, uint32_t *len_o wo_schema *wo_schema_decode(const uint8_t *payload, uint32_t len); void wo_schema_free(wo_schema *sc); +/* databasev2 12: what boot decided about one stored class. `new_cid` is where + * its records go; WO_SCHEMA_NONE means POISONED — the class cannot be + * migrated, and `poison` says why. A poison only bites when a record of the + * class is actually met: no rows, no verdict. */ +typedef struct wo_mig_class { + uint32_t new_cid; /* WO_SCHEMA_NONE = poisoned */ + char *poison; /* malloc'd reason; NULL unless poisoned */ + uint32_t old_field_cnt; + int32_t *fmap; /* old field index -> new slot, -1 = deleted */ + int changed; /* own field set differs (add and/or delete) */ +} wo_mig_class; +typedef struct wo_mig_plan { + uint32_t old_class_cnt; + wo_mig_class *classes; + /* 1 = every stored class keeps its cid and its shape: replay as-is, no + * transcode. New classes in the binary do not break identity — they have + * no records, and the head record refreshes at the next compaction. */ + int identity; +} wo_mig_plan; + +/* Diff the log's stored schema against the compiled one, classes matched by + * NAME, fields by NAME — so pure declaration reordering is identity apart + * from the cid map. Returns 0 with *plan filled (free with + * wo_mig_plan_free), -1 on OOM. Refusals are expressed as per-class poisons, + * not errors: retype, same-kind delete+add (a disguised rename), a vanished + * class, changed flags, and any class that EMBEDS (owned/container fields) + * a class whose shape changed — its old records encode the old sub-shape, + * which v1 does not rewrite recursively. */ +int wo_schema_diff(const wo_schema *oldsc, const wo_schema *newsc, wo_mig_plan *plan); +void wo_mig_plan_free(wo_mig_plan *plan); + /* Adopt `sc` as this log's compiled schema (encoded and owned by the wal). */ int wo_wal_set_schema(wo_wal *w, const wo_schema *sc); /* A fresh, empty log gets the schema as its first record — durable before diff --git a/runtime/test/test_wal.c b/runtime/test/test_wal.c index a3e6b0f..355f3b0 100644 --- a/runtime/test/test_wal.c +++ b/runtime/test/test_wal.c @@ -1008,6 +1008,141 @@ static void test_should_compact_absolute_and_ceiling(void) { T_EQ(wo_wal_should_compact(2048, 1024, floor_b, 2), 0); /* not yet */ } +/* ---- databasev2 12: the boot diff --------------------------------------- */ + +#define SF(nm, k) {(const uint8_t *)nm, (uint32_t)(sizeof nm - 1), k, WO_SCHEMA_NONE, WO_SCHEMA_NONE} +#define SFC(nm, k, fc) {(const uint8_t *)nm, (uint32_t)(sizeof nm - 1), k, fc, WO_SCHEMA_NONE} +#define SC(nm, fl, arr) {(const uint8_t *)nm, (uint32_t)(sizeof nm - 1), fl, \ + (uint32_t)(sizeof arr / sizeof arr[0]), arr} + +static void test_schema_diff_verdicts(void) { + /* base: A { n: scalar, t: text }, B { part: owned->A } */ + wo_schema_field a_f[] = {SF("n", WO_K_SCALAR), SF("t", WO_K_TEXT)}; + wo_schema_field b_f[] = {SFC("part", WO_K_OWNED, 0)}; + wo_schema_class base_c[] = {SC("A", 0, a_f), SC("B", 0, b_f)}; + wo_schema base = {2, base_c, NULL}; + wo_mig_plan pl; + + /* 1. identical -> identity */ + T_EQ(wo_schema_diff(&base, &base, &pl), 0); + T_EQ(pl.identity, 1); + T_CHECK(pl.classes[0].poison == NULL && pl.classes[1].poison == NULL); + wo_mig_plan_free(&pl); + + /* 2. classes reordered -> not identity, cids remapped BY NAME, and the + owned reference (a different NUMBER now) is recognised by name too */ + { + wo_schema_field b2_f[] = {SFC("part", WO_K_OWNED, 1)}; /* A is cid 1 now */ + wo_schema_class swap_c[] = {SC("B", 0, b2_f), SC("A", 0, a_f)}; + wo_schema swp = {2, swap_c, NULL}; + T_EQ(wo_schema_diff(&base, &swp, &pl), 0); + T_EQ(pl.identity, 0); + T_EQ(pl.classes[0].new_cid, 1u); /* old A -> new cid 1 */ + T_EQ(pl.classes[1].new_cid, 0u); /* old B -> new cid 0 */ + T_CHECK(pl.classes[0].poison == NULL && pl.classes[1].poison == NULL); + T_CHECK(pl.classes[0].changed == 0 && pl.classes[1].changed == 0); + wo_mig_plan_free(&pl); + } + + /* 3. added field -> changed, surviving map intact, B poisoned (embeds A) */ + { + wo_schema_field a3_f[] = {SF("n", WO_K_SCALAR), SF("t", WO_K_TEXT), + SF("extra", WO_K_SCALAR)}; + wo_schema_class c3[] = {SC("A", 0, a3_f), SC("B", 0, b_f)}; + wo_schema n3 = {2, c3, NULL}; + T_EQ(wo_schema_diff(&base, &n3, &pl), 0); + T_EQ(pl.identity, 0); + T_CHECK(pl.classes[0].poison == NULL && pl.classes[0].changed == 1); + T_EQ(pl.classes[0].fmap[0], 0); + T_EQ(pl.classes[0].fmap[1], 1); + T_CHECK(pl.classes[1].poison != NULL); /* embeds a changed class */ + wo_mig_plan_free(&pl); + } + + /* 4. deleted field -> fmap -1; different-shape delete+add migrates */ + { + wo_schema_field a4_f[] = {SF("n", WO_K_SCALAR), SF("blob", WO_K_BYTES)}; + wo_schema_class c4[] = {SC("A", 0, a4_f), SC("B", 0, b_f)}; + wo_schema n4 = {2, c4, NULL}; /* t: Text deleted, blob: Bytes added */ + T_EQ(wo_schema_diff(&base, &n4, &pl), 0); + T_CHECK(pl.classes[0].poison == NULL && pl.classes[0].changed == 1); + T_EQ(pl.classes[0].fmap[0], 0); + T_EQ(pl.classes[0].fmap[1], -1); + wo_mig_plan_free(&pl); + } + + /* 5. SAME-shape delete+add -> the rename ambiguity poison */ + { + wo_schema_field a5_f[] = {SF("n", WO_K_SCALAR), SF("headline", WO_K_TEXT)}; + wo_schema_class c5[] = {SC("A", 0, a5_f), SC("B", 0, b_f)}; + wo_schema n5 = {2, c5, NULL}; + T_EQ(wo_schema_diff(&base, &n5, &pl), 0); + T_CHECK(pl.classes[0].poison != NULL); + T_CHECK(strstr(pl.classes[0].poison, "two separate steps") != NULL); + wo_mig_plan_free(&pl); + } + + /* 6. retype -> poison naming the field */ + { + wo_schema_field a6_f[] = {SF("n", WO_K_FLOAT), SF("t", WO_K_TEXT)}; + wo_schema_class c6[] = {SC("A", 0, a6_f), SC("B", 0, b_f)}; + wo_schema n6 = {2, c6, NULL}; + T_EQ(wo_schema_diff(&base, &n6, &pl), 0); + T_CHECK(pl.classes[0].poison != NULL && + strstr(pl.classes[0].poison, "`n`") != NULL); + wo_mig_plan_free(&pl); + } + + /* 7. vanished class -> poison; the other class (embedding nothing that + changed shape) is untouched */ + { + wo_schema_class c7[] = {SC("A", 0, a_f)}; + wo_schema n7 = {1, c7, NULL}; + T_EQ(wo_schema_diff(&base, &n7, &pl), 0); + T_CHECK(pl.classes[1].new_cid == WO_SCHEMA_NONE && + pl.classes[1].poison != NULL); + T_CHECK(pl.classes[0].poison == NULL); + wo_mig_plan_free(&pl); + } + + /* 8. flags change -> poison (v1 migrates fields, not storage modes) */ + { + wo_schema_class c8[] = {SC("A", WO_CLASSF_RESIDENT_KEYS, a_f), SC("B", 0, b_f)}; + wo_schema n8 = {2, c8, NULL}; + T_EQ(wo_schema_diff(&base, &n8, &pl), 0); + T_CHECK(pl.classes[0].poison != NULL && + strstr(pl.classes[0].poison, "storage") != NULL); + wo_mig_plan_free(&pl); + } + + /* 9. a NEW class in the binary does not break identity: it has no records */ + { + wo_schema_field n_f[] = {SF("x", WO_K_SCALAR)}; + wo_schema_class c9[] = {SC("A", 0, a_f), SC("B", 0, b_f), SC("C", 0, n_f)}; + wo_schema n9 = {3, c9, NULL}; + T_EQ(wo_schema_diff(&base, &n9, &pl), 0); + T_EQ(pl.identity, 1); + wo_mig_plan_free(&pl); + } + + /* 10. the embed closure is transitive: C owns B, B owns A, A changed -> + both B and C poisoned */ + { + wo_schema_field cB[] = {SFC("a", WO_K_OWNED, 0)}; + wo_schema_field cC[] = {SFC("b", WO_K_OWNED, 1)}; + wo_schema_class oc[] = {SC("A", 0, a_f), SC("B", 0, cB), SC("C", 0, cC)}; + wo_schema oldsc = {3, oc, NULL}; + wo_schema_field a10[] = {SF("n", WO_K_SCALAR)}; /* t deleted */ + wo_schema_class nc[] = {SC("A", 0, a10), SC("B", 0, cB), SC("C", 0, cC)}; + wo_schema newsc = {3, nc, NULL}; + T_EQ(wo_schema_diff(&oldsc, &newsc, &pl), 0); + T_CHECK(pl.classes[0].poison == NULL && pl.classes[0].changed == 1); + T_CHECK(pl.classes[1].poison != NULL); + T_CHECK(pl.classes[2].poison != NULL); + wo_mig_plan_free(&pl); + } +} + /* ---- databasev2 12: the schema record ---------------------------------- */ /* a hand-built two-class schema exercising every payload field */ @@ -2810,6 +2945,7 @@ int main(void) { test_keys_resident_update_field(); test_keys_resident_update_indexed(); test_keys_resident_indexed_across_flatten(); + test_schema_diff_verdicts(); test_schema_roundtrip(); test_schema_fresh_log(); test_schema_compaction_adopts_legacy();