feat(db2-migrate): the boot diff — name-keyed, poisons instead of errors

- wo_schema_diff matches classes and fields by NAME, so declaration
  reordering is identity apart from the cid map — the silent
  cid-renumbering hole closes as a side effect
- owned-field references (fclass) compare by the NAME the number
  resolves to, never the number: a raw compare would false-poison
  retype on every pure reorder
- refusals are per-class POISONS carried in the plan, not diff errors:
  a poison bites only when a record of the class is met, so a retyped
  class with no stored rows never blocks a boot
- poison set: retype, same-shape delete+add (a disguised rename, one
  reading destroys a column), vanished class, storage-flag change, and
  the embed closure — any class whose old records carry values of a
  class whose shape changed, iterated to a fixpoint
- identity plans skip the rewrite entirely; a NEW class in the binary
  does not break identity (no records; the head refreshes at the next
  compaction)
- ten verdict tests; test_wal 5778 pass, 0 fail

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
(cherry picked from commit 63a063b822af381a10c2e599c58cf3455d9c5bf7)
This commit is contained in:
shoney.arickathil 2026-08-31 21:25:11 +02:00
parent f07295b3c0
commit df09158b7f
3 changed files with 364 additions and 0 deletions

View file

@ -7,6 +7,7 @@
#include <time.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
@ -547,6 +548,202 @@ void wo_schema_free(wo_schema *sc) {
free(sc);
}
/* ---- databasev2 12: the boot diff ------------------------------------- */
static int sc_name_eq(const uint8_t *a, uint32_t al, const uint8_t *b, uint32_t bl) {
return al == bl && (al == 0 || memcmp(a, b, al) == 0);
}
static uint32_t sc_find_class(const wo_schema *sc, const uint8_t *name, uint32_t len) {
for (uint32_t c = 0; c < sc->class_cnt; c++)
if (sc_name_eq(sc->classes[c].name, sc->classes[c].name_len, name, len)) return c;
return WO_SCHEMA_NONE;
}
static uint32_t sc_find_field(const wo_schema_class *k, const uint8_t *name, uint32_t len) {
for (uint32_t f = 0; f < k->field_cnt; f++)
if (sc_name_eq(k->fields[f].name, k->fields[f].name_len, name, len)) return f;
return WO_SCHEMA_NONE;
}
/* fclass words number classes in their OWN schema, so under reordering the
* same referenced class carries different numbers — equality is by the NAME
* the number resolves to, never the number itself */
static int sc_ref_eq(const wo_schema *osc, uint32_t ofc, const wo_schema *nsc, uint32_t nfc) {
if (ofc == WO_SCHEMA_NONE || nfc == WO_SCHEMA_NONE) return ofc == nfc;
if (ofc >= osc->class_cnt || nfc >= nsc->class_cnt) return 0;
return sc_name_eq(osc->classes[ofc].name, osc->classes[ofc].name_len,
nsc->classes[nfc].name, nsc->classes[nfc].name_len);
}
static int sc_field_shape_eq(const wo_schema *osc, const wo_schema_field *of,
const wo_schema *nsc, const wo_schema_field *nf) {
return of->kind == nf->kind && of->felem == nf->felem &&
sc_ref_eq(osc, of->fclass, nsc, nf->fclass);
}
#if defined(__GNUC__)
__attribute__((format(printf, 1, 2)))
#endif
static char *sc_poisonf(const char *fmt, ...) {
char buf[512];
va_list ap;
va_start(ap, fmt);
vsnprintf(buf, sizeof buf, fmt, ap);
va_end(ap);
return strdup(buf);
}
void wo_mig_plan_free(wo_mig_plan *plan) {
if (!plan->classes) return;
for (uint32_t c = 0; c < plan->old_class_cnt; c++) {
free(plan->classes[c].poison);
free(plan->classes[c].fmap);
}
free(plan->classes);
plan->classes = NULL;
}
int wo_schema_diff(const wo_schema *osc, const wo_schema *nsc, wo_mig_plan *plan) {
memset(plan, 0, sizeof *plan);
plan->old_class_cnt = osc->class_cnt;
plan->classes = calloc(osc->class_cnt ? osc->class_cnt : 1, sizeof *plan->classes);
if (!plan->classes) return -1;
for (uint32_t c = 0; c < osc->class_cnt; c++) {
const wo_schema_class *ok = &osc->classes[c];
wo_mig_class *mc = &plan->classes[c];
mc->old_field_cnt = ok->field_cnt;
mc->new_cid = sc_find_class(nsc, ok->name, ok->name_len);
if (mc->new_cid == WO_SCHEMA_NONE) {
mc->poison = sc_poisonf("class `%.*s` has stored rows this binary no "
"longer declares — restore the class, or delete "
"WO_DATA if the rows are expendable",
(int)ok->name_len, (const char *)ok->name);
continue;
}
const wo_schema_class *nk = &nsc->classes[mc->new_cid];
if (ok->flags != nk->flags) {
mc->new_cid = WO_SCHEMA_NONE;
mc->poison = sc_poisonf("class `%.*s` changed its storage declaration "
"(durable/resident) with rows in the log — v1 "
"migrates fields, not storage modes",
(int)ok->name_len, (const char *)ok->name);
continue;
}
mc->fmap = malloc((ok->field_cnt ? ok->field_cnt : 1) * sizeof *mc->fmap);
if (!mc->fmap) return -1;
uint32_t deleted = 0;
for (uint32_t f = 0; f < ok->field_cnt; f++) {
const wo_schema_field *of = &ok->fields[f];
uint32_t nf = sc_find_field(nk, of->name, of->name_len);
if (nf == WO_SCHEMA_NONE) {
mc->fmap[f] = -1;
deleted++;
continue;
}
if (!sc_field_shape_eq(osc, of, nsc, &nk->fields[nf])) {
free(mc->fmap);
mc->fmap = NULL;
mc->new_cid = WO_SCHEMA_NONE;
mc->poison = sc_poisonf("class `%.*s`: field `%.*s` changed its type "
"— v1 has no conversions; add a new field "
"and backfill instead",
(int)ok->name_len, (const char *)ok->name,
(int)of->name_len, (const char *)of->name);
break;
}
mc->fmap[f] = (int32_t)nf;
}
if (!mc->fmap) continue; /* poisoned above */
uint32_t added = 0;
for (uint32_t f = 0; f < nk->field_cnt; f++)
if (sc_find_field(ok, nk->fields[f].name, nk->fields[f].name_len) ==
WO_SCHEMA_NONE)
added++;
mc->changed = (deleted > 0 || added > 0);
/* a deleted and an added field of the SAME shape in one step is
* byte-for-byte indistinguishable from a rename, and the two
* readings differ by exactly one column of destroyed data */
if (deleted && added) {
for (uint32_t f = 0; f < ok->field_cnt && mc->fmap; f++) {
if (mc->fmap[f] != -1) continue;
for (uint32_t g = 0; g < nk->field_cnt; g++) {
if (sc_find_field(ok, nk->fields[g].name, nk->fields[g].name_len) !=
WO_SCHEMA_NONE)
continue;
if (sc_field_shape_eq(osc, &ok->fields[f], nsc, &nk->fields[g])) {
free(mc->fmap);
mc->fmap = NULL;
mc->new_cid = WO_SCHEMA_NONE;
mc->poison = sc_poisonf(
"class `%.*s`: `%.*s` was deleted and a field of the "
"same type added — a rename and a delete+add are "
"indistinguishable here and one of them destroys data. "
"Deploy the delete and the add as two separate steps",
(int)ok->name_len, (const char *)ok->name,
(int)ok->fields[f].name_len,
(const char *)ok->fields[f].name);
break;
}
}
if (!mc->fmap) break;
}
}
}
/* the embed closure: a class whose old records EMBED (owned or container
* values of) a class whose shape changed cannot be transcoded — the
* nested bytes are in the OLD sub-shape and v1 does not rewrite value
* trees recursively. Iterate to a fixpoint so chains of embedding
* poison through. */
for (int again = 1; again;) {
again = 0;
for (uint32_t c = 0; c < osc->class_cnt; c++) {
wo_mig_class *mc = &plan->classes[c];
if (mc->poison) continue;
for (uint32_t f = 0; f < osc->classes[c].field_cnt; f++) {
const wo_schema_field *of = &osc->classes[c].fields[f];
if (of->kind != WO_K_OWNED && of->kind != WO_K_MULTI &&
of->kind != WO_K_MAP)
continue;
int tainted = 0;
if (of->fclass != WO_SCHEMA_NONE && of->fclass < osc->class_cnt) {
const wo_mig_class *ref = &plan->classes[of->fclass];
tainted = ref->poison != NULL || ref->changed;
} else if (of->kind == WO_K_OWNED) {
/* an owned field with no recorded target class: assume the
* worst whenever anything at all changed */
for (uint32_t x = 0; x < osc->class_cnt && !tainted; x++)
tainted = plan->classes[x].poison != NULL ||
plan->classes[x].changed;
}
if (tainted) {
free(mc->fmap);
mc->fmap = NULL;
mc->new_cid = WO_SCHEMA_NONE;
mc->poison = sc_poisonf(
"class `%.*s`: field `%.*s` embeds a class whose shape "
"changed — its stored values carry the old sub-shape, "
"which v1 does not rewrite. Migrate the embedded class "
"on its own first",
(int)osc->classes[c].name_len,
(const char *)osc->classes[c].name, (int)of->name_len,
(const char *)of->name);
again = 1;
break;
}
}
}
}
plan->identity = 1;
for (uint32_t c = 0; c < osc->class_cnt; c++) {
const wo_mig_class *mc = &plan->classes[c];
if (mc->poison) continue; /* a poison alone never forces a rewrite */
if (mc->new_cid != c || mc->changed) {
plan->identity = 0;
break;
}
}
return 0;
}
int wo_wal_set_schema(wo_wal *w, const wo_schema *sc) {
uint8_t *p;
uint32_t len;

View file

@ -161,6 +161,37 @@ int wo_schema_encode(const wo_schema *sc, uint8_t **payload_out, uint32_t *len_o
wo_schema *wo_schema_decode(const uint8_t *payload, uint32_t len);
void wo_schema_free(wo_schema *sc);
/* databasev2 12: what boot decided about one stored class. `new_cid` is where
* its records go; WO_SCHEMA_NONE means POISONED — the class cannot be
* migrated, and `poison` says why. A poison only bites when a record of the
* class is actually met: no rows, no verdict. */
typedef struct wo_mig_class {
uint32_t new_cid; /* WO_SCHEMA_NONE = poisoned */
char *poison; /* malloc'd reason; NULL unless poisoned */
uint32_t old_field_cnt;
int32_t *fmap; /* old field index -> new slot, -1 = deleted */
int changed; /* own field set differs (add and/or delete) */
} wo_mig_class;
typedef struct wo_mig_plan {
uint32_t old_class_cnt;
wo_mig_class *classes;
/* 1 = every stored class keeps its cid and its shape: replay as-is, no
* transcode. New classes in the binary do not break identity — they have
* no records, and the head record refreshes at the next compaction. */
int identity;
} wo_mig_plan;
/* Diff the log's stored schema against the compiled one, classes matched by
* NAME, fields by NAME — so pure declaration reordering is identity apart
* from the cid map. Returns 0 with *plan filled (free with
* wo_mig_plan_free), -1 on OOM. Refusals are expressed as per-class poisons,
* not errors: retype, same-kind delete+add (a disguised rename), a vanished
* class, changed flags, and any class that EMBEDS (owned/container fields)
* a class whose shape changed — its old records encode the old sub-shape,
* which v1 does not rewrite recursively. */
int wo_schema_diff(const wo_schema *oldsc, const wo_schema *newsc, wo_mig_plan *plan);
void wo_mig_plan_free(wo_mig_plan *plan);
/* Adopt `sc` as this log's compiled schema (encoded and owned by the wal). */
int wo_wal_set_schema(wo_wal *w, const wo_schema *sc);
/* A fresh, empty log gets the schema as its first record — durable before

View file

@ -1008,6 +1008,141 @@ static void test_should_compact_absolute_and_ceiling(void) {
T_EQ(wo_wal_should_compact(2048, 1024, floor_b, 2), 0); /* not yet */
}
/* ---- databasev2 12: the boot diff --------------------------------------- */
#define SF(nm, k) {(const uint8_t *)nm, (uint32_t)(sizeof nm - 1), k, WO_SCHEMA_NONE, WO_SCHEMA_NONE}
#define SFC(nm, k, fc) {(const uint8_t *)nm, (uint32_t)(sizeof nm - 1), k, fc, WO_SCHEMA_NONE}
#define SC(nm, fl, arr) {(const uint8_t *)nm, (uint32_t)(sizeof nm - 1), fl, \
(uint32_t)(sizeof arr / sizeof arr[0]), arr}
static void test_schema_diff_verdicts(void) {
/* base: A { n: scalar, t: text }, B { part: owned->A } */
wo_schema_field a_f[] = {SF("n", WO_K_SCALAR), SF("t", WO_K_TEXT)};
wo_schema_field b_f[] = {SFC("part", WO_K_OWNED, 0)};
wo_schema_class base_c[] = {SC("A", 0, a_f), SC("B", 0, b_f)};
wo_schema base = {2, base_c, NULL};
wo_mig_plan pl;
/* 1. identical -> identity */
T_EQ(wo_schema_diff(&base, &base, &pl), 0);
T_EQ(pl.identity, 1);
T_CHECK(pl.classes[0].poison == NULL && pl.classes[1].poison == NULL);
wo_mig_plan_free(&pl);
/* 2. classes reordered -> not identity, cids remapped BY NAME, and the
owned reference (a different NUMBER now) is recognised by name too */
{
wo_schema_field b2_f[] = {SFC("part", WO_K_OWNED, 1)}; /* A is cid 1 now */
wo_schema_class swap_c[] = {SC("B", 0, b2_f), SC("A", 0, a_f)};
wo_schema swp = {2, swap_c, NULL};
T_EQ(wo_schema_diff(&base, &swp, &pl), 0);
T_EQ(pl.identity, 0);
T_EQ(pl.classes[0].new_cid, 1u); /* old A -> new cid 1 */
T_EQ(pl.classes[1].new_cid, 0u); /* old B -> new cid 0 */
T_CHECK(pl.classes[0].poison == NULL && pl.classes[1].poison == NULL);
T_CHECK(pl.classes[0].changed == 0 && pl.classes[1].changed == 0);
wo_mig_plan_free(&pl);
}
/* 3. added field -> changed, surviving map intact, B poisoned (embeds A) */
{
wo_schema_field a3_f[] = {SF("n", WO_K_SCALAR), SF("t", WO_K_TEXT),
SF("extra", WO_K_SCALAR)};
wo_schema_class c3[] = {SC("A", 0, a3_f), SC("B", 0, b_f)};
wo_schema n3 = {2, c3, NULL};
T_EQ(wo_schema_diff(&base, &n3, &pl), 0);
T_EQ(pl.identity, 0);
T_CHECK(pl.classes[0].poison == NULL && pl.classes[0].changed == 1);
T_EQ(pl.classes[0].fmap[0], 0);
T_EQ(pl.classes[0].fmap[1], 1);
T_CHECK(pl.classes[1].poison != NULL); /* embeds a changed class */
wo_mig_plan_free(&pl);
}
/* 4. deleted field -> fmap -1; different-shape delete+add migrates */
{
wo_schema_field a4_f[] = {SF("n", WO_K_SCALAR), SF("blob", WO_K_BYTES)};
wo_schema_class c4[] = {SC("A", 0, a4_f), SC("B", 0, b_f)};
wo_schema n4 = {2, c4, NULL}; /* t: Text deleted, blob: Bytes added */
T_EQ(wo_schema_diff(&base, &n4, &pl), 0);
T_CHECK(pl.classes[0].poison == NULL && pl.classes[0].changed == 1);
T_EQ(pl.classes[0].fmap[0], 0);
T_EQ(pl.classes[0].fmap[1], -1);
wo_mig_plan_free(&pl);
}
/* 5. SAME-shape delete+add -> the rename ambiguity poison */
{
wo_schema_field a5_f[] = {SF("n", WO_K_SCALAR), SF("headline", WO_K_TEXT)};
wo_schema_class c5[] = {SC("A", 0, a5_f), SC("B", 0, b_f)};
wo_schema n5 = {2, c5, NULL};
T_EQ(wo_schema_diff(&base, &n5, &pl), 0);
T_CHECK(pl.classes[0].poison != NULL);
T_CHECK(strstr(pl.classes[0].poison, "two separate steps") != NULL);
wo_mig_plan_free(&pl);
}
/* 6. retype -> poison naming the field */
{
wo_schema_field a6_f[] = {SF("n", WO_K_FLOAT), SF("t", WO_K_TEXT)};
wo_schema_class c6[] = {SC("A", 0, a6_f), SC("B", 0, b_f)};
wo_schema n6 = {2, c6, NULL};
T_EQ(wo_schema_diff(&base, &n6, &pl), 0);
T_CHECK(pl.classes[0].poison != NULL &&
strstr(pl.classes[0].poison, "`n`") != NULL);
wo_mig_plan_free(&pl);
}
/* 7. vanished class -> poison; the other class (embedding nothing that
changed shape) is untouched */
{
wo_schema_class c7[] = {SC("A", 0, a_f)};
wo_schema n7 = {1, c7, NULL};
T_EQ(wo_schema_diff(&base, &n7, &pl), 0);
T_CHECK(pl.classes[1].new_cid == WO_SCHEMA_NONE &&
pl.classes[1].poison != NULL);
T_CHECK(pl.classes[0].poison == NULL);
wo_mig_plan_free(&pl);
}
/* 8. flags change -> poison (v1 migrates fields, not storage modes) */
{
wo_schema_class c8[] = {SC("A", WO_CLASSF_RESIDENT_KEYS, a_f), SC("B", 0, b_f)};
wo_schema n8 = {2, c8, NULL};
T_EQ(wo_schema_diff(&base, &n8, &pl), 0);
T_CHECK(pl.classes[0].poison != NULL &&
strstr(pl.classes[0].poison, "storage") != NULL);
wo_mig_plan_free(&pl);
}
/* 9. a NEW class in the binary does not break identity: it has no records */
{
wo_schema_field n_f[] = {SF("x", WO_K_SCALAR)};
wo_schema_class c9[] = {SC("A", 0, a_f), SC("B", 0, b_f), SC("C", 0, n_f)};
wo_schema n9 = {3, c9, NULL};
T_EQ(wo_schema_diff(&base, &n9, &pl), 0);
T_EQ(pl.identity, 1);
wo_mig_plan_free(&pl);
}
/* 10. the embed closure is transitive: C owns B, B owns A, A changed ->
both B and C poisoned */
{
wo_schema_field cB[] = {SFC("a", WO_K_OWNED, 0)};
wo_schema_field cC[] = {SFC("b", WO_K_OWNED, 1)};
wo_schema_class oc[] = {SC("A", 0, a_f), SC("B", 0, cB), SC("C", 0, cC)};
wo_schema oldsc = {3, oc, NULL};
wo_schema_field a10[] = {SF("n", WO_K_SCALAR)}; /* t deleted */
wo_schema_class nc[] = {SC("A", 0, a10), SC("B", 0, cB), SC("C", 0, cC)};
wo_schema newsc = {3, nc, NULL};
T_EQ(wo_schema_diff(&oldsc, &newsc, &pl), 0);
T_CHECK(pl.classes[0].poison == NULL && pl.classes[0].changed == 1);
T_CHECK(pl.classes[1].poison != NULL);
T_CHECK(pl.classes[2].poison != NULL);
wo_mig_plan_free(&pl);
}
}
/* ---- databasev2 12: the schema record ---------------------------------- */
/* a hand-built two-class schema exercising every payload field */
@ -2810,6 +2945,7 @@ int main(void) {
test_keys_resident_update_field();
test_keys_resident_update_indexed();
test_keys_resident_indexed_across_flatten();
test_schema_diff_verdicts();
test_schema_roundtrip();
test_schema_fresh_log();
test_schema_compaction_adopts_legacy();