diff --git a/.gitignore b/.gitignore index c60e62e..395c5ca 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,7 @@ __pycache__/ dist/ docs/examples/*/target/ .wo-deps/ + +# Obsidian vault state (developer-local) +docs/.obsidian/ +docs/Untitled.base diff --git a/database/src/CODE-LOGIC.md b/database/src/CODE-LOGIC.md index ad5987e..119d1c0 100644 --- a/database/src/CODE-LOGIC.md +++ b/database/src/CODE-LOGIC.md @@ -61,3 +61,21 @@ rather than acknowledging what disk never got. - `just oop-e2e`, `just log-watcher` — regression that linking the engine into wovm changed nothing observable (it is dead code until Task 3 wires the first builtin). + +## The slot-level surface (arc stage 3, 2026-08-21) + +- **Why it exists:** the transparent DB actor executes a worker's + statement on the owner shard, and VM heaps are never read cross-shard — + so the requester encodes to engine slots on its own thread and the owner + executes from slots, exactly the shape WAL replay already used. +- `wo_db_val_encode` exposes the in-gate for the RPC marshaler; + `wo_db_val_clone` deep-copies an engine value (a get-field reply must + outlive the row: a later serialized statement may free the slot); + `wo_row_insert_slots` / `wo_row_update_field_slot` are the pre-encoded + twins of insert/update (slot values consumed either way — installed on + success, freed on failure); `wo_db_exec_req` (db.c) mirrors + `wo_builtin_db` case for case with slot inputs and plain outputs, so a + worker sees byte-identical traps and messages. +- The update refactor extracted `row_apply_field_slot` (the post-encode + half: unique shadow-check, index fix-up, slot swap) shared by both + entry points — the VM-value path's behavior is unchanged bit for bit. diff --git a/database/src/db.c b/database/src/db.c index 3f40d88..ad1460a 100644 --- a/database/src/db.c +++ b/database/src/db.c @@ -1,5 +1,6 @@ #include "db.h" +#include #include #include "cont.h" @@ -161,3 +162,176 @@ int wo_builtin_db(wo_vm *vm, uint64_t *R, uint32_t ins, const char **msg) { return WO_T_DB; } } + +/* ---- arc stage 3: the owner-shard executor ------------------------------ + * Mirrors the switch above case for case, with slot inputs and plain + * outputs — every trap code and message a worker sees is byte-identical to + * what the same statement would produce on the primary. */ +void wo_db_exec_req(wo_vm *vm, wo_db_req *q) { + wo_db *db = (wo_db *)vm->rt.db; + wo_wal *w = (wo_wal *)vm->rt.wal; + const char *m = "db failed"; + q->status = 0; + q->msg = ""; + if (!db) { + q->status = WO_T_DB; + q->msg = "database engine not initialized"; + goto out; + } + switch (q->op) { + case WO_B_DB_INSERT: { + int ek = 0; + uint64_t id = wo_row_insert_slots(db, q->cid, q->slots, &m, &ek); + if (!id) { + q->status = ek == DB_ERR_UNIQUE ? WO_T_UNIQUE + : ek == DB_ERR_OOM ? WO_T_OOM + : WO_T_DB; + q->msg = m; + break; + } + if (w) { + if (wo_wal_append_insert(w, db, q->cid, id) != 0 || wo_wal_commit(w) != 0) { + wo_row_remove(db, q->cid, id); + q->status = WO_T_IO; + q->msg = "wal commit failed"; + break; + } + } + q->result = id; + break; + } + case WO_B_DB_UPDATE_FIELD: { + int ek = 0; + if (wo_row_update_field_slot(db, q->cid, q->id, q->field, q->slots[0], &m, &ek) != 0) { + q->status = ek == DB_ERR_UNIQUE ? WO_T_UNIQUE : ek == DB_ERR_OOM ? WO_T_OOM : WO_T_DB; + q->msg = m; + break; + } + if (w) { + if (wo_wal_append_update(w, db, q->cid, q->id) != 0 || wo_wal_commit(w) != 0) { + q->status = WO_T_IO; + q->msg = "wal commit failed"; + break; + } + } + break; + } + case WO_B_DB_DELETE: { + if (wo_row_has_referrers(db, q->cid, q->id)) { + q->status = WO_T_FK; + q->msg = "row is still referenced (restrict)"; + break; + } + if (wo_row_remove(db, q->cid, q->id) != 0) { + q->status = WO_T_DB; + q->msg = "no such row"; + break; + } + if (w) { + if (wo_wal_append_remove(w, q->cid, q->id) != 0 || wo_wal_commit(w) != 0) { + q->status = WO_T_IO; + q->msg = "wal commit failed"; + break; + } + } + break; + } + case WO_B_DB_SCAN: + case WO_B_DB_PROBE: { + if (q->cid >= db->class_cnt) { + q->status = WO_T_DB; + q->msg = "no such class"; + break; + } + db_table *t = &db->tables[q->cid]; + uint64_t *out = NULL; + uint32_t n = 0, cap = 0; + if (t->row_size && (q->op == WO_B_DB_SCAN || q->index < t->index_cnt)) { + uint32_t col = 0; + uint8_t kind = 0; + if (q->op == WO_B_DB_PROBE) { + col = t->indexes[q->index].cols[0]; + kind = db->classes[q->cid].kinds[col]; + } + uint32_t total = t->slab_cnt * DB_SLAB_ROWS; + for (uint32_t g = 0; g < total; g++) { + if (!(t->bitmap[g >> 6] & (1ull << (g & 63)))) continue; + db_row *row = + (db_row *)(t->slabs[g / DB_SLAB_ROWS] + (size_t)(g % DB_SLAB_ROWS) * t->row_size); + if (q->op == WO_B_DB_PROBE) { + int eq; + if (kind == WO_K_TEXT || kind == WO_K_BYTES) { + /* both sides engine-encoded: the key was encoded on + * the requester's thread, the slot lives here */ + const db_text *want = (const db_text *)(uintptr_t)q->slots[0]; + const db_text *have = (const db_text *)(uintptr_t)row->slots[col]; + eq = (!want && !have) || + (want && have && want->len == have->len && + memcmp(want->bytes, have->bytes, have->len) == 0); + } else + eq = row->slots[col] == q->slots[0]; + if (!eq) continue; + } + if (n == cap) { + uint32_t ncap = cap ? cap * 2 : 16; + uint64_t *no = realloc(out, (size_t)ncap * 8u); + if (!no) { + free(out); + out = NULL; + q->status = WO_T_OOM; + q->msg = "out of memory"; + break; + } + out = no; + cap = ncap; + } + out[n++] = row->id; + } + } + if (!q->status) { + q->ids = out; + q->id_cnt = n; + } + break; + } + case WO_B_DB_GET_FIELD: { + if (q->cid >= db->class_cnt || q->field >= db->classes[q->cid].field_cnt) { + q->status = WO_T_DB; + q->msg = "no such field"; + break; + } + db_row *row = wo_row_ptr(db, q->cid, q->id); + if (!row) { + q->status = WO_T_DB; + q->msg = "no such row"; + break; + } + int ok = 1; + q->val_kind = db->classes[q->cid].kinds[q->field]; + q->val = wo_db_val_clone(db->classes, q->val_kind, row->slots[q->field], &ok); + if (!ok) { + q->status = WO_T_OOM; + q->msg = "out of memory"; + } + break; + } + default: + q->status = WO_T_DB; + q->msg = "unknown db builtin"; + break; + } +out: + /* slot VALUES were consumed by the ops above (insert/update install or + * free them); the PROBE key is ours to free, the array always is. (The + * requester only encodes a key for an index its identical class table + * declares, so a keyed request always finds its kind here.) */ + if (db && q->op == WO_B_DB_PROBE && q->slots && q->cid < db->class_cnt) { + db_table *t = &db->tables[q->cid]; + if (t->row_size && q->index < t->index_cnt) + wo_db_val_free(db, db->classes[q->cid].kinds[t->indexes[q->index].cols[0]], + q->slots[0]); + } + free(q->slots); + q->slots = NULL; + q->slot_cnt = 0; +} diff --git a/database/src/db.h b/database/src/db.h index 95eb97d..db3a0c4 100644 --- a/database/src/db.h +++ b/database/src/db.h @@ -21,4 +21,36 @@ int wo_builtin_db(wo_vm *vm, uint64_t *R, uint32_t ins, const char **msg); +/* ---- arc stage 3: one marshaled DB statement (the transparent DB actor). + * A worker shard fills the request on ITS thread — args pre-encoded into + * engine slots, since VM heaps are never read cross-shard — and ships it + * to shard 0 in an envelope; the owner executes it via wo_db_exec_req and + * ships it back. Ownership: `slots` VALUES pass to the owner (consumed by + * the op), the array and every reply buffer pass back to the requester. + * The envelope handoff (mutex + eventfd) orders `done` on both sides. */ +typedef struct wo_db_req { + /* request */ + uint32_t op; /* WO_B_DB_INSERT..WO_B_DB_PROBE */ + uint32_t cid, field, index; + uint64_t id; + uint64_t *slots; /* INSERT: field_cnt; UPDATE: 1; PROBE: 1 (the key) */ + uint32_t slot_cnt; + /* routing */ + uint32_t from_shard; + void *fiber; /* the parked wo_fiber*, opaque to the engine */ + int done; + /* reply */ + int status; /* 0 ok, else the WO_T_* the local path would trap */ + const char *msg; /* static literal, safe cross-thread */ + uint64_t result; /* INSERT: the new id */ + uint64_t *ids; /* SCAN/PROBE: malloc'd id list */ + uint32_t id_cnt; + uint8_t val_kind; /* GET_FIELD: a cloned engine value */ + uint64_t val; +} wo_db_req; + +/* Execute one marshaled statement on the OWNER shard (vm = shard 0's; its + * rt.db/rt.wal are the engine). Fills the reply fields; never traps. */ +void wo_db_exec_req(wo_vm *vm, wo_db_req *q); + #endif /* WO_DB_H */ diff --git a/database/src/table.c b/database/src/table.c index 82f60fa..3b2899e 100644 --- a/database/src/table.c +++ b/database/src/table.c @@ -591,6 +591,56 @@ uint64_t wo_row_insert(wo_db *db, uint32_t class_id, const uint64_t *vals, return r->id; } +uint64_t wo_row_insert_slots(wo_db *db, uint32_t class_id, const uint64_t *slots, + const char **msg, int *err_kind) { + if (err_kind) *err_kind = DB_ERR_MISC; + db_table *t = table_of(db, class_id); + const wo_classdesc *c = class_id < db->class_cnt ? &db->classes[class_id] : NULL; + if (!t || !c) { + /* slot kinds unknowable without the class: the values leak rather + than die by the wrong kind (defensive; the requester validated) */ + *msg = "no such class"; + return 0; + } + uint32_t g = slot_alloc(t); + if (g == UINT32_MAX) { + for (uint32_t j = 0; j < c->field_cnt; j++) db_val_free(c->kinds[j], slots[j]); + if (err_kind) *err_kind = DB_ERR_OOM; + *msg = "out of memory growing a table"; + return 0; + } + db_row *r = slot_row(t, g); + r->class_id = class_id; + r->flags = 0; + memcpy(r->slots, slots, (size_t)c->field_cnt * 8u); + r->id = t->next_id; + t->next_id += db->nshards; + if (hput(t, r->id, (uint64_t)g + 1) != 0) { + for (uint32_t j = 0; j < c->field_cnt; j++) db_val_free(c->kinds[j], r->slots[j]); + t->next_id -= db->nshards; + if (t->free_cnt < t->free_cap) t->free_slots[t->free_cnt++] = g; + if (err_kind) *err_kind = DB_ERR_OOM; + *msg = "out of memory indexing a row"; + return 0; + } + t->bitmap[g >> 6] |= 1ull << (g & 63); + t->count++; + int irc = idx_add_row(db, t, r); + if (irc != 0) { + t->bitmap[g >> 6] &= ~(1ull << (g & 63)); + hdel(t, r->id); + t->count--; + t->next_id -= db->nshards; /* the id was never observable: reclaim it */ + for (uint32_t j = 0; j < c->field_cnt; j++) db_val_free(c->kinds[j], r->slots[j]); + if (t->free_cnt < t->free_cap) t->free_slots[t->free_cnt++] = g; + if (err_kind) *err_kind = irc; + *msg = irc == DB_ERR_UNIQUE ? "unique index violation" : "out of memory indexing a row"; + return 0; + } + if (err_kind) *err_kind = DB_ERR_NONE; + return r->id; +} + db_row *wo_row_ptr(wo_db *db, uint32_t class_id, uint64_t id) { if (class_id >= db->class_cnt) return NULL; db_table *t = &db->tables[class_id]; @@ -645,12 +695,101 @@ void wo_db_val_free(wo_db *db, uint8_t kind, uint64_t v) { db_val_free(kind, v); } +uint64_t wo_db_val_encode(const wo_classdesc *classes, uint8_t kind, uint64_t vm_val, + int *ok, const char **msg) { + return db_val_encode(classes, kind, vm_val, ok, msg); +} + +/* Deep engine-to-engine copy; shapes mirror db_val_free's recursion. */ +uint64_t wo_db_val_clone(const wo_classdesc *classes, uint8_t kind, uint64_t v, int *ok) { + *ok = 1; + if (!v) return 0; + switch (kind) { + case WO_K_SCALAR: + case WO_K_FLOAT: return v; + case WO_K_TEXT: + case WO_K_BYTES: { + const db_text *s = (const db_text *)(uintptr_t)v; + db_text *t = malloc(sizeof(db_text) + s->len); + if (!t) goto oom; + t->len = s->len; + memcpy(t->bytes, s->bytes, s->len); + return (uint64_t)(uintptr_t)t; + } + case WO_K_OWNED: { + const db_rec *s = (const db_rec *)(uintptr_t)v; + const wo_classdesc *c = &classes[s->class_id]; + db_rec *r = malloc(sizeof(db_rec) + (size_t)c->field_cnt * 8u); + if (!r) goto oom; + r->class_id = s->class_id; + r->_pad = 0; + for (uint32_t i = 0; i < c->field_cnt; i++) { + r->slots[i] = wo_db_val_clone(classes, c->kinds[i], s->slots[i], ok); + if (!*ok) { + for (uint32_t j = 0; j < i; j++) db_val_free(c->kinds[j], r->slots[j]); + free(r); + return 0; + } + } + return (uint64_t)(uintptr_t)r; + } + case WO_K_MULTI: { + const db_multi *s = (const db_multi *)(uintptr_t)v; + db_multi *d = malloc(sizeof(db_multi) + (size_t)s->len * 8u); + if (!d) goto oom; + d->elem_kind = s->elem_kind; + d->len = s->len; + for (uint32_t i = 0; i < s->len; i++) { + d->items[i] = wo_db_val_clone(classes, s->elem_kind, s->items[i], ok); + if (!*ok) { + for (uint32_t j = 0; j < i; j++) db_val_free(d->elem_kind, d->items[j]); + free(d); + return 0; + } + } + return (uint64_t)(uintptr_t)d; + } + case WO_K_MAP: { + const db_map *s = (const db_map *)(uintptr_t)v; + db_map *d = malloc(sizeof(db_map) + (size_t)s->len * 16u); + if (!d) goto oom; + d->key_kind = s->key_kind; + d->val_kind = s->val_kind; + d->len = s->len; + for (uint32_t i = 0; i < s->len; i++) { + d->kv[2 * i] = wo_db_val_clone(classes, s->key_kind, s->kv[2 * i], ok); + uint64_t dv = 0; + if (*ok) dv = wo_db_val_clone(classes, s->val_kind, s->kv[2 * i + 1], ok); + d->kv[2 * i + 1] = dv; + if (!*ok) { + for (uint32_t j = 0; j <= i; j++) { + db_val_free(d->key_kind, d->kv[2 * j]); + db_val_free(d->val_kind, d->kv[2 * j + 1]); + } + free(d); + return 0; + } + } + return (uint64_t)(uintptr_t)d; + } + default: return v; /* GCREF never stored; nothing to clone */ + } +oom: + *ok = 0; + return 0; +} + uint64_t wo_val_decode_vm(wo_db *db, wo_rt *rt, uint8_t kind, uint64_t engine_val, int *ok, const char **msg) { (void)db; return db_val_decode(rt, kind, engine_val, ok, msg); } +static int row_apply_field_slot(wo_db *db, db_table *t, const wo_classdesc *c, + db_row *r, uint32_t class_id, uint64_t id, + uint32_t field, uint64_t nv, const char **msg, + int *err_kind); + int wo_row_update_field(wo_db *db, uint32_t class_id, uint64_t id, uint32_t field, uint64_t vm_val, const char **msg, int *err_kind) { if (err_kind) *err_kind = DB_ERR_MISC; @@ -671,6 +810,16 @@ int wo_row_update_field(wo_db *db, uint32_t class_id, uint64_t id, uint32_t fiel if (err_kind) *err_kind = DB_ERR_BADKIND; return -1; } + return row_apply_field_slot(db, t, c, r, class_id, id, field, nv, msg, err_kind); +} + +/* The post-encode half of an update: unique shadow-check, index fix-up, + * slot swap. Consumes [nv] (installed on success, freed on failure) — + * shared by the VM-value wrapper above and the RPC slot path. */ +static int row_apply_field_slot(wo_db *db, db_table *t, const wo_classdesc *c, + db_row *r, uint32_t class_id, uint64_t id, + uint32_t field, uint64_t nv, const char **msg, + int *err_kind) { /* indexes containing this column: unique checks against the NEW value run first, against a shadow of the row, before anything mutates */ uint64_t old = r->slots[field]; @@ -738,6 +887,31 @@ int wo_row_update_field(wo_db *db, uint32_t class_id, uint64_t id, uint32_t fiel return 0; } +int wo_row_update_field_slot(wo_db *db, uint32_t class_id, uint64_t id, uint32_t field, + uint64_t slot, const char **msg, int *err_kind) { + if (err_kind) *err_kind = DB_ERR_MISC; + /* bounds first: the RPC requester validated cid/field to encode at all, + so these are defensive; the slot's kind is unknowable on a class + violation and the value leaks rather than dies by the wrong kind */ + if (class_id >= db->class_cnt) { + *msg = "no such class"; + return -1; + } + const wo_classdesc *c = &db->classes[class_id]; + if (field >= c->field_cnt) { + *msg = "no such field"; + return -1; + } + db_row *r = wo_row_ptr(db, class_id, id); + if (!r) { + db_val_free(c->kinds[field], slot); + *msg = "no such row"; + return -1; + } + return row_apply_field_slot(db, &db->tables[class_id], c, r, class_id, id, + field, slot, msg, err_kind); +} + int wo_row_has_referrers(wo_db *db, uint32_t class_id, uint64_t id) { if (!id) return 0; for (uint32_t c = 0; c < db->class_cnt; c++) { diff --git a/database/src/table.h b/database/src/table.h index cf0dd62..b130bfa 100644 --- a/database/src/table.h +++ b/database/src/table.h @@ -193,4 +193,32 @@ uint64_t wo_val_decode_vm(wo_db *db, wo_rt *rt, uint8_t kind, uint64_t engine_va * (0 ok, -1). */ int wo_row_raw_commit(wo_db *db, uint32_t class_id, db_row *r); +/* ---- arc stage 3: the slot-level surface the transparent DB RPC uses ---- + * VM heaps are never read cross-shard (a worker's GC writes header mark + * bits concurrently), so the REQUESTER shard encodes its VM values into + * engine-owned slots on its own thread and ships those; the OWNER shard + * executes from slots. Everything here is thread-agnostic: it touches only + * the wo_db it is handed and engine-owned mallocs. */ + +/* Encode one VM value into an engine slot on the caller's thread (the + * in-gate, split out of wo_row_insert for the RPC path). */ +uint64_t wo_db_val_encode(const wo_classdesc *classes, uint8_t kind, uint64_t vm_val, + int *ok, const char **msg); + +/* Deep-copy one engine value — a GET_FIELD reply must outlive the row it + * was read from (a later statement may free the row's slot). */ +uint64_t wo_db_val_clone(const wo_classdesc *classes, uint8_t kind, uint64_t v, int *ok); + +/* Insert from PRE-ENCODED slots (field_cnt of them). Ownership of the slot + * VALUES transfers: installed on success, freed on failure. New id, or 0 + * with *msg / *err_kind set exactly as wo_row_insert sets them. */ +uint64_t wo_row_insert_slots(wo_db *db, uint32_t class_id, const uint64_t *slots, + const char **msg, int *err_kind); + +/* Update one field from a PRE-ENCODED slot value (consumed either way: + * installed on success, freed on failure). Same contract as + * wo_row_update_field after its encode. */ +int wo_row_update_field_slot(wo_db *db, uint32_t class_id, uint64_t id, uint32_t field, + uint64_t slot, const char **msg, int *err_kind); + #endif /* WO_TABLE_H */ diff --git a/docs/00-code-review.md b/docs/00-code-review.md index 1674f2e..8123503 100644 --- a/docs/00-code-review.md +++ b/docs/00-code-review.md @@ -4,7 +4,7 @@ This document was a gap analysis of what the `woc` front end needs before the log-watcher sample compiles. Its findings were extracted on 2026-08-10 into [`superpowers/specs/2026-08-10-logwatcher-gap-closure-design.md`](superpowers/specs/2026-08-10-logwatcher-gap-closure-design.md) and the plans it amends; its Phase 1–4 roadmap is retired in favour of the -approved story iterations. See [`00-status.md`](00-status.md) for current +approved story iterations. See [`00-status.md`](stories/00-status.md) for current status. ## Standing critique (undated, author unrecorded) diff --git a/docs/00-dependency-graph.md b/docs/00-dependency-graph.md index 850e701..bcd83ca 100644 --- a/docs/00-dependency-graph.md +++ b/docs/00-dependency-graph.md @@ -1,6 +1,6 @@ # Dependency graphs — iterations and framework features -> Companion to [00-status.md](00-status.md) (states live THERE; this page +> Companion to [00-status.md](stories/00-status.md) (states live THERE; this page > carries the edges). An arrow `A --> B` means **A must exist before B**; > a dashed arrow is a scope DIRECTIVE, not a technical dependency. Use it > to pick the next implementation: anything whose incoming arrows are all @@ -33,10 +33,10 @@ flowchart TD I9c["20 cross-program tables (half-built)"]:::open I9d["21 keypair attach auth (half-built; crypto+handshake already on its branch)"]:::open I9e["22 durability + throughput baseline"]:::open - I8["8 shard-actor runtime"]:::open + I8["8 shard-actor runtime ✅ 2026-08-21"]:::done I9f["23 io_uring group-commit"]:::open I10["10 HTTP service layer (lowers onto the framework)"]:::open - I11["11 fibers"]:::open + I11["11 fibers ✅ 2026-08-21"]:::done I12["12 blue-green deploy"]:::open I13["13 metaprogramming @derive"]:::open I14["14 skillhost workload (demoted)"]:::open diff --git a/docs/00-principles.md b/docs/00-principles.md index 24ee993..6c107e2 100644 --- a/docs/00-principles.md +++ b/docs/00-principles.md @@ -54,8 +54,9 @@ Cross-shard work is a message send that moves ownership. There is no `Arc>` anywhere and never will be. *Why:* sharing mutable state buys contention, locks, and heisenbugs; moving ownership buys linear scaling and per-shard GC. -*Enforced by:* [plan 09](plan/09-concurrency-scaleout.md) (shipped on the -Rust runtime), [the shard-actor plan](superpowers/plans/2026-08-01-shard-actor-vm-runtime.md). +*Enforced by:* [the shard-fiber arc plan](superpowers/plans/2026-08-20-shard-fiber-arc.md) +(stages 1+2 landed; supersedes the discarded 2026-08-01 shard-actor plan +and the Rust-era plan 09, removed with that track 2026-08-18). ## 6. The runtime never stops @@ -74,8 +75,10 @@ before acknowledgment; boot replays the log. Mirrors (Postgres) are reconstructible backups that reads and acks never depend on. *Why:* one source of truth with predictable latency; durability is a sequential append, not a storage engine bolted to the side. -*Enforced by:* [plan 11](plan/11-wal-and-recovery.md), -[plan 16](plan/16-postgres-mirror.md) (mirror-is-backup doctrine). +*Enforced by:* [the db-engine binding plan](superpowers/plans/2026-08-01-db-engine-binding.md) +(typed WAL + boot replay, shipped); the mirror-is-backup doctrine is +recorded in [`plan/discarded.md`](plan/discarded.md) (the Rust-era WAL +and mirror plans 11/16 were removed with that track 2026-08-18). ## 8. Samples force the grammar @@ -84,7 +87,8 @@ directory is the de facto integration suite, and new surface is proven by re-expressing real workloads (blog, ecommerce, pricing, log-watcher). *Why:* grammars designed in the abstract grow features nobody needs and miss the ones real programs demand. -*Enforced by:* [the blog sample](examples/blog/README.md), +*Enforced by:* [the web-app sample](examples/web-app/README.md) +(the blog sample left with the Rust track), the sample-workload acceptance in [the systems-track spec](superpowers/specs/2026-08-01-systems-track-design.md). ## 9. Linux is the target diff --git a/docs/08-project-structure.md b/docs/08-project-structure.md index f388b8a..8aeb388 100644 --- a/docs/08-project-structure.md +++ b/docs/08-project-structure.md @@ -2,7 +2,7 @@ Canonical map of the repository: what every root directory is and who writes to it. Companion to [`CLAUDE.md`](../CLAUDE.md) (working rules), the status board -([`00-status.md`](00-status.md)), and the story arc +([`00-status.md`](stories/00-status.md)), and the story arc ([`stories/language-runtime-database/00-story.md`](stories/language-runtime-database/00-story.md)). writeonce is **one compiled language, one runtime, one embedded database, one diff --git a/docs/examples/db-actor/main.wo b/docs/examples/db-actor/main.wo new file mode 100644 index 0000000..e879608 --- /dev/null +++ b/docs/examples/db-actor/main.wo @@ -0,0 +1,62 @@ +use time + +-- db-actor — arc stage 3's acceptance workload: the database is an +-- actor on the owner shard (shard 0); a spawned actor placed on ANY +-- shard reads and writes it through transparent RPC. Before stage 3 a +-- worker-shard insert traps WO_T_DB ("database engine not +-- initialized"); after, this program's output is shard-placement- +-- independent: two writer lines and one exact main line. + +@table(name: "notes", index: [tag]) +class Note { + tag: Text + val: Int +} + +class Job { + n: Int +} + +-- Each writer inserts one row, then scans the whole table. Placement is +-- round-robin, so with two writers at default shards one lands off the +-- primary — the RPC path under test. +class Writer { + pad: Int + fn receive(msg: Job) { + insert Note { tag: "w", val: msg.n }; + let total = 0; + for x in from n in Note select n { + total = total + x.val; + } + print("writer ${msg.n} sees sum ${total}"); + } +} + +fn main() -> Int { + let a: actor Job = spawn Writer { pad: 0 }; + let b: actor Job = spawn Writer { pad: 1 }; + send(a, Job { n: 1 }); + send(b, Job { n: 2 }); + -- no request/response surface yet (iteration 31): poll until both rows + -- landed, then give the writers' own prints a beat before main returns + -- (main-return reaps every other fiber, mid-print included) + let tries = 0; + let count = 0; + while count < 2 and tries < 200 { + time.sleep(10); + count = 0; + for x in from n in Note select n { + count = count + 1; + } + tries = tries + 1; + } + time.sleep(1000); + count = 0; + let total = 0; + for x in from n in Note select n { + count = count + 1; + total = total + x.val; + } + print("main sees ${count} rows, sum ${total}"); + return 0; +} diff --git a/docs/examples/db-actor/wo.toml b/docs/examples/db-actor/wo.toml new file mode 100644 index 0000000..a939d9d --- /dev/null +++ b/docs/examples/db-actor/wo.toml @@ -0,0 +1,6 @@ +name = "db-actor" +version = "0.1.0" +description = "arc stage 3 proof: actors on worker shards read and write the database through the DB actor" + +[runtime] +wo = ">= 0.1" diff --git a/docs/examples/writeonce-framework/README.md b/docs/examples/writeonce-framework/README.md index 590e510..64baab9 100644 --- a/docs/examples/writeonce-framework/README.md +++ b/docs/examples/writeonce-framework/README.md @@ -17,7 +17,9 @@ writeonce-framework = { git = "https://github.com/shoneyj/writeonce-framework", Connection policy: **pipelined requests are served on one connection; idle connections close after the response** — on a single-threaded server a parked keep-alive connection would block `accept` and starve every - other client, so closing is the correct shape until shards/fibers (8/11). + other client, so closing is the correct shape until fiber-per-connection + serving lands (the arc — 8/11 — landed 2026-08-21; the serve-loop slice + that consumes it is iteration 24's). A proxy in front simply reconnects. - **Router** (`router/`): method + path table with `:param` captures into `req.params`; first match wins; a known path with the wrong method is @@ -51,7 +53,8 @@ writeonce-framework = { git = "https://github.com/shoneyj/writeonce-framework", ## Honest limits (v1, all deliberate) - **Single-threaded, blocking** — one request at a time. Concurrency arrives - underneath this same surface with the shard/fiber iterations (8/11). + underneath this same surface now that the arc (8/11) has landed + (2026-08-21); the switch itself rides iteration 24's serving slice. - **TLS: none, anywhere.** Deploy behind nginx/caddy; the proxy terminates TLS+ALPN and gives browsers HTTP/2 while this backend speaks HTTP/1.1 keep-alive. See the web-app sample's README for the nginx sketch. @@ -77,7 +80,7 @@ first (pure `.wo` cannot express it yet). | Item | State | | --- | --- | | HTTP/1.1 parsing | 🔶 parses + 400-and-survive; STRICT ambiguity rejection (duplicate/conflicting `Content-Length`, oversize checks beyond BODY_MAX) not audited — hardening slice | -| Keep-alive | ✅ pipelined-serve / close-when-idle (starvation-honest until 8/11) | +| Keep-alive | ✅ pipelined-serve / close-when-idle (arc landed 2026-08-21; retirement of close-when-idle rides iteration 24's fiber-per-connection slice) | | Read/write/idle timeouts | 🔧 `net` has no timeout surface — runtime seam, then a framework knob | | Request size limits | ✅ BODY_MAX bounds headers AND body | | Unix socket binding | 🔧 `net.listen` is TCP-only — runtime seam | @@ -102,7 +105,7 @@ first (pure `.wo` cannot express it yet). | Content negotiation | 🔶 `media_type(req)` covers the request side; `Accept`-driven response negotiation ⬜ | | Trusted-proxy client IP | 🔶 `X-Forwarded-For/-Proto` parsing is expressible (candidate slice); VERIFYING the peer is the trusted proxy needs a peer-address runtime seam 🔧 | | Status/header setting · redirects | ✅ builders + `set_header` | -| Lazy body streaming + backpressure · streaming responses · explicit commit point | ⏸ 8/11 — whole bodies, one write, by design | +| Lazy body streaming + backpressure · streaming responses · explicit commit point | ⏸ UNBLOCKED by the arc (8/11 landed 2026-08-21) — stays parked until its own slice | | ETag + conditional requests | ⬜ candidate; wants the crypto slice's hashing | ### Context & middleware diff --git a/docs/in-progress/2026-08-21-db-bench.md b/docs/in-progress/2026-08-21-db-bench.md new file mode 100644 index 0000000..63d7db1 --- /dev/null +++ b/docs/in-progress/2026-08-21-db-bench.md @@ -0,0 +1,36 @@ +# In progress — iteration 22: db-bench, the measurement backbone + +> **Status: 🔄 in progress** (spec + plan approved 2026-08-21) — second +> slice of the concurrency chain **✅ stage 3 → 22 → 31 → 24 → 23 → 32**. +> Board: [../stories/00-status.md](../stories/00-status.md). +> +> One marker doc per active slice; deleted when the slice lands. + +## What + +Execute [`superpowers/plans/2026-08-21-db-bench.md`](../superpowers/plans/2026-08-21-db-bench.md) +(spec: [`2026-08-21-db-bench-design.md`](../superpowers/specs/2026-08-21-db-bench-design.md), +approved): the `time.ticks` µs builtin, the `docs/examples/db-bench` +sample (seed/read/query/write/mix/msgrate/verify), the campaign driver +with relative gates vs `bench/baseline.json`, the restart proof and +kill -9 battery at both shard counts, and the first committed baseline. + +## Why now + +Nothing performance-shaped is sourced until this runs — and the arc owes +its before/after. One campaign now covers single- AND multi-shard +honestly (stage 3 landed), prices the RPC, and produces the mutex-inbox +number stage-2's deviation waits on. + +## Story + +- [22 — durability, throughput, scale](../stories/language-runtime-database/in-progress/22-durability-throughput-scale.md) + +## Definition of done + +- Plan Tasks 1–6 checked; `just db-bench` exit 0 twice in a row; + gate-bites smoke proven (doctored results FAIL); baseline committed + with rationale; arc delta + msgrate copied into story 8; full battery + green. +- Story 22 → done/; board standup written from the actual numbers; this + file deleted. Next slice: iteration 31 (actor lifecycle). diff --git a/docs/plan/compiler/2026-08-01-haxe-parity-language.md b/docs/plan/compiler/2026-08-01-haxe-parity-language.md index f4ff3e2..79c9709 100644 --- a/docs/plan/compiler/2026-08-01-haxe-parity-language.md +++ b/docs/plan/compiler/2026-08-01-haxe-parity-language.md @@ -1,6 +1,6 @@ # Haxe-Parity Language Adoptions Implementation Plan -> **Status: ✅ COMPLETE 2026-08-20** (story iteration 5, branch `language-surface-strictness`) — every task closed. Tasks 1–4 ✅ (modules, small surface, switch expressions, records+variants); Task 5 ✅ try/catch (2026-08-14); Task 6 ✅ `?T` forced handling (2026-08-18, WO-E211/212/213 + narrowing); Task 7 ✅ statics + `pub(read)` syntax (2026-08-14), write enforcement WO-E219 and `using` extensions with WO-E220 collision (2026-08-20 — compile-time rewrite to a free-fn call, owner/emit untouched); Task 8 ✅ reject rows WO-E105 (2026-08-18) and `#if` build flags (`woc -D name`, token-level, WO-E003 misuse; 2026-08-20). `is`/`throw` cut, `abstract` rejected. Board: [00-status.md](../../00-status.md) +> **Status: ✅ COMPLETE 2026-08-20** (story iteration 5, branch `language-surface-strictness`) — every task closed. Tasks 1–4 ✅ (modules, small surface, switch expressions, records+variants); Task 5 ✅ try/catch (2026-08-14); Task 6 ✅ `?T` forced handling (2026-08-18, WO-E211/212/213 + narrowing); Task 7 ✅ statics + `pub(read)` syntax (2026-08-14), write enforcement WO-E219 and `using` extensions with WO-E220 collision (2026-08-20 — compile-time rewrite to a free-fn call, owner/emit untouched); Task 8 ✅ reject rows WO-E105 (2026-08-18) and `#if` build flags (`woc -D name`, token-level, WO-E003 misuse; 2026-08-20). `is`/`throw` cut, `abstract` rejected. Board: [00-status.md](../../stories/00-status.md) > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. > diff --git a/docs/plan/compiler/2026-08-01-wob-emit-e2e-single-binary.md b/docs/plan/compiler/2026-08-01-wob-emit-e2e-single-binary.md index 63d7c43..0485221 100644 --- a/docs/plan/compiler/2026-08-01-wob-emit-e2e-single-binary.md +++ b/docs/plan/compiler/2026-08-01-wob-emit-e2e-single-binary.md @@ -1,6 +1,6 @@ # Bytecode Emit + End-to-End Corpus + Single Binary Implementation Plan -> **Status: ✅ done** (story iteration 4) — Tasks 1–6 + 8 shipped: bytecode emitter, disassembler (`--dump-bc`), three-kind conformance harness (`just oop-e2e`), `woc build` single-binary output, `WO-E405`. Task 7 (parity harness against the Rust runtime) **deferred by explicit user decision** — the two stacks diverge by design. Milestone-1 acceptance: all five criteria met. Board: [00-status.md](../../00-status.md) +> **Status: ✅ done** (story iteration 4) — Tasks 1–6 + 8 shipped: bytecode emitter, disassembler (`--dump-bc`), three-kind conformance harness (`just oop-e2e`), `woc build` single-binary output, `WO-E405`. Task 7 (parity harness against the Rust runtime) **deferred by explicit user decision** — the two stacks diverge by design. Milestone-1 acceptance: all five criteria met. Board: [00-status.md](../../stories/00-status.md) > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. > diff --git a/docs/plan/compiler/2026-08-01-woc-compiler-front.md b/docs/plan/compiler/2026-08-01-woc-compiler-front.md index d1cad18..4093c5e 100644 --- a/docs/plan/compiler/2026-08-01-woc-compiler-front.md +++ b/docs/plan/compiler/2026-08-01-woc-compiler-front.md @@ -1,6 +1,6 @@ # woc Compiler Front (OCaml) Implementation Plan -> **Status: ✅ done** (story iteration 3) — Tasks 1–8 shipped: dune scaffold, `diag`, newline-significant lexer, declaration + statement/expression parser with multi-error recovery, typechecker (field kinds, `?T` plumbing, W201, E225, E214), MVS ownership pass with the four emitter tables, driver with directory discovery. `?T` semantics and eight other `WO-E2xx` codes stayed unenforced — carried as named known gaps, not silently owed. Board: [00-status.md](../../00-status.md) +> **Status: ✅ done** (story iteration 3) — Tasks 1–8 shipped: dune scaffold, `diag`, newline-significant lexer, declaration + statement/expression parser with multi-error recovery, typechecker (field kinds, `?T` plumbing, W201, E225, E214), MVS ownership pass with the four emitter tables, driver with directory discovery. `?T` semantics and eight other `WO-E2xx` codes stayed unenforced — carried as named known gaps, not silently owed. Board: [00-status.md](../../stories/00-status.md) > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. > diff --git a/docs/plan/compiler/2026-08-14-logwatcher-executable.md b/docs/plan/compiler/2026-08-14-logwatcher-executable.md index 0fe6a3d..c9b6e0f 100644 --- a/docs/plan/compiler/2026-08-14-logwatcher-executable.md +++ b/docs/plan/compiler/2026-08-14-logwatcher-executable.md @@ -3,7 +3,7 @@ > **Status: ✅ done 2026-08-15** (story iteration 7) — all six tasks landed: > the sample compiles, runs, stops on SIGTERM, holds RSS and descriptors flat > under sustained load in all three modes, and the soak that proves it is in -> the acceptance script. Board: [00-status.md](../../00-status.md) +> the acceptance script. Board: [00-status.md](../../stories/00-status.md) > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. > diff --git a/docs/plan/compiler/2026-08-15-employee-relations-query.md b/docs/plan/compiler/2026-08-15-employee-relations-query.md index 37de242..966752a 100644 --- a/docs/plan/compiler/2026-08-15-employee-relations-query.md +++ b/docs/plan/compiler/2026-08-15-employee-relations-query.md @@ -3,7 +3,7 @@ > **Status: ⬜ pending** (story iteration 9b) — blocked on iteration 9's engine > plan ([`2026-08-01-db-engine-binding.md`](../../superpowers/plans/2026-08-01-db-engine-binding.md)): > Tasks 3–6 below consume its row storage, WAL, indexes and select subset. -> Board: [00-status.md](../../00-status.md) +> Board: [00-status.md](../../stories/00-status.md) > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. > diff --git a/docs/plan/discarded.md b/docs/plan/discarded.md index ca189cf..90765a7 100644 --- a/docs/plan/discarded.md +++ b/docs/plan/discarded.md @@ -5,7 +5,7 @@ exists so a settled question is not re-proposed. If you want to revisit an entry, argue against the reason recorded here — do not re-open it as if it were new. -Status board: [`00-status.md`](../00-status.md) · Doctrine: [`../00-principles.md`](../00-principles.md) +Status board: [`00-status.md`](../stories/00-status.md) · Doctrine: [`../00-principles.md`](../00-principles.md) ## Language surface @@ -54,4 +54,5 @@ Status board: [`00-status.md`](../00-status.md) · Doctrine: [`../00-principles. | **Raw code in plan documents** | Plans carry concept, reason, and required behavior in words; the executor writes the code. | | **`##ui` / `.htmlx` LiveView frontend track** | 2026-08-17: removed the 9-doc `exploration/ui/` design set, the `14-mvc-ui-implementation` plan, and the `ui-htmlx-live` plan. All were built on the non-advancing Rust runtime (`.dev/reference/crates/wo-htmlx`, `cargo run`, WebSocket live-patches) and contradict the current woc/wovm direction. The 13d pricing-UI row went with them. Revisit only if a UI story is re-opened on the woc/wovm stack. | | **Old-runtime "front door" + v1 design docs** | 2026-08-17: removed `writeonce-pl.md`, `runtime/wo-language.md`, `future-scope/ai-agents-content-management.md`, the numbered v1 set `02-recovery`/`03-data`/`04-ui`/`05-datalayer`/`06-markdown-render`/`07-ssl`, and `runtime/database/05-go-sdk.md`. They pitched the old Rust `wo` runtime (REST + LiveView + SQL/Cypher) as the current language and contradicted the shipped woc/wovm toolchain. | +| **The 2026-08-01 shard-actor plan (epoll-based)** | 2026-08-21: [`superpowers/plans/2026-08-01-shard-actor-vm-runtime.md`](../superpowers/plans/2026-08-01-shard-actor-vm-runtime.md) marked discarded, file kept as reference. Superseded by the arc plan of record ([`2026-08-20-shard-fiber-arc.md`](../superpowers/plans/2026-08-20-shard-fiber-arc.md), stages 1+2 landed): io_uring is a MUST and the epoll-based approach is discarded — the old plan's "epoll now / io_uring later" premise is inverted, and its substrate (`runtime/wo-rt.c`) left with the Rust track 2026-08-18. | | **The entire Rust `wo` runtime track** | 2026-08-18: removed `crates/` (the Stage-2 Rust runtime), `Cargo.toml`/`Cargo.lock`, `prototypes/` (wo-rt-c stale duplicate + wo-db C++ ref), the `rt-c-*` justfile recipes, the Rust engineering plans (`docs/plan/05..16`, `docs/plan/done/`), and `docs/runtime/` (the old runtime overview + 7-phase DB design series + async/fibers/gc/surreal essays). It was the prior, abandoned architecture — fully independent of the woc/wovm stack. Master now reflects only the current single-language project; the removed track lives in git history if ever needed as reference. Kept: the syscall/postgres/assembly/c-runtime **exploration studies** (they fed the current C runtime) and the discarded/learnings registers. | diff --git a/docs/plan/exploration/c-runtime/00-plan.md b/docs/plan/exploration/c-runtime/00-plan.md index a459188..f39af61 100644 --- a/docs/plan/exploration/c-runtime/00-plan.md +++ b/docs/plan/exploration/c-runtime/00-plan.md @@ -1,6 +1,6 @@ # wo-rt-c roadmap — multi-threaded io_uring RAM database runtime, in C -> **Status: ✅ done** — phases A–F all shipped with measured exit evidence below. Board: [00-status.md](../../../00-status.md) +> **Status: ✅ done** — phases A–F all shipped with measured exit evidence below. Board: [00-status.md](../../../stories/00-status.md) **Context sources:** [`prototypes/wo-rt-c/wo-rt.c`](../../../../runtime/wo-rt.c) (phase 0 — the single-threaded epoll baseline), [`../../09-concurrency-scaleout.md`](../../09-concurrency-scaleout.md) (the thread-per-core doctrine every phase here miniaturizes), [`../../10-storage-foundations.md`](../../10-storage-foundations.md) / [`11-wal-and-recovery.md`](../../11-wal-and-recovery.md) / [`12-engine-disk-cutover.md`](../../12-engine-disk-cutover.md) (the storage track), kernel reference cards [`../linux/07-io_uring.md`](../linux/07-io_uring.md), [`08-mmap.md`](../linux/08-mmap.md), [`09-fallocate.md`](../linux/09-fallocate.md), [`12-pwrite-fsync.md`](../linux/12-pwrite-fsync.md), [`02-eventfd.md`](../linux/02-eventfd.md). diff --git a/docs/plan/exploration/fibers/00-fibers.md b/docs/plan/exploration/fibers/00-fibers.md index f1265d1..66e8e0b 100644 --- a/docs/plan/exploration/fibers/00-fibers.md +++ b/docs/plan/exploration/fibers/00-fibers.md @@ -3,8 +3,8 @@ > Exploration/reference note (no status banner by board convention). > The normative decisions live in the arc spec > ([`2026-08-20-shard-fiber-arc-design.md`](../../../superpowers/specs/2026-08-20-shard-fiber-arc-design.md)) -> and iterations [8](../../../stories/language-runtime-database/refine/08-shard-actor-runtime.md) / -> [11](../../../stories/language-runtime-database/refine/11-fibers.md); this +> and iterations [8](../../../stories/language-runtime-database/done/08-shard-actor-runtime.md) / +> [11](../../../stories/language-runtime-database/done/11-fibers.md); this > page explains the WHY at doctrine depth. Written 2026-08-20, when this > file was also the target of a dangling reference from iteration 11 — > it exists now. diff --git a/docs/plan/learnings.md b/docs/plan/learnings.md index de6d854..d998ff1 100644 --- a/docs/plan/learnings.md +++ b/docs/plan/learnings.md @@ -3,7 +3,7 @@ What the work actually taught, independent of whether it shipped. Recorded so the same wall is not hit twice. Newest first within each section. -Status board: [`00-status.md`](../00-status.md) · Rejections: [`discarded.md`](discarded.md) +Status board: [`00-status.md`](../stories/00-status.md) · Rejections: [`discarded.md`](discarded.md) ## Testing and verification diff --git a/docs/plan/oop-vm/03-concurrency-coroutines.md b/docs/plan/oop-vm/03-concurrency-coroutines.md new file mode 100644 index 0000000..2c6d3f6 --- /dev/null +++ b/docs/plan/oop-vm/03-concurrency-coroutines.md @@ -0,0 +1,160 @@ +# Stackless coroutines — the concurrency contract (fibers, parking, no `async`) + +Normative reference for the concurrency iterations (the 8+11 arc and its +chain). Landed by the arc's stages 1+2 (branch `concurrency-arc`, +2026-08-20); the mechanism lives in `runtime/src/vm.h` (`wo_fiber`, +`wo_vm`), `runtime/src/vm.c` (scheduler), `runtime/src/park.c` (I/O +plane). Shard/actor rules live in the arc spec +([`2026-08-20-shard-fiber-arc-design.md`](../../superpowers/specs/2026-08-20-shard-fiber-arc-design.md)) +— this doc is the coroutine core plus its seams. + +## 1. The model — a coroutine is interpreter state, not a stack + +A fiber IS the VM's per-execution state, made per-fiber: one heap +struct (`wo_fiber`) holding the register window, the frame stack, the +catch stack, the saved resume pc, and the park descriptor. That is the +whole coroutine. + +**Stackless** means: no native C stack per fiber, ever. The interpreter +runs every fiber on the one OS thread stack of its shard; suspending a +fiber never saves a C stack because there is nothing on it — suspension +exists only at VM boundaries: + +- the dispatch loop (reduction budget hits zero), and +- inside a blocking builtin (the builtin fills the park descriptor and + returns to the scheduler). + +C code between those boundaries never yields; therefore no +`makecontext`/`ucontext`, no split stacks, no stack copying, no guard +pages. Consequences the design buys: + +| property | why it follows | +| --- | --- | +| spawn is one allocation | a fiber is a calloc'd struct, not an 8 MiB mapping | +| deterministic replay | suspension points are exact VM instructions, not signal arrival | +| trivial GC rooting | a fiber's roots are its registers + frames arrays — walkable structs | +| no FFI hazard | there is no foreign frame that could hold a suspended C stack | + +## 2. No `async` keyword — the no-coloring rule + +There is ONE function type. The keyword `async` (and `await`) is a +**permanently rejected surface**, not deferred (story 11's +out-of-scope). The rule that replaces it: + +- **Blocking builtins park instead of block on server shards.** The same + `net`/`time` call that blocks the thread in program mode hands its fd + or deadline to the shard's I/O plane and parks the calling fiber; the + shard serves other fibers meanwhile; the fiber resumes with the + result. Same source text, no annotation, no second color of function. +- **Program mode stays single-fiber and genuinely blocking** — the + log-watcher needs nothing more; the parked path is the serving path. +- Preemption is by **reduction budget**, never signals or safepoint + interrupts: the dispatch loop decrements a countdown and parks the + fiber at zero. Erlang's shape. + +Precedent survey (kernel evidence, BEAM adopted, Go stack copying and +Tokio coloring rejected): +[`docs/plan/exploration/fibers/00-fibers.md`](../exploration/fibers/00-fibers.md). + +## 3. The fiber state machine + +States (`wo_fib_state`): **RUNNABLE → PARKED → RUNNABLE → … → DONE**. + +- **RUNNABLE** — on the shard's FIFO run queue (`qhead`/`qtail`, + intrusive `next` link). FIFO is the v1 fairness policy; no priorities. +- **PARKED** — on the parked list (`pnext` link), registered with the + I/O plane as one wait (fd readiness or deadline). +- **DONE** — returned or trapped; reaped. + +Fixed points of the machine: + +- **Fiber 0 is main**, embedded in the `wo_vm` (never allocated, never + reaped before shutdown). Spawned fibers are calloc'd; `nfibers` + counts them. +- **`vm->cur` is the live fiber** — every interpreter access reads + through it. One live fiber per shard at any instant. +- **Trap isolation**: an uncaught trap kills the trapping fiber alone — + it unwinds through its own drop maps and goes DONE; the shard and the + other fibers continue. +- **Main-return reap**: when main returns, remaining fibers are + unwound (drop maps run — parked fibers die as cleanly as trapped + ones) and the program ends. +- **Actor delivery fibers**: an actor executes messages on a fiber with + `fiber->actor` set; one message at a time (the actor guarantee); + `cur_msg` is runtime-owned and dropped after the receive call + returns. Actor rules beyond delivery are the arc spec's. + +## 4. Suspension and resume — the exact protocols + +**Reduction budget.** `budget0` reductions per slice (`WO_REDUCTIONS`, +default 4000). The countdown decrements at loop **BACK-EDGES ONLY, +after the jump lands** — not at the "same three sites as the GC". The +distinction is load-bearing: a pre-instruction decrement re-executes +the jump into the same decrement at budget 1 and livelocks (pinned by +the deterministic fiber corpus). At zero the fiber re-queues RUNNABLE +at the FIFO tail and the next fiber runs. + +**Parking on an fd** (accept/read/write not ready): the builtin fills +the park descriptor — `park_fd`, `park_events` (POLLIN/POLLOUT), +`park_done = 0` — and the plane arms a wait. `park_done = 0` means +resume **RE-EXECUTES the builtin**: the retry runs the same call now +that the fd is ready. A partial `net.write`'s progress crosses the +retry in `park_wr_at` (the write offset) — re-execution continues the +write, never restarts it. + +**Parking on a deadline** (`time.sleep`): `park_fd = -1`, +`park_deadline` set, the result preset before parking, `park_done = 1` +— resume **continues PAST the builtin**. `park_ts` must outlive the +ring submission (the TIMEOUT op reads it asynchronously). + +**Parking on a reply** (stage 3, the DB actor): `park_fd = +WO_PARK_INBOX` (-2) — the fiber joins the parked list with NO plane +wait at all; the wake is `wo_io_unpark` from the shard's envelope +drain when the reply lands. `park_done = 0`: resume re-executes the +builtin, which consumes the answer. Deadline scans key on +`park_fd == -1` EXACTLY — an inbox park must never read as a deadline. + +**The I/O plane** (`park.c`): one event loop per shard — parked fibers' +waits and the shard's inbox eventfd on the SAME loop. io_uring FIRST +(raw `io_uring_setup`/`io_uring_enter`, POLL_ADD + TIMEOUT at the Linux +5.4 op floor, libc-only); epoll is the portability fallback behind a +startup probe (seccomp'd containers routinely deny io_uring), forced +either way with `WO_IO=uring|epoll` so CI proves both paths on one +kernel. Note: the epoll fallback here predates the 2026-08-21 +epoll-discard directive for PLANS — the runtime keeps the probe until a +removal slice says otherwise. + +**Stop**: the stop flag interrupting the plane's wait unwinds +EVERYTHING — every fiber, parked included, through its drop maps; a +stop is not a trap and cannot be caught. + +## 5. Memory and GC seams + +- **Roots**: `vm_gc_roots` iterates ALL fibers' registers + frames — + parked fibers' frames are roots exactly as the live frame stack is. + Nothing live is collected while its only reference sits in a parked + fiber. +- **Drops**: every exit path (return, trap, stop-unwind, main-return + reap) runs the same drop maps; ASan-zero-leaks is the standing gate. +- **Open question** (tracked in story 11): how a parked fiber's borrow + state interacts with the shard's GC safepoints — settles with stage 3 + or the collector's next pass. + +## 6. Seams — pointers, not content + +| concern | where it is normative | +| --- | --- | +| shards, envelopes, ownership-move sends, WO-E221/E222, placement | [arc spec](../../superpowers/specs/2026-08-20-shard-fiber-arc-design.md) + [arc plan deviations](../../superpowers/plans/2026-08-20-shard-fiber-arc.md) | +| request/response, bounded mailboxes, actor death, timers | [iteration 31](../../stories/language-runtime-database/refine/31-actor-lifecycle.md) — not built yet | +| the DB actor (stage 3) | [story 8's guarantee contract](../../stories/language-runtime-database/done/08-shard-actor-runtime.md) — landed 2026-08-21 | +| builtin ids and their park behavior | [`08-builtin-surface.md`](08-builtin-surface.md) | + +## 7. Rejected alternatives — settled, argue against the reason + +| rejected | reason | +| --- | --- | +| **`async`/`await`, function coloring** | splits the world into two function types and infects every caller; the park-under-blocking-API posture serves the same need with zero surface. Permanent. | +| **Stackful coroutines (`ucontext`/`makecontext`, per-fiber C stacks)** | pays a stack (or guard-page games) per fiber, breaks the one-allocation spawn, and reintroduces foreign-frame suspension the GC would have to scan blind. | +| **Go-style segmented/copied stacks** | stack copying needs precise pointer maps for native frames — a moving-stack machinery this VM does not need because fibers never own native frames. | +| **Signal/safepoint preemption** | signals arrive between ANY two instructions — kills deterministic replay and demands async-signal-safe everything; the reduction budget preempts at exact VM points. | +| **Per-instruction budget decrement** | measurable dispatch cost for zero fairness gain; back-edges bound every loop already (and the pre-instruction variant livelocks at budget 1). | diff --git a/docs/plan/oop-vm/README.md b/docs/plan/oop-vm/README.md index a9fa364..fa8af28 100644 --- a/docs/plan/oop-vm/README.md +++ b/docs/plan/oop-vm/README.md @@ -7,7 +7,7 @@ The normative contract documents both stacks cite. Landed by their named plan ta | `00-wob-format.md` | `.wob` bytecode format (compiler↔VM) | 1 | | `01-error-catalog.md` | every `WO-E###` code | 2, grows 3/8 | | `02-corpus.md` | how to add conformance fixtures | 3 | -| `03-shard-actor.md` | shard ownership, mailboxes, send-as-move | 4 | +| `03-concurrency-coroutines.md` | stackless fibers, park/resume protocols, reduction budget, the no-`async` rule | arc (was plan 4's slot; that plan ✖ discarded 2026-08-21) | | `04-db-binding.md` | row format, WAL records, query subset | 5 | | `05-http-service.md` | route section, trap→HTTP table, JSON subset | 6 | | `06-ui-live.md` | delta frames, subscribe protocol, wo:live | 7 | diff --git a/docs/00-status.md b/docs/stories/00-status.md similarity index 61% rename from docs/00-status.md rename to docs/stories/00-status.md index 515a21f..9f37158 100644 --- a/docs/00-status.md +++ b/docs/stories/00-status.md @@ -1,6 +1,6 @@ # Status board — what is done, what is next -Edges live in [00-dependency-graph.md](00-dependency-graph.md) — mermaid +Edges live in [00-dependency-graph.md](../00-dependency-graph.md) — mermaid graphs of iteration and feature dependencies; anything with all-green incoming arrows is startable. This board carries the STATES. @@ -11,21 +11,80 @@ folders** — a doc stays where it was authored when its work lands; only its banner and this board change. ONE exception by directive (2026-08-20): story iteration files move physically — landed ones into `stories/language-runtime-database/done/`, brainstorm-needing ones into -`refine/`; in-flight/parked stay at the root. Every plan and phase doc opens with a +`refine/`, held ones into `hold/`, the active slice's stories into +`stories/language-runtime-database/in-progress/` (both 2026-08-21); +ready ones stay at the root. Second exception (2026-08-21): the active +slice's one marker doc lives in `docs/in-progress/` and is deleted when +the slice lands. Every plan and phase doc opens with a `> **Status:**` banner linking back here; normative contracts (`plan/oop-vm/`), exploration studies, reference docs and the discarded/learnings registers carry none by design. Update this board in the same change that finishes work — move the item to done with _what actually landed_, set the next in-progress item, and record any -rejection in [`discarded.md`](plan/discarded.md) with its reason. +rejection in [`discarded.md`](../plan/discarded.md) with its reason. Statuses: ✅ **done** · 🔄 **in progress** · ⬜ **pending** · ⏸ **hold** +Story files carry YAML frontmatter (`iteration`/`status`/`chain`) — the +machine-readable truth behind this board; live Obsidian Dataview views: +[`board-views.md`](board-views.md) (Kanban = view only, never edits status). + --- ## ▶ NEXT PLAN +**The concurrency + fiber chain — ✅ stage 3 → 22 → 31 → 24 → 23 → 32** +(directive 2026-08-21). Active slice: **iteration 22, the measurement +backbone** — spec + plan approved 2026-08-21 +([spec](../superpowers/specs/2026-08-21-db-bench-design.md) · +[plan](../superpowers/plans/2026-08-21-db-bench.md) · +[marker](../in-progress/2026-08-21-db-bench.md)); the four forks settled +as their leanings, plus `time.ticks` (µs clock) as the one runtime +addition and a new `db-bench` sample as the vehicle. + +**Implemented last time (2026-08-21):** the arc's **stage 3 — the +transparent DB actor landed, the arc is COMPLETE** (stories +[8](language-runtime-database/done/08-shard-actor-runtime.md) + +[11](language-runtime-database/done/11-fibers.md) → done/). A worker +shard's DB statement marshals to shard 0 (requester-side slot encode), +executes serialized on the owner, and the fiber resumes with the +materialized reply — `WO_T_DB` off the primary is gone. NEW gate +`just db-actor` 8/0; ASan/TSan clean; WO_DATA pair proves worker writes +are ack-after-durable and replay. + +**Key findings:** a latent stage-1 bug — io_uring ring params were ONE +shared static, rewritten by every shard's lazy init while others read +offsets from it: submits landed at garbage offsets and parked fibers +LOST WAKES (~1/20 hangs at default cores). Per-vm params fixed it; a +short `io_uring_enter` submit is now a loud trap. Also: single-binary +gates embed the runtime — rebuild the SAMPLE, not just wovm, or you +debug a stale binary. + +**Learned from the last iteration:** the owner thread must never read a +requester's VM heap (concurrent mark-bit writes = TSan race) — marshal +by ENCODING on the requester's thread, execute from slots replay-style; +a plane-less park (`WO_PARK_INBOX`) + envelope wake is all an RPC reply +needs; a busy shard adopting its inbox once per reduction slice bounds +request latency. + +**Dependencies unblocked:** 22's multi-shard campaign (the store is +correct under shards now); 24's serving model (fiber-per-connection has +a database it can touch from any shard); the framework ledger rows the +arc gates stay ⏸ until their own slices. + +**Next steps:** 22 (baselines single- AND multi-shard + the mutex-inbox +number; precursor recorded in story 8: remote insert ≈8µs/op RAM-only) +→ 31 (lifecycle) → 24 (chat) → 23 (io_uring group-commit) → 32 (WAL +checkpoint). Held tail resumes on its own precedence notes. + +**`.dev/reference` used:** `linux` (io_uring uapi struct layouts and the +"single event loop" card — both load-bearing in the ring-params fix). + +--- + +### Landed 2026-08-20 — framework v1 (the previous NEXT PLAN) + **Framework v1 — a polished micro-framework (routing, middleware, `Req`/`Resp`), nothing MVC-scale.** Directive 2026-08-20: iteration 17 (library kind + `internal/`) is **parked** — spec + plan approved and ready @@ -61,14 +120,15 @@ sees through Interp; same corpus pin). Body-parsing hooks: all three ✅. Scope split (2026-08-20): the surface above plus the remaining transport/ routing/security gaps is **framework v1**, tracked item-by-item in the -[framework README's status ledger](examples/writeonce-framework/README.md) +[framework README's status ledger](../examples/writeonce-framework/README.md) (✅/🔶/⬜/⏸/🔧 per feature — timeouts and Unix sockets need `net` runtime seams, crypto hashes need C builtins since the language has no bitwise operators, streaming/cancellation park behind 8/11). The memory-rich -features are **framework v2** = iteration 18 (spec APPROVED 2026-08-20, -plan next): TTL cache, @table flags, durable job queue with -drain-on-request, `transaction { }` over the WAL's staged batch. After 18, -the order resumes at 20/21. Edges: [00-dependency-graph.md](00-dependency-graph.md). +features are **framework v2** = iteration 18 (⏸ HELD 2026-08-21 with spec +approved + plan authored intact): TTL cache, @table flags, durable job +queue with drain-on-request, `transaction { }` over the WAL's staged +batch. The pending order is now the concurrency chain (see *Pending* +below). Edges: [00-dependency-graph.md](../00-dependency-graph.md). --- @@ -125,8 +185,8 @@ itself, and all six landed: 14 600 KiB across 601 686 requests in 90 s once past its ~1200-request quarantine warm-up. -Plan: [`plan/compiler/2026-08-14-logwatcher-executable.md`](plan/compiler/2026-08-14-logwatcher-executable.md) · -Story slice: [`docs/stories/language-runtime-database/done/07-logwatcher-proof.md`](stories/language-runtime-database/done/07-logwatcher-proof.md) +Plan: [`plan/compiler/2026-08-14-logwatcher-executable.md`](../plan/compiler/2026-08-14-logwatcher-executable.md) · +Story slice: [`docs/stories/language-runtime-database/done/07-logwatcher-proof.md`](language-runtime-database/done/07-logwatcher-proof.md) **Deferred by name, with the measurement that says so:** @@ -147,45 +207,48 @@ and run log-watcher_ — is met; the database engine (9/9b), deps (15), and the web framework (16) landed on top of it. The goal is now the framework as a polished micro-framework (17 parked; see the NEXT PLAN above and "Implementation order" under Pending). (The prior Rust `wo` runtime was -removed from the repo 2026-08-18 — see [`discarded.md`](plan/discarded.md).) +removed from the repo 2026-08-18 — see [`discarded.md`](../plan/discarded.md).) --- ## Stories -[`docs/stories/language-runtime-database/`](stories/language-runtime-database/00-story.md) +[`docs/stories/language-runtime-database/`](language-runtime-database/00-story.md) — one language, one runtime, one database, one binary. Twelve iterations, each an unsplittable slice with Given/When/Then acceptance and a pointer to the plan that sequences its tasks. Read one, approve, then the next starts. | # | Iteration | State | | --- | -------------------------------------------------------------------------------------------- | ---------------------------- | ---- | -| 1 | [Principles doc](stories/language-runtime-database/done/01-principles-doc.md) | ✅ | -| 2 | [VM core (`wovm`)](stories/language-runtime-database/done/02-vm-core.md) | ✅ | -| 3 | [Compiler front (`woc`)](stories/language-runtime-database/done/03-compiler-front.md) | ✅ (known gaps below) | -| 4 | [Single binary end-to-end](stories/language-runtime-database/done/04-single-binary-e2e.md) | ✅ (known gaps below) | -| 5 | [Language surface](stories/language-runtime-database/done/05-language-surface.md) | 🔄 grammar done; **`?T` forced handling ✅ + reject rows ✅ + WO-E205 ✅ (2026-08-18)**; `pub(read)`/`using`/`#if` still ⏸ | -| 6 | [Program mode + stdlib](stories/language-runtime-database/done/06-program-mode-stdlib.md) | ✅ (the surface log-watcher uses) | -| 7 | [log-watcher proof](stories/language-runtime-database/done/07-logwatcher-proof.md) | ✅ **landed 2026-08-15** — executable, not merely compilable: zero ASan leaks in all three modes, SIGTERM ends parked syscalls, fds flat, `LW_SOAK` gate; `just log-watcher` 7/0 | -| 7b | [Inferred GC + mark-sweep](stories/language-runtime-database/done/07b-inferred-gc-mark-sweep.md) | ✅ **landed 2026-08-18** — `@gc` gone (WO-E104), GC-ness inferred, RC replaced by incremental mark-sweep, `.wob` v4; supersedes iteration 2's RC memory model | -| 8 | [Shard-actor runtime](stories/language-runtime-database/refine/08-shard-actor-runtime.md) | ⬜ | -| 9 | [Database engine](stories/language-runtime-database/done/09-database-engine.md) | 🔄 engine complete (storage/WAL/indexes/insert-update-delete); reads land with 9b | -| 9b | [`@table`, relations, query](stories/language-runtime-database/done/09b-table-relations-query.md) | 🔄 query surface + relations + FK done (branch query-surface); group-by parked | -| 19 | [Float + Bytes](stories/language-runtime-database/done/19-missing-scalar-types.md) | ✅ **landed 2026-08-20** — `.wob` v5: Float constant tag, field kinds 6/7, opcodes 34-41 (IEEE-quiet f64), builtins 70-83. Full stack: literals, arithmetic, `@table` column, WAL bit-exact replay, json fractions in / shortest-round-trip out, `?Float` reserved-NaN nil, total-order index (NaN last, `-0.0` == `+0.0`), Bytes + base64. No implicit Int/Float mixing (WO-E201); `float`/`trunc` are the only bridges. Proof: web-app price is a real Float (`{"price":9.99}`), `just web-app` 23/0; corpus 103/0 | -| 20 | [Cross-program tables](stories/language-runtime-database/refine/20-cross-program-tables.md) | 🔄 channel done (branch ipc-attach); manifest+binding pending | -| 21 | [Keypair attach auth](stories/language-runtime-database/refine/21-keypair-attach-auth.md) | 🔄 crypto+handshake done (branch keypair-auth); manifest pending | -| 22 | [Durability, throughput, scale](stories/language-runtime-database/refine/22-durability-throughput-scale.md) | ⬜ needs a spec first | -| 23 | [io_uring group-commit](stories/language-runtime-database/refine/23-io-uring-commit.md) | ⬜ after 8 + 22 | -| 27 | [Query grammar corpus](stories/language-runtime-database/refine/27-query-grammar-corpus.md) | ⬜ needs a spec first | -| 10 | [HTTP service layer](stories/language-runtime-database/25-http-service.md) | ⬜ | Hold | -| 11 | [Fibers](stories/language-runtime-database/refine/11-fibers.md) | ⬜ | Hold | -| 12 | [Blue-green deploy](stories/language-runtime-database/26-blue-green-deploy.md) | ⬜ | Hold | -| 13 | [Compile-time metaprogramming](stories/language-runtime-database/refine/29-compile-time-metaprogramming.md) | ⬜ needs a spec first | -| 14 | [skillhost host workload](stories/language-runtime-database/refine/28-skillhost-host-workload.md) | ⬜ gaps recorded (branch query-grammar found skillhost needs no new query grammar); each gap a candidate iteration | -| 15 | [deps: `wo.toml [deps]`](stories/language-runtime-database/done/15-deps-package-manager.md) | ✅ **landed 2026-08-18** (branch web-framework): [deps] inline tables, git-binary fetch, wo.lock pinning, offline-when-locked, --update-deps, WO-E106/E107; `just deps-accept` 8/0 | -| 16 | [web framework](stories/language-runtime-database/done/16-web-framework.md) | ✅ **landed 2026-08-19** — writeonce-framework (HTTP/1.1 + router + Handler/Middleware) consumed by web-app through [deps]; h2c parked (§C) behind 8/23/11. **v1 polish landed 2026-08-20** (branch framework-v1): get/post/put/delete_ helpers, 405+Allow, HEAD, Logging middleware, set_header; `just web-app` 16/0; fixed the interp-borrowed-field emitter crash en route. **Auth-in-core landed 2026-08-20**: http/auth.wo (Bearer/Basic, ct_eq, req.principal), web-app dogfoods BearerAuth, gate 17/0 | -| 17 | [library projects + `internal/`](stories/language-runtime-database/done/17-library-projects-internal.md) | ✅ **landed 2026-08-20** — `kind = "library"` in `wo.toml` (default `program`, so every existing manifest is byte-identical; unknown value = WO-E109 exit 2); `woc ` on a library runs the FULL pipeline entry-less and writes nothing, retiring iteration 16's `--emit` workaround; the no-entry build error names the kind; lib+bin dual works. Go's `internal/` rule as **WO-E108** at the consumer's own `use`, dep-boundary-only — the library imports its own interior freely. Framework reorganized: `internal/{parse,serve}.wo` behind the line, `http/form.wo` split out to keep `media_type`/`form_values` public. Driver-only change; VM/`.wob`/GC untouched. `just web-app` **26/0** (3 new checks), every standing gate unchanged | -| 18 | [framework v2: memory-rich features](stories/language-runtime-database/hold/18-memory-db-features.md) | 🔄 **spec APPROVED 2026-08-20, plan next** ([spec](superpowers/specs/2026-08-20-memory-db-features-design.md)): TTL cache + @table flags + durable job queue (drain-on-request) + `transaction { }` over the WAL's staged batch; pub/sub REJECTED until 8/11 | +| 1 | [Principles doc](language-runtime-database/done/01-principles-doc.md) | ✅ | +| 2 | [VM core (`wovm`)](language-runtime-database/done/02-vm-core.md) | ✅ | +| 3 | [Compiler front (`woc`)](language-runtime-database/done/03-compiler-front.md) | ✅ (known gaps below) | +| 4 | [Single binary end-to-end](language-runtime-database/done/04-single-binary-e2e.md) | ✅ (known gaps below) | +| 5 | [Language surface](language-runtime-database/done/05-language-surface.md) | 🔄 grammar done; **`?T` forced handling ✅ + reject rows ✅ + WO-E205 ✅ (2026-08-18)**; `pub(read)`/`using`/`#if` still ⏸ | +| 6 | [Program mode + stdlib](language-runtime-database/done/06-program-mode-stdlib.md) | ✅ (the surface log-watcher uses) | +| 7 | [log-watcher proof](language-runtime-database/done/07-logwatcher-proof.md) | ✅ **landed 2026-08-15** — executable, not merely compilable: zero ASan leaks in all three modes, SIGTERM ends parked syscalls, fds flat, `LW_SOAK` gate; `just log-watcher` 7/0 | +| 7b | [Inferred GC + mark-sweep](language-runtime-database/done/07b-inferred-gc-mark-sweep.md) | ✅ **landed 2026-08-18** — `@gc` gone (WO-E104), GC-ness inferred, RC replaced by incremental mark-sweep, `.wob` v4; supersedes iteration 2's RC memory model | +| 8 | [Shard-actor runtime](language-runtime-database/done/08-shard-actor-runtime.md) | ✅ **landed 2026-08-21** — the arc complete: stages 1+2 (fibers/budget/actors/io_uring plane, shards, envelopes, WO-E222) + stage 3's transparent DB actor (`just db-actor` 8/0, ASan/TSan clean, WAL replay pair) | +| 9 | [Database engine](language-runtime-database/done/09-database-engine.md) | 🔄 engine complete (storage/WAL/indexes/insert-update-delete); reads land with 9b | +| 9b | [`@table`, relations, query](language-runtime-database/done/09b-table-relations-query.md) | 🔄 query surface + relations + FK done (branch query-surface); group-by parked | +| 19 | [Float + Bytes](language-runtime-database/done/19-missing-scalar-types.md) | ✅ **landed 2026-08-20** — `.wob` v5: Float constant tag, field kinds 6/7, opcodes 34-41 (IEEE-quiet f64), builtins 70-83. Full stack: literals, arithmetic, `@table` column, WAL bit-exact replay, json fractions in / shortest-round-trip out, `?Float` reserved-NaN nil, total-order index (NaN last, `-0.0` == `+0.0`), Bytes + base64. No implicit Int/Float mixing (WO-E201); `float`/`trunc` are the only bridges. Proof: web-app price is a real Float (`{"price":9.99}`), `just web-app` 23/0; corpus 103/0 | +| 11 | [Fibers](language-runtime-database/done/11-fibers.md) | ✅ **landed 2026-08-21** with the arc (`just fibers` 10/0); fs-park re-scoped out of v1, disclosed in the story | +| 22 | [Durability, throughput, scale](language-runtime-database/in-progress/22-durability-throughput-scale.md) | 🔄 **spec + plan approved 2026-08-21, executing** — db-bench sample + campaign gates; forks settled | +| 31 | [Actor lifecycle](language-runtime-database/refine/31-actor-lifecycle.md) | ⬜ needs a spec first — third in chain (story written 2026-08-21) | +| 24 | [chat: WebSocket workload](language-runtime-database/refine/24-chat-websocket-workload.md) | ⬜ fourth in chain — the arc's acceptance; after 31 | +| 23 | [io_uring group-commit](language-runtime-database/refine/23-io-uring-commit.md) | ⬜ fifth in chain, after stage 3 + 22 | +| 32 | [WAL checkpoint](language-runtime-database/refine/32-wal-checkpoint.md) | ⬜ last in chain, after 23 — disk reclamation + bounded replay (story written 2026-08-21) | +| 20 | [Cross-program tables](language-runtime-database/hold/20-cross-program-tables.md) | ⏸ hold (2026-08-21); channel done (branch ipc-attach keeps its manifest) | +| 21 | [Keypair attach auth](language-runtime-database/hold/21-keypair-attach-auth.md) | ⏸ hold (2026-08-21); crypto+handshake done (branch keypair-auth keeps its manifest) | +| 25 | [HTTP service layer](../superpowers/plans/2026-08-01-http-service-layer.md) | ⏸ hold (2026-08-21) — story file removed; the plan doc remains | +| 26 | [Blue-green deploy](language-runtime-database/hold/26-blue-green-deploy.md) | ⏸ hold (2026-08-21) | +| 27 | [Query grammar corpus](language-runtime-database/hold/27-query-grammar-corpus.md) | ⏸ hold (2026-08-21) | +| 28 | [skillhost host workload](language-runtime-database/hold/28-skillhost-host-workload.md) | ⏸ hold (2026-08-21); gaps recorded (branch query-grammar found skillhost needs no new query grammar) | +| 29 | [Compile-time metaprogramming](language-runtime-database/hold/29-compile-time-metaprogramming.md) | ⏸ hold (2026-08-21) | +| 15 | [deps: `wo.toml [deps]`](language-runtime-database/done/15-deps-package-manager.md) | ✅ **landed 2026-08-18** (branch web-framework): [deps] inline tables, git-binary fetch, wo.lock pinning, offline-when-locked, --update-deps, WO-E106/E107; `just deps-accept` 8/0 | +| 16 | [web framework](language-runtime-database/done/16-web-framework.md) | ✅ **landed 2026-08-19** — writeonce-framework (HTTP/1.1 + router + Handler/Middleware) consumed by web-app through [deps]; h2c parked (§C) behind 8/23/11. **v1 polish landed 2026-08-20** (branch framework-v1): get/post/put/delete_ helpers, 405+Allow, HEAD, Logging middleware, set_header; `just web-app` 16/0; fixed the interp-borrowed-field emitter crash en route. **Auth-in-core landed 2026-08-20**: http/auth.wo (Bearer/Basic, ct_eq, req.principal), web-app dogfoods BearerAuth, gate 17/0 | +| 17 | [library projects + `internal/`](language-runtime-database/done/17-library-projects-internal.md) | ✅ **landed 2026-08-20** — `kind = "library"` in `wo.toml` (default `program`, so every existing manifest is byte-identical; unknown value = WO-E109 exit 2); `woc ` on a library runs the FULL pipeline entry-less and writes nothing, retiring iteration 16's `--emit` workaround; the no-entry build error names the kind; lib+bin dual works. Go's `internal/` rule as **WO-E108** at the consumer's own `use`, dep-boundary-only — the library imports its own interior freely. Framework reorganized: `internal/{parse,serve}.wo` behind the line, `http/form.wo` split out to keep `media_type`/`form_values` public. Driver-only change; VM/`.wob`/GC untouched. `just web-app` **26/0** (3 new checks), every standing gate unchanged | +| 18 | [framework v2: memory-rich features](language-runtime-database/hold/18-memory-db-features.md) | ⏸ hold (2026-08-21); spec approved + plan authored, both held intact ([spec](../superpowers/specs/2026-08-20-memory-db-features-design.md), [plan](../superpowers/plans/2026-08-20-framework-v2-memory-features.md)): TTL cache + @table flags + durable job queue (drain-on-request) + `transaction { }` over the WAL's staged batch; pub/sub rejection expired with the arc (8/11 landed 2026-08-21) — revisit on unhold | --- @@ -193,12 +256,13 @@ that sequences its tasks. Read one, approve, then the next starts. | Track | Item | Where | | -------- | --------------------------------------------------------------------------- | ---------------------------------------------------------- | -| Language | 🔄 [iteration 36 — operator parity](stories/language-runtime-database/in-progress/36-operator-parity.md): `not`, bitwise `& \| ^ << >>`, hex/binary/`_` literals, compound assigns — CODE LANDED 2026-08-22 (branch operator-parity, `.wob` v6, all gates green; reference project `.dev/reference/go` drove the design). Awaiting the developer's MANUAL pass on `docs/examples/operators/` (no test fixtures by directive); unblocks story 34's pure-`.wo` HMAC question | [plan](superpowers/plans/2026-08-22-operator-parity.md) | +| Language | 🔄 [iteration 36 — operator parity](language-runtime-database/in-progress/36-operator-parity.md): `not`, bitwise `& \| ^ << >>`, hex/binary/`_` literals, compound assigns — CODE LANDED 2026-08-22 (branch operator-parity, `.wob` v6, all gates green; reference project `.dev/reference/go` drove the design). Awaiting the developer's MANUAL pass on `docs/examples/operators/` (no test fixtures by directive); unblocks story 34's pure-`.wo` HMAC question | [plan](../superpowers/plans/2026-08-22-operator-parity.md) | | Language | the framework v1-polish slice landed 2026-08-20 (branch framework-v1, awaiting merge); next per the order: brainstorm 20/21's forks | [order](#implementation-order-re-sequenced-2026-08-20--code-review-pass) | +| Runtime | **iteration 22: db-bench** — spec + plan approved 2026-08-21; executing | [marker](../in-progress/2026-08-21-db-bench.md) · [plan](../superpowers/plans/2026-08-21-db-bench.md) | -Off-goal work is parked; the goal (2026-08-20) is the web framework as a -polished micro-framework v1 — iteration 17 (library kind + `internal/`) is -parked with its spec + plan ready on branch `library-internal`. +The active slice's marker doc lives in [`in-progress/`](../in-progress/) — +one file, deleted when the slice lands. Everything else pending is the +concurrency chain (see *Pending* below); the held tail is in `hold/`. ### Landed 2026-08-14 — the compile-and-run milestone @@ -243,14 +307,14 @@ Gates at the end of that session: corpus 71/0, `woc` runtest 565/0, every | Status | Item | Doc | What actually landed | | ------ | ------------------------------------ | ---------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| ✅ | Principles | [`../00-principles.md`](00-principles.md) | 13 principles, each with a why and a link to the doc that enforces it | -| ✅ | `wovm` VM core | [plan 1](superpowers/plans/2026-08-01-wob-format-and-vm-core.md) | `.wob` v1 loader with full static validation, register interpreter (computed-goto + ISO-C fallback), arena with size-class free lists, borrow word, RC + budgeted Bacon–Rajan cycle collector, drop-map trap unwinding, containers, builtins, ICALL, CLI. 13 suites × 2 dispatch flavors + CLI smoke, ASan/UBSan clean | -| ✅ | `.wob` format contract | [`oop-vm/00-wob-format.md`](plan/oop-vm/00-wob-format.md) | Normative; twinned with `runtime/src/wob.h` | -| ✅ | `woc` compiler front | [plan 2](plan/compiler/2026-08-01-woc-compiler-front.md) | Tasks 1–8: dune scaffold, `diag` (WO-E codes, two-site related errors, ordered dedup), newline-significant lexer at rt parity, declaration + statement/expression parser with skip-on-block and multi-error recovery, typechecker (field kinds, `?T` plumbing, W201, E225, E214), MVS ownership pass with the four emitter tables, driver with directory discovery + cross-file programs. 14 + 264 checks | -| ✅ | Error catalog | [`oop-vm/01-error-catalog.md`](plan/oop-vm/01-error-catalog.md) | 14 emitted codes + 10 reserved, each with the reason it is not yet emitted | -| ✅ | log-watcher `.wo` sample | [`../examples/log-watcher/`](examples/log-watcher/README.md) | Eight-file port authored docs-first with its `.hx` mapping table; compiles for real in iteration 7 | -| ✅ | Scalar cleanup | [`discarded.md`](plan/discarded.md) | `Money`/`SKU`/`Float` and the abstract allowlist removed; `abstract` flipped adopt → reject | -| ✅ | `woc` emitter, corpus, single binary | [plan 3](plan/compiler/2026-08-01-wob-emit-e2e-single-binary.md) | Tasks 1–6 + 8 (Task 7, a parity harness against the Rust runtime, **deferred by explicit user decision** — the two stacks diverge by design). Bytecode emitter (`emit.ml`) + disassembler (`disasm.ml`, `--dump-bc`); three-kind conformance harness (`scripts/oop-e2e.sh`, `just oop-e2e`) over `tests/corpus/{run,compile-fail,trap,gc}`; pricing-demo + ownership/trap corpora (19 fixtures); `@gc` cycle collector's post-exit pump (`WO_GC_BUDGET`/`WO_GC_TRACE`) + 2 gc fixtures (`gc/held-cycle` retired — see criterion-3 closure below); `woc build` single-binary output + relocation/corrupt-trailer smoke; `WO-E405` closing criterion 3's ASan leak (entry must return `Int`); `just oop-accept` wiring all five spec criteria + both unit gates into one command. 14 + 399 compiler checks; `oop-e2e` 25/25 against the release `wovm`. **Milestone-1 acceptance gate is fully green — all five criteria met** (see the dated acceptance note in `docs/superpowers/specs/2026-08-01-oop-compiler-vm-design.md`) | +| ✅ | Principles | [`../00-principles.md`](../00-principles.md) | 13 principles, each with a why and a link to the doc that enforces it | +| ✅ | `wovm` VM core | [plan 1](../superpowers/plans/2026-08-01-wob-format-and-vm-core.md) | `.wob` v1 loader with full static validation, register interpreter (computed-goto + ISO-C fallback), arena with size-class free lists, borrow word, RC + budgeted Bacon–Rajan cycle collector, drop-map trap unwinding, containers, builtins, ICALL, CLI. 13 suites × 2 dispatch flavors + CLI smoke, ASan/UBSan clean | +| ✅ | `.wob` format contract | [`oop-vm/00-wob-format.md`](../plan/oop-vm/00-wob-format.md) | Normative; twinned with `runtime/src/wob.h` | +| ✅ | `woc` compiler front | [plan 2](../plan/compiler/2026-08-01-woc-compiler-front.md) | Tasks 1–8: dune scaffold, `diag` (WO-E codes, two-site related errors, ordered dedup), newline-significant lexer at rt parity, declaration + statement/expression parser with skip-on-block and multi-error recovery, typechecker (field kinds, `?T` plumbing, W201, E225, E214), MVS ownership pass with the four emitter tables, driver with directory discovery + cross-file programs. 14 + 264 checks | +| ✅ | Error catalog | [`oop-vm/01-error-catalog.md`](../plan/oop-vm/01-error-catalog.md) | 14 emitted codes + 10 reserved, each with the reason it is not yet emitted | +| ✅ | log-watcher `.wo` sample | [`../examples/log-watcher/`](../examples/log-watcher/README.md) | Eight-file port authored docs-first with its `.hx` mapping table; compiles for real in iteration 7 | +| ✅ | Scalar cleanup | [`discarded.md`](../plan/discarded.md) | `Money`/`SKU`/`Float` and the abstract allowlist removed; `abstract` flipped adopt → reject | +| ✅ | `woc` emitter, corpus, single binary | [plan 3](../plan/compiler/2026-08-01-wob-emit-e2e-single-binary.md) | Tasks 1–6 + 8 (Task 7, a parity harness against the Rust runtime, **deferred by explicit user decision** — the two stacks diverge by design). Bytecode emitter (`emit.ml`) + disassembler (`disasm.ml`, `--dump-bc`); three-kind conformance harness (`scripts/oop-e2e.sh`, `just oop-e2e`) over `tests/corpus/{run,compile-fail,trap,gc}`; pricing-demo + ownership/trap corpora (19 fixtures); `@gc` cycle collector's post-exit pump (`WO_GC_BUDGET`/`WO_GC_TRACE`) + 2 gc fixtures (`gc/held-cycle` retired — see criterion-3 closure below); `woc build` single-binary output + relocation/corrupt-trailer smoke; `WO-E405` closing criterion 3's ASan leak (entry must return `Int`); `just oop-accept` wiring all five spec criteria + both unit gates into one command. 14 + 399 compiler checks; `oop-e2e` 25/25 against the release `wovm`. **Milestone-1 acceptance gate is fully green — all five criteria met** (see the dated acceptance note in `docs/superpowers/specs/2026-08-01-oop-compiler-vm-design.md`) | **Known gaps carried out of iteration 3** — recorded, not silently owed: @@ -258,7 +322,7 @@ Gates at the end of that session: corpus 71/0, `woc` runtest 565/0, every `?T`; the semantics do not exist (`WO-E211`/`E212`/`E213` declared, never emitted — a probe returning `?Int` as `Int` exits 0). Owned by iteration 5, plan 8 Task 6, which is that iteration's first task because it blocks the - log-watcher port. See [`compiler/nullable-types-implementation.md`](plan/compiler/nullable-types-implementation.md). + log-watcher port. See [`compiler/nullable-types-implementation.md`](../plan/compiler/nullable-types-implementation.md). - **Structural interface satisfaction is not checked** (`WO-E205` dead), along with type mismatch, bad arity, and unknown-fn (`E201`/`E203`/`E204`) — all named in plan 2 Task 6's own must-fail list. Gaps in shipped work, catalogued @@ -279,7 +343,7 @@ Gates at the end of that session: corpus 71/0, `woc` runtest 565/0, every special case in `owner.ml`'s `analyze_call`, so a `@gc` key or value handed to `set` is under-counted and the collector can free it while the map still points at it. Nothing in the corpus exercises this yet. See - [`oop-vm/08-builtin-surface.md`](plan/oop-vm/08-builtin-surface.md). + [`oop-vm/08-builtin-surface.md`](../plan/oop-vm/08-builtin-surface.md). **Known gaps carried out of the 2026-08-14 compile-and-run milestone** — recorded, not silently owed: @@ -302,7 +366,7 @@ recorded, not silently owed: used to answer `{"isError":true,"text":"tool failed: not a text value"}`; all four MCP tools now return `isError:false` with correct payloads. `OWNED`/`GCREF` elements still move, and `set`'s `@gc` retention gap is still - open (see [`oop-vm/08-builtin-surface.md`](plan/oop-vm/08-builtin-surface.md)). + open (see [`oop-vm/08-builtin-surface.md`](../plan/oop-vm/08-builtin-surface.md)). - **A blocking `accept`/`read` swallows SIGTERM.** `env.stopping()` installs a handler that only sets a flag, and `net.accept`/`net.read` retry on `EINTR`, so a server parked in `accept` never observes it: a plain TERM does not stop @@ -330,7 +394,7 @@ recorded, not silently owed: not fixture pairs; `tests/corpus/` still gates every pre-existing behavior (71 checks, 0 failures). - **E201/E203 and seven other `WO-E2xx` codes remain declared but unemitted** - — see [`oop-vm/01-error-catalog.md`](plan/oop-vm/01-error-catalog.md). + — see [`oop-vm/01-error-catalog.md`](../plan/oop-vm/01-error-catalog.md). - **CLOSED — milestone-1's ASan gate (`just oop-accept`) failing on `gc/held-cycle`.** Root cause (Task 8's finding, restated): `main.c`'s entry-method return value (`uint64_t ret`, `src/main.c:158`) is stored @@ -355,80 +419,88 @@ recorded, not silently owed: The C proving-ground work (`exploration/c-runtime/`, phases A–F: 859k reads/s, 618k durable commits/s) fed the current C runtime and remains as an -[exploration study](plan/exploration/c-runtime/00-plan.md). +[exploration study](../plan/exploration/c-runtime/00-plan.md). --- ## Pending -### Implementation order (re-sequenced 2026-08-20 — code-review pass) +### Implementation order (re-sequenced 2026-08-21 — concurrency chain) -Replaces the framework-goal ordering. Basis: the verified findings in -[`00-code-review.md`](00-code-review.md) — measure before optimizing, close -correctness holes before adding surface, stop stacking features on -unmeasured ground. IDs below are post-renumber; the authoritative table with -per-row reasoning is -[`00-story.md`](stories/language-runtime-database/00-story.md). +Everything still pending IS the runtime-concurrency chain. Basis: the +2026-08-20 code-review pass (measure before optimizing, close correctness +holes before adding surface), amended 2026-08-21 by developer decision: +**stage 3 before 22** — correctness first, then one benchmark campaign +covers single- and multi-shard. The authoritative table with per-row +reasoning is [`00-story.md`](language-runtime-database/00-story.md). -Dependency rules that still force the shape: 23 explicitly after 8 + 22; -21's plan folds into 20's; 26 only after 9 + 25; 11 rides 8's shard -scheduler; h2c parked behind 8/23/11. +Dependency rules that force the shape: 23 after stage 3 + 22 (the ring is +the arc's, the baseline is 22's); 24 after 31 (chat is dishonest without +lifecycle); h2c stays parked behind the chain; the held tail keeps its own +precedence notes for resumption. -1. **22** — the measurement backbone, and now first: it has never run, so - every performance claim on this project is unsourced. No - `bench/baseline.json`, no `just db-bench`; `runtime/bench/` is the - retired C prototype's harness. -2. **8+11 stage 3** — transparent DB RPC, then 22 re-run for the - concurrency delta. Reframed as a correctness fix: worker VMs are - zero-initialized, so a DB statement off the primary traps `WO_T_DB`. - A multi-shard program that touches the database is broken today. -3. **30** (new) — observability, CI, fuzz: runtime counters + a profiler - hook, 22's harness run per change instead of by hand, a fuzz target on - the parser and `.wob` loader. No iteration covered any of this. -4. **19** — Float + Bytes; small, and it gates 24 (WS frames) and the - crypto fork (digests). -5. **31** (new) — actor lifecycle: request/response (`send` is one-way and - callers `sleep` to await), bounded mailboxes (the FIFO only grows), - actor death/supervision, timers beyond `time.sleep`. -6. **24** — chat, the arc's acceptance; honest only after 19 + 31. -7. **23** — io_uring group-commit; explicitly after 8 + 22. -8. **25** — HTTP service layer; `service` blocks lower onto the framework - instead of a parallel stack. -9. **18** — framework v2 (transaction{} + cache/flags/jobs); spec APPROVED - 2026-08-20 but **demoted from first**: more surface on a framework with - one consumer, and its cache stores `Text` because there are no generics. -10. **27, then 26** — query grammar from corpora (likely collapses to - "confirm `len(query)` + add `exists`"), then blue-green. -11. **20 then 21** — demoted hard: new distribution surface while there is - no TLS, no crypto primitives, and the multi-shard DB still traps. The - half-done branches (ipc-attach, keypair-auth) keep their manifests. -12. **28, then 29 + parked drain** — skillhost is no longer the driving - workload; then metaprogramming (spec first), group-by, ADT roster, - WO-E225, held by the 2026-08-08 scope directive. +1. ✅ **8+11 stage 3** — landed 2026-08-21 (`just db-actor` 8/0; arc + complete, stories in done/). Was: transparent DB actor. A correctness fix, not an + optimization: worker VMs are zero-initialized, so a DB statement off + the primary traps `WO_T_DB` — a multi-shard program touching the + database is broken today. Plan of record: + [`2026-08-20-shard-fiber-arc.md`](../superpowers/plans/2026-08-20-shard-fiber-arc.md) + (stages 1+2 landed 2026-08-20, branch `concurrency-arc`). +2. 🔄 **22** — IN PROGRESS (spec + plan approved 2026-08-21) — the + measurement backbone: restart-persistence proof + baseline + benchmark (durable + RAM-only), single- AND multi-shard in one + campaign, plus the stage-2 mutex-inbox number (rings only if the mutex + costs). It has never run — no `bench/baseline.json`, no `just db-bench`; + the arc's stages 1+2 delta is recorded retroactively. +3. **31** — actor lifecycle + ([story](language-runtime-database/refine/31-actor-lifecycle.md), + written 2026-08-21): request/response (`send` is one-way and callers + `sleep` to await), bounded mailboxes (the FIFO only grows), actor + death/supervision, timers beyond `time.sleep`. +4. **24** — chat, the arc's acceptance; honest only after 31 (19 landed + 2026-08-20 — Bytes carries the frames). +5. **23** — io_uring group-commit; the WAL's WRITE+FSYNC chains ride the + arc's per-shard ring (T4); after 22's baseline — the payoff, measured. +6. **32** — WAL checkpoint + ([story](language-runtime-database/refine/32-wal-checkpoint.md), + written 2026-08-21): the WAL is append-only forever — snapshot + + truncate reclaims disk and bounds replay; after 23 (composes with + group-commit), policy set by 22's aged-store numbers. + +**30** — observability, CI, fuzz: named 2026-08-20, still row-only (no +story file); slots in when scheduled — nothing in the chain depends on it. + +⏸ **Held** (2026-08-21, developer decision): 18, 20, 21, 25, 26, 27, 28, +29 — stories in +[`hold/`](language-runtime-database/hold/) (25's story file +removed; its [plan doc](../superpowers/plans/2026-08-01-http-service-layer.md) +remains). Half-done branches (ipc-attach, keypair-auth) keep their +manifests. ✅ **17** — landed 2026-08-20 (unparked and executed): `kind = "library"`, check mode, and the `internal/` dep boundary (WO-E108). Driver-only. +✅ **19** — landed 2026-08-20: Float + Bytes, `.wob` v5. ### Language track — sequenced, on the critical path | # | Item | Plan | | --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------- | -| 5 | Haxe-parity language surface — **`?T` forced handling first**, then switch expressions, records, enum payloads, try/catch, statics, `using`, modules, `is`, `pub(read)`, `#if` | [plan 8](plan/compiler/2026-08-01-haxe-parity-language.md) | -| 6 | Program mode + systems stdlib — `fn main`, exit codes, `fs`/`proc`/`net`/`time`/`json` | [plan 9](superpowers/plans/2026-08-01-program-mode-stdlib.md) | -| 7 | log-watcher proof — the sample compiles and detects a silent death live | [plan 10](superpowers/plans/2026-08-01-log-watcher-sample.md) | -| 8 | Shard-actor runtime | [plan 4](superpowers/plans/2026-08-01-shard-actor-vm-runtime.md) | -| 9 | Database engine binding | [plan 5](superpowers/plans/2026-08-01-db-engine-binding.md) | -| 9b | `@table` + relations + language-integrated query — comprehension queries, `ref`/`backlink` navigation, GroupBy aggregates; acceptance: new `docs/examples/employee` sample | [spec](superpowers/specs/2026-08-15-table-relations-query-design.md) · [plan](plan/compiler/2026-08-15-employee-relations-query.md) | +| 5 | Haxe-parity language surface — **`?T` forced handling first**, then switch expressions, records, enum payloads, try/catch, statics, `using`, modules, `is`, `pub(read)`, `#if` | [plan 8](../plan/compiler/2026-08-01-haxe-parity-language.md) | +| 6 | Program mode + systems stdlib — `fn main`, exit codes, `fs`/`proc`/`net`/`time`/`json` | [plan 9](../superpowers/plans/2026-08-01-program-mode-stdlib.md) | +| 7 | log-watcher proof — the sample compiles and detects a silent death live | [plan 10](../superpowers/plans/2026-08-01-log-watcher-sample.md) | +| 8 | Shard-actor runtime | [arc plan](../superpowers/plans/2026-08-20-shard-fiber-arc.md) (plan 4 ✖ discarded 2026-08-21 — epoll-based) | +| 9 | Database engine binding | [plan 5](../superpowers/plans/2026-08-01-db-engine-binding.md) | +| 9b | `@table` + relations + language-integrated query — comprehension queries, `ref`/`backlink` navigation, GroupBy aggregates; acceptance: new `docs/examples/employee` sample | [spec](../superpowers/specs/2026-08-15-table-relations-query-design.md) · [plan](../plan/compiler/2026-08-15-employee-relations-query.md) | | 20 | Cross-program tables — attach to a running program's database (IPC string in wo.toml, manifest-granted rights, owner stays the single writer) | **no spec yet** — four open forks recorded in the iteration; brainstorm before planning | | 21 | Keypair attach auth — mutual challenge–response, grants name public keys, uid superseded | **no spec yet** — four forks recorded; plan folds into 20's | | 22 | Durability + throughput + scale — restart-persistence, read/write benchmark, ~1M rows; the gate every later optimization re-runs | **no spec yet** — four forks recorded; the measurement backbone | | 23 | io_uring group-commit write path — batched durability overlapped on shard threads, fsync fallback | **no spec yet** — brainstorm after iterations 8 + 22 | | 27 | Query grammar from real embedded-DB corpora — whole-query count + correlated exists, driven by the skillhost SQL catalogue; add only what a corpus uses | **no spec yet** — three forks; may collapse to "confirm len(query) + add exists" | | 14 | skillhost host workload — port skillhost (MCP host + confined script runner) to writeonce; drives the missing host capabilities into the open (bounded subprocess, stdin/stdout transport, fs metadata, FFI-vs-out-of-process) | **no spec yet** — gaps recorded in the iteration; each gap brainstormed on demand, bounded-subprocess first | -| 17 | library projects + dependency privacy — `wo.toml` kind = "library" (checkable without entry, dual lib+bin) + Go-style `internal/` at the [deps] boundary; framework reorg demonstrates both | ✅ **landed 2026-08-20** — [spec](superpowers/specs/2026-08-20-library-kind-internal-design.md) · [plan](superpowers/plans/2026-08-20-library-kind-internal.md) | -| 10 | HTTP service layer | [plan 6](superpowers/plans/2026-08-01-http-service-layer.md) | -| 11 | Fibers | vision §3, [blue-green exploration](plan/exploration/blue-green-vm/00-vision.md) | -| 12 | Blue-green deploy | [spec](superpowers/specs/2026-08-03-blue-green-vm-design.md) — plan authored after iterations 9 + 25 | +| 17 | library projects + dependency privacy — `wo.toml` kind = "library" (checkable without entry, dual lib+bin) + Go-style `internal/` at the [deps] boundary; framework reorg demonstrates both | ✅ **landed 2026-08-20** — [spec](../superpowers/specs/2026-08-20-library-kind-internal-design.md) · [plan](../superpowers/plans/2026-08-20-library-kind-internal.md) | +| 10 | HTTP service layer | [plan 6](../superpowers/plans/2026-08-01-http-service-layer.md) | +| 11 | Fibers | vision §3, [blue-green exploration](../plan/exploration/blue-green-vm/00-vision.md) | +| 12 | Blue-green deploy | [spec](../superpowers/specs/2026-08-03-blue-green-vm-design.md) — plan authored after iterations 9 + 25 | ### Language track — parked until after iteration 26 @@ -438,7 +510,7 @@ log-watcher proof. - `WO-W201` `@gc`-suggestion refinement beyond the self-reference heuristic - `WO-E225` broadened to `ref`/`multi`/`map` element types and fn signatures - ADT container roster adoption (Stack, Queue, Set, Tree, Graph, …) — see the - roster in [`compiler/nullable-types-implementation.md`](plan/compiler/nullable-types-implementation.md) + roster in [`compiler/nullable-types-implementation.md`](../plan/compiler/nullable-types-implementation.md) - Web framework as a `.wo` library; UI (`##ui` SSR + live patches); script-based destructive migrations; MCP/agent wrapper over the management plane - `throw` (explicit raise) — cut 2026-08-10, 0 uses in the driving workload @@ -455,13 +527,13 @@ implementation plan, the 7-of-7 `ui-htmlx-live` plan, and the 9-doc `plan/exploration/ui/` design set — was **removed**. It was built entirely on the non-advancing Rust runtime (`.dev/reference/crates/wo-htmlx`, `cargo run`, WebSocket live-patches) and contradicts the current woc/wovm direction. Recorded -in [`discarded.md`](plan/discarded.md). +in [`discarded.md`](../plan/discarded.md). --- ## Discarded -Settled rejections with their reasons live in [`discarded.md`](plan/discarded.md) — +Settled rejections with their reasons live in [`discarded.md`](../plan/discarded.md) — inheritance, `abstract` newtypes, `Money`/`SKU`/`Float`, `Dynamic`/`cast`/ `macro`/`extern`, AOT-to-C, Menhir, shared mutable engine state, external deployer daemon, destructive migrations in v1, and more. Argue against the @@ -469,7 +541,7 @@ recorded reason rather than re-opening an entry as new. ## Learnings -What attempts taught, shipped or not, in [`learnings.md`](plan/learnings.md) — +What attempts taught, shipped or not, in [`learnings.md`](../plan/learnings.md) — plumbed-is-not-enforced, vacuously-passing goldens, exit-0-with-wrong-output, the malloc-path ASan trick, deferred checks that never reach the runtime, validate-once-at-the-boundary, and reference-implement-in-C-first. diff --git a/docs/stories/board-views.md b/docs/stories/board-views.md new file mode 100644 index 0000000..604a7e1 --- /dev/null +++ b/docs/stories/board-views.md @@ -0,0 +1,66 @@ +# Live board views (Obsidian Dataview) + +Every story iteration file carries YAML frontmatter — **the frontmatter +is the source of truth**: + +```yaml +--- +iteration: "8" # immutable id (string: "7b", "9b" exist) +status: in-progress # done | in-progress | refine | hold — mirrors its folder +chain: 1 # concurrency-chain position, chain stories only (1–6) +--- +``` + +The folder move IS the status change: moving a story between `done/`, +`in-progress/`, `refine/`, `hold/` must update its `status:` in the same +change — the two never disagree. The prose board +([`00-status.md`](00-status.md)) stays the standup narrative; these +queries are the live views over the same facts. + +Adjust the `FROM` path to your vault root (queries below assume the +vault opens at the repo root). + +## Everything not done, chain order first + +```dataview +TABLE iteration, status, chain +FROM "docs/stories/language-runtime-database" +WHERE status != "done" +SORT chain ASC, iteration ASC +``` + +## Grouped by status (the kanban lanes, as data) + +```dataview +TABLE rows.file.link AS story, rows.iteration AS iteration +FROM "docs/stories/language-runtime-database" +WHERE status != "done" +GROUP BY status +``` + +## The concurrency chain, in execution order + +```dataview +TABLE iteration, status +FROM "docs/stories/language-runtime-database" +WHERE chain +SORT chain ASC +``` + +## Active right now + +```dataview +LIST +FROM "docs/stories/language-runtime-database" +WHERE status = "in-progress" +``` + +## Kanban caveat + +The Kanban plugin stores board state in its own markdown file — a +second copy of status. To keep frontmatter the single source of truth: +**use Dataview for querying; treat any Kanban board as a VIEW, never +the place status is edited.** Status changes happen by moving the story +file between folders + updating its `status:` key (one commit); a +Kanban card drag that only rewrites the Kanban file is a lie the next +query won't see. diff --git a/docs/stories/language-runtime-database/00-story.md b/docs/stories/language-runtime-database/00-story.md index 2d27a31..5a01301 100644 --- a/docs/stories/language-runtime-database/00-story.md +++ b/docs/stories/language-runtime-database/00-story.md @@ -71,6 +71,20 @@ adding surface, and stop stacking features on unmeasured ground. or timers behind `spawn`/`send`. - **18, 20, 21 demoted.** All three add surface; none answer a named gap. +RE-SEQUENCED 2026-08-21 (third pass — the concurrency chain). Everything +still pending IS the runtime-concurrency chain; order: +**stage 3 → 22 → 31 → 24 → 23 → 32**. Changes from the second pass: + +- **The arc's stage 3 moves ahead of 22** — correctness before + measurement: a multi-shard program touching the database traps + `WO_T_DB` today, and fixing that first lets one benchmark campaign + cover single- and multi-shard honestly. +- **31 has its story file** ([refine/31-actor-lifecycle.md](refine/31-actor-lifecycle.md)); + 30 stays a row until it is scheduled. +- **Holds landed** (developer decision, 2026-08-21): 18, 20, 21, 26, 27, + 28, 29 moved to `hold/`; 25's story file removed (its plan doc remains + in superpowers). 19 landed 2026-08-20. + | Seq | # | Iteration | Delivers | | --- | --- | --- | --- | | 1 | 1 | [Principles doc](done/01-principles-doc.md) | `docs/00-principles.md` — the doctrine page every later slice links back to | @@ -85,21 +99,22 @@ adding surface, and stop stacking features on unmeasured ground. | 10 | 9b | [`@table`, relations, query](done/09b-table-relations-query.md) | `@table` real storage; `ref`/`backlink`/`multi`; compiler-checked queries | | 11 | 15 | [deps: `wo.toml [deps]`](done/15-deps-package-manager.md) | exact-rev git deps + `wo.lock` + `.wo-deps`; flat-only, offline once locked | | 12 | 16 | [web framework](done/16-web-framework.md) | the `.wo` framework v1 (router, middleware, auth, all three body hooks) consumed via `[deps]` | -| 13 | 22 | [Durability, throughput, scale](refine/22-durability-throughput-scale.md) | restart-persistence proof, benchmarks, ~1M rows — the baseline the arc and 23 sign against. **Promoted to first pending (was seq 18)**: it has never run, so every performance claim on this project is currently unsourced. *(was 9e)* | -| 14 | 8+11 | [Shard-actor runtime](refine/08-shard-actor-runtime.md) · [Fibers](refine/11-fibers.md) | THE ARC (stages 1+2 landed 2026-08-20: fibers/budget/actors/io_uring plane; pinned shards, envelope sends, home-routed frees, WO-E222); stage 3 = transparent DB RPC + 22 re-run. **Stage 3 is a correctness hole, not an optimization**: worker shards are zero-initialized, so a DB statement off the primary traps `WO_T_DB`. | +| 13 | 8+11 | [Shard-actor runtime](done/08-shard-actor-runtime.md) · [Fibers](done/11-fibers.md) | ✅ **THE ARC LANDED 2026-08-21** — stages 1+2 (fibers/budget/actors/io_uring plane; pinned shards, envelope sends, home-routed frees, WO-E222) + stage 3's transparent DB actor: worker statements marshal to shard 0, ack-after-owner-fsync, materialized replies (`just db-actor` 8/0, ASan/TSan, WAL replay pair). fs-park re-scoped out (disclosed in story 11). | +| 14 | 22 | [Durability, throughput, scale](in-progress/22-durability-throughput-scale.md) | 🔄 **spec + plan approved 2026-08-21, executing** — db-bench sample (`time.ticks` µs clock, seed/read/query/write/mix/msgrate/verify), campaign gates vs `bench/baseline.json`, restart + kill -9 proofs at both shard counts; the arc's delta and the mutex-inbox number come out of the first run. *(was 9e)* | | 15 | 30 | Observability, CI, fuzz *(no story file yet)* | **NEW** — runtime counters + a profiler hook, 22's harness wired to run per change instead of by hand, and a fuzz target on the parser and `.wob` loader. The whole proof-maturity gap had no iteration to point at. | | 16 | 19 | [Float + Bytes](done/19-missing-scalar-types.md) | **LANDED 2026-08-20** — `.wob` v5; the full stack: IEEE-quiet f64 through literals/VM/@table/WAL/json + Bytes as the binary carrier, no implicit mixing, total-order indexes. Unblocks 24 (WS frames) and the crypto fork (digests). *(was 20)* | -| 17 | 31 | Actor lifecycle *(no story file yet)* | **NEW** — request/response (today `send` is one-way and callers `sleep` to await), bounded mailboxes with backpressure (today the FIFO just grows), actor death/supervision, and timers beyond `time.sleep`. 24 cannot be written honestly without these. | +| 17 | 31 | [Actor lifecycle](refine/31-actor-lifecycle.md) | request/response (today `send` is one-way and callers `sleep` to await), bounded mailboxes with backpressure (today the FIFO just grows), actor death/supervision, and timers beyond `time.sleep`. 24 cannot be written honestly without these. *(story written 2026-08-21)* | | 18 | 24 | [chat: WebSocket workload](refine/24-chat-websocket-workload.md) | the arc's acceptance: WS upgrade + frames (SHA-1 via crypto fork, Bytes via 19), rooms/broadcast, 1k clients, drain-clean. *(was 19)* | | 19 | 23 | [io_uring group-commit](refine/23-io-uring-commit.md) | WAL WRITE+FSYNC chains on the arc's per-shard rings; fsync fallback kept (after 22 + the arc). *(was 9f)* | -| 20 | 25 | [HTTP service layer](25-http-service.md) | `service` blocks lower onto the framework (after 9b + 20 by their own precedence notes). *(was 10)* | -| 21 | 18 | [framework v2: memory-rich features](hold/18-memory-db-features.md) | spec+plan approved: TTL cache, @table flags, durable job queue, `transaction { }` over the WAL's staged batch. **Demoted from seq 14**: more surface on a framework with one consumer, and the cache still stores `Text` because there are no generics | -| 22 | 27 | [Query grammar corpus](refine/27-query-grammar-corpus.md) | grow the query grammar from real corpora; likely collapses to "confirm `len(query)` + add `exists`"; precedes 28. *(was 9g)* | -| 23 | 26 | [Blue-green deploy](26-blue-green-deploy.md) | two VM slots, in-runtime compile, atomic switch, resident rollback (plan authored after 9 + 25). *(was 12)* | -| 24 | 20 | [Cross-program tables](refine/20-cross-program-tables.md) | attach to a running program's database over local IPC; owner stays the single writer (channel half-built). **Demoted from seq 16**: new distribution surface while there is no TLS, no crypto, and the multi-shard DB still traps. *(was 9c)* | -| 25 | 21 | [Keypair attach auth](refine/21-keypair-attach-auth.md) | program identity is a keypair; mutual challenge–response at attach (crypto half-built; plan folds into 20's). **Demoted with 20** — and it needs crypto primitives that do not exist. *(was 9d)* | -| 26 | 28 | [skillhost host workload](refine/28-skillhost-host-workload.md) | host-shaped driving workload naming runtime gaps — demoted with the framework goal. *(was 14)* | -| 27 | 29 | [Compile-time metaprogramming](refine/29-compile-time-metaprogramming.md) | `@derive(...)` from class-table metadata; held with the parked drain by the 2026-08-08 scope directive. *(was 13)* | +| 20 | 32 | [WAL checkpoint](refine/32-wal-checkpoint.md) | **NEW 2026-08-21** (stage-3 guarantee refinement found the hole) — the WAL is append-only forever: snapshot + truncate reclaims disk and bounds replay time; every durability guarantee byte-identical; crash mid-checkpoint recovers from the previous snapshot + full tail. After 23 (composes with group-commit); RAM slot-reuse already contracted in `04-db-binding.md`. | +| 21 | 25 | [HTTP service layer](../../superpowers/plans/2026-08-01-http-service-layer.md) | `service` blocks lower onto the framework (after 9b + 20 by their own precedence notes). **HELD 2026-08-21** — story file removed; the plan doc remains. *(was 10)* | +| 22 | 18 | [framework v2: memory-rich features](hold/18-memory-db-features.md) | spec+plan approved: TTL cache, @table flags, durable job queue, `transaction { }` over the WAL's staged batch. **Demoted from seq 14**: more surface on a framework with one consumer, and the cache still stores `Text` because there are no generics | +| 23 | 27 | [Query grammar corpus](hold/27-query-grammar-corpus.md) | grow the query grammar from real corpora; likely collapses to "confirm `len(query)` + add `exists`"; precedes 28. *(was 9g)* | +| 24 | 26 | [Blue-green deploy](hold/26-blue-green-deploy.md) | two VM slots, in-runtime compile, atomic switch, resident rollback (plan authored after 9 + 25). *(was 12)* | +| 25 | 20 | [Cross-program tables](hold/20-cross-program-tables.md) | attach to a running program's database over local IPC; owner stays the single writer (channel half-built). **Demoted from seq 16**: new distribution surface while there is no TLS, no crypto, and the multi-shard DB still traps. *(was 9c)* | +| 26 | 21 | [Keypair attach auth](hold/21-keypair-attach-auth.md) | program identity is a keypair; mutual challenge–response at attach (crypto half-built; plan folds into 20's). **Demoted with 20** — and it needs crypto primitives that do not exist. *(was 9d)* | +| 27 | 28 | [skillhost host workload](hold/28-skillhost-host-workload.md) | host-shaped driving workload naming runtime gaps — demoted with the framework goal. *(was 14)* | +| 28 | 29 | [Compile-time metaprogramming](hold/29-compile-time-metaprogramming.md) | `@derive(...)` from class-table metadata; held with the parked drain by the 2026-08-08 scope directive. *(was 13)* | | ✅ | 17 | [library projects + `internal/`](done/17-library-projects-internal.md) | **LANDED 2026-08-20** — `kind = "library"` + entry-less check mode (retires the `--emit` workaround) and Go's `internal/` rule as WO-E108 at the consumer's `use`; driver-only, VM/GC untouched. `just web-app` 26/0 | @@ -131,11 +146,9 @@ list, and a pointer to the plan document that already sequences its tasks. nothing MVC-scale — and iteration 17 (library kind + `internal/`) is **parked** with spec + plan ready on branch `library-internal`. The v1 slices landed 2026-08-20 (`just web-app` 21/0 after polish, auth, - form, multipart). Implementation order for everything still pending: - **18 (spec approved)** → 20/21 → 22 → 8 → 23 → 11 (+ h2c unparks) → - 10 → 12 → 27 → 14 → 13 + parked drain; 17 parked, slots anywhere after - 16 on directive. The iterations table above carries this order - row-by-row; edges live in + form, multipart). The implementation-order list this note once carried + is superseded — the iterations table above is the authority + (re-sequenced 2026-08-20 twice, then 2026-08-21); edges live in [`docs/00-dependency-graph.md`](../../00-dependency-graph.md). - **Iteration 7b (inserted 2026-08-11)** sits after the critical path deliberately: it delays nothing on the log-watcher line, and it must precede diff --git a/docs/stories/language-runtime-database/25-http-service.md b/docs/stories/language-runtime-database/25-http-service.md deleted file mode 100644 index 625daf5..0000000 --- a/docs/stories/language-runtime-database/25-http-service.md +++ /dev/null @@ -1,53 +0,0 @@ -# Iteration 25 — HTTP service layer - -> Format: fiberloom `product/story-iteration-template`. Part of -> [Story — one language, one runtime, one database, one binary](00-story.md). - -## Goals - -- The single binary serves: `service rest` blocks compile to routes and the - runtime answers HTTP from its shards — hand-rolled HTTP and JSON, zero - libraries, per the dependency doctrine. -- The new stack reaches feature parity with the Rust runtime's Stage 2 - REST surface — the concrete bar the retirement decision measures - against. - -## Acceptance Criteria - -- What to achieve? - - **Given** the blog sample's `service` declarations, - - **when** the compiled binary boots and `reference/rest/blog.rest` - runs against it, - - **then** every request in the smoke file answers as documented — - including the intentional 501/405/404 responses. -- What to achieve? - - **Given** a method call arriving over REST that traps (borrow - violation, abort, unique violation), - - **when** the response returns, - - **then** exactly one trap-to-HTTP table governs the mapping (e.g. - conflict-shaped 409s) and the structured error body carries the trap - record — the same trap surface everywhere, now over the wire. -- What to achieve? - - **Given** transports declared at boot, - - **when** the runtime starts under systemd socket activation or with - zero listeners, - - **then** both boot correctly — a runtime is not tied to a port. - -## Out Of Scope - -- WebSocket/live subscriptions and UI (the parked `##ui` story). -- The management plane endpoints (iteration 11 builds them on this - machinery). - -## Info - -- Route declarations ride the `.wob` format as a versioned section — a - coordinated format bump, the pattern later sections follow. -- The C reference's phase-C HTTP machine is the porting source. - -## Proposed Solution - -- Execute the existing plan: `docs/superpowers/plans/2026-08-01-http-service-layer.md` - (per-shard http module, hand-rolled JSON codec, service-block route - section, Stage-2-parity CRUD, method RPC with the trap table, blog.rest - smoke). diff --git a/docs/stories/language-runtime-database/done/01-principles-doc.md b/docs/stories/language-runtime-database/done/01-principles-doc.md index ddccfb8..6bc2cd4 100644 --- a/docs/stories/language-runtime-database/done/01-principles-doc.md +++ b/docs/stories/language-runtime-database/done/01-principles-doc.md @@ -1,3 +1,8 @@ +--- +iteration: "1" +status: done +--- + # Iteration 1 — the principles doc > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/done/02-vm-core.md b/docs/stories/language-runtime-database/done/02-vm-core.md index 9855480..c396b0d 100644 --- a/docs/stories/language-runtime-database/done/02-vm-core.md +++ b/docs/stories/language-runtime-database/done/02-vm-core.md @@ -1,3 +1,8 @@ +--- +iteration: "2" +status: done +--- + # Iteration 2 — VM core (`wovm`) > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/done/03-compiler-front.md b/docs/stories/language-runtime-database/done/03-compiler-front.md index 4ebbefa..c7fdbfb 100644 --- a/docs/stories/language-runtime-database/done/03-compiler-front.md +++ b/docs/stories/language-runtime-database/done/03-compiler-front.md @@ -1,3 +1,8 @@ +--- +iteration: "3" +status: done +--- + # Iteration 3 — compiler front (`woc`) > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/done/04-single-binary-e2e.md b/docs/stories/language-runtime-database/done/04-single-binary-e2e.md index 9886c87..919de08 100644 --- a/docs/stories/language-runtime-database/done/04-single-binary-e2e.md +++ b/docs/stories/language-runtime-database/done/04-single-binary-e2e.md @@ -1,3 +1,8 @@ +--- +iteration: "4" +status: done +--- + # Iteration 4 — single binary end-to-end > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/done/05-language-surface.md b/docs/stories/language-runtime-database/done/05-language-surface.md index 11db327..eb860ed 100644 --- a/docs/stories/language-runtime-database/done/05-language-surface.md +++ b/docs/stories/language-runtime-database/done/05-language-surface.md @@ -1,3 +1,8 @@ +--- +iteration: "5" +status: done +--- + # Iteration 5 — language surface (Haxe-parity adoptions) > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/done/06-program-mode-stdlib.md b/docs/stories/language-runtime-database/done/06-program-mode-stdlib.md index 14dc9da..ea2d2aa 100644 --- a/docs/stories/language-runtime-database/done/06-program-mode-stdlib.md +++ b/docs/stories/language-runtime-database/done/06-program-mode-stdlib.md @@ -1,3 +1,8 @@ +--- +iteration: "6" +status: done +--- + # Iteration 6 — program mode + systems stdlib > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/done/07-logwatcher-proof.md b/docs/stories/language-runtime-database/done/07-logwatcher-proof.md index e998c80..80dcf47 100644 --- a/docs/stories/language-runtime-database/done/07-logwatcher-proof.md +++ b/docs/stories/language-runtime-database/done/07-logwatcher-proof.md @@ -1,3 +1,8 @@ +--- +iteration: "7" +status: done +--- + # Iteration 7 — log-watcher proof workload > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/done/07b-inferred-gc-mark-sweep.md b/docs/stories/language-runtime-database/done/07b-inferred-gc-mark-sweep.md index 5ea8ec0..8f6d0ca 100644 --- a/docs/stories/language-runtime-database/done/07b-inferred-gc-mark-sweep.md +++ b/docs/stories/language-runtime-database/done/07b-inferred-gc-mark-sweep.md @@ -1,3 +1,8 @@ +--- +iteration: "7b" +status: done +--- + # Iteration 7b — inferred GC + incremental mark-sweep > Format: fiberloom `product/story-iteration-template`. Part of @@ -92,7 +97,7 @@ - **Gated by the benchmark (2026-08-15):** this is the "implement garbage collection" lever of the performance arc — tri-color mark-sweep replacing RC changes the write path's tail latency, so landing it means re-running - iteration [22](../refine/22-durability-throughput-scale.md) and recording the + iteration [22](../in-progress/22-durability-throughput-scale.md) and recording the delta (does tracing help or hurt p99 under write load?). - **Constraint added by the database track (2026-08-15):** a GC-managed value in a `@table` field is a compile error (the engine/heap bulkhead — 9b diff --git a/docs/stories/language-runtime-database/done/08-shard-actor-runtime.md b/docs/stories/language-runtime-database/done/08-shard-actor-runtime.md new file mode 100644 index 0000000..0d9cbe9 --- /dev/null +++ b/docs/stories/language-runtime-database/done/08-shard-actor-runtime.md @@ -0,0 +1,177 @@ +--- +iteration: "8" +status: done +chain: 1 +--- + +# Iteration 8 — shard-actor runtime (the 8+11 concurrency arc, part 1) + +> Format: fiberloom `product/story-iteration-template`. Part of +> [Story — one language, one runtime, one database, one binary](../00-story.md). +> +> **✅ LANDED 2026-08-21** — the arc is complete. Stage 3 closed the +> `WO_T_DB` hole: worker-shard DB statements marshal to the owner shard +> (requester-side slot encode, serialized owner execution, materialized +> reply, ack-after-owner-fsync). Proof: `just db-actor` 8/0 (NEW gate, +> `docs/examples/db-actor`), ASan/TSan clean, full battery green, WAL +> replay pair. The three stage-3 criteria below hold; the heavier +> concurrent-load truth is iteration 22's campaign. 22's minimal +> precursor (RAM-only): 500 remote inserts ≈4ms (~8µs/RPC round-trip) +> vs local ≈0ms; 50 remote scans ≈2–4ms. En route, a latent stage-1 bug +> fell: shared io_uring params raced by lazy worker init lost park wakes +> (~1/20 hangs) — params are per-vm now, short submits trap loud. +> +> **REFINED 2026-08-20** (developer decisions, no code): iterations 8 and +> 11 are **one arc** — the scheduler, fibers on it, then serving — because +> the database-ownership decision below makes a fiberless multi-shard +> server block a whole thread per cross-shard call. The arc's driving +> workload is **iteration 24 (chat: WebSocket pub/sub)**. The pre-existing +> plan (`docs/superpowers/plans/2026-08-01-shard-actor-vm-runtime.md`) +> predates inferred GC (7b), the unified surface, and the DB decision — +> it is a source of ideas, NOT the plan of record (✖ DISCARDED +> 2026-08-21: epoll-based; io_uring is a must). +> +> **RE-SEQUENCED 2026-08-21** (developer decision): the brainstorm → +> spec → plan happened. The arc's plan of record is +> [`2026-08-20-shard-fiber-arc.md`](../../../superpowers/plans/2026-08-20-shard-fiber-arc.md) +> (spec: [`2026-08-20-shard-fiber-arc-design.md`](../../../superpowers/specs/2026-08-20-shard-fiber-arc-design.md)), +> and **stages 1+2 LANDED 2026-08-20** on branch `concurrency-arc` +> (T1–T6, seven disclosed deviations recorded in the plan). Remaining +> scope: **stage 3, the transparent DB actor** — a correctness fix, not +> an optimization: worker VMs are zero-initialized, so a DB statement +> off the primary shard traps `WO_T_DB`. Concurrency-chain order: +> **stage 3 → 22 → 31 → 24 → 23 → 32**. + +## Settled decisions (2026-08-20) + +1. **The database is an actor.** The engine (`wo_db` + WAL) lives on one + owner shard; every query/write from another shard is a message send, + and results come back materialized (queries already copy rows out — + the model was built for this). Doctrine-pure: no lock, no shared + mutable state. Consequence, accepted deliberately: callers must PARK + while the reply travels, which is why 8 and 11 ship as one arc. + (Rejected: a coarse engine lock — bends the doctrine and caps write + scaling anyway; partitioned tables — the real scale answer, but it + waits for a measured need, not v1.) +2. **One concurrency surface: everything is an actor address.** `spawn` + returns an address whether the spawnee lands on this shard (a fiber) + or another (placement policy's call); `send` always moves ownership; + a same-heap send skips the ring and is cheap. The language never + shows a fiber-vs-actor split. (This dissolves iteration 11's + "handle vs address" open question.) +3. **Driving workload: chat** (iteration 24) — rooms, broadcast, N + concurrent WebSocket clients, one binary. The arc's acceptance is the + chat sample's, not only synthetic corpora. +4. **Order — SUPERSEDED 2026-08-21.** Originally 22 → the 8+11 arc → 23; + in fact the arc's stages 1+2 landed before 22 ever ran (accepted + deviation — the before/after delta is owed and lands as 22's + multi-shard pass). Current order: **stage 3 → 22 → 31 → 24 → 23 → 32**. + 23 still waits for the arc's tick boundary and 22's baseline. + +## Goals + +- The runtime scales past one core the doctrine way: pinned + thread-per-core shards, each owning its own heap and event loop; + cross-shard communication is a message send that **moves ownership** — + shared mutable state never exists. +- The language grows `spawn`/`send` (the unified address surface); + garbage collection stays per-shard (7b's collector is already + per-shard by construction), so no global pause appears at any core + count. +- The database keeps its single-writer truth by BEING an actor on its + owner shard. + +## Acceptance Criteria + +- What to achieve? + - **Given** a program spawning actors across shards, + - **when** an owned object is sent to another shard, + - **then** the sender can no longer touch it (compile-time move), the + receiver owns it, and its eventual free routes back to its + allocation-home arena. +- What to achieve? + - **Given** debug builds with shard-ownership asserts, + - **when** the deterministic actor corpus runs under ASan and TSan, + - **then** zero races, zero leaks, and identical output across runs. +- What to achieve? + - **Given** a reference to a TRACED object (GC-ness is inferred since + 7b — there is no `@gc` to write), + - **when** code attempts to send it cross-shard, + - **then** the compiler rejects it: aliased references cannot cross + heap boundaries, and the diagnostic names the inferred-traced class + and why it is traced. (This criterion originally said `@gc`; + restated 2026-08-20 in inference terms — same rule, current + language.) +- What to achieve? + - **Given** handlers on serving shards querying and writing through + the DB-owner shard, + - **when** the employee/web-app matrices run multi-shard, + - **then** every answer is byte-identical to the single-shard run and + the WAL's ack-after-durable contract is unchanged. +- What to achieve? (stage-3 refinement, 2026-08-21) + - **Given** a worker shard issuing a write through the DB actor, + - **when** the process is killed between the worker's send and the + owner's commit, + - **then** the write was never acknowledged AND replay shows no + partial state — a write RPC is exactly one owner-shard commit, + and the ack crosses shards only after the owner's fsync. +- What to achieve? (stage-3 refinement, 2026-08-21) + - **Given** a multi-shard boot, + - **when** workers start serving, + - **then** WAL replay has already completed on the primary, and no + worker ever opens the WAL or the data directory (asserted in + debug builds). +- What to achieve? (stage-3 refinement, 2026-08-21) + - **Given** concurrent workers hammering reads and writes at one + table, + - **when** the deterministic multi-shard corpus runs under TSan, + - **then** no torn read exists — every statement sees the serialized + moment its envelope executes on the owner shard (replies are + materialized copies). Full guarantee map: the contract table in + *Info* below. + +## Out Of Scope + +- Cross-shard transactions (2PC) — the DB actor serializes writers, so + iteration 18's `transaction { }` is unaffected; distributing it is a + later story. +- Fiber details beyond the shared scheduler substrate — part 2 + ([iteration 11](11-fibers.md)) owns them. +- WebSocket framing — the framework's (iteration 24's) job. + +## Info + +**Guarantee contract** (stage-3 refinement 2026-08-21; moved here from +the slice's marker doc when it landed): + +| property | state | +| --- | --- | +| Atomicity | per-statement ✅ (WAL record replays whole-or-not-at-all); multi-statement = `transaction { }`, iteration 18, ⏸ held. Stage 3: a worker write RPC is exactly ONE owner-shard commit — a crash between send and commit leaves no ack and no partial state. | +| Durability | ✅ fsync-per-commit, ack-after-durable; the ack crosses shards only AFTER the owner's fsync (`just db-actor`'s WAL pair). Power-loss rides fdatasync semantics; 22's kill battery is the scripted proof. | +| Crash recovery | ✅ boot replay, torn-tail drop, index rebuild; replay completes on the primary before any worker serves (main.c boots the engine before the shards). 22 scripts the restart proof. | +| Concurrency control | ✅ stage 3 — the DB actor serializes every statement; replies are materialized copies, no torn read by construction. Cross-statement snapshots arrive with 18. | +| Space reclamation | RAM ✅ (deleted rows free their slot — ids never reused, slots are); disk ✖ → [story 32](../refine/32-wal-checkpoint.md), end of chain. | + +- The C proving ground (`docs/plan/exploration/c-runtime/`, phases A–F: + epoll loops, eventfd mail) is the substrate this lifts into `wovm`. + (Path restated 2026-08-20; the old `runtime/wo-rt.c` reference was + stale — that tree was removed with the Rust runtime.) +- The VM's object header has carried a shard id since iteration 2 — no + relayout. +- **Gated by the benchmark:** landing the arc means re-running + [22](../in-progress/22-durability-throughput-scale.md) at the concurrency + scale it unlocks and recording the before/after delta; it is also + where [23](../refine/23-io-uring-commit.md) gets a thread to overlap + durability against. + +## Proposed Solution + +Execute stage 3 of the plan of record +([`2026-08-20-shard-fiber-arc.md`](../../../superpowers/plans/2026-08-20-shard-fiber-arc.md)): +the DB-actor migration — engine calls off the owner shard become message +sends with parked replies, closing the `WO_T_DB` hole. Stages 1+2 +(scheduler, fibers, unified spawn/send, WO-E222 traced-send rejection, +cross-shard envelopes with home-routed frees) landed 2026-08-20. After +stage 3: 22 measures, then iteration 24 proves the arc. Actor lifecycle +(request/response, backpressure, supervision, timers) is deliberately +NOT the arc's scope — it is [iteration 31](../refine/31-actor-lifecycle.md). diff --git a/docs/stories/language-runtime-database/done/09-database-engine.md b/docs/stories/language-runtime-database/done/09-database-engine.md index 0cebd2c..6826a66 100644 --- a/docs/stories/language-runtime-database/done/09-database-engine.md +++ b/docs/stories/language-runtime-database/done/09-database-engine.md @@ -1,3 +1,8 @@ +--- +iteration: "9" +status: done +--- + # Iteration 9 — database engine > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/done/09b-table-relations-query.md b/docs/stories/language-runtime-database/done/09b-table-relations-query.md index 1da4ac4..5c0bc03 100644 --- a/docs/stories/language-runtime-database/done/09b-table-relations-query.md +++ b/docs/stories/language-runtime-database/done/09b-table-relations-query.md @@ -1,3 +1,8 @@ +--- +iteration: "9b" +status: done +--- + # Iteration 9b — `@table`, relations, and language-integrated query > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/refine/11-fibers.md b/docs/stories/language-runtime-database/done/11-fibers.md similarity index 61% rename from docs/stories/language-runtime-database/refine/11-fibers.md rename to docs/stories/language-runtime-database/done/11-fibers.md index 3984fa9..dfe5133 100644 --- a/docs/stories/language-runtime-database/refine/11-fibers.md +++ b/docs/stories/language-runtime-database/done/11-fibers.md @@ -1,8 +1,24 @@ +--- +iteration: "11" +status: done +chain: 1 +--- + # Iteration 11 — fibers (the 8+11 concurrency arc, part 2) > Format: fiberloom `product/story-iteration-template`. Part of > [Story — one language, one runtime, one database, one binary](../00-story.md). > +> **✅ LANDED 2026-08-21** with the arc's stage 3 (fiber substance landed +> stages 1+2; stage 3 added the plane-less reply park `WO_PARK_INBOX` — +> a fiber parked on the DB actor's reply, woken by the envelope drain). +> RE-SCOPED at close, disclosed: the goal's "blocking builtins park" +> covers `net`/`time`; **`fs` still blocks the shard thread** — v1 +> semantics, deliberately: program mode is single-fiber by design, and +> no serving workload reads files mid-request yet. fs-park becomes real +> when a workload demands it (candidate rider on 23's ring work). The +> criteria below are met under that re-scope. +> > **REFINED 2026-08-20** (developer decisions, no code): 8 and 11 ship as > **one arc** — the DB becomes an actor on an owner shard > ([iteration 8](08-shard-actor-runtime.md)'s decision), so serving @@ -10,7 +26,7 @@ > The spawn-surface question below is SETTLED: one unified actor-address > surface (`spawn` returns an address, fiber or remote alike; `send` > moves ownership; same-heap sends take the cheap path). The arc's -> driving workload is [iteration 24: chat](24-chat-websocket-workload.md) +> driving workload is [iteration 24: chat](../refine/24-chat-websocket-workload.md) > — fiber-per-WebSocket-connection is the serving model that retires the > framework's close-when-idle keep-alive policy. > @@ -20,9 +36,21 @@ > time delivery, main-return reap, fiber-trap isolation, and parked > `net`/`time` builtins on the io_uring-first per-shard I/O plane > (`WO_IO=uring|epoll`, epoll fallback proven). Demonstrated by -> `docs/examples/fibers` (`just fibers` 8/0). The shard-context criteria -> (TID assertions, cross-shard sends, blue-green drain reuse) close with -> the arc's stage 2. +> `docs/examples/fibers` (`just fibers` 8/0). +> +> **STAGE-2 SUBSTANCE LANDED 2026-08-20** (branch `concurrency-arc`, +> T5–T6): pinned thread-per-core shards, envelope sends with ownership +> move, round-robin placement, home-routed frees, WO-E222 on EVERY +> spawn/send (placement makes any actor potentially remote), TID-verified +> shard context. Deviations disclosed in the plan of record +> ([`2026-08-20-shard-fiber-arc.md`](../../../superpowers/plans/2026-08-20-shard-fiber-arc.md)): +> the inbox is a mutex-guarded list + eventfd (rings arrive only if +> iteration 22 measures the mutex as a cost), the deterministic corpus +> pins `WO_SHARDS=1`, two TSan races and one teardown SEGV fixed. +> +> **RE-SEQUENCED 2026-08-21**: what remains for the chain is the arc's +> stage 3 (transparent DB actor — [iteration 8](08-shard-actor-runtime.md)), +> then measurement. Order: **stage 3 → 22 → 31 → 24 → 23 → 32**. ## Goals @@ -83,6 +111,9 @@ ## Info +- Normative contract (added 2026-08-21): stackless coroutine model, + park/resume protocols, the no-`async` rule — + [`docs/plan/oop-vm/03-concurrency-coroutines.md`](../../../plan/oop-vm/03-concurrency-coroutines.md). - Research note: [`docs/plan/exploration/fibers/00-fibers.md`](../../../plan/exploration/fibers/00-fibers.md) — kernel's-eye evidence (task_struct costs, CFS collapse at high task counts) and the precedent survey (BEAM reductions adopted; Go stack @@ -90,15 +121,20 @@ matches the stdlib posture). - Vision origin: [blue-green vision §3](../../../plan/exploration/blue-green-vm/00-vision.md); iteration 8's scheduler is the substrate this extends. -- Open questions to settle in the spec — REDUCED 2026-08-20: the spawn - surface is settled (the unified actor address, iteration 8 decision 2). - Still open for the arc's spec: budget size and check granularity, - parked-fiber drop semantics, run-queue fairness (FIFO v1), and how a - parked fiber's borrow state interacts with the shard's GC safepoints. +- Open questions — REDUCED AGAIN 2026-08-21: the spawn surface, budget + size/granularity (back-edge accounting — see the plan's livelock + deviation), run-queue fairness (FIFO), and parked-fiber drop semantics + (fiber-trap isolation + main-return reap) are all settled by the landed + implementation. Still open: how a parked fiber's borrow state interacts + with the shard's GC safepoints — tracked for stage 3 / the collector's + next pass. Lifecycle surface (request/response, backpressure, + supervision, timers) is [iteration 31](../refine/31-actor-lifecycle.md)'s, not + this story's. ## Proposed Solution -- No implementation plan exists yet — this iteration starts with the - brainstorming → spec → writing-plans chain (the superpowers path every - prior iteration followed), then executes that plan. The research note - above is the brainstorm's entry material. +- The plan exists and its fiber stages are landed: + [`2026-08-20-shard-fiber-arc.md`](../../../superpowers/plans/2026-08-20-shard-fiber-arc.md) + (stages 1+2 complete 2026-08-20). This story closes when the arc's + stage 3 lands and iteration 22 records the delta; the chat workload + ([iteration 24](../refine/24-chat-websocket-workload.md)) is the proof. diff --git a/docs/stories/language-runtime-database/done/15-deps-package-manager.md b/docs/stories/language-runtime-database/done/15-deps-package-manager.md index 9c6ecfc..1e9449a 100644 --- a/docs/stories/language-runtime-database/done/15-deps-package-manager.md +++ b/docs/stories/language-runtime-database/done/15-deps-package-manager.md @@ -1,3 +1,8 @@ +--- +iteration: "15" +status: done +--- + # Iteration 15 — dependencies: `wo.toml [deps]`, git fetch, `wo.lock` > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/done/16-web-framework.md b/docs/stories/language-runtime-database/done/16-web-framework.md index 154419f..7629122 100644 --- a/docs/stories/language-runtime-database/done/16-web-framework.md +++ b/docs/stories/language-runtime-database/done/16-web-framework.md @@ -1,3 +1,8 @@ +--- +iteration: "16" +status: done +--- + # Iteration 16 — the web framework: a `.wo` library, HTTP/1.1 behind a proxy > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/done/17-library-projects-internal.md b/docs/stories/language-runtime-database/done/17-library-projects-internal.md index db9b555..3485b22 100644 --- a/docs/stories/language-runtime-database/done/17-library-projects-internal.md +++ b/docs/stories/language-runtime-database/done/17-library-projects-internal.md @@ -1,3 +1,8 @@ +--- +iteration: "17" +status: done +--- + # Iteration 17 — library projects and dependency privacy (`kind`, `internal/`) > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/done/19-missing-scalar-types.md b/docs/stories/language-runtime-database/done/19-missing-scalar-types.md index c6ad5d1..b947d9a 100644 --- a/docs/stories/language-runtime-database/done/19-missing-scalar-types.md +++ b/docs/stories/language-runtime-database/done/19-missing-scalar-types.md @@ -1,3 +1,8 @@ +--- +iteration: "19" +status: done +--- + # Iteration 19 — the missing scalar types: Float and Bytes > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/hold/18-memory-db-features.md b/docs/stories/language-runtime-database/hold/18-memory-db-features.md index 36116ee..84872c4 100644 --- a/docs/stories/language-runtime-database/hold/18-memory-db-features.md +++ b/docs/stories/language-runtime-database/hold/18-memory-db-features.md @@ -1,3 +1,8 @@ +--- +iteration: "18" +status: hold +--- + # Iteration 18 — framework v2: memory-rich features over the embedded database > **Scope label (2026-08-20): this iteration is FRAMEWORK V2.** Framework diff --git a/docs/stories/language-runtime-database/hold/20-cross-program-tables.md b/docs/stories/language-runtime-database/hold/20-cross-program-tables.md index cd76390..e7091df 100644 --- a/docs/stories/language-runtime-database/hold/20-cross-program-tables.md +++ b/docs/stories/language-runtime-database/hold/20-cross-program-tables.md @@ -1,3 +1,8 @@ +--- +iteration: "20" +status: hold +--- + # Iteration 20 — cross-program tables: attach to a running program's database > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/hold/21-keypair-attach-auth.md b/docs/stories/language-runtime-database/hold/21-keypair-attach-auth.md index 39c74eb..6fe5010 100644 --- a/docs/stories/language-runtime-database/hold/21-keypair-attach-auth.md +++ b/docs/stories/language-runtime-database/hold/21-keypair-attach-auth.md @@ -1,3 +1,8 @@ +--- +iteration: "21" +status: hold +--- + # Iteration 21 — keypair authentication for cross-program attach > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/hold/26-blue-green-deploy.md b/docs/stories/language-runtime-database/hold/26-blue-green-deploy.md index a28c751..47e9f76 100644 --- a/docs/stories/language-runtime-database/hold/26-blue-green-deploy.md +++ b/docs/stories/language-runtime-database/hold/26-blue-green-deploy.md @@ -1,3 +1,8 @@ +--- +iteration: "26" +status: hold +--- + # Iteration 26 — blue-green in-runtime deployment > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/hold/27-query-grammar-corpus.md b/docs/stories/language-runtime-database/hold/27-query-grammar-corpus.md index 5573de4..03716ad 100644 --- a/docs/stories/language-runtime-database/hold/27-query-grammar-corpus.md +++ b/docs/stories/language-runtime-database/hold/27-query-grammar-corpus.md @@ -1,3 +1,8 @@ +--- +iteration: "27" +status: hold +--- + # Iteration 27 — query grammar, driven by real embedded-DB corpora > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/hold/28-skillhost-host-workload.md b/docs/stories/language-runtime-database/hold/28-skillhost-host-workload.md index d1ed877..2214f2c 100644 --- a/docs/stories/language-runtime-database/hold/28-skillhost-host-workload.md +++ b/docs/stories/language-runtime-database/hold/28-skillhost-host-workload.md @@ -1,3 +1,8 @@ +--- +iteration: "28" +status: hold +--- + # Iteration 28 — skillhost: a host-shaped workload, and the capability gaps it exposes > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/hold/29-compile-time-metaprogramming.md b/docs/stories/language-runtime-database/hold/29-compile-time-metaprogramming.md index 63cb956..a42107b 100644 --- a/docs/stories/language-runtime-database/hold/29-compile-time-metaprogramming.md +++ b/docs/stories/language-runtime-database/hold/29-compile-time-metaprogramming.md @@ -1,3 +1,8 @@ +--- +iteration: "29" +status: hold +--- + # Iteration 29 — compile-time metaprogramming (derive from the class table) > Format: fiberloom `product/story-iteration-template`. Part of diff --git a/docs/stories/language-runtime-database/refine/22-durability-throughput-scale.md b/docs/stories/language-runtime-database/in-progress/22-durability-throughput-scale.md similarity index 76% rename from docs/stories/language-runtime-database/refine/22-durability-throughput-scale.md rename to docs/stories/language-runtime-database/in-progress/22-durability-throughput-scale.md index 3c433e2..38afc89 100644 --- a/docs/stories/language-runtime-database/refine/22-durability-throughput-scale.md +++ b/docs/stories/language-runtime-database/in-progress/22-durability-throughput-scale.md @@ -1,3 +1,9 @@ +--- +iteration: "22" +status: in-progress +chain: 2 +--- + # Iteration 22 — durability proof, throughput, and scale under load > Format: fiberloom `product/story-iteration-template`. Part of @@ -10,7 +16,27 @@ > performance work, because each of those must be gated by re-running THIS > iteration's benchmark and showing the number moved the right way. > -> **No spec exists yet.** The forks in *Info* are genuine decisions. +> ~~**No spec exists yet.** The forks in *Info* are genuine decisions.~~ +> +> **SPEC APPROVED 2026-08-21** — the four forks below are SETTLED as +> their recorded leanings (developer confirmation), plus two new +> decisions: the vehicle is a NEW sample `docs/examples/db-bench` +> (employee stays a teaching sample) and `time.ticks` (CLOCK_MONOTONIC +> µs) is the iteration's one runtime addition. Spec: +> [`2026-08-21-db-bench-design.md`](../../../superpowers/specs/2026-08-21-db-bench-design.md) +> · plan: [`2026-08-21-db-bench.md`](../../../superpowers/plans/2026-08-21-db-bench.md) +> — **in progress** (second slice of the chain). +> +> **RE-SEQUENCED 2026-08-21** (developer decision): runs AFTER the arc's +> stage 3 — the transparent DB actor is a correctness hole (a multi-shard +> program touching the database traps `WO_T_DB` today), and fixing it +> first lets ONE benchmark campaign cover single- and multi-shard +> honestly. The arc's stages 1+2 landed 2026-08-20 unmeasured; their +> delta is recorded retroactively against this iteration's first +> baseline. New measurement target since stage 2: the mutex-guarded +> inbox + eventfd (the plan's deviation — lock-free rings arrive only if +> this number says the mutex costs). Chain order: +> **stage 3 → 22 → 31 → 24 → 23 → 32**. ## Goals @@ -122,15 +148,18 @@ exists to close. - **Brainstorm the spec**, settling the four forks; then a plan whose first task is the harness and the baseline file, because nothing downstream means anything without them. -- **Sequence the whole performance arc around this iteration:** - 1. 9b lands → employee compiles and runs → **22 restart-persistence** and - **22 baseline benchmark** (single-thread, both durable and RAM-only). - 2. **7b** (inferred GC + mark-sweep) → re-run 22, record the delta (does - tracing change the write path's tail latency?). - 3. **8** (shard-actor, thread-per-core) → re-run 22 at the connection/ - concurrency scale it unlocks, record the delta. - 4. **23** (io_uring group-commit) → re-run 22's durable write number, record - the delta against the fsync-per-commit baseline — the payoff. +- **Sequence the performance chain around this iteration** (rewritten + 2026-08-21 — the first version predated 7b and the arc landing first): + 1. Already landed unmeasured: 9b, **7b** (inferred GC + mark-sweep, + 2026-08-18), the arc's **stages 1+2** (fibers + shards, 2026-08-20). + Their deltas are owed retroactively against the first baseline. + 2. Arc **stage 3** (transparent DB actor) lands → **22 runs**: + restart-persistence proof + baseline benchmark, durable and + RAM-only, single- AND multi-shard, plus the mutex-inbox number. + 3. **31** (actor lifecycle), then **24** (chat) → re-run the + concurrency-facing numbers at the connection scale chat unlocks. + 4. **23** (io_uring group-commit) → re-run 22's durable write number, + record the delta against the fsync-per-commit baseline — the payoff. - The benchmark harness and its baseline live under `bench/` (or the existing `runtime/bench/`), and `just` gets a `db-bench` recipe kept off the fast path, exactly like `log-watcher::soak`. diff --git a/docs/stories/language-runtime-database/refine/08-shard-actor-runtime.md b/docs/stories/language-runtime-database/refine/08-shard-actor-runtime.md deleted file mode 100644 index 89309e0..0000000 --- a/docs/stories/language-runtime-database/refine/08-shard-actor-runtime.md +++ /dev/null @@ -1,110 +0,0 @@ -# Iteration 8 — shard-actor runtime (the 8+11 concurrency arc, part 1) - -> Format: fiberloom `product/story-iteration-template`. Part of -> [Story — one language, one runtime, one database, one binary](../00-story.md). -> -> **REFINED 2026-08-20** (developer decisions, no code): iterations 8 and -> 11 are **one arc** — the scheduler, fibers on it, then serving — because -> the database-ownership decision below makes a fiberless multi-shard -> server block a whole thread per cross-shard call. The arc's driving -> workload is **iteration 24 (chat: WebSocket pub/sub)**. The pre-existing -> plan (`docs/superpowers/plans/2026-08-01-shard-actor-vm-runtime.md`) -> predates inferred GC (7b), the unified surface, and the DB decision — -> it is a source of ideas, NOT the plan of record; the arc starts with a -> fresh brainstorm → spec → plan. - -## Settled decisions (2026-08-20) - -1. **The database is an actor.** The engine (`wo_db` + WAL) lives on one - owner shard; every query/write from another shard is a message send, - and results come back materialized (queries already copy rows out — - the model was built for this). Doctrine-pure: no lock, no shared - mutable state. Consequence, accepted deliberately: callers must PARK - while the reply travels, which is why 8 and 11 ship as one arc. - (Rejected: a coarse engine lock — bends the doctrine and caps write - scaling anyway; partitioned tables — the real scale answer, but it - waits for a measured need, not v1.) -2. **One concurrency surface: everything is an actor address.** `spawn` - returns an address whether the spawnee lands on this shard (a fiber) - or another (placement policy's call); `send` always moves ownership; - a same-heap send skips the ring and is cheap. The language never - shows a fiber-vs-actor split. (This dissolves iteration 11's - "handle vs address" open question.) -3. **Driving workload: chat** (iteration 24) — rooms, broadcast, N - concurrent WebSocket clients, one binary. The arc's acceptance is the - chat sample's, not only synthetic corpora. -4. **Order: 22 → the 8+11 arc → 23.** The io_uring write path waits for - the arc (its batch boundary is the shard tick) and for 22's baseline. - -## Goals - -- The runtime scales past one core the doctrine way: pinned - thread-per-core shards, each owning its own heap and event loop; - cross-shard communication is a message send that **moves ownership** — - shared mutable state never exists. -- The language grows `spawn`/`send` (the unified address surface); - garbage collection stays per-shard (7b's collector is already - per-shard by construction), so no global pause appears at any core - count. -- The database keeps its single-writer truth by BEING an actor on its - owner shard. - -## Acceptance Criteria - -- What to achieve? - - **Given** a program spawning actors across shards, - - **when** an owned object is sent to another shard, - - **then** the sender can no longer touch it (compile-time move), the - receiver owns it, and its eventual free routes back to its - allocation-home arena. -- What to achieve? - - **Given** debug builds with shard-ownership asserts, - - **when** the deterministic actor corpus runs under ASan and TSan, - - **then** zero races, zero leaks, and identical output across runs. -- What to achieve? - - **Given** a reference to a TRACED object (GC-ness is inferred since - 7b — there is no `@gc` to write), - - **when** code attempts to send it cross-shard, - - **then** the compiler rejects it: aliased references cannot cross - heap boundaries, and the diagnostic names the inferred-traced class - and why it is traced. (This criterion originally said `@gc`; - restated 2026-08-20 in inference terms — same rule, current - language.) -- What to achieve? - - **Given** handlers on serving shards querying and writing through - the DB-owner shard, - - **when** the employee/web-app matrices run multi-shard, - - **then** every answer is byte-identical to the single-shard run and - the WAL's ack-after-durable contract is unchanged. - -## Out Of Scope - -- Cross-shard transactions (2PC) — the DB actor serializes writers, so - iteration 18's `transaction { }` is unaffected; distributing it is a - later story. -- Fiber details beyond the shared scheduler substrate — part 2 - ([iteration 11](11-fibers.md)) owns them. -- WebSocket framing — the framework's (iteration 24's) job. - -## Info - -- The C proving ground (`docs/plan/exploration/c-runtime/`, phases A–F: - epoll loops, eventfd mail) is the substrate this lifts into `wovm`. - (Path restated 2026-08-20; the old `runtime/wo-rt.c` reference was - stale — that tree was removed with the Rust runtime.) -- The VM's object header has carried a shard id since iteration 2 — no - relayout. -- **Gated by the benchmark:** landing the arc means re-running - [22](22-durability-throughput-scale.md) at the concurrency - scale it unlocks and recording the before/after delta; it is also - where [23](23-io-uring-commit.md) gets a thread to overlap - durability against. - -## Proposed Solution - -Fresh brainstorm → spec → plan for the WHOLE arc (8+11), staged: the -pinned-worker scheduler + shard-stamped heaps + MPSC mailboxes + the -unified spawn/send surface and the traced-send rejection; fibers on that -scheduler (part 2's document); the DB-actor migration; then iteration 24 -proves it. The 2026-08-01 plan is reference material for the mailbox and -heap-stamping shapes only. diff --git a/docs/stories/language-runtime-database/refine/23-io-uring-commit.md b/docs/stories/language-runtime-database/refine/23-io-uring-commit.md index 53ba52f..ab36ca3 100644 --- a/docs/stories/language-runtime-database/refine/23-io-uring-commit.md +++ b/docs/stories/language-runtime-database/refine/23-io-uring-commit.md @@ -1,3 +1,9 @@ +--- +iteration: "23" +status: refine +chain: 5 +--- + # Iteration 23 — io_uring group-commit write path > Format: fiberloom `product/story-iteration-template`. Part of @@ -22,8 +28,13 @@ > accumulated; (4) startup auto-probe + an env override so CI proves both > paths on one kernel — AMENDED: the override is the arc-wide > `WO_IO=uring|epoll` (the arc's T4 owns the probe and the per-shard -> ring; `WO_WAL_MODE` is subsumed). Position: AFTER the 8+11 arc (order -> settled 2026-08-20: 9e → 8+11 → 9f). AMENDED 2026-08-20 (io_uring-first +> ring; `WO_WAL_MODE` is subsumed). Position — RE-SEQUENCED 2026-08-21: +> FIFTH in the concurrency chain (32, WAL checkpoint, follows it — +> added 2026-08-21), **stage 3 → 22 → 31 → 24 → 23 → 32** +> (supersedes the 2026-08-20 old-id ordering "9e → 8+11 → 9f"); the +> per-shard ring already exists (arc T4 landed 2026-08-20, +> `WO_IO=uring|epoll`) — this iteration adds the WAL's WRITE+FSYNC +> chains to it. AMENDED 2026-08-20 (io_uring-first > directive): the WAL's WRITE+FSYNC chains ride the SAME per-shard ring > T4 creates for fiber parking — one event loop per shard, readiness ops > and durability ops together, exactly the linux reference project's diff --git a/docs/stories/language-runtime-database/refine/24-chat-websocket-workload.md b/docs/stories/language-runtime-database/refine/24-chat-websocket-workload.md index aa0cdbc..51a870f 100644 --- a/docs/stories/language-runtime-database/refine/24-chat-websocket-workload.md +++ b/docs/stories/language-runtime-database/refine/24-chat-websocket-workload.md @@ -1,3 +1,9 @@ +--- +iteration: "24" +status: refine +chain: 4 +--- + # Iteration 24 — chat: the WebSocket pub/sub driving workload > Format: fiberloom `product/story-iteration-template`. Part of @@ -6,6 +12,12 @@ > **Inserted 2026-08-20** (concurrency-chain refinement): the 8+11 arc's > driving workload, the role log-watcher played for iterations 3–7. Needs > its spec AFTER the arc's — it lands at the arc's end and proves it. +> +> **RE-SEQUENCED 2026-08-21**: fourth in the chain, +> **stage 3 → 22 → 31 → 24 → 23 → 32** — chat cannot be written honestly +> before [iteration 31](31-actor-lifecycle.md) (request/response, +> bounded mailboxes, actor death, timers). Iteration 19 LANDED +> 2026-08-20, so Bytes is available for frame parse/serialize. ## Why this iteration exists @@ -54,11 +66,15 @@ proxy — the proxy story extends to WS pass-through, documented). ## Info -- Dependencies: the 8+11 arc (fibers + cross-shard send), the crypto - builtins fork (SHA-1 for the upgrade handshake — note: the ledger's - crypto slice lists SHA-256/512; the WS handshake specifically needs - SHA-1, so the builtin set must include it), and the framework's parse - seam (upgrade is an HTTP request until it isn't). +- Dependencies: the 8+11 arc (fibers + cross-shard send — stages 1+2 + landed 2026-08-20, stage 3 pending), [iteration 31](31-actor-lifecycle.md) + (request/response, backpressure, death, timers — the mechanisms rooms + and presence are made of), iteration 19 (LANDED 2026-08-20 — Bytes + carries the frames), the crypto builtins fork (SHA-1 for the upgrade + handshake — note: the ledger's crypto slice lists SHA-256/512; the WS + handshake specifically needs SHA-1, so the builtin set must include + it), and the framework's parse seam (upgrade is an HTTP request until + it isn't). - Unparks on landing: the framework ledger's WebSocket/pub-sub rows and the iteration-18 rejection note ("pub/sub REJECTED until 8/11"). diff --git a/docs/stories/language-runtime-database/refine/31-actor-lifecycle.md b/docs/stories/language-runtime-database/refine/31-actor-lifecycle.md new file mode 100644 index 0000000..2df4e1a --- /dev/null +++ b/docs/stories/language-runtime-database/refine/31-actor-lifecycle.md @@ -0,0 +1,112 @@ +--- +iteration: "31" +status: refine +chain: 3 +--- + +# Iteration 31 — actor lifecycle: request/response, backpressure, death, timers + +> Format: fiberloom `product/story-iteration-template`. Part of +> [Story — one language, one runtime, one database, one binary](../00-story.md). +> +> **Inserted 2026-08-21** (concurrency-chain re-sequence; the iteration +> was named as "new 31" in the 2026-08-20 code-review re-sequence — this +> is its story file). Third in the chain, +> **stage 3 → 22 → 31 → 24 → 23 → 32**: chat +> ([iteration 24](24-chat-websocket-workload.md)) cannot be written +> honestly without these four mechanisms. + +## Why this iteration exists + +The arc's stages 1+2 shipped `spawn`/`send` mechanism without lifecycle: +`send` is one-way and callers `sleep`-poll to await an answer; the +mailbox FIFO grows without bound (a hot sender can exhaust a shard's +memory); an actor that traps dies silently (nobody learns, nothing +restarts, its mailbox rots); and there is no timer surface beyond a +fiber blocking in `time.sleep`. Every real serving program — chat first — +is made of request/response turns, bounded queues, death notices, and +deadlines. Without this iteration the arc is a demo, not a runtime. + +## Goals + +- **Request/response over one-way sends.** A caller can send and park + until the reply arrives — one surface, no `sleep`-polling, no second + concurrency vocabulary. Ownership rules unchanged: the request moves, + the reply moves back. +- **Bounded mailboxes with a stated backpressure policy.** A mailbox has + a cap; what happens at the cap (park the sender vs error) is decided by + the spec, one policy, doctrine-pure — no silent unbounded growth + anywhere in the runtime. +- **Actor death is observable.** A trap or normal exit produces a signal + another actor can receive; a fiber-trap already isolates (stage 1) — + this makes the fact of death deliverable, so a supervisor CAN be + written in `.wo`. +- **Timers as messages.** A deadline or interval delivers to a mailbox + like any other send, riding the shard's existing io_uring/epoll + timeout plumbing (arc T4) — no new event loop. + +## Acceptance Criteria (draft — the spec refines) + +- What to achieve? + - **Given** an actor that answers requests, + - **when** a caller awaits the reply, + - **then** the caller's fiber parks (the shard serves other fibers, + TID-verified), resumes with the moved reply, and never busy-waits. +- What to achieve? + - **Given** a mailbox at its cap, + - **when** another send arrives, + - **then** the stated backpressure policy fires deterministically, + memory stays bounded (RSS flat under a hot-sender soak), and no + message is silently dropped. +- What to achieve? + - **Given** an actor that traps mid-message, + - **when** it dies, + - **then** its drop maps run (ASan zero leaks), a death signal + reaches the observer that asked for one, and a supervisor written + in `.wo` can respawn it. +- What to achieve? + - **Given** a timer armed by an actor, + - **when** it fires, + - **then** the actor receives it as an ordinary message on its own + shard, and cancelling before expiry means it never delivers. + +## Out Of Scope + +- Supervision TREES / OTP-scale restart policy — a `.wo` library once + death signals exist, not runtime policy. +- Priorities and custom scheduling — the reduction budget stays the only + fairness mechanism. +- Distributed (cross-process) supervision — no network layer exists. +- Changing the ownership-move rule or the unified address surface — + iteration 8's decisions stand. + +## Info + +Forks the spec must settle: + +1. **The request/response surface.** A reply-address baked into the + message shape vs a runtime-level call that parks — and what the + compiler checks (does a request type name its reply type?). +2. **The backpressure policy at the cap.** Park the sender (natural with + fibers, risks deadlock cycles) vs fail the send (explicit, pushes + handling to the program). One policy, stated; not configurable per + mailbox in v1. +3. **The death-signal shape.** Erlang's link (bidirectional, dies + together) vs monitor (one-way notice) — likely monitor-only v1. +4. **The timer surface.** Builtin (`time.after` delivering a message) vs + actor-spawned sleeper fiber — and cancellation semantics. + +Sources: the 2026-08-20 code-review findings (the gaps this iteration +answers), the arc plan's stage-2 deviations +([`2026-08-20-shard-fiber-arc.md`](../../../superpowers/plans/2026-08-20-shard-fiber-arc.md) +— the unbounded FIFO is deviation 4's mutex inbox), and BEAM precedent +already surveyed in +[`docs/plan/exploration/fibers/00-fibers.md`](../../../plan/exploration/fibers/00-fibers.md). + +## Proposed Solution + +Brainstorm → spec → plan after the arc's stage 3 lands and iteration 22 +has its baseline (the mailbox-cap and inbox-ring decisions want 22's +mutex number). The spec is written against iteration 24's needs — chat +names the lifecycle mechanisms it consumes, this iteration names chat as +its first honest consumer. diff --git a/docs/stories/language-runtime-database/refine/32-wal-checkpoint.md b/docs/stories/language-runtime-database/refine/32-wal-checkpoint.md new file mode 100644 index 0000000..cff827d --- /dev/null +++ b/docs/stories/language-runtime-database/refine/32-wal-checkpoint.md @@ -0,0 +1,98 @@ +--- +iteration: "32" +status: refine +chain: 6 +--- + +# Iteration 32 — WAL checkpoint: disk space reclamation and bounded replay + +> Format: fiberloom `product/story-iteration-template`. Part of +> [Story — one language, one runtime, one database, one binary](../00-story.md). +> +> **Inserted 2026-08-21** (stage-3 guarantee refinement found the hole): +> the WAL is append-only FOREVER — no checkpoint, no truncation exists +> in the engine or anywhere on the roadmap. Disk grows without bound and +> replay time grows with history, so restart cost rises with every write +> the program ever made. RAM reclamation already exists (deleted rows +> free their slot — [`04-db-binding.md`](../../../plan/oop-vm/04-db-binding.md): +> "Ids are never reused; slots are"); this iteration is the DISK half. +> LAST in the concurrency chain: +> **stage 3 → 22 → 31 → 24 → 23 → 32** — it wants 22's measured +> replay/restart numbers to justify its policy and must compose with +> 23's group-commit write path. + +## Goals + +- **Disk space is reclaimed.** A checkpoint writes the live store as a + snapshot and truncates the WAL behind it; deleted rows and + overwritten versions stop occupying disk forever. +- **Replay is bounded.** Startup replays snapshot + WAL tail, not the + program's whole write history — restart time becomes a function of + store size, not store age. +- **Every existing guarantee holds byte-for-byte.** Ack-after-durable, + replay-whole-or-not-at-all, torn-tail drop, ids never reused — a + checkpoint changes where bytes live, never what an ack means. A crash + DURING checkpoint recovers from the previous snapshot + full tail: + the old WAL is not truncated until the new snapshot is durable. + +## Acceptance Criteria (draft — the spec refines) + +- **Given** a store with N rows after many writes and deletes, **when** + a checkpoint completes, **then** disk usage reflects the live rows + (plus the WAL tail), and a restart replays snapshot + tail to the + byte-identical store. +- **Given** kill -9 at ANY instant during a checkpoint, **when** the + process restarts, **then** recovery produces the same consistent + store as if the checkpoint had never started — no acknowledged write + lost, no partial snapshot ever read. +- **Given** the iteration-22 restart benchmark re-run after checkpoint + lands, **when** replay time is measured on an aged store, **then** + the bounded-replay improvement is recorded as a before/after delta. +- **Given** writes arriving while a checkpoint runs (the DB actor + serializes statements; the checkpoint must not stall them beyond the + stated budget), **when** the mixed load completes, **then** every ack + held its durability contract and the tail contains exactly the + post-snapshot writes. + +## Out Of Scope + +- MVCC / multi-version reads — the store is update-in-place RAM; "old + versions" exist only as WAL history, which is exactly what truncation + reclaims. +- Incremental/streaming backup, point-in-time recovery — a snapshot is + a recovery artifact here, not a backup product. +- Cross-shard checkpoint coordination — the WAL is owner-shard-only + (stage 3's rule); one shard, one checkpoint. +- Compression, dedup, tiering — measure first (22), add only what a + number justifies. + +## Info + +Forks the spec must settle: + +1. **Snapshot format** — a row-image dump of the live store (simple, + O(live rows)) vs a rewritten-compacted WAL (reuses replay machinery, + O(live rows) too but stays in one format). Leaning: row-image dump + in the WAL's existing record grammar, so replay needs no second + decoder. +2. **Trigger policy** — size threshold (WAL bytes vs snapshot bytes + ratio), boot-time compaction, explicit call, or some mix. Leaning: + ratio threshold checked at commit, plus manual trigger for tests; + decided against 22's numbers. +3. **Write availability during checkpoint** — stop-the-world dump + (simplest; the DB actor just runs one long "statement") vs + fork-and-dump vs incremental copy. Leaning: measure the + stop-the-world pause on the 1M-row store first (22); complexity only + if the pause breaks a stated budget. +4. **Composition with 23** — the snapshot's durability barrier rides + the same per-shard ring (WRITE+FSYNC chain, then the truncate); + ordering vs in-flight group commits must be stated normatively in + [`04-db-binding.md`](../../../plan/oop-vm/04-db-binding.md)'s WAL + section. + +## Proposed Solution + +Brainstorm → spec → plan after 23 lands (the write path it composes +with) using 22's aged-store replay numbers as the policy input; extend +`04-db-binding.md`'s WAL section with the snapshot format the way the +record grammar is documented today. diff --git a/docs/superpowers/plans/2026-08-01-db-engine-binding.md b/docs/superpowers/plans/2026-08-01-db-engine-binding.md index 9bebc83..11de1bb 100644 --- a/docs/superpowers/plans/2026-08-01-db-engine-binding.md +++ b/docs/superpowers/plans/2026-08-01-db-engine-binding.md @@ -1,6 +1,6 @@ # DB Engine Binding Implementation Plan -> **Status: 🔄 in progress — Tasks 1–4 done; Task 5 engine half done; Task 6 incremental. The language read surface is the 9b plan's (recorded deviation at Task 5); the engine itself is COMPLETE for single-shard: storage, WAL+replay, insert/update/delete execution, indexes, unique.** (story iteration 9) — class-shaped tables, typed WAL + recovery, `insert`/`select` execution. Story iteration 9b (`@table` relations + language-integrated query) follows it and needs a spec brainstormed first. Board: [00-status.md](../../00-status.md) +> **Status: 🔄 in progress — Tasks 1–4 done; Task 5 engine half done; Task 6 incremental. The language read surface is the 9b plan's (recorded deviation at Task 5); the engine itself is COMPLETE for single-shard: storage, WAL+replay, insert/update/delete execution, indexes, unique.** (story iteration 9) — class-shaped tables, typed WAL + recovery, `insert`/`select` execution. Story iteration 9b (`@table` relations + language-integrated query) follows it and needs a spec brainstormed first. Board: [00-status.md](../../stories/00-status.md) > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. > diff --git a/docs/superpowers/plans/2026-08-01-http-service-layer.md b/docs/superpowers/plans/2026-08-01-http-service-layer.md index 859b514..998d4f5 100644 --- a/docs/superpowers/plans/2026-08-01-http-service-layer.md +++ b/docs/superpowers/plans/2026-08-01-http-service-layer.md @@ -1,6 +1,6 @@ # HTTP Service Layer Implementation Plan -> **Status: ⬜ pending** (story iteration 25) — `service` blocks route to VM methods; REST parity with the shipped Rust Stage 2 runtime. Board: [00-status.md](../../00-status.md) +> **Status: ⏸ hold (2026-08-21, developer decision)** (story iteration 25) — `service` blocks route to VM methods; REST parity with the shipped Rust Stage 2 runtime. Board: [00-status.md](../../stories/00-status.md) > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. > diff --git a/docs/superpowers/plans/2026-08-01-log-watcher-sample.md b/docs/superpowers/plans/2026-08-01-log-watcher-sample.md index 7e1003f..8da22cc 100644 --- a/docs/superpowers/plans/2026-08-01-log-watcher-sample.md +++ b/docs/superpowers/plans/2026-08-01-log-watcher-sample.md @@ -1,6 +1,6 @@ # log-watcher .wo Sample Implementation Plan -> **Status: ⬜ pending** (story iteration 7 — the acceptance gate) — the eight-file `.wo` sample compiles clean and detects a silent death live. Blocked on iterations 5–6. The sample is already authored ([`docs/examples/log-watcher/`](../../examples/log-watcher/README.md)) and currently reports 93 diagnostics, down from 307. Board: [00-status.md](../../00-status.md) +> **Status: ⬜ pending** (story iteration 7 — the acceptance gate) — the eight-file `.wo` sample compiles clean and detects a silent death live. Blocked on iterations 5–6. The sample is already authored ([`docs/examples/log-watcher/`](../../examples/log-watcher/README.md)) and currently reports 93 diagnostics, down from 307. Board: [00-status.md](../../stories/00-status.md) > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. > diff --git a/docs/superpowers/plans/2026-08-01-program-mode-stdlib.md b/docs/superpowers/plans/2026-08-01-program-mode-stdlib.md index 185e1e2..daaafec 100644 --- a/docs/superpowers/plans/2026-08-01-program-mode-stdlib.md +++ b/docs/superpowers/plans/2026-08-01-program-mode-stdlib.md @@ -1,6 +1,6 @@ # Program Mode + Systems Stdlib Implementation Plan -> **Status: ⬜ pending** (story iteration 6 — next after iteration 5) — `fn main`, exit codes, and the `fs`/`proc`/`net`/`time`/`json`/`env` builtin surface. Blocked on plan 8; the six stdlib namespaces already typecheck as UNKNOWN-BUT-RESERVED, so a qualified call compiles today and only fails at emission (`WO-E406`). Board: [00-status.md](../../00-status.md) +> **Status: ⬜ pending** (story iteration 6 — next after iteration 5) — `fn main`, exit codes, and the `fs`/`proc`/`net`/`time`/`json`/`env` builtin surface. Blocked on plan 8; the six stdlib namespaces already typecheck as UNKNOWN-BUT-RESERVED, so a qualified call compiles today and only fails at emission (`WO-E406`). Board: [00-status.md](../../stories/00-status.md) > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. > diff --git a/docs/superpowers/plans/2026-08-01-shard-actor-vm-runtime.md b/docs/superpowers/plans/2026-08-01-shard-actor-vm-runtime.md index b871073..eb4c1dc 100644 --- a/docs/superpowers/plans/2026-08-01-shard-actor-vm-runtime.md +++ b/docs/superpowers/plans/2026-08-01-shard-actor-vm-runtime.md @@ -1,6 +1,16 @@ # Shard-Actor VM Runtime Implementation Plan -> **Status: ⬜ pending** (story iteration 8) — thread-per-core shards, per-shard heaps, ownership-move messaging. Must follow story iteration 7b (inferred GC + incremental mark-sweep): the collector settles before shards multiply. Board: [00-status.md](../../00-status.md) +> **Status: ✖ DISCARDED 2026-08-21** (developer decision) — superseded by +> the arc plan of record, +> [`2026-08-20-shard-fiber-arc.md`](2026-08-20-shard-fiber-arc.md), whose +> stages 1+2 landed 2026-08-20. This plan's premise is inverted twice: it +> builds on "epoll now / io_uring when the loop module ports phase C" +> (the io_uring-first directive made the ring primary, and the epoll-based +> approach is discarded), and it ports patterns from `runtime/wo-rt.c`, a +> tree removed 2026-08-18 with the Rust track. Kept as historical +> reference for the mailbox-ring and heap-stamping idea shapes only. +> Recorded in [`plan/discarded.md`](../../plan/discarded.md). Board: +> [00-status.md](../../stories/00-status.md) > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. > diff --git a/docs/superpowers/plans/2026-08-01-wob-format-and-vm-core.md b/docs/superpowers/plans/2026-08-01-wob-format-and-vm-core.md index c99545a..6259646 100644 --- a/docs/superpowers/plans/2026-08-01-wob-format-and-vm-core.md +++ b/docs/superpowers/plans/2026-08-01-wob-format-and-vm-core.md @@ -1,6 +1,6 @@ # .wob Format + wovm VM Core Implementation Plan -> **Status: ✅ done** (story iteration 2) — `.wob` v1 loader with full static validation, register interpreter (computed-goto + ISO-C fallback), arena with size-class free lists, borrow word, RC + budgeted Bacon–Rajan cycle collector, drop-map trap unwinding, containers, builtins, ICALL, CLI. 13 suites × 2 dispatch flavors + CLI smoke, ASan/UBSan clean. The format contract itself lives on in [`plan/oop-vm/00-wob-format.md`](../../plan/oop-vm/00-wob-format.md). Board: [00-status.md](../../00-status.md) +> **Status: ✅ done** (story iteration 2) — `.wob` v1 loader with full static validation, register interpreter (computed-goto + ISO-C fallback), arena with size-class free lists, borrow word, RC + budgeted Bacon–Rajan cycle collector, drop-map trap unwinding, containers, builtins, ICALL, CLI. 13 suites × 2 dispatch flavors + CLI smoke, ASan/UBSan clean. The format contract itself lives on in [`plan/oop-vm/00-wob-format.md`](../../plan/oop-vm/00-wob-format.md). Board: [00-status.md](../../stories/00-status.md) > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. > diff --git a/docs/superpowers/plans/2026-08-20-framework-v2-memory-features.md b/docs/superpowers/plans/2026-08-20-framework-v2-memory-features.md index 89443bd..2177975 100644 --- a/docs/superpowers/plans/2026-08-20-framework-v2-memory-features.md +++ b/docs/superpowers/plans/2026-08-20-framework-v2-memory-features.md @@ -1,7 +1,9 @@ # Iteration 18 — framework v2 (transaction{} + cache/flags/jobs): implementation plan -> **Status: ready to execute (2026-08-20).** Board: -> [docs/00-status.md](../../00-status.md). +> **Status: ⏸ hold (2026-08-21, developer decision)** — story iteration 18 +> sits in `stories/language-runtime-database/hold/`; plan was ready to +> execute (2026-08-20) and stays intact for resumption. Board: +> [docs/00-status.md](../../stories/00-status.md). > **For agentic workers:** REQUIRED SUB-SKILL: Use > superpowers:subagent-driven-development (recommended) or diff --git a/docs/superpowers/plans/2026-08-20-library-kind-internal.md b/docs/superpowers/plans/2026-08-20-library-kind-internal.md index 7133f2d..d461e73 100644 --- a/docs/superpowers/plans/2026-08-20-library-kind-internal.md +++ b/docs/superpowers/plans/2026-08-20-library-kind-internal.md @@ -6,7 +6,7 @@ > after this plan was written: `http/parse.wo` was SPLIT rather than moved > whole (its `media_type`/`form_values` are public surface the web-app calls), > and the gate reads 26/0 rather than 17/0 (`just web-app` was already at 23 -> before this iteration). Board: [docs/00-status.md](../../00-status.md). +> before this iteration). Board: [docs/00-status.md](../../stories/00-status.md). > **For agentic workers:** REQUIRED SUB-SKILL: Use > superpowers:subagent-driven-development (recommended) or diff --git a/docs/superpowers/plans/2026-08-20-shard-fiber-arc.md b/docs/superpowers/plans/2026-08-20-shard-fiber-arc.md index 78738af..b523345 100644 --- a/docs/superpowers/plans/2026-08-20-shard-fiber-arc.md +++ b/docs/superpowers/plans/2026-08-20-shard-fiber-arc.md @@ -1,8 +1,18 @@ # The 8+11 concurrency arc — implementation plan (staged) -> **Status: STAGES 1 AND 2 COMPLETE 2026-08-20** (branch `concurrency-arc`, -> T1–T4 landed + the `docs/examples/fibers` demo and its `just fibers` -> gate, 8/0). Stages 2–3 pending. Execution deviations, disclosed: +> **Status: ✅ ARC COMPLETE — STAGE 3 LANDED 2026-08-21** (branch +> `concurrency-arc-stage3`, T7 executed; T8 is this closeout). The +> transparent DB actor is live: a worker shard's DB statement marshals to +> shard 0, parks, resumes with the materialized reply — `WO_T_DB` off the +> primary is gone. Proof: NEW gate `just db-actor` 8/0 (multi-shard ×3 + +> both forced backends + single-shard byte-exact + WO_DATA replay pair), +> ASan/TSan 6/6 on the RPC path, full battery green. Stage-3 deviations +> are disclosed at Task 7; stage 1+2 history below stands. 22's minimal +> precursor recorded in the stories (RAM-only: 500 remote inserts ≈4ms +> vs local ≈0ms — real numbers are 22's). +> +> Stages 1+2 completed 2026-08-20 (branch `concurrency-arc`, T1–T6 + the +> `docs/examples/fibers` demo and its gate). Execution deviations, disclosed: > (1) the reduction budget decrements at loop BACK-EDGES ONLY, after the > jump lands — the spec's "same three sites as the GC" wording had a > livelock at budget 1 (pre-instruction save re-executes the jump into @@ -22,7 +32,7 @@ > (7) two TSan-caught races fixed (late-init memset vs concurrent push; > wake-efd read outside the lock) and one teardown SEGV (routed frees > during teardown are now no-ops: arenas die wholesale). -> Board: [docs/00-status.md](../../00-status.md). +> Board: [docs/00-status.md](../../stories/00-status.md). > **For agentic workers:** REQUIRED SUB-SKILL: Use > superpowers:subagent-driven-development (recommended) or @@ -37,8 +47,8 @@ every standing gate green before the next begins. **Architecture:** see the spec (normative): [`../specs/2026-08-20-shard-fiber-arc-design.md`](../specs/2026-08-20-shard-fiber-arc-design.md). -Stories: [8](../../stories/language-runtime-database/refine/08-shard-actor-runtime.md) · -[11](../../stories/language-runtime-database/refine/11-fibers.md). +Stories: [8](../../stories/language-runtime-database/done/08-shard-actor-runtime.md) · +[11](../../stories/language-runtime-database/done/11-fibers.md). **Tech Stack:** C11 libc-only (`wovm`), OCaml stdlib-only (`woc`), bash gates; TSan added to the corpus harness at stage 2. @@ -185,29 +195,68 @@ transitively-traced check), corpus + TSan. ## Stage 3 — the DB actor + closing the arc +> **Guarantee obligations (2026-08-21 refinement, developer-approved)** +> — Tasks 7–8 build against these, in addition to their own checkboxes: +> (1) a worker write RPC is exactly ONE owner-shard commit; the ack +> crosses shards only AFTER the owner's fsync — a kill between send and +> commit leaves no ack and no partial state; (2) workers never open the +> WAL or data directory (debug-build assert); replay completes on the +> primary before any worker serves; (3) statements are serialized by the +> DB actor — replies are materialized copies, no torn reads under the +> concurrent multi-shard corpus (TSan). The five-property map lives in +> [story 8's guarantee contract](../../stories/language-runtime-database/done/08-shard-actor-runtime.md); disk +> space reclamation is story 32, not this stage. + ### Task 7 — transparent DB RPC **Files:** `runtime/src/builtin.c` (db cases marshal when not on shard 0), `database/src/` untouched (the engine never learns), `runtime/src/vm.c` (request/reply parking). -- [ ] Non-owner DB builtins marshal statement + args to shard 0, park, +- [x] Non-owner DB builtins marshal statement + args to shard 0, park, resume with materialized reply; `transaction { }` travels as one unit - (18's staged batch stays owner-side). -- [ ] `just employee` + `just web-app` at default cores, answers - byte-identical to N=1 (iteration 8's criterion 4, the arc's headline - proof). Commit. + (18's staged batch stays owner-side — nothing to do until 18 unholds). + DEVIATIONS, disclosed: (1) "database/src untouched" bent to + "database/src gains thread-agnostic slot-level entry points" + (wo_db_val_encode/clone, wo_row_insert_slots, wo_row_update_field_slot, + wo_db_exec_req) — the owner thread must never read a requester's VM + heap (concurrent mark-bit writes = TSan race), so the REQUESTER encodes + args to engine slots and the owner executes from slots, replay-style; + (2) the reply park is a new plane-less park (`WO_PARK_INBOX`), woken by + the DB_RESP envelope (envelope kinds 3/4; `wo_io_unpark` exported); + resume re-executes the builtin, which consumes the reply; (3) a busy + shard adopts its inbox once per reduction slice, bounding a request's + wait on a computing primary; (4) main.c boots the engine + replay + BEFORE `wo_engine_start` (the replay-before-serve obligation — it also + publishes the engine's class table to worker threads by the spawn); + (5) EN ROUTE, a latent stage-1 bug fixed: io_uring ring params were ONE + file static, rewritten by every shard's lazy init while other shards + read offsets from it — submits landed at garbage offsets and parked + fibers lost wakes (~1/20 hangs at default cores). Params now live + per-vm (`io_params`), and a short `io_uring_enter` submit is a loud + trap, never a success. +- [x] Verified: `just db-actor` (NEW gate, 8/0 — worker-shard actors + insert/scan/get through the DB actor; multi-shard set-asserted ×3 + + both forced backends; single-shard byte-exact; WO_DATA pair proves a + worker's write is ack-after-durable and replays). ASan 6/6 and TSan + 6/6 clean on the RPC path; 60/60 hang-free at default cores. + `just employee` + `just web-app` at default cores green (byte-identical + to N=1). Commit. ### Task 8 — the arc's closeout -- [ ] 22's benchmark re-run (or its minimal precursor if 22 has not - landed: the employee read/write loop timed) single- vs multi-shard; - numbers recorded in the stories. -- [ ] Stories 8 + 11 landing banners; board rows; graph node classes; - framework README ledger rows that the arc unblocks (streaming etc. - stay ⏸ until their own slices — the arc UNBLOCKS, it does not build - them); CODE-LOGIC files (vm fiber model, shard/mailbox model, DB RPC). -- [ ] Full battery once more after doc edits. Commit. +- [x] 22's minimal precursor (22 has not landed): a timed 500-insert + + 50-scan loop, local vs spawned-actor, RAM-only — remote inserts ≈4ms + for 500 (~8µs/RPC round-trip incl. park/resume), local ≈0ms; scans + ≈2–4ms per 50. Recorded in story 8's landing banner; honest numbers + with p50/p99 are 22's campaign. +- [x] Stories 8 + 11 landed with banners (11 carries the fs-park + re-scope, disclosed); board standup + rows + pending list flipped; + graph nodes I8/I11 → done; framework README ledger rows note the arc + UNBLOCKED them (streaming/keep-alive retirement ride iteration 24; + rows stay ⏸); CODE-LOGIC: runtime/src gains the DB-actor + ring-params + section, database/src the slot-surface section. +- [x] Full battery once more after doc edits. Commit. ## Success criteria diff --git a/docs/superpowers/plans/2026-08-21-db-bench.md b/docs/superpowers/plans/2026-08-21-db-bench.md new file mode 100644 index 0000000..b950611 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-db-bench.md @@ -0,0 +1,224 @@ +# Iteration 22 — db-bench: durability proof, throughput, scale (implementation plan) + +> **Status: ready to execute (2026-08-21).** Board: +> [docs/00-status.md](../../stories/00-status.md). + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps +> use checkbox (`- [ ]`) syntax for tracking. +> +> **Style rule (user convention):** concept, reason, and required +> behavior in words plus verification commands only — no implementation +> or test code blocks; the executor writes the code. + +**Goal:** the measurement backbone — a `.wo` benchmark sample, a campaign +driver, a committed baseline contract, and the durability proofs, so +every later optimization signs a measured before/after. + +**Architecture:** four artifacts (spec §1): `docs/examples/db-bench` +(load generator, pure `.wo`), `scripts/db-bench.sh` (campaign driver + +gates), `bench/baseline.json` (the contract), `just db-bench` / +`just db-bench-quick`. One runtime addition: the `time.ticks` builtin +(CLOCK_MONOTONIC microseconds) — everything else is sample + script. + +**Tech Stack:** `.wo` (generator), bash + python3 (driver/gate — the +linkcheck.py precedent), C11 libc-only (one builtin case), OCaml +stdlib-only (one stdlib-table row). + +**Spec:** [`../specs/2026-08-21-db-bench-design.md`](../specs/2026-08-21-db-bench-design.md) +(approved 2026-08-21, normative — the four forks + decisions 5/6 live +there). Story: +[`22-durability-throughput-scale.md`](../../stories/language-runtime-database/in-progress/22-durability-throughput-scale.md). + +## Global Constraints + +- Branch `db-bench` off the current arc line; commits local only, never + push. +- Gates that stay green after every task: `just woc-test`, + `just oop-e2e`, `just deps-accept`, `just web-app`, + `just log-watcher`, `just employee`, `just fibers`, `just db-actor`. +- The ONLY runtime change is the `time.ticks` builtin (spec decision 6); + everything else must not touch `runtime/src` or `database/src`. +- `bench/results/` is gitignored; `bench/baseline.json` is tracked and + changes only with a commit that says why. +- The headline numbers come from compiled `.wo` end to end; the C-API + microbench is attribution-only and never gated (spec §5). + +--- + +## Task 1 — the `time.ticks` builtin (the honest clock) + +**Files:** +- Modify: `runtime/src/wob.h` (new builtin id 84, `WO_B_MAX` bump), + `runtime/src/sysio.c` (the case, beside `WO_B_TIME_SLEEP`), + `compiler/src/types.ml` (the stdlib table row beside + `m "time" "sleep" 1 46`; grep for the sibling tables in `owner.ml`/ + `emit.ml` that list stdlib names and mirror the row wherever `sleep` + appears), `docs/plan/oop-vm/08-builtin-surface.md` + + `docs/plan/oop-vm/07-systems-stdlib.md` (the contract rows). +- Test: a corpus fixture `tests/corpus/run/time-ticks` and one line in + the existing runtime/compiler suites only if their tables enumerate + builtins. + +**Interfaces:** +- Produces: `time.ticks()` — zero args, returns Int microseconds from + CLOCK_MONOTONIC (never wall clock: it must be immune to NTP steps; + the difference of two calls is a duration). Later tasks time every + operation with it. + +- [ ] Corpus fixture first: two `time.ticks()` calls around a spin loop; + assert the difference is non-negative and the second call is >= the + first (exact values are machine noise — the fixture asserts ordering + and that the builtin exists). Verify it FAILS today (unknown builtin). +- [ ] Wire the id (84), the sysio case (clock_gettime MONOTONIC, + seconds*1e6 + nsec/1e3, as Int), the compiler table rows, the two + contract-doc rows (name, arity 0, return Int µs, monotonic-not-wall + wording). +- [ ] Verify: fixture green under `just oop-e2e`; full battery green. + Commit. + +## Task 2 — db-bench sample: tables, serial modes, per-op stats + +**Files:** +- Create: `docs/examples/db-bench/wo.toml`, + `docs/examples/db-bench/types.wo` (the two related tables — a parent + with a `@unique` Text column, a child with `ref` parent + two indexed + columns; the employee shape, spec §2), + `docs/examples/db-bench/main.wo` (argv mode dispatch, employee's + pattern), `docs/examples/db-bench/README.md` (what each mode measures + and the output-line contract). + +**Interfaces:** +- Consumes: `time.ticks()` from Task 1. +- Produces: modes `seed N`, `read N`, `query N`, `write N`, `verify`; + every measured mode prints exactly one line per operation class in + the spec's contract: ` `. + `verify` recounts, checksums contents, runs one indexed probe, exits + nonzero on mismatch; `seed`/`write` print an acknowledged high-water + line (`acked `) the crash battery reads (spec §4). + +- [ ] Tables + `seed`/`verify` first: seed writes N children (parents + amortized 1:100), verify recomputes count + a Int-sum checksum over an + indexed column + probes one known unique parent. Prove the pair by + hand: seed 10k, verify exits 0; corrupt expectation (verify 10k+1) + exits 1. +- [ ] Per-op timing: a fixed-size reservoir (spec §2 — honest, not + clever) collecting per-op durations from `time.ticks`; p50/p99 by + sorting the reservoir at report time; ops/sec from total ticks. +- [ ] `read`/`query`/`write` modes over a seeded store, each emitting + the contract line; a malformed mode prints usage and exits 2 + (employee's shape). +- [ ] Verify: run all modes by hand single-shard (`WO_SHARDS=1`), lines + parse (field count + numeric), `just` battery untouched. Commit. + +## Task 3 — mix + msgrate: the concurrent modes + +**Files:** +- Modify: `docs/examples/db-bench/main.wo` (+ a `types.wo` message + class), README rows. + +**Interfaces:** +- Consumes: Task 2's tables, stats, output contract. +- Produces: `mix N C` — C spawned actors each running the 90/10 + read/write mix, N total ops, one contract line for reads and one for + writes plus the `acked` high-water; `msgrate N` — two actors + ping-ponging N messages, printing `msgrate `. Under + `WO_SHARDS=1` everything is same-shard (fibers); at default cores + placement spreads the actors and every DB statement rides the stage-3 + RPC — no bench code may check the shard count (transparency is the + point). +- No request/response surface exists (iteration 31): main drives + completion the db-actor way — actors bump rows a completion `verify` + can count; main sleep-polls the store until the expected count, then + settles. Document that as the coordination idiom this side of 31. + +- [ ] `mix`: prove single-shard first (deterministic-ish, one thread), + then default cores; both emit parseable lines; TSan flavor of the + binary runs one short mix clean. +- [ ] `msgrate`: same proof shape; single- and multi-shard runs both + print (same-heap vs mutex-inbox comparison, spec §2). +- [ ] Verify: hand runs at both shard counts + TSan; battery. Commit. + +## Task 4 — the campaign driver, gate, and recipes + +**Files:** +- Create: `scripts/db-bench.sh` (driver), `scripts/db-bench-gate.py` + (JSON compare — python3, the linkcheck.py precedent), + `bench/baseline.json` (placeholder schema, values filled by Task 5), + `bench/results/.gitignore`. +- Modify: `justfile` (recipes `db-bench`, `db-bench-quick`), + `.gitignore` if bench/results needs a root rule. + +**Interfaces:** +- Consumes: the sample's line contract + `acked` lines. +- Produces: one timestamped JSON per campaign under `bench/results/` + (structure: flavor → shards → mode → {count, ops_sec, p50us, p99us}); + gate exit 0/1 with a per-metric summary tail (value, baseline, delta, + tolerance); the baseline schema: per metric `{value, tolerance_pct, + floor}` (spec fork 2). + +- [ ] Driver: for flavor in ram/durable × shards in 1/default — seed, + read, query, write, mix (+ msgrate once per shard count); durable + runs under a temp `WO_DATA`; RSS + fd sampled during mix, failing on + LW_SOAK tolerances (growth > 256 KiB resident or any fd growth). +- [ ] Durability teeth in the driver (spec §4): restart proof + (seed → clean stop → rerun `verify`), crash battery (kill -9 + mid-`write` K times per shard count, restart, `verify` against the + last `acked` high-water; K lives in baseline.json). +- [ ] Gate: compares every metric to baseline (relative tolerance + + absolute floor), prints the standup tail, exit nonzero on breach; + `--write-baseline` records a run as the new contract. +- [ ] `db-bench-quick`: seconds-long counts, loose gate (floors only) — + the CI-shaped smoke. +- [ ] Verify: quick mode end-to-end green against a freshly written + baseline; battery. Commit. + +## Task 5 — the first campaign: baseline, deltas, gate-bites proof + +**Files:** +- Modify: `bench/baseline.json` (the first honest full run's values + + chosen tolerances + floors + K), + `docs/stories/language-runtime-database/done/08-shard-actor-runtime.md` + (the arc's recorded delta: single- vs multi-shard columns), + `docs/examples/db-bench/README.md` (the reference-machine numbers). + +- [ ] Full campaign on this machine; inspect the tail; commit the + baseline with a message that says it IS the first contract. +- [ ] Gate-bites smoke (spec acceptance): doctor a copy of the results + (halve one ops/sec) and run the gate against it — must FAIL; then the + real results — must PASS. Record the procedure in the README. +- [ ] Copy the arc delta + msgrate numbers into story 8's record and + the mutex-inbox note (arc plan deviation 4 references it). +- [ ] Verify: `just db-bench` exit 0 twice in a row (repeatability); + battery. Commit. + +## Task 6 — closeout + +**Files:** +- Modify: story 22 (landing banner, criteria check, frontmatter + `status: done`, file moves refine/→done/ with link sweep), the board + (standup entry: implemented/findings/learned/unblocked/next/.dev-ref; + In-progress row; pending list; stories table), `00-story.md` row, + `00-dependency-graph.md` node, the spec's status banner (APPROVED → + landed), `docs/in-progress/` marker deleted. + +- [ ] Docs synced (the standup answers written from the actual numbers); + links verified (the session's link-check loop); frontmatter matches + folders. +- [ ] Full battery + `just db-bench-quick` once more after doc edits. + Commit. + +## Self-review notes + +- Spec coverage: §1→T4 artifacts + T1 clock; §2 modes→T2/T3; §3 + campaign+baseline→T4/T5; §4 durability→T4 driver + T5 run; §5 + microbench→deliberately NOT a task until a regression needs blaming + (YAGNI — the spec calls it attribution-only; first need creates it, + recorded here so the omission is a decision, not a gap); §6 + acceptance→T5 (gate-bites, repeatability) + T6. +- The only interface later tasks depend on from T1 is `time.ticks()` + returning Int µs; T2's line contract is quoted verbatim where T4 + parses it. +- No code blocks by user convention; every step names its verification + command or observable. diff --git a/docs/superpowers/specs/2026-08-03-blue-green-vm-design.md b/docs/superpowers/specs/2026-08-03-blue-green-vm-design.md index e5a7e6f..26b724d 100644 --- a/docs/superpowers/specs/2026-08-03-blue-green-vm-design.md +++ b/docs/superpowers/specs/2026-08-03-blue-green-vm-design.md @@ -1,7 +1,7 @@ # Blue/Green VM deployment — design spec **Date:** 2026-08-03 -**Status:** approved (2026-08-03); implementation plan deferred until plans 5 + 6 ship +**Status:** approved (2026-08-03); ⏸ on hold (2026-08-21, developer decision — story iteration 26 moved to `stories/language-runtime-database/hold/`); implementation plan deferred until plans 5 + 6 ship **Scope:** the writeonce runtime's in-process deployment subsystem **Supersedes:** §5–§6 of [`docs/plan/exploration/blue-green-vm/00-vision.md`](../../plan/exploration/blue-green-vm/00-vision.md) diff --git a/docs/superpowers/specs/2026-08-10-logwatcher-gap-closure-design.md b/docs/superpowers/specs/2026-08-10-logwatcher-gap-closure-design.md index 4c8917c..cd2e8b0 100644 --- a/docs/superpowers/specs/2026-08-10-logwatcher-gap-closure-design.md +++ b/docs/superpowers/specs/2026-08-10-logwatcher-gap-closure-design.md @@ -166,7 +166,7 @@ against the sample. | [plan 3](../../plan/compiler/2026-08-01-wob-emit-e2e-single-binary.md) | Unchanged. | | [plan 10](../plans/2026-08-01-log-watcher-sample.md) | Acceptance gains the diagnostic-count gate: 307 → 0. | | [`01-error-catalog.md`](../../plan/oop-vm/01-error-catalog.md) | `WO-E205` re-filed as **reachable but unenforced** — corrected 2026-08-11, see § 5 — with its repro; `WO-E208`/`E210`/`E211`–`E213` keep their existing reserved entries. | -| [`docs/00-status.md`](../../00-status.md) | NEXT PLAN gains the milestone-grammar-only note; pending list gains the three cuts under the parked section. | +| [`docs/00-status.md`](../../stories/00-status.md) | NEXT PLAN gains the milestone-grammar-only note; pending list gains the three cuts under the parked section. | | `docs/00-code-review.md` | Reduced to a stub: one paragraph saying its findings landed here and in the plans, pointing at `docs/00-status.md`. | ## Error handling diff --git a/docs/superpowers/specs/2026-08-18-web-framework-design.md b/docs/superpowers/specs/2026-08-18-web-framework-design.md index c5aa9bf..5323190 100644 --- a/docs/superpowers/specs/2026-08-18-web-framework-design.md +++ b/docs/superpowers/specs/2026-08-18-web-framework-design.md @@ -7,9 +7,10 @@ library, HTTP/1.1 behind a TLS-terminating reverse proxy, and (C) the parked HTTP/2 path. Three sub-projects; A and B are the fundable ones, C is a recorded successor. -**Relates to:** iteration 25 (`service` blocks — this framework becomes their -lowering target, not a rival), iterations 8/23/11 (the concurrency work h2c -waits for), `docs/plan/discarded.md` (FFI reject row — load-bearing here). +**Relates to:** iteration 25 (⏸ on hold since 2026-08-21; `service` blocks — +this framework becomes their lowering target, not a rival), iterations +8/23/11 (the concurrency work h2c waits for), `docs/plan/discarded.md` (FFI +reject row — load-bearing here). ## Decisions locked during brainstorming diff --git a/docs/superpowers/specs/2026-08-20-library-kind-internal-design.md b/docs/superpowers/specs/2026-08-20-library-kind-internal-design.md index 16f595e..d458203 100644 --- a/docs/superpowers/specs/2026-08-20-library-kind-internal-design.md +++ b/docs/superpowers/specs/2026-08-20-library-kind-internal-design.md @@ -6,7 +6,7 @@ > settled in > [the iteration](../../stories/language-runtime-database/done/17-library-projects-internal.md) > (four forks + impact analysis); this spec makes them buildable. The plan -> follows after review. Board: [docs/00-status.md](../../00-status.md). +> follows after review. Board: [docs/00-status.md](../../stories/00-status.md). > > Per repo convention this spec carries concept, reason, and required > behavior in words only — no implementation code. diff --git a/docs/superpowers/specs/2026-08-20-memory-db-features-design.md b/docs/superpowers/specs/2026-08-20-memory-db-features-design.md index fb42675..a178bd8 100644 --- a/docs/superpowers/specs/2026-08-20-memory-db-features-design.md +++ b/docs/superpowers/specs/2026-08-20-memory-db-features-design.md @@ -4,11 +4,14 @@ > the framework README's status ledger; this spec is what the embedded > store adds on top of it. > -> **Status: APPROVED 2026-08-20** (developer review). Plan next. -> Decisions were settled in +> **Status: APPROVED 2026-08-20** (developer review); ⏸ on hold +> (2026-08-21, developer decision — story iteration 18 sits in +> `stories/language-runtime-database/hold/`). Decisions were settled in > [the iteration](../../stories/language-runtime-database/hold/18-memory-db-features.md); -> this spec makes them buildable. The plan follows after review. -> Board: [docs/00-status.md](../../00-status.md). +> this spec makes them buildable. The plan is authored +> ([framework v2 plan](../plans/2026-08-20-framework-v2-memory-features.md)) +> and held with it. +> Board: [docs/00-status.md](../../stories/00-status.md). > > Per repo convention: concept, reason, and required behavior in words > only — no implementation code. diff --git a/docs/superpowers/specs/2026-08-20-shard-fiber-arc-design.md b/docs/superpowers/specs/2026-08-20-shard-fiber-arc-design.md index f26319d..36c7d65 100644 --- a/docs/superpowers/specs/2026-08-20-shard-fiber-arc-design.md +++ b/docs/superpowers/specs/2026-08-20-shard-fiber-arc-design.md @@ -1,11 +1,11 @@ # The 8+11 concurrency arc — shards, fibers, actors: design > **Status: spec, awaiting review (2026-08-20).** The arc's decisions were -> settled in [iteration 8](../../stories/language-runtime-database/refine/08-shard-actor-runtime.md) +> settled in [iteration 8](../../stories/language-runtime-database/done/08-shard-actor-runtime.md) > (refined) and the brainstorm of 2026-08-20 (this document's Decisions). > Covers iterations 8 AND 11 as one arc; iteration 24 (chat) is its > acceptance workload and gets its own spec after this one. The plan -> follows after review. Board: [docs/00-status.md](../../00-status.md). +> follows after review. Board: [docs/00-status.md](../../stories/00-status.md). > > Per repo convention: concept, reason, and required behavior in words > only — no implementation code. diff --git a/docs/superpowers/specs/2026-08-21-db-bench-design.md b/docs/superpowers/specs/2026-08-21-db-bench-design.md new file mode 100644 index 0000000..2199a8a --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-db-bench-design.md @@ -0,0 +1,172 @@ +# Iteration 22 — durability proof, throughput, and scale under load (design) + +**Date:** 2026-08-21 +**Status:** ✅ APPROVED 2026-08-21 (developer review) — plan ready: +[`2026-08-21-db-bench.md`](../plans/2026-08-21-db-bench.md). +Board: [docs/00-status.md](../../stories/00-status.md) +**Scope:** the measurement backbone — a benchmark workload in `.wo`, a +campaign driver script, a tracked baseline contract, and the durability +proofs (restart persistence + crash battery), single- AND multi-shard. +**Relates to:** [story 22](../../stories/language-runtime-database/in-progress/22-durability-throughput-scale.md) +(the four forks settled below), the landed arc +([story 8's guarantee contract](../../stories/language-runtime-database/done/08-shard-actor-runtime.md) +— the stage-3 delta this iteration records), iteration 23 (the durable +write number it exists to beat), iteration 32 (the aged-store replay +number its policy wants), stage-2 deviation 4 (the mutex-inbox number). + +## Decisions locked during brainstorming (2026-08-21) + +The story's four forks are SETTLED as their recorded leanings (developer +confirmation, the iteration-23 precedent): + +1. **The load generator is compiled `.wo`.** The headline numbers cross + the whole stack — lexer to WAL — because that is the layer a + language-integrated query pays. A C-API microbench exists ONLY to + attribute a regression to engine vs lowering; it is never the quoted + number. +2. **Gates are relative, against a committed baseline.** Fail when a + metric is worse than `bench/baseline.json` by more than its recorded + tolerance (default 15%, tunable per metric in the file). One loud + ABSOLUTE floor per class — a deliberately low catastrophic-regression + tripwire that fails even on a slow machine. The baseline refreshes + only by a commit that says why. +3. **The scale axis is rows, not connections.** "A million users read + and write" means ~1M rows seeded, a bounded-concurrency 90/10 + read/write mix sustained for a stated duration, throughput above the + floor, p99 under the ceiling, RSS flat. Connection scale belongs to + iteration 24's serving workload. +4. **Both durability flavors run and publish, labeled.** `ram` (no + `WO_DATA`) is the engine's ceiling; `durable` (fsync-per-commit) is + the number an operator plans against. The gap between them is + exactly what iteration 23 exists to close — this iteration prices it. + +New decisions (post-arc reality): + +5. **The vehicle is a NEW sample, `docs/examples/db-bench`** — the + employee sample stays a teaching sample; the multi-shard campaign + needs actor writers, which employee is deliberately not. +6. **One runtime addition, disclosed: `time.ticks`** — CLOCK_MONOTONIC + microseconds as an Int. `time.now` is wall-clock milliseconds and + cannot rank microsecond-scale operations; honest per-op p50/p99 from + inside `.wo` needs this clock. It joins the stdlib contract doc like + every builtin; no other runtime change is in scope. + +## 1. Shape — four artifacts + +- **`docs/examples/db-bench`** — the load generator, pure `.wo`, + mode-dispatched on argv exactly as the employee sample is. +- **`scripts/db-bench.sh`** — the campaign driver: environment setup + (`WO_SHARDS`, `WO_DATA`, `WO_IO`), the kill -9 battery, RSS/fd + sampling (the `LW_SOAK` discipline: resident growth beyond 256 KiB or + any descriptor growth fails), result collection into JSON, and the + relative/absolute gate evaluation. +- **`bench/baseline.json`** — the tracked contract: per metric, the + baseline value, its tolerance, and the absolute floor. The first + honest run writes it; every later run is judged against it. +- **`just db-bench`** — runs the campaign, off the fast path (like + `log-watcher::soak`); `just db-bench-quick` runs a seconds-long smoke + of the same modes for CI-shaped sanity, gated loosely. + +## 2. The workload — db-bench's modes + +Two related tables in the employee shape (a parent with a `@unique` +text column, a child with `ref` parent + two indexed columns) so reads, +indexed probes, FK checks, and unique maintenance are all priced. + +Modes, each printing one machine-parsable line per operation class — +` `: + +- **`seed N`** — N child rows (parents amortized), timed inserts. +- **`read N`** — N point-reads by id over the seeded store. +- **`query N`** — N indexed probes (the `where` path). +- **`write N`** — N mixed inserts + field updates. +- **`mix N C`** — the sustained load: 90/10 read/write from C + concurrent actors for N total operations. Single-shard, actors are + fibers on the primary; multi-shard, placement spreads them and every + DB statement rides the stage-3 RPC. +- **`msgrate N`** — the mutex-inbox number: two actors on different + shards ping-pong N messages; reports msgs/sec. This is the number + stage-2's deviation 4 waits on before lock-free rings earn their + complexity (single-shard run included for the same-heap comparison). +- **`verify`** — recount + content checksum + one indexed probe against + expected values; exit nonzero on any mismatch. The restart and crash + proofs are `seed`/`write` runs bracketing a `verify` across process + boundaries. + +Timing is per-operation via `time.ticks`, aggregated in `.wo` +(count, ops/sec, p50, p99 from a fixed-size reservoir — the harness +must stay honest, not clever; if the reservoir ever dominates cost the +spec's answer is a bigger batch, not a fancier sampler). + +## 3. The campaign — what one `just db-bench` run produces + +For each flavor `ram` and `durable`, for each shard count 1 and +default-cores: seed, read, query, write, mix — plus `msgrate` once per +shard count. Results land in one JSON file (per-run, timestamped, in +`bench/results/`, gitignored except the baseline), then the gate +compares against `bench/baseline.json` and fails on any tolerance +breach. The stdout tail is the standup summary: one line per metric +with the baseline delta. + +The FIRST full run on the reference machine writes the baseline and +sets the scale-target numbers (fork 3's floor/ceiling become recorded +values, not prose). The arc's owed before/after is recorded the same +way: the single- vs multi-shard columns of the same table ARE the +delta, stated in the results and copied into story 8's record. + +## 4. Durability — proven by a restart, not asserted + +- **Restart persistence:** `seed` under `WO_DATA`, clean stop, restart, + `verify` — counts, contents, id continuation past the persisted + maximum, and an indexed probe (the index was rebuilt on replay). The + employee gate's existing seed-twice check stays as the second witness. +- **Crash battery:** kill -9 mid-`write` under `WO_DATA`, restart, + `verify` in acknowledged-writes mode: every operation the bench + recorded as acknowledged (it prints a running high-water mark for + exactly this) is present; no partial row is visible. K repetitions + (K recorded in the baseline file), run at BOTH shard counts — the + multi-shard rounds fire the stage-3 obligation under load: a kill + between a worker's send and the owner's commit must leave no ack and + no partial state. +- **Sustained-run hygiene:** during `mix`, the driver samples RSS and + descriptor counts; growth beyond the LW_SOAK tolerances fails the + campaign regardless of throughput. + +## 5. The C-API microbench — attribution only + +A `runtime/test`-shaped harness driving `wo_row_insert`/`wo_row_ptr`/ +probe loops directly, printing the same line format. Run manually when +a headline regression needs blaming (engine vs lowering); never gated, +never quoted. Kept deliberately minimal — one file, no options beyond +counts. + +## 6. Acceptance criteria + +- **Given** a fresh checkout on the reference machine, **when** + `just db-bench` runs to completion, **then** it produces the JSON + results, evaluates every gate against `bench/baseline.json`, and + exits 0 with the summary tail. +- **Given** the restart proof, **when** seed/stop/restart/verify runs + under `WO_DATA`, **then** verify exits 0 (counts, contents, id + continuation, index probe). +- **Given** the crash battery, **when** kill -9 lands mid-write K times + at each shard count, **then** every acknowledged write is present + after replay and no partial state is ever visible — zero tolerance. +- **Given** the multi-shard campaign, **when** the same modes run at + `WO_SHARDS=1` and default cores, **then** both columns publish and + the arc's delta is recorded in story 8; a >tolerance regression in + the single-shard column against baseline fails the gate. +- **Given** a deliberate engine slowdown (a manual smoke, documented in + the plan), **when** the gate runs against the committed baseline, + **then** it FAILS — the contract must be shown to bite before the + iteration closes. + +## Out of scope + +- The optimizations themselves (23's group commit, 32's checkpoint, + ring inboxes) — this iteration prices, they change. +- Cross-host / distributed load; connection-count scale (iteration 24). +- Micro-optimizing the harness; a perfect load generator is not the + deliverable — an honest, repeatable one is. +- A cost-based query planner; group-by (parked with 9b). +- Any runtime change beyond the `time.ticks` builtin. diff --git a/justfile b/justfile index 943fdd7..65396b7 100644 --- a/justfile +++ b/justfile @@ -54,6 +54,12 @@ web-app: fibers: ./scripts/fibers-accept.sh +# db-actor: arc stage 3's gate (docs/examples/db-actor) — worker-shard +# actors read/write the database through the transparent DB actor; WAL +# replay pair included. `just db-actor` runs it. +db-actor: + ./scripts/db-actor-accept.sh + # install-accept: extract the dist tarball to a temp prefix, PATH it, and prove # `woc version` + a from-scratch project build+run (self-located wovm) + the # wo-constraint refusal all work — the "tarball install actually works" gate. diff --git a/runtime/src/CODE-LOGIC.md b/runtime/src/CODE-LOGIC.md index 8df47d3..6379457 100644 --- a/runtime/src/CODE-LOGIC.md +++ b/runtime/src/CODE-LOGIC.md @@ -200,3 +200,35 @@ layout. - **The loader validates the five opcodes as plain three-register forms** — the count is a register, not an immediate, so there is nothing to range-check at load time. + +## The transparent DB actor (arc stage 3, 2026-08-21) + +- **The database is an actor on shard 0.** A worker shard's DB builtin + never touches an engine (its `rt.db` is NULL, asserted at serve entry): + `wo_db_rpc` (vm.c) marshals the statement, ships it in a kind-3 envelope + to the primary's inbox, and parks the fiber; the primary executes it + serialized inside its inbox drain (`wo_vm_adopt` case 3, `wo_db_exec_req`) + and ships the same request back as a kind-4 reply, which unparks the + fiber; the builtin RE-EXECUTES and consumes the answer. +- **VM heaps never cross shards.** The requester ENCODES its argument + values into engine slots on its own thread (`wo_db_val_encode`) — an + owner-side read of a requester's Text would race that shard's collector + writing header mark bits. Replies come back as plain ids (scan/probe), a + deep-cloned engine value (get-field, decoded into the requester's arena), + or a bare id (insert). Traps and messages are byte-identical to the local + path; the ack crosses shards only after the owner's WAL commit. +- **The reply park holds no plane wait.** `WO_PARK_INBOX` (park_fd -2) + joins the parked list only; the wake is `wo_io_unpark` from the envelope + drain. Deadline scans key on park_fd == -1 EXACTLY — a -2 must never be + read as a deadline. A busy shard adopts its inbox once per reduction + slice, so a computing primary bounds a worker's DB latency to one slice. +- **Ring params are per-vm (`wo_vm.io_params`) — never share them.** They + were one file static; a worker's lazy `wo_vm_init` memset+refilled it + while another shard read ring offsets out of it, submits landed at + garbage offsets, and parked fibers lost their wakes (~1/20 hangs at + default cores, found by stage 3's cross-shard traffic). A short + `io_uring_enter` submit is a failure, never a success — that check is + what turns any relapse into a loud WO_T_IO instead of a silent hang. +- Proof: `just db-actor` (docs/examples/db-actor — multi-shard set ×3, + both forced backends, single-shard byte-exact, WO_DATA replay pair); + ASan/TSan clean on the RPC path. diff --git a/runtime/src/builtin.c b/runtime/src/builtin.c index c30c765..563fb40 100644 --- a/runtime/src/builtin.c +++ b/runtime/src/builtin.c @@ -171,7 +171,15 @@ int wo_builtin(wo_vm *vm, uint64_t *R, uint32_t ins, const char **msg) { if (C == WO_B_JSON_ENCODE || C == WO_B_JSON_DECODE) return wo_builtin_json(vm, R, ins, msg); if (C >= WO_B_SYS_FIRST && C <= WO_B_PROC_RUN) return wo_builtin_sys(vm, R, ins, msg); - if (C >= WO_B_DB_INSERT && C <= WO_B_DB_PROBE) return wo_builtin_db(vm, R, ins, msg); + if (C >= WO_B_DB_INSERT && C <= WO_B_DB_PROBE) { + /* arc stage 3: the database is an actor on shard 0. A worker shard + * has no engine by design — its statement marshals, parks, resumes + * with the materialized reply. The primary (and every single-shard + * or test build) keeps the direct path bit for bit. */ + if (!vm->rt.db && !vm->is_primary && wo_eng.nshards > 1) + return wo_db_rpc(vm, R, ins, msg); + return wo_builtin_db(vm, R, ins, msg); + } switch (C) { case WO_B_SPAWN: { /* arc: R[B] = the moved-in instance, R[B+1] = receive's method diff --git a/runtime/src/main.c b/runtime/src/main.c index 63e10b5..af72a83 100644 --- a/runtime/src/main.c +++ b/runtime/src/main.c @@ -178,31 +178,17 @@ int main(int argc, char **argv) { return 2; } wo_tls_set(&VM); - /* the arc's stage 2: all cores by default (the brave landing), one - * pinned worker vm per extra core; WO_SHARDS caps or forces it */ - { - long cores = sysconf(_SC_NPROCESSORS_ONLN); - uint32_t nshards = cores > 0 ? (uint32_t)cores : 1; - const char *se = getenv("WO_SHARDS"); - if (se && se[0]) { - unsigned long v = strtoul(se, NULL, 10); - if (v >= 1 && v <= WO_MAX_SHARDS) nshards = (uint32_t)v; - } - if (nshards > WO_MAX_SHARDS) nshards = WO_MAX_SHARDS; - wo_eng.shards = SHARDS; - if (wo_engine_start(&mod, heap_mb << 20, nshards) != 0) { - fprintf(stderr, "wovm: cannot start %u shards\n", nshards); - wo_engine_stop(); - wo_vm_destroy(&VM); - wo_module_free(&mod); - return 2; - } - } /* The database engine boots with the VM: every class IS a table. * Durability is opt-in — WO_DATA= opens /shard-0.wal, * replays it before the entry runs (boot-before-listeners doctrine), * and every insert commits before it acknowledges. Without WO_DATA - * the engine runs RAM-only, which is what the corpus expects. */ + * the engine runs RAM-only, which is what the corpus expects. + * Arc stage 3 obligation: this whole block runs BEFORE the worker + * shards spawn — replay completes before anything can serve, and the + * engine's immutable class-table pointer is published to the worker + * threads by the spawn itself. The engine and WAL stay the PRIMARY's + * alone (rt.db/rt.wal are never set on a worker); workers reach them + * through the DB actor's message path. */ if (wo_db_init(&DB, mod.classes, mod.class_cnt, 0, 1) != 0) { fprintf(stderr, "wovm: cannot initialize the database engine\n"); wo_vm_destroy(&VM); @@ -230,6 +216,28 @@ int main(int argc, char **argv) { } VM.rt.wal = &WAL; } + /* the arc's stage 2: all cores by default (the brave landing), one + * pinned worker vm per extra core; WO_SHARDS caps or forces it */ + { + long cores = sysconf(_SC_NPROCESSORS_ONLN); + uint32_t nshards = cores > 0 ? (uint32_t)cores : 1; + const char *se = getenv("WO_SHARDS"); + if (se && se[0]) { + unsigned long v = strtoul(se, NULL, 10); + if (v >= 1 && v <= WO_MAX_SHARDS) nshards = (uint32_t)v; + } + if (nshards > WO_MAX_SHARDS) nshards = WO_MAX_SHARDS; + wo_eng.shards = SHARDS; + if (wo_engine_start(&mod, heap_mb << 20, nshards) != 0) { + fprintf(stderr, "wovm: cannot start %u shards\n", nshards); + wo_engine_stop(); + if (VM.rt.wal) wo_wal_close(&WAL); + wo_db_destroy(&DB); + wo_vm_destroy(&VM); + wo_module_free(&mod); + return 2; + } + } /* Program mode: an entry that declares one parameter gets the program's * OWN arguments as a `multi Text` — not the program name, and not the diff --git a/runtime/src/park.c b/runtime/src/park.c index 8c201e4..e13770d 100644 --- a/runtime/src/park.c +++ b/runtime/src/park.c @@ -72,30 +72,43 @@ typedef struct { struct io_uring_sqe *sqes; } rings; -static struct io_uring_params g_params; /* offsets survive init */ +/* Ring params live INSIDE each vm (vm.h io_params, opaque bytes) — arc + * stage 3 fix: this WAS one shared static ("offsets survive init"), and a + * worker's LAZY uring_init memset+refilled it on the worker thread while + * another shard was reading ring offsets out of it — submits landed at + * garbage offsets, the kernel saw no sqe (enter returned 0, treated as + * ok), and a parked fiber's TIMEOUT silently never existed. Per-vm storage + * ends the race by construction (a vm's params are only ever touched by + * its own thread); the short-submit check below turns any relapse into a + * loud trap instead of a lost wake. NOTE: not indexed by shard_id — the + * lazy wo_vm_init runs while the worker's shard_id is transiently 0. */ +_Static_assert(sizeof(struct io_uring_params) <= sizeof(((wo_vm *)0)->io_params), + "io_params too small"); static rings ring_ptrs(const wo_vm *vm) { + const struct io_uring_params *p = (const struct io_uring_params *)vm->io_params; rings r; uint8_t *sq = (uint8_t *)vm->io_sq, *cq = (uint8_t *)vm->io_cq; - r.sq_head = (uint32_t *)(sq + g_params.sq_off.head); - r.sq_tail = (uint32_t *)(sq + g_params.sq_off.tail); - r.sq_mask = (uint32_t *)(sq + g_params.sq_off.ring_mask); - r.sq_array = (uint32_t *)(sq + g_params.sq_off.array); - r.cq_head = (uint32_t *)(cq + g_params.cq_off.head); - r.cq_tail = (uint32_t *)(cq + g_params.cq_off.tail); - r.cq_mask = (uint32_t *)(cq + g_params.cq_off.ring_mask); - r.cqes = (struct io_uring_cqe *)(cq + g_params.cq_off.cqes); + r.sq_head = (uint32_t *)(sq + p->sq_off.head); + r.sq_tail = (uint32_t *)(sq + p->sq_off.tail); + r.sq_mask = (uint32_t *)(sq + p->sq_off.ring_mask); + r.sq_array = (uint32_t *)(sq + p->sq_off.array); + r.cq_head = (uint32_t *)(cq + p->cq_off.head); + r.cq_tail = (uint32_t *)(cq + p->cq_off.tail); + r.cq_mask = (uint32_t *)(cq + p->cq_off.ring_mask); + r.cqes = (struct io_uring_cqe *)(cq + p->cq_off.cqes); r.sqes = (struct io_uring_sqe *)vm->io_sqes; return r; } static int uring_init(wo_vm *vm) { - memset(&g_params, 0, sizeof g_params); - long fd = syscall(SYS_io_uring_setup, 64u, &g_params); + struct io_uring_params *p = (struct io_uring_params *)vm->io_params; + memset(p, 0, sizeof *p); + long fd = syscall(SYS_io_uring_setup, 64u, p); if (fd < 0) return -1; - size_t sq_len = g_params.sq_off.array + g_params.sq_entries * sizeof(uint32_t); - size_t cq_len = g_params.cq_off.cqes + g_params.cq_entries * sizeof(struct io_uring_cqe); - size_t sqes_len = g_params.sq_entries * sizeof(struct io_uring_sqe); + size_t sq_len = p->sq_off.array + p->sq_entries * sizeof(uint32_t); + size_t cq_len = p->cq_off.cqes + p->cq_entries * sizeof(struct io_uring_cqe); + size_t sqes_len = p->sq_entries * sizeof(struct io_uring_sqe); void *sq = mmap(NULL, sq_len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, (int)fd, IORING_OFF_SQ_RING); void *cq = mmap(NULL, cq_len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, (int)fd, @@ -128,7 +141,9 @@ static int uring_submit(wo_vm *vm, const struct io_uring_sqe *sqe) { r.sq_array[idx] = idx; __atomic_store_n(r.sq_tail, tail + 1, __ATOMIC_RELEASE); long rc = syscall(SYS_io_uring_enter, vm->io_fd, 1u, 0u, 0u, NULL, 0); - return rc < 0 ? -1 : 0; + /* a short submit is a LOST WAKE, never a success (the g_params race + * above hid behind rc >= 0 for a whole debugging session) */ + return rc == 1 ? 0 : -1; } /* ---- backend-neutral helpers ------------------------------------------ */ @@ -160,6 +175,10 @@ static void wake(wo_vm *vm, wo_fiber *fb) { vm->qtail = fb; } +void wo_io_unpark(wo_vm *vm, wo_fiber *fb) { + if (fb->state == WO_FIB_PARKED) wake(vm, fb); +} + /* ---- API --------------------------------------------------------------- */ int wo_io_init(wo_vm *vm) { @@ -192,6 +211,9 @@ int wo_io_arm(wo_vm *vm, wo_fiber *fb) { fb->pnext = vm->parked; vm->parked = fb; vm->nparked++; + /* arc stage 3: an inbox-wait fiber holds no plane wait at all — the + * wake is wo_io_unpark from the envelope drain */ + if (fb->park_fd == WO_PARK_INBOX) return 0; if (vm->io_kind == 0) { struct io_uring_sqe sqe; memset(&sqe, 0, sizeof sqe); @@ -305,7 +327,7 @@ int wo_io_wait(wo_vm *vm) { int timeout = -1; int64_t now = now_ms(); for (wo_fiber *fb = vm->parked; fb; fb = fb->pnext) - if (fb->park_fd < 0) { + if (fb->park_fd == -1) { /* deadline waits only, never INBOX */ int64_t rel = fb->park_deadline - now; if (rel < 0) rel = 0; if (timeout < 0 || rel < timeout) timeout = (int)rel; @@ -333,7 +355,7 @@ int wo_io_wait(wo_vm *vm) { wo_fiber *fb = vm->parked; while (fb) { wo_fiber *nx = fb->pnext; - if (fb->park_fd < 0 && fb->park_deadline <= now) { + if (fb->park_fd == -1 && fb->park_deadline <= now) { wake(vm, fb); woke = 1; } diff --git a/runtime/src/park.h b/runtime/src/park.h index 934efe7..60390c0 100644 --- a/runtime/src/park.h +++ b/runtime/src/park.h @@ -22,9 +22,15 @@ int wo_io_init(wo_vm *vm); void wo_io_destroy(wo_vm *vm); /* Register the just-parked fiber's wait (fb->park_* already filled by the - * builtin). 0 ok; -1 = arming failed (caller traps the builtin as IO). */ + * builtin). 0 ok; -1 = arming failed (caller traps the builtin as IO). + * park_fd == WO_PARK_INBOX joins the parked list with NO plane wait — the + * wake arrives as an inbox envelope (arc stage 3's DB reply). */ int wo_io_arm(wo_vm *vm, wo_fiber *fb); +/* Wake one parked fiber from OUTSIDE the plane (the inbox path: the DB + * actor's reply). parked list -> run queue. */ +void wo_io_unpark(wo_vm *vm, wo_fiber *fb); + /* Block until at least one parked fiber wakes; woken fibers move to the * run queue. 0 = something woke; WO_IO_STOP = the stop flag interrupted * the wait (caller unwinds everything); -1 = fatal backend error. */ diff --git a/runtime/src/vm.c b/runtime/src/vm.c index a9de7fb..b2a06ac 100644 --- a/runtime/src/vm.c +++ b/runtime/src/vm.c @@ -12,6 +12,11 @@ #include "gc.h" #include "park.h" +#include + +#include "db.h" /* arc stage 3: the transparent DB RPC (wo_db_req) */ +#include "table.h" /* slot encode/decode for the RPC marshaling */ + #include #include #include @@ -94,6 +99,28 @@ static int wo_vm_adopt(wo_vm *vm) { case 2: /* a home-routed free: this arena owns the object */ wo_drop_obj(&vm->rt, (wo_hdr *)(uintptr_t)e->payload); break; + case 3: { /* arc stage 3: a marshaled DB statement — WE are the DB + * actor (only shard 0 ever receives these). Execute + * serialized, right here on the owner thread, then ship + * the same request back as the reply. */ + wo_db_req *q = (wo_db_req *)(uintptr_t)e->payload; + assert(vm->is_primary && "DB requests route to shard 0 only"); + wo_db_exec_req(vm, q); + q->done = 1; + wo_envelope *re = calloc(1, sizeof *re); + if (re) { + re->kind = 4; + re->payload = e->payload; + inbox_push_to(q->from_shard, re); + } /* OOM: the requester stays parked until stop — leak, not UB */ + break; + } + case 4: { /* the DB actor's reply: wake the requesting fiber; the + * re-executed builtin consumes the request */ + wo_db_req *q = (wo_db_req *)(uintptr_t)e->payload; + wo_io_unpark(vm, (wo_fiber *)q->fiber); + break; + } } free(e); n++; @@ -113,6 +140,190 @@ void wo_route_free(wo_hdr *h) { inbox_push_to(h->shard_id, e); } +/* ---- arc stage 3: the requester half of the transparent DB RPC --------- + * A worker shard's DB builtin lands here (its rt.db is NULL by design): + * the args are ENCODED into engine slots on THIS thread — VM heaps are + * never read cross-shard — the request rides an envelope to shard 0, and + * the fiber parks with no plane wait (WO_PARK_INBOX). The reply unparks + * the fiber, the builtin RE-EXECUTES, lands here again, and consumes the + * answer. Every status/msg pair is the one the local path would trap. */ +int wo_db_rpc(wo_vm *vm, uint64_t *R, uint32_t ins, const char **msg) { + uint32_t A = wo_ins_a(ins), B = wo_ins_b(ins), C = wo_ins_c(ins); + wo_fiber *fb = vm->cur; + wo_db_req *q = (wo_db_req *)fb->dbreq; + const wo_classdesc *classes = vm->mod->classes; + + if (q && q->done) { /* the reply: consume it and finish the builtin */ + fb->dbreq = NULL; + int rc = q->status; + if (rc) { + *msg = q->msg; + } else { + switch (C) { + case WO_B_DB_INSERT: R[A] = q->result; break; + case WO_B_DB_UPDATE_FIELD: + case WO_B_DB_DELETE: R[A] = 0; break; + case WO_B_DB_SCAN: + case WO_B_DB_PROBE: { + wo_multi *ids = wo_multi_new(&vm->rt, WO_K_SCALAR); + if (!ids) rc = WO_T_OOM; + for (uint32_t i = 0; !rc && i < q->id_cnt; i++) + if (wo_multi_push(ids, q->ids[i]) != 0) rc = WO_T_OOM; + if (!rc) R[A] = (uint64_t)(uintptr_t)ids; + else *msg = "out of memory"; + break; + } + case WO_B_DB_GET_FIELD: { + int ok = 1; + uint64_t v = wo_val_decode_vm(NULL, &vm->rt, q->val_kind, q->val, &ok, msg); + wo_db_val_free(NULL, q->val_kind, q->val); + q->val = 0; + if (!ok) rc = WO_T_OOM; + else R[A] = v; + break; + } + default: + *msg = "unknown db builtin"; + rc = WO_T_DB; + } + } + if (q->val) wo_db_val_free(NULL, q->val_kind, q->val); + free(q->ids); + free(q); + return rc; + } + + /* first entry: marshal on OUR thread, ship, park */ + q = calloc(1, sizeof *q); + if (!q) { + *msg = "out of memory"; + return WO_T_OOM; + } + q->op = C; + q->from_shard = vm->shard_id; + q->fiber = fb; + int ok = 1; + switch (C) { + case WO_B_DB_INSERT: { + q->cid = (uint32_t)R[B]; + if (q->cid >= vm->mod->class_cnt) { + free(q); + *msg = "no such class"; + return WO_T_DB; + } + const wo_classdesc *c = &classes[q->cid]; + q->slots = calloc(c->field_cnt ? c->field_cnt : 1, 8); + if (!q->slots) { + free(q); + *msg = "out of memory"; + return WO_T_OOM; + } + q->slot_cnt = c->field_cnt; + for (uint32_t i = 0; i < c->field_cnt; i++) { + q->slots[i] = wo_db_val_encode(classes, c->kinds[i], R[B + 1 + i], &ok, msg); + if (!ok) { /* GCREF (the compiler's reject, defensively) or OOM */ + for (uint32_t j = 0; j < i; j++) + wo_db_val_free(NULL, c->kinds[j], q->slots[j]); + free(q->slots); + free(q); + return WO_T_DB; + } + } + break; + } + case WO_B_DB_UPDATE_FIELD: { + q->cid = (uint32_t)R[B]; + q->id = R[B + 1]; + q->field = (uint32_t)R[B + 2]; + if (q->cid >= vm->mod->class_cnt || q->field >= classes[q->cid].field_cnt) { + free(q); + *msg = "no such field"; + return WO_T_DB; + } + q->slots = calloc(1, 8); + if (!q->slots) { + free(q); + *msg = "out of memory"; + return WO_T_OOM; + } + q->slot_cnt = 1; + q->slots[0] = + wo_db_val_encode(classes, classes[q->cid].kinds[q->field], R[B + 3], &ok, msg); + if (!ok) { + free(q->slots); + free(q); + return WO_T_DB; + } + break; + } + case WO_B_DB_DELETE: + q->cid = (uint32_t)R[B]; + q->id = R[B + 1]; + break; + case WO_B_DB_SCAN: + q->cid = (uint32_t)R[B]; + break; + case WO_B_DB_GET_FIELD: + q->cid = (uint32_t)R[B]; + q->id = R[B + 1]; + q->field = (uint32_t)R[B + 2]; + break; + case WO_B_DB_PROBE: { + q->cid = (uint32_t)R[B]; + q->index = (uint32_t)R[B + 1]; + /* the key's kind comes from the class table's index metadata — + * identical on every shard (one module). An index the metadata + * does not know ships keyless; the owner answers empty, exactly + * as the local path does. */ + if (q->cid < vm->mod->class_cnt && classes[q->cid].idx_meta && + q->index < classes[q->cid].idx_cnt) { + const uint32_t *p = classes[q->cid].idx_meta; + for (uint32_t k = 0; k < q->index; k++) p += 2 + p[1]; + uint8_t kind = classes[q->cid].kinds[p[2]]; + q->slots = calloc(1, 8); + if (!q->slots) { + free(q); + *msg = "out of memory"; + return WO_T_OOM; + } + q->slot_cnt = 1; + q->slots[0] = wo_db_val_encode(classes, kind, R[B + 2], &ok, msg); + if (!ok) { + free(q->slots); + free(q); + return WO_T_DB; + } + } + break; + } + default: + free(q); + *msg = "unknown db builtin"; + return WO_T_DB; + } + wo_envelope *e = calloc(1, sizeof *e); + if (!e) { + if (q->slot_cnt && C == WO_B_DB_INSERT) { + const wo_classdesc *c = &classes[q->cid]; + for (uint32_t j = 0; j < c->field_cnt; j++) + wo_db_val_free(NULL, c->kinds[j], q->slots[j]); + } else if (q->slot_cnt && C == WO_B_DB_UPDATE_FIELD) { + wo_db_val_free(NULL, classes[q->cid].kinds[q->field], q->slots[0]); + } /* a PROBE key leaks on this path: kind recompute not worth it */ + free(q->slots); + free(q); + *msg = "out of memory"; + return WO_T_OOM; + } + fb->dbreq = q; + e->kind = 3; + e->payload = (uint64_t)(uintptr_t)q; + inbox_push_to(0, e); + fb->park_fd = WO_PARK_INBOX; + fb->park_done = 0; /* resume RE-EXECUTES the builtin: the consume path */ + return WO_SYS_PARKED; +} + /* A worker's whole life in T5: pinned, parked on its wake eventfd until * shutdown. T6 gives it an inbox to adopt fibers from and the serve loop * that runs them. */ @@ -821,6 +1032,11 @@ static int vm_run(wo_vm *vm, uint64_t *ret, wo_err *err) { do { \ if (--vm->budget <= 0) { \ vm->budget = vm->budget0; \ + /* arc stage 3: a busy shard still serves its inbox once per \ + * slice — bounds a DB request's wait on a computing primary \ + * to one reduction budget */ \ + if (INBOX_READY[vm->shard_id % WO_ENG_MAX_SHARDS]) \ + (void)wo_vm_adopt(vm); \ if (vm->qhead) { \ vm->cur->frames[vm->cur->depth - 1].pc = pc; \ fib_enqueue(vm, vm->cur); \ @@ -1385,6 +1601,10 @@ dispatch: * stopped (1), or a fatal error (-1). */ int wo_vm_serve(wo_vm *vm) { tls_vm = vm; + /* arc stage 3 obligation: a worker NEVER holds the engine or the WAL — + * its DB statements marshal to shard 0 (wo_db_rpc). Replay finished on + * the primary before wo_engine_start spawned this thread. */ + assert(!vm->rt.db && !vm->rt.wal); if (!vm->qhead) return 2; vm->cur = fib_dequeue(vm); vm->budget = vm->budget0; diff --git a/runtime/src/vm.h b/runtime/src/vm.h index bf87791..2c8c96b 100644 --- a/runtime/src/vm.h +++ b/runtime/src/vm.h @@ -79,8 +79,16 @@ typedef struct wo_fiber { * borrows — the RUNTIME owns it and drops it after the call returns. */ struct wo_actor *actor; uint64_t cur_msg; + /* arc stage 3: the in-flight DB request while parked on the DB actor's + * reply (a wo_db_req*, opaque here; vm.c owns the protocol) */ + void *dbreq; } wo_fiber; +/* arc stage 3: park_fd sentinel — PARKED with NO plane wait; the wake is + * an inbox envelope (the DB actor's reply). Excluded from the deadline + * scans, which key on park_fd == -1 exactly. */ +#define WO_PARK_INBOX (-2) + /* An actor: moved-in state, its receive method, a FIFO mailbox, and at * most one delivery fiber at a time (one message at a time — the actor * guarantee). Actors live until program end (v1: no actor death). */ @@ -126,6 +134,12 @@ typedef struct wo_vm { int io_fd; /* ring fd or epoll fd */ void *io_sq, *io_cq, *io_sqes; /* uring mmaps (NULL under epoll) */ size_t io_sq_len, io_cq_len, io_sqes_len; + /* THIS ring's io_uring_params (opaque bytes; park.c owns the type). + * Arc stage 3 fix: a single file-static params was rewritten by every + * shard's lazy init while other shards read ring offsets out of it — + * submits landed at garbage offsets and parked fibers lost their + * wakes. Per-vm storage ends the race by construction. */ + unsigned char io_params[256]; } wo_vm; /* arc: the spawn/send builtins' runtime halves (vm.c owns the scheduler). */ @@ -146,14 +160,23 @@ typedef struct wo_engine { extern wo_engine wo_eng; /* the process's one engine (vm.c) */ -/* inbox envelope kinds (arc T6) */ +/* inbox envelope kinds (arc T6; 3/4 = stage 3's transparent DB RPC) */ typedef struct wo_envelope { struct wo_envelope *next; - int kind; /* 0 = SEND (actor, payload), 1 = SPAWN-ADOPT (actor), 2 = FREE (payload = wo_hdr*) */ + int kind; /* 0 = SEND (actor, payload), 1 = SPAWN-ADOPT (actor), + * 2 = FREE (payload = wo_hdr*), + * 3 = DB_REQ (payload = wo_db_req*, to shard 0), + * 4 = DB_RESP (payload = wo_db_req*, back to the requester) */ struct wo_actor *actor; uint64_t payload; } wo_envelope; +/* arc stage 3: the requester half of the transparent DB RPC (vm.c). Called + * by the builtin dispatcher on a worker shard whose rt.db is NULL: first + * entry marshals + parks (WO_SYS_PARKED), the re-execution after the reply + * consumes it. */ +int wo_db_rpc(wo_vm *vm, uint64_t *R, uint32_t ins, const char **msg); + /* the shard whose thread we are on (thread-local; obj.c stamps and gc.c * routes with it). NULL only before main's vm exists. */ wo_vm *wo_tls_vm(void); diff --git a/scripts/db-actor-accept.sh b/scripts/db-actor-accept.sh new file mode 100755 index 0000000..ae8ed9c --- /dev/null +++ b/scripts/db-actor-accept.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# scripts/db-actor-accept.sh — arc stage 3's gate: actors on worker shards +# read and write the database through the transparent DB actor. Multi-shard +# output is asserted as a SET (scheduling orders the writer lines); the +# main line and the single-shard run are exact. The WO_DATA pair proves a +# worker's write rides the owner's WAL and replays. +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WOC="$ROOT/compiler/_build/default/bin/woc" +WOVM="$ROOT/runtime/wovm" +DIR="$ROOT/docs/examples/db-actor" + +pass=0; fail=0 +ok() { echo "ok $1"; pass=$((pass + 1)); } +bad() { echo "FAIL $1 -- $2"; fail=$((fail + 1)); } + +if [[ ! -x "$WOC" || ! -x "$WOVM" ]]; then + echo "db-actor-accept: build woc and wovm first" >&2; exit 1 +fi + +if "$WOC" build "$DIR" -o "$DIR/target/db-actor" --runtime "$WOVM" >/dev/null 2>&1; then + ok "builds" +else + bad "build" "woc failed"; echo "db-actor-accept: 1 checks, 1 failures"; exit 1 +fi + +check_set() { # name [env pairs...] + local name="$1"; shift + local out + out="$(env "$@" timeout 30 "$DIR/target/db-actor" 2>&1)" + if printf '%s' "$out" | grep -q "writer 1 sees sum" \ + && printf '%s' "$out" | grep -q "writer 2 sees sum" \ + && printf '%s' "$out" | grep -q "^main sees 2 rows, sum 3$" ; then + ok "$name: both writers wrote and read cross-shard; main exact" + else + bad "$name" "$(printf '%s' "$out" | tr '\n' '|')" + fi +} + +# multi-shard (default = all cores): the RPC path under test, three rounds +check_set "multi #1" +check_set "multi #2" +check_set "multi #3" +# forced backends: the reply park is plane-independent +check_set "multi uring" WO_IO=uring +check_set "multi epoll" WO_IO=epoll + +# single-shard: byte-exact — the local path is untouched +sout="$(WO_SHARDS=1 timeout 30 "$DIR/target/db-actor" 2>&1)" +want=$'writer 1 sees sum 1\nwriter 2 sees sum 3\nmain sees 2 rows, sum 3' +if [[ "$sout" == "$want" ]]; then + ok "single-shard byte-exact" +else + bad "single-shard" "$(printf '%s' "$sout" | tr '\n' '|')" +fi + +# durability through the RPC: a worker's insert commits on the owner's WAL +# before the ack; a restart replays it (2 rows, then 2+2) +DATA="$(mktemp -d)" +r1="$(WO_DATA="$DATA" timeout 30 "$DIR/target/db-actor" 2>&1 | tail -1)" +r2="$(WO_DATA="$DATA" timeout 30 "$DIR/target/db-actor" 2>&1 | tail -1)" +rm -rf "$DATA" +if [[ "$r1" == "main sees 2 rows, sum 3" && "$r2" == "main sees 4 rows, sum 6" ]]; then + ok "WAL: worker writes ack-after-durable, replay doubles the store" +else + bad "WAL replay" "r1=$r1 r2=$r2" +fi + +echo +echo "db-actor-accept: $((pass + fail)) checks, $fail failures" +[[ $fail -eq 0 ]] || exit 1