diff --git a/database/src/wal.c b/database/src/wal.c index 19e797f..5fb28b5 100644 --- a/database/src/wal.c +++ b/database/src/wal.c @@ -315,6 +315,7 @@ int wo_wal_open(wo_wal *w, const char *path, uint64_t prealloc) { if ((size_t)snprintf(tmp, sizeof tmp, "%s%s", path, WO_WAL_TMP_SUFFIX) < sizeof tmp) (void)unlink(tmp); } + w->prealloc = prealloc; if (prealloc) { /* best-effort: a filesystem without fallocate still works */ (void)posix_fallocate(w->fd, 0, (off_t)prealloc); @@ -502,7 +503,14 @@ int wo_wal_compact(wo_wal *w, wo_db *db) { (void)unlink(tmp); /* a stale one would otherwise be appended to */ wo_wal nw; - if (wo_wal_open(&nw, tmp, 0) != 0) return -1; + /* THE REPLACEMENT MUST BE PREALLOCATED LIKE THE ORIGINAL. The WAL is + * preallocated so appends never extend the file, which is precisely what + * makes fdatasync sufficient as the ack barrier — no file-size metadata + * has to reach disk for an acked record to be readable. Opening the + * replacement with prealloc 0 silently removed that property, and the + * crash battery caught it: records acked shortly before a kill went + * missing, with the log otherwise intact and self-consistent. */ + if (wo_wal_open(&nw, tmp, w->prealloc) != 0) return -1; /* one INSERT per live row, in the existing grammar, through the existing * append path — so replay needs no second decoder and ids are preserved diff --git a/database/src/wal.h b/database/src/wal.h index e219a14..442e81c 100644 --- a/database/src/wal.h +++ b/database/src/wal.h @@ -68,6 +68,11 @@ typedef struct wo_wal { * log against THIS rather than an estimate of the live set — estimating * would mean estimating Text, and the compactor knows the true number. */ uint64_t compacted_bytes; + /* databasev2 3: the preallocation this log was opened with. Compaction + * MUST give the replacement the same one: the WAL is preallocated so that + * appends never extend the file, which is what lets fdatasync alone be the + * ack barrier. A replacement without it silently weakens durability. */ + uint64_t prealloc; } wo_wal; /* Open (create if missing) and preallocate [prealloc] bytes (best-effort; diff --git a/runtime/test/test_wal.c b/runtime/test/test_wal.c index 1be2996..ce39ab4 100644 --- a/runtime/test/test_wal.c +++ b/runtime/test/test_wal.c @@ -358,6 +358,156 @@ static void test_compact_refuses_with_staged_records(void) { wo_rt_destroy(&rt); } +/* databasev2 3 Task 4: kill -9 DURING compaction. + * + * The existing battery is insert-only, so its "records >= acks" oracle is + * exactly what compaction is allowed to break: collapsing history is the point. + * The invariant that survives is the ACKED LIVE SET — every id acked as + * inserted and not later acked as deleted must be present with its acked value, + * and every id acked as deleted must be absent. Both the pre-compaction and the + * post-compaction log satisfy that identically, which is precisely the + * "never a mixture" property the design is shaped around. + * + * The child deletes as it goes so HISTORY accumulates while the live set stays + * small — without that, compaction would have nothing to collapse and the test + * would prove nothing. */ +#define CK_DELETED UINT64_MAX + +static void ck_ack(int fd, uint64_t id, uint64_t val) { + uint64_t rec[2] = {id, val}; + if (write(fd, rec, sizeof rec) != (ssize_t)sizeof rec) _exit(0); /* parent gone */ +} + +static void compact_battery_child(const char *path, int ack_fd) { + wo_rt rt; + wo_db db; + wo_wal w; + if (wo_rt_init(&rt, 1 << 20, CLASSES, 1) != 0) _exit(9); + if (wo_db_init(&db, CLASSES, 1, 0, 1) != 0) _exit(9); + if (wo_wal_open(&w, path, 1 << 20) != 0) _exit(9); + const char *msg = ""; + uint64_t live[512]; + size_t nlive = 0; + for (uint64_t i = 0;; i++) { + uint64_t val = i * 7 + 3; + wo_str *s = wo_str_new(&rt, "r", 1); + uint64_t vals[2] = {val, (uint64_t)(uintptr_t)s}; + uint64_t id = wo_row_insert(&db, 0, vals, &msg, NULL); + wo_str_free(&rt, s); + if (!id) _exit(9); + if (wo_wal_append_insert(&w, &db, 0, id) != 0) _exit(9); + if (wo_wal_commit(&w) != 0) _exit(9); /* durable BEFORE the ack */ + ck_ack(ack_fd, id, val); + if (nlive < 512) live[nlive++] = id; + + /* drop the oldest so history grows while the live set does not */ + if (nlive > 16) { + uint64_t victim = live[0]; + memmove(live, live + 1, (nlive - 1) * sizeof live[0]); + nlive--; + /* INTENT FIRST, deliberately. An ack after the commit would race: + * a kill between them leaves the row legitimately gone on disk + * while the last ack still says "inserted", and the parent would + * demand a row the engine was right to remove. Announcing intent + * makes the row's fate simply UNKNOWN to the parent, which is the + * honest thing to assert about it. */ + ck_ack(ack_fd, victim, CK_DELETED); + if (wo_row_remove(&db, 0, victim) != 0) _exit(9); + if (wo_wal_append_remove(&w, 0, victim) != 0) _exit(9); + if (wo_wal_commit(&w) != 0) _exit(9); + } + /* compact often, so a kill has a real chance of landing inside one */ + if (i % 24 == 23) (void)wo_wal_compact(&w, &db); + } +} + +static void test_compact_crash_battery(void) { + int rounds = 40; /* it is a RACE: one green run proves very little */ + for (int round = 0; round < rounds; round++) { + char path[128], tmp[160]; + snprintf(path, sizeof path, "%s/ckcrash-%d.wal", g_dir, round); + snprintf(tmp, sizeof tmp, "%s%s", path, WO_WAL_TMP_SUFFIX); + int pipefd[2]; + T_EQ(pipe(pipefd), 0); + pid_t pid = fork(); + T_CHECK(pid >= 0); + if (pid == 0) { + close(pipefd[0]); + compact_battery_child(path, pipefd[1]); + _exit(0); + } + close(pipefd[1]); + /* vary the instant so kills land before, inside and after rewrites */ + struct timespec ts = {0, (7 + round * 3) * 1000000L}; + while (nanosleep(&ts, &ts) != 0) {} + kill(pid, SIGKILL); + int status; + waitpid(pid, &status, 0); + + /* replay the acks into the expected live set, in order */ + uint64_t ids[65536], vals[65536]; + size_t n = 0; + for (;;) { + uint64_t rec[2]; + ssize_t r = read(pipefd[0], rec, sizeof rec); + if (r != (ssize_t)sizeof rec) break; + if (n < 65536) { ids[n] = rec[0]; vals[n] = rec[1]; n++; } + } + close(pipefd[0]); + T_CHECK(n > 0); /* the child got at least one commit out */ + + wo_rt rt; + T_EQ(wo_rt_init(&rt, 1 << 22, CLASSES, 1), 0); + wo_db db; + T_EQ(wo_db_init(&db, CLASSES, 1, 0, 1), 0); + int64_t ck_recs = wo_wal_check(path, NULL); + int64_t ck_applied = wo_wal_replay(path, &db); + T_CHECK(ck_applied >= 0); /* never reported as corruption */ + + /* A stale temp may well EXIST after a kill inside compaction — that is + * the expected debris. The guarantee is that the next OPEN removes it + * and never reads it, so that is what gets asserted here; checking + * merely for its absence after a replay would be asserting something + * the design never promised (wo_wal_replay does not open the WAL). */ + { + wo_wal probe; + T_EQ(wo_wal_open(&probe, path, 1 << 20), 0); + T_CHECK(access(tmp, F_OK) != 0); + wo_wal_close(&probe); + } + + const char *msg = ""; + int bad = 0, checked = 0; + for (size_t k = 0; k < n && !bad; k++) { + if (vals[k] == CK_DELETED) continue; /* intent: fate is unknown */ + /* an id ever announced for deletion may legally be gone */ + int doomed = 0; + for (size_t j = 0; j < n; j++) + if (ids[j] == ids[k] && vals[j] == CK_DELETED) { doomed = 1; break; } + if (doomed) continue; + uint64_t out[2]; + int rc = wo_row_read(&db, &rt, 0, ids[k], out, &msg); + if (0) { + } else if (rc != 0 || out[0] != vals[k]) { + bad = 1; /* an acked insert is missing or wrong */ + fprintf(stderr, "CKDIAG round=%d id=%llu rc=%d got=%llu want=%llu ack#%zu/%zu " + "log_records=%lld replay_applied=%lld\n", + round, (unsigned long long)ids[k], rc, + rc == 0 ? (unsigned long long)out[0] : 0ull, + (unsigned long long)vals[k], k, n, + (long long)ck_recs, (long long)ck_applied); + } else { + wo_str_free(&rt, (wo_str *)(uintptr_t)out[1]); + } + checked++; + } + T_CHECK(checked > 0); + T_CHECK(!bad); + wo_db_destroy(&db); + wo_rt_destroy(&rt); + } +} + static void test_torn_tail(void) { char path[128]; snprintf(path, sizeof path, "%s/torn.wal", g_dir); @@ -576,6 +726,7 @@ int main(void) { test_torn_tail(); test_float_bytes_replay(); test_crash_battery(); + test_compact_crash_battery(); /* leave the dir for a failed run's forensics only */ if (!t_fail) { char cmd[128];