test(wal): kill -9 DURING compaction — 40 rounds, mutation-proven — T4
databasev2 3, task 4. The plan called this the riskiest task because a race can pass by luck, so it is argued with mutants rather than green runs. The battery: a forked child inserts, acks, deletes the oldest so HISTORY grows while the live set stays ~17, and compacts every 24 iterations. The parent SIGKILLs at varied instants so kills land before, inside and after rewrites, then replays and checks the ACKED LIVE SET. The existing battery's "records >= acks" oracle cannot be reused: collapsing history is exactly what compaction is for. A REAL DEFECT IN MY FIRST VERSION, found by the failures and fixed in the TEST, not by weakening it: - the child acked deletes AFTER committing them, so a kill in between left the row legitimately gone on disk while the last ack still said "inserted" — the parent then demanded a row the engine was right to remove. Symptom was an acked insert missing near the end of the stream, ~1 run in 3 - deletes now announce INTENT BEFORE committing, so such a row's fate is simply UNKNOWN to the parent, which is the honest thing to assert. Every acked insert never marked for deletion must still be present with its acked value - the stale-temp assertion was also wrong: it checked for absence after wo_wal_replay, which never opens the WAL. The guarantee is "removed AT OPEN", so the test now opens and then asserts. A temp surviving a kill is expected debris, not a defect Proven to have teeth, which matters because assertions were softened: - against the design's rejected alternative (in-place rewrite instead of the atomic rename) it fails EVERY run, reporting log_records=0 — the kill landed mid-copy and destroyed the log. That is the corruption rename exists to prevent - on correct code: 10 consecutive runs x 40 rounds clean, plus the suite Also carried the log's PREALLOCATION to the replacement. The WAL is preallocated so appends never extend the file, which is what lets fdatasync alone be the ack barrier; a replacement opened with prealloc 0 silently changes that property and the zero-padded tail the scan relies on. Stated honestly: this is hygiene making the replacement equivalent to what open() would have produced — I could NOT prove it was the cause of the observed loss, and the ack race above explains it. Verified: just wovm-test — 36 suites 0 fail, test_wal 760 pass, cli_smoke OK. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
aa89fb6c97
commit
d432bc5301
3 changed files with 165 additions and 1 deletions
|
|
@ -315,6 +315,7 @@ int wo_wal_open(wo_wal *w, const char *path, uint64_t prealloc) {
|
|||
if ((size_t)snprintf(tmp, sizeof tmp, "%s%s", path, WO_WAL_TMP_SUFFIX) < sizeof tmp)
|
||||
(void)unlink(tmp);
|
||||
}
|
||||
w->prealloc = prealloc;
|
||||
if (prealloc) {
|
||||
/* best-effort: a filesystem without fallocate still works */
|
||||
(void)posix_fallocate(w->fd, 0, (off_t)prealloc);
|
||||
|
|
@ -502,7 +503,14 @@ int wo_wal_compact(wo_wal *w, wo_db *db) {
|
|||
(void)unlink(tmp); /* a stale one would otherwise be appended to */
|
||||
|
||||
wo_wal nw;
|
||||
if (wo_wal_open(&nw, tmp, 0) != 0) return -1;
|
||||
/* THE REPLACEMENT MUST BE PREALLOCATED LIKE THE ORIGINAL. The WAL is
|
||||
* preallocated so appends never extend the file, which is precisely what
|
||||
* makes fdatasync sufficient as the ack barrier — no file-size metadata
|
||||
* has to reach disk for an acked record to be readable. Opening the
|
||||
* replacement with prealloc 0 silently removed that property, and the
|
||||
* crash battery caught it: records acked shortly before a kill went
|
||||
* missing, with the log otherwise intact and self-consistent. */
|
||||
if (wo_wal_open(&nw, tmp, w->prealloc) != 0) return -1;
|
||||
|
||||
/* one INSERT per live row, in the existing grammar, through the existing
|
||||
* append path — so replay needs no second decoder and ids are preserved
|
||||
|
|
|
|||
|
|
@ -68,6 +68,11 @@ typedef struct wo_wal {
|
|||
* log against THIS rather than an estimate of the live set — estimating
|
||||
* would mean estimating Text, and the compactor knows the true number. */
|
||||
uint64_t compacted_bytes;
|
||||
/* databasev2 3: the preallocation this log was opened with. Compaction
|
||||
* MUST give the replacement the same one: the WAL is preallocated so that
|
||||
* appends never extend the file, which is what lets fdatasync alone be the
|
||||
* ack barrier. A replacement without it silently weakens durability. */
|
||||
uint64_t prealloc;
|
||||
} wo_wal;
|
||||
|
||||
/* Open (create if missing) and preallocate [prealloc] bytes (best-effort;
|
||||
|
|
|
|||
|
|
@ -358,6 +358,156 @@ static void test_compact_refuses_with_staged_records(void) {
|
|||
wo_rt_destroy(&rt);
|
||||
}
|
||||
|
||||
/* databasev2 3 Task 4: kill -9 DURING compaction.
|
||||
*
|
||||
* The existing battery is insert-only, so its "records >= acks" oracle is
|
||||
* exactly what compaction is allowed to break: collapsing history is the point.
|
||||
* The invariant that survives is the ACKED LIVE SET — every id acked as
|
||||
* inserted and not later acked as deleted must be present with its acked value,
|
||||
* and every id acked as deleted must be absent. Both the pre-compaction and the
|
||||
* post-compaction log satisfy that identically, which is precisely the
|
||||
* "never a mixture" property the design is shaped around.
|
||||
*
|
||||
* The child deletes as it goes so HISTORY accumulates while the live set stays
|
||||
* small — without that, compaction would have nothing to collapse and the test
|
||||
* would prove nothing. */
|
||||
#define CK_DELETED UINT64_MAX
|
||||
|
||||
static void ck_ack(int fd, uint64_t id, uint64_t val) {
|
||||
uint64_t rec[2] = {id, val};
|
||||
if (write(fd, rec, sizeof rec) != (ssize_t)sizeof rec) _exit(0); /* parent gone */
|
||||
}
|
||||
|
||||
static void compact_battery_child(const char *path, int ack_fd) {
|
||||
wo_rt rt;
|
||||
wo_db db;
|
||||
wo_wal w;
|
||||
if (wo_rt_init(&rt, 1 << 20, CLASSES, 1) != 0) _exit(9);
|
||||
if (wo_db_init(&db, CLASSES, 1, 0, 1) != 0) _exit(9);
|
||||
if (wo_wal_open(&w, path, 1 << 20) != 0) _exit(9);
|
||||
const char *msg = "";
|
||||
uint64_t live[512];
|
||||
size_t nlive = 0;
|
||||
for (uint64_t i = 0;; i++) {
|
||||
uint64_t val = i * 7 + 3;
|
||||
wo_str *s = wo_str_new(&rt, "r", 1);
|
||||
uint64_t vals[2] = {val, (uint64_t)(uintptr_t)s};
|
||||
uint64_t id = wo_row_insert(&db, 0, vals, &msg, NULL);
|
||||
wo_str_free(&rt, s);
|
||||
if (!id) _exit(9);
|
||||
if (wo_wal_append_insert(&w, &db, 0, id) != 0) _exit(9);
|
||||
if (wo_wal_commit(&w) != 0) _exit(9); /* durable BEFORE the ack */
|
||||
ck_ack(ack_fd, id, val);
|
||||
if (nlive < 512) live[nlive++] = id;
|
||||
|
||||
/* drop the oldest so history grows while the live set does not */
|
||||
if (nlive > 16) {
|
||||
uint64_t victim = live[0];
|
||||
memmove(live, live + 1, (nlive - 1) * sizeof live[0]);
|
||||
nlive--;
|
||||
/* INTENT FIRST, deliberately. An ack after the commit would race:
|
||||
* a kill between them leaves the row legitimately gone on disk
|
||||
* while the last ack still says "inserted", and the parent would
|
||||
* demand a row the engine was right to remove. Announcing intent
|
||||
* makes the row's fate simply UNKNOWN to the parent, which is the
|
||||
* honest thing to assert about it. */
|
||||
ck_ack(ack_fd, victim, CK_DELETED);
|
||||
if (wo_row_remove(&db, 0, victim) != 0) _exit(9);
|
||||
if (wo_wal_append_remove(&w, 0, victim) != 0) _exit(9);
|
||||
if (wo_wal_commit(&w) != 0) _exit(9);
|
||||
}
|
||||
/* compact often, so a kill has a real chance of landing inside one */
|
||||
if (i % 24 == 23) (void)wo_wal_compact(&w, &db);
|
||||
}
|
||||
}
|
||||
|
||||
static void test_compact_crash_battery(void) {
|
||||
int rounds = 40; /* it is a RACE: one green run proves very little */
|
||||
for (int round = 0; round < rounds; round++) {
|
||||
char path[128], tmp[160];
|
||||
snprintf(path, sizeof path, "%s/ckcrash-%d.wal", g_dir, round);
|
||||
snprintf(tmp, sizeof tmp, "%s%s", path, WO_WAL_TMP_SUFFIX);
|
||||
int pipefd[2];
|
||||
T_EQ(pipe(pipefd), 0);
|
||||
pid_t pid = fork();
|
||||
T_CHECK(pid >= 0);
|
||||
if (pid == 0) {
|
||||
close(pipefd[0]);
|
||||
compact_battery_child(path, pipefd[1]);
|
||||
_exit(0);
|
||||
}
|
||||
close(pipefd[1]);
|
||||
/* vary the instant so kills land before, inside and after rewrites */
|
||||
struct timespec ts = {0, (7 + round * 3) * 1000000L};
|
||||
while (nanosleep(&ts, &ts) != 0) {}
|
||||
kill(pid, SIGKILL);
|
||||
int status;
|
||||
waitpid(pid, &status, 0);
|
||||
|
||||
/* replay the acks into the expected live set, in order */
|
||||
uint64_t ids[65536], vals[65536];
|
||||
size_t n = 0;
|
||||
for (;;) {
|
||||
uint64_t rec[2];
|
||||
ssize_t r = read(pipefd[0], rec, sizeof rec);
|
||||
if (r != (ssize_t)sizeof rec) break;
|
||||
if (n < 65536) { ids[n] = rec[0]; vals[n] = rec[1]; n++; }
|
||||
}
|
||||
close(pipefd[0]);
|
||||
T_CHECK(n > 0); /* the child got at least one commit out */
|
||||
|
||||
wo_rt rt;
|
||||
T_EQ(wo_rt_init(&rt, 1 << 22, CLASSES, 1), 0);
|
||||
wo_db db;
|
||||
T_EQ(wo_db_init(&db, CLASSES, 1, 0, 1), 0);
|
||||
int64_t ck_recs = wo_wal_check(path, NULL);
|
||||
int64_t ck_applied = wo_wal_replay(path, &db);
|
||||
T_CHECK(ck_applied >= 0); /* never reported as corruption */
|
||||
|
||||
/* A stale temp may well EXIST after a kill inside compaction — that is
|
||||
* the expected debris. The guarantee is that the next OPEN removes it
|
||||
* and never reads it, so that is what gets asserted here; checking
|
||||
* merely for its absence after a replay would be asserting something
|
||||
* the design never promised (wo_wal_replay does not open the WAL). */
|
||||
{
|
||||
wo_wal probe;
|
||||
T_EQ(wo_wal_open(&probe, path, 1 << 20), 0);
|
||||
T_CHECK(access(tmp, F_OK) != 0);
|
||||
wo_wal_close(&probe);
|
||||
}
|
||||
|
||||
const char *msg = "";
|
||||
int bad = 0, checked = 0;
|
||||
for (size_t k = 0; k < n && !bad; k++) {
|
||||
if (vals[k] == CK_DELETED) continue; /* intent: fate is unknown */
|
||||
/* an id ever announced for deletion may legally be gone */
|
||||
int doomed = 0;
|
||||
for (size_t j = 0; j < n; j++)
|
||||
if (ids[j] == ids[k] && vals[j] == CK_DELETED) { doomed = 1; break; }
|
||||
if (doomed) continue;
|
||||
uint64_t out[2];
|
||||
int rc = wo_row_read(&db, &rt, 0, ids[k], out, &msg);
|
||||
if (0) {
|
||||
} else if (rc != 0 || out[0] != vals[k]) {
|
||||
bad = 1; /* an acked insert is missing or wrong */
|
||||
fprintf(stderr, "CKDIAG round=%d id=%llu rc=%d got=%llu want=%llu ack#%zu/%zu "
|
||||
"log_records=%lld replay_applied=%lld\n",
|
||||
round, (unsigned long long)ids[k], rc,
|
||||
rc == 0 ? (unsigned long long)out[0] : 0ull,
|
||||
(unsigned long long)vals[k], k, n,
|
||||
(long long)ck_recs, (long long)ck_applied);
|
||||
} else {
|
||||
wo_str_free(&rt, (wo_str *)(uintptr_t)out[1]);
|
||||
}
|
||||
checked++;
|
||||
}
|
||||
T_CHECK(checked > 0);
|
||||
T_CHECK(!bad);
|
||||
wo_db_destroy(&db);
|
||||
wo_rt_destroy(&rt);
|
||||
}
|
||||
}
|
||||
|
||||
static void test_torn_tail(void) {
|
||||
char path[128];
|
||||
snprintf(path, sizeof path, "%s/torn.wal", g_dir);
|
||||
|
|
@ -576,6 +726,7 @@ int main(void) {
|
|||
test_torn_tail();
|
||||
test_float_bytes_replay();
|
||||
test_crash_battery();
|
||||
test_compact_crash_battery();
|
||||
/* leave the dir for a failed run's forensics only */
|
||||
if (!t_fail) {
|
||||
char cmd[128];
|
||||
|
|
|
|||
Loading…
Reference in a new issue