(* lexer.ml — tokenizer for `.wo` source (milestone-1 OOP subset). Ported from crates/rt/src/lexer.rs; behavior kept identical wherever it defines what the language literally is: - newline tokens are emitted, never filtered, and consecutive newlines collapse to a single Newline token — exactly rt's `out.last() == Some(Newline) => don't push another` rule; - `--` starts a line comment that runs to (not including) the next newline; - single- or double-quoted strings support the same backslash escapes as rt: n, t, backslash, or either quote character, each backslash-prefixed; anything else verbatim. `\r` and `\0` were added 2026-08-14 (a program writing HTTP needs CRLF, and rt never had to); - integer literals are plain runs of ASCII digits; - identifiers may contain internal dashes, exactly like rt's read_ident_chars (crates/rt/src/lexer.rs) — so `foo-bar` lexes as one identifier, not `foo`, Dash, `bar`. This is a faithful port of an rt quirk, not a milestone-1 design choice: Task 5's expression parser must therefore require whitespace around a binary minus that immediately follows an identifier (`a - b`, not `a-b`) to avoid the ambiguity, exactly as rt's schema layer already does. Flagged here for whoever picks up Task 5. Divergence from rt, deliberate: rt's `tokenize` returns `anyhow::Result` and bails (stops the whole file) on the first bad byte or malformed punctuation shape (a bare `!` not followed by `=`, a `$` with no name, ...). This front end's diagnostics contract is multi-error (compiler/src/diag.ml — every later stage accumulates into one Collector rather than stopping at the first problem), so every one of those cases becomes: report one WO-E001 unknown- character diagnostic at the offending position, skip exactly that one byte, and keep lexing. One bad byte must never stop the whole file (Task 3 brief's Unknown character decision). *) let unknown_char_code = Diag.lexing_prefix ^ "01" (* WO-E001 *) (* rt's equivalent site (crates/rt/src/lexer.rs:106) bails outright: a backslash as the very last byte of the file, with no character left to escape, loses information silently otherwise (the intended escaped character is simply gone, not skip-and-continue like the unknown- character case). Ported as its own code rather than reusing WO-E001 because the shape is different (a truncated escape, not a byte the lexer doesn't recognize at all) and diag.ml's convention is one code per distinct lexing situation. Deliberately NOT applied to a plain unterminated string: a string that runs off the end of the file with no dangling backslash (say, a quote-open let-binding with nothing after it and no closing quote) — rt does not bail there either, its scanning loop just stops when peek returns None and emits whatever was collected as the Str token. Reporting nothing in that case is intentional rt parity, not an oversight; pinned by the plain-unterminated-string-reports-nothing assertion in compiler/test/runner.ml. *) let unterminated_escape_code = Diag.lexing_prefix ^ "02" (* WO-E002 *) let directive_code = Diag.lexing_prefix ^ "03" (* WO-E003: #if/#else/#end misuse *) (* haxe-parity Task 8: build flags. `woc -D name` fills this before any tokenize call; undefined flags are false. A module-level ref because the compiler is a single-shot process — tests that care set it explicitly and reset to empty. *) module StringSet = Set.Make (String) let defines : StringSet.t ref = ref StringSet.empty type lexer = { src : string; len : int; mutable pos : int; mutable line : int; mutable col : int; } let make src = { src; len = String.length src; pos = 0; line = 1; col = 1 } let peek lx = if lx.pos < lx.len then Some lx.src.[lx.pos] else None let peek_at lx n = let i = lx.pos + n in if i < lx.len then Some lx.src.[i] else None let advance lx = match peek lx with | None -> None | Some c -> lx.pos <- lx.pos + 1; if c = '\n' then begin lx.line <- lx.line + 1; lx.col <- 1 end else lx.col <- lx.col + 1; Some c let is_digit c = c >= '0' && c <= '9' let is_alpha c = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') let is_ident_start c = is_alpha c || c = '_' let is_ident_cont c = is_alpha c || is_digit c || c = '_' || c = '-' (* Consumes a run of identifier characters starting at the lexer's current position (caller has already confirmed is_ident_start on the character at that position) and returns the collected text. Mirrors rt's read_ident_chars, dash-continuation included (see module doc). *) let read_ident_chars lx = let start = lx.pos in let continue_ = ref true in while !continue_ do match peek lx with | Some c when is_ident_cont c -> ignore (advance lx) | _ -> continue_ := false done; String.sub lx.src start (lx.pos - start) (* The milestone-1 keyword set, exact per the Task 3 brief: type class interface fn let mut take return if else while for in true false, plus uppercase-only INSERT/SELECT. Deliberately absent: self, me, subscribe, receive, and lowercase insert/select — those fall through to the `_ -> None` case below and lex as plain Ident, matching rt and the CLAUDE.md gotcha this task exists to preserve. `use`/`pub` (haxe-parity Task 1, modules) added on top of that set. *) let keyword_kind = function | "type" -> Some Token.KwType | "class" -> Some Token.KwClass | "interface" -> Some Token.KwInterface | "fn" -> Some Token.KwFn | "let" -> Some Token.KwLet | "mut" -> Some Token.KwMut | "take" -> Some Token.KwTake | "return" -> Some Token.KwReturn | "if" -> Some Token.KwIf | "else" -> Some Token.KwElse | "while" -> Some Token.KwWhile | "for" -> Some Token.KwFor | "in" -> Some Token.KwIn | "true" -> Some Token.KwTrue | "false" -> Some Token.KwFalse | "use" -> Some Token.KwUse | "spawn" -> Some Token.KwSpawn | "using" -> Some Token.KwUsing | "pub" -> Some Token.KwPub | "break" -> Some Token.KwBreak | "continue" -> Some Token.KwContinue | "do" -> Some Token.KwDo | "const" -> Some Token.KwConst | "and" -> Some Token.KwAnd | "or" -> Some Token.KwOr | "not" -> Some Token.KwNot | "inline" -> Some Token.KwInline | "switch" -> Some Token.KwSwitch | "case" -> Some Token.KwCase | "default" -> Some Token.KwDefault | "typedef" -> Some Token.KwTypedef | "try" -> Some Token.KwTry | "catch" -> Some Token.KwCatch | "nil" -> Some Token.KwNil | "as" -> Some Token.KwAs | "INSERT" -> Some Token.KwInsert | "SELECT" -> Some Token.KwSelect | _ -> None (* haxe-parity Task 2: scans the raw source of one `${...}` interpolation body, starting right after the `{` (caller already consumed `$` and `{`). Returns that raw, unlexed text -- the parser re-tokenizes it as a full expression (parser.ml's own desugar-to-Concat step; this is a mechanical extraction only, no semantics). Tracks brace depth so a nested `{}` (a constructor literal inside an interpolation, `${Point{x:1}.x}`) doesn't end the scan early, and skips a nested string literal verbatim (honoring its own backslash escapes) so a quote or brace *inside* that nested string can't confuse either count. Runs off the end of the file the same silent way an unterminated outer string does -- the caller's own EOF handling picks up right after. *) let read_interp_expr lx = let buf = Buffer.create 16 in let depth = ref 0 in let continue_ = ref true in while !continue_ do match peek lx with | None -> continue_ := false | Some '}' when !depth = 0 -> ignore (advance lx); continue_ := false | Some ('{' as c) -> incr depth; Buffer.add_char buf c; ignore (advance lx) | Some ('}' as c) -> decr depth; Buffer.add_char buf c; ignore (advance lx) | Some (('"' | '\'') as q) -> Buffer.add_char buf q; ignore (advance lx); let scanning = ref true in while !scanning do match peek lx with | None -> scanning := false | Some c when c = q -> Buffer.add_char buf c; ignore (advance lx); scanning := false | Some '\\' -> ( Buffer.add_char buf '\\'; ignore (advance lx); match peek lx with | Some c -> Buffer.add_char buf c; ignore (advance lx) | None -> scanning := false) | Some c -> Buffer.add_char buf c; ignore (advance lx) done | Some c -> Buffer.add_char buf c; ignore (advance lx) done; Buffer.contents buf (* haxe-parity Task 8: the #if filter, run over the in-order token list at the end of tokenize. A `#if ` section is kept when the flag is defined AND every enclosing section is kept; `#else` flips the section; `#end` closes it. Nesting allowed; flag NAMES only (no expression language — the spec's limit); undefined flags are false. Misuse is WO-E003: a #if without a flag name, a second #else, a stray #else/#end, or a #if left open at end of file. Eof always survives so the parser still terminates after a reported error. *) let preprocess (collector : Diag.Collector.t) ~(file : string) (toks : Token.t list) : Token.t list = let err line col msg = Diag.Collector.add collector (Diag.error ~code:directive_code ~file ~line ~col ~message:msg ()) in (* frame: (emitting, seen_else, opening line, opening col) *) let stack : (bool * bool * int * int) list ref = ref [] in let emitting () = List.for_all (fun (e, _, _, _) -> e) !stack in let out = ref [] in let rec go = function | [] -> ( match !stack with | (_, _, l, c) :: _ -> err l c "#if left open — missing #end" | [] -> ()) | { Token.kind = Token.HashIf; line; col } :: rest -> ( match rest with | { Token.kind = Token.Ident flag; _ } :: rest2 -> stack := (StringSet.mem flag !defines, false, line, col) :: !stack; go rest2 | _ -> err line col "#if needs a flag name (`#if portable`)"; stack := (false, false, line, col) :: !stack; go rest) | { Token.kind = Token.HashElse; line; col } :: rest -> (match !stack with | (e, false, l, c) :: tl -> stack := (not e, true, l, c) :: tl | (_, true, _, _) :: _ -> err line col "second #else in one #if section" | [] -> err line col "#else outside any #if"); go rest | { Token.kind = Token.HashEnd; line; col } :: rest -> (match !stack with | _ :: tl -> stack := tl | [] -> err line col "#end outside any #if"); go rest | ({ Token.kind = Token.Eof; _ } as t) :: rest -> out := t :: !out; go rest | t :: rest -> if emitting () then out := t :: !out; go rest in go toks; List.rev !out let tokenize (collector : Diag.Collector.t) ~(file : string) (src : string) : Token.t list = let lx = make src in let out = ref [] in let emit kind line col = out := { Token.kind; line; col } :: !out in let last_is_newline () = match !out with | { Token.kind = Token.Newline; _ } :: _ -> true | _ -> false in let report_unknown line col c = Diag.Collector.add collector (Diag.error ~code:unknown_char_code ~file ~line ~col ~message:(Printf.sprintf "unknown character '%c'" c) ()) in let report_unterminated_escape line col = Diag.Collector.add collector (Diag.error ~code:unterminated_escape_code ~file ~line ~col ~message:"unterminated string escape" ()) in let running = ref true in while !running do match peek lx with | None -> running := false | Some c -> ( let line = lx.line and col = lx.col in if c = '-' && peek_at lx 1 = Some '-' then begin (* line comment: -- ... EOL (EOL itself is left for the next iteration to turn into its own Newline token). *) let scanning = ref true in while !scanning do match peek lx with | Some '\n' | None -> scanning := false | Some _ -> ignore (advance lx) done end else if c = '\n' then begin ignore (advance lx); if not (last_is_newline ()) then emit Token.Newline line col end else if c = ' ' || c = '\t' || c = '\r' then ignore (advance lx) else if c = '"' || c = '\'' then begin let quote = c in ignore (advance lx); let buf = Buffer.create 16 in (* haxe-parity Task 2: segments accumulate here only when at least one `${...}` is actually found (flush_text below); a plain string never touches `parts` at all, so it emits the exact same `Token.Str` it always did -- see the `match !parts` dispatch after the loop. *) let parts = ref [] in let flush_text () = parts := Token.SText (Buffer.contents buf) :: !parts; Buffer.clear buf in let scanning = ref true in while !scanning do match peek lx with | None -> (* Plain unterminated string (ran off the end of the file with no closing quote and no dangling backslash): rt does not bail here either, it just stops and emits whatever was collected. No diagnostic, deliberately — see unterminated_escape_code's doc comment above. *) scanning := false | Some c when c = quote -> ignore (advance lx); scanning := false | Some '$' when peek_at lx 1 = Some '{' -> (* Unescaped `${` -- `\$` never reaches here, it is fully consumed by the backslash branch below, one dispatch earlier, so this is always a genuine interpolation start, never an escaped `$` that happens to be followed by `{`. *) flush_text (); ignore (advance lx); (* '$' *) ignore (advance lx); (* '{' *) parts := Token.SExpr (read_interp_expr lx) :: !parts | Some '\\' -> ( (* Captured before advancing: this is the backslash's own position, so a dangling-escape diagnostic points at the `\` itself rather than wherever the scan happens to stop. *) let esc_line = lx.line and esc_col = lx.col in ignore (advance lx); match advance lx with | Some 'n' -> Buffer.add_char buf '\n' | Some 't' -> Buffer.add_char buf '\t' (* `\r` — added 2026-08-14: without it a program cannot write CRLF at all, and the driving workload's HTTP server needs it (its `index_of(buf, "\r\n\r\n")` was searching for a literal backslash-r, so it never found a header terminator). *) | Some 'r' -> Buffer.add_char buf '\r' | Some '0' -> Buffer.add_char buf '\000' | Some '\\' -> Buffer.add_char buf '\\' | Some '"' -> Buffer.add_char buf '"' | Some '\'' -> Buffer.add_char buf '\'' (* `\$` -- not one of the escapes above, so it falls into this catch-all exactly like any other unrecognized backslash sequence, producing a literal `$` that the `$` dispatch above never sees (it already advanced past it). *) | Some other -> Buffer.add_char buf other | None -> report_unterminated_escape esc_line esc_col; scanning := false) | Some other -> ignore (advance lx); Buffer.add_char buf other done; flush_text (); (match List.rev !parts with | [] -> emit (Token.Str "") line col | [ Token.SText s ] -> emit (Token.Str s) line col | segs -> emit (Token.InterpStr segs) line col) end else if is_digit c then begin (* iteration 19: one scanner for both numeric worlds. The integer run is scanned into a buffer as well as accumulated, because a fraction or an exponent turns the whole thing into a Float and OCaml's float_of_string wants the original text. A digit run stays an Int unless it is followed by: - '.' AND a digit -> `1.5`. The digit requirement is what keeps `0..10` a range (Dot Dot after Int 0) and leaves any future `1.method()` reachable; without it `0..10` would lex as Float 0. followed by `.10`. - 'e'/'E' with an optional sign AND a digit -> `2e10`. Checked before consuming, so `2eggs` is still Int 2 then Ident. *) (* `c` is PEEKED, not consumed — the loop below reads it. Adding it to the buffer here as well would count the first digit twice. *) (* iteration 36: hex (`0x`) and binary (`0b`) Int literals, and `_` digit separators in every integer form. The prefix commits only when the character AFTER it is a real digit of that base, so `0x` followed by anything else stays Int 0 + Ident — a parse error at its own position, no new lexer diagnostic. Accumulation uses OCaml's native int (63-bit): a full-width 64-bit literal like 0xFFFFFFFFFFFFFFFF is out of reach — all-ones is spelled -1. The float path below is untouched: neither prefix can reach it (a fraction/exponent needs the decimal branch), and `_` is consumed only between digits of an integer run. *) let is_hex_digit ch = is_digit ch || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') in let hex_val ch = if is_digit ch then Char.code ch - Char.code '0' else if ch >= 'a' && ch <= 'f' then Char.code ch - Char.code 'a' + 10 else Char.code ch - Char.code 'A' + 10 in let scan_prefixed base is_base_digit digit_val = (* consumes the peeked '0' and the prefix char, then the run *) ignore (advance lx); ignore (advance lx); let n = ref 0 in let scanning = ref true in while !scanning do match peek lx with | Some d when is_base_digit d -> n := (!n * base) + digit_val d; ignore (advance lx) | Some '_' when (match peek_at lx 1 with | Some d -> is_base_digit d | None -> false) -> ignore (advance lx) | _ -> scanning := false done; emit (Token.Int !n) line col in match (c, peek_at lx 1, peek_at lx 2) with | '0', Some ('x' | 'X'), Some d when is_hex_digit d -> scan_prefixed 16 is_hex_digit hex_val | '0', Some ('b' | 'B'), Some ('0' | '1') -> scan_prefixed 2 (fun ch -> ch = '0' || ch = '1') (fun ch -> Char.code ch - Char.code '0') | _ -> let buf = Buffer.create 16 in let n = ref 0 in let scanning = ref true in while !scanning do match peek lx with | Some d when is_digit d -> n := (!n * 10) + (Char.code d - Char.code '0'); Buffer.add_char buf d; ignore (advance lx) | Some '_' when (match peek_at lx 1 with | Some d -> is_digit d | None -> false) -> (* separator only BETWEEN digits: `1_` stops the run and the `_` lexes as its own ident, a parse error at its position *) ignore (advance lx) | _ -> scanning := false done; let is_float = ref false in (match (peek lx, peek_at lx 1) with | Some '.', Some d when is_digit d -> is_float := true; Buffer.add_char buf '.'; ignore (advance lx); let frac = ref true in while !frac do match peek lx with | Some d when is_digit d -> Buffer.add_char buf d; ignore (advance lx) | _ -> frac := false done | _ -> ()); (* exponent, on an integer run (`2e10`) or after a fraction (`1.5e-3`) *) (match (peek lx, peek_at lx 1, peek_at lx 2) with | Some ('e' | 'E'), Some d, _ when is_digit d -> is_float := true | Some ('e' | 'E'), Some ('+' | '-'), Some d when is_digit d -> is_float := true | _ -> ()); if !is_float then begin (match peek lx with | Some (('e' | 'E') as e) -> Buffer.add_char buf e; ignore (advance lx); (match peek lx with | Some (('+' | '-') as s) -> Buffer.add_char buf s; ignore (advance lx) | _ -> ()); let ex = ref true in while !ex do match peek lx with | Some d when is_digit d -> Buffer.add_char buf d; ignore (advance lx) | _ -> ex := false done | _ -> ()); (* float_of_string cannot fail here: the buffer is a well-formed decimal by construction. Overflow is not an error either — it yields infinity, which is a legitimate Float per IEEE quiet semantics (`1e400` is `inf`, not a compile error). *) emit (Token.Float (float_of_string (Buffer.contents buf))) line col end else emit (Token.Int !n) line col end else if c = '#' then begin (* haxe-parity Task 8: `#if` / `#else` / `#end` build-flag directives. Names only — anything else after '#' is WO-E003. *) ignore (advance lx); let name = match peek lx with | Some d when is_ident_start d -> read_ident_chars lx | _ -> "" in match name with | "if" -> emit Token.HashIf line col | "else" -> emit Token.HashElse line col | "end" -> emit Token.HashEnd line col | other -> Diag.Collector.add collector (Diag.error ~code:directive_code ~file ~line ~col ~message: (Printf.sprintf "unknown directive `#%s` — the build-flag directives are #if , #else, #end" other) ()) end else if is_ident_start c then begin let name = read_ident_chars lx in let kind = match keyword_kind name with Some k -> k | None -> Token.Ident name in emit kind line col end else match c with | '{' -> ignore (advance lx); emit Token.LBrace line col | '}' -> ignore (advance lx); emit Token.RBrace line col | '(' -> ignore (advance lx); emit Token.LParen line col | ')' -> ignore (advance lx); emit Token.RParen line col | '[' -> ignore (advance lx); emit Token.LBracket line col | ']' -> ignore (advance lx); emit Token.RBracket line col | ',' -> ignore (advance lx); emit Token.Comma line col | ';' -> ignore (advance lx); emit Token.Semicolon line col | ':' -> ignore (advance lx); emit Token.Colon line col | '.' -> ( ignore (advance lx); match peek lx with | Some '.' -> ignore (advance lx); emit Token.DotDot line col | _ -> emit Token.Dot line col) | '?' -> ignore (advance lx); emit Token.Question line col | '@' -> ignore (advance lx); emit Token.At line col | '|' -> ignore (advance lx); emit Token.Pipe line col | '-' -> ( ignore (advance lx); match peek lx with | Some '>' -> ignore (advance lx); emit Token.Arrow line col | Some '=' -> ignore (advance lx); emit Token.MinusEq line col | _ -> emit Token.Dash line col) | '+' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.PlusEq line col | _ -> emit Token.Plus line col) | '*' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.StarEq line col | _ -> emit Token.Star line col) | '/' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.SlashEq line col | _ -> emit Token.Slash line col) | '%' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.PercentEq line col | _ -> emit Token.Percent line col) (* iteration 36: the bitwise operators. `&` and `^` were unknown characters before this; `|` (Pipe, above) is reused in expression position by the parser. No `&=`/`^=`/`<<=`/`>>=` — bitwise compound assigns are out of scope per the story. *) | '&' -> ignore (advance lx); emit Token.Amp line col | '^' -> ignore (advance lx); emit Token.Caret line col | '=' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.EqEq line col | Some '>' -> ignore (advance lx); emit Token.FatArrow line col | _ -> emit Token.Eq line col) | '!' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.NotEq line col | _ -> report_unknown line col '!') | '<' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.LtEq line col | Some '<' -> ignore (advance lx); emit Token.Shl line col | _ -> emit Token.Lt line col) | '>' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.GtEq line col | Some '>' -> ignore (advance lx); emit Token.Shr line col | _ -> emit Token.Gt line col) | other -> ignore (advance lx); report_unknown line col other) done; emit Token.Eof lx.line lx.col; preprocess collector ~file (List.rev !out)