(* lexer.ml — tokenizer for `.wo` source (milestone-1 OOP subset). Ported from crates/rt/src/lexer.rs; behavior kept identical wherever it defines what the language literally is: - newline tokens are emitted, never filtered, and consecutive newlines collapse to a single Newline token — exactly rt's `out.last() == Some(Newline) => don't push another` rule; - `--` starts a line comment that runs to (not including) the next newline; - single- or double-quoted strings support the same backslash escapes as rt: n, t, backslash, or either quote character, each backslash-prefixed; anything else verbatim. `\r` and `\0` were added 2026-08-14 (a program writing HTTP needs CRLF, and rt never had to); - integer literals are plain runs of ASCII digits; - identifiers may contain internal dashes, exactly like rt's read_ident_chars (crates/rt/src/lexer.rs) — so `foo-bar` lexes as one identifier, not `foo`, Dash, `bar`. This is a faithful port of an rt quirk, not a milestone-1 design choice: Task 5's expression parser must therefore require whitespace around a binary minus that immediately follows an identifier (`a - b`, not `a-b`) to avoid the ambiguity, exactly as rt's schema layer already does. Flagged here for whoever picks up Task 5. Divergence from rt, deliberate: rt's `tokenize` returns `anyhow::Result` and bails (stops the whole file) on the first bad byte or malformed punctuation shape (a bare `!` not followed by `=`, a `$` with no name, ...). This front end's diagnostics contract is multi-error (compiler/src/diag.ml — every later stage accumulates into one Collector rather than stopping at the first problem), so every one of those cases becomes: report one WO-E001 unknown- character diagnostic at the offending position, skip exactly that one byte, and keep lexing. One bad byte must never stop the whole file (Task 3 brief's Unknown character decision). *) let unknown_char_code = Diag.lexing_prefix ^ "01" (* WO-E001 *) (* rt's equivalent site (crates/rt/src/lexer.rs:106) bails outright: a backslash as the very last byte of the file, with no character left to escape, loses information silently otherwise (the intended escaped character is simply gone, not skip-and-continue like the unknown- character case). Ported as its own code rather than reusing WO-E001 because the shape is different (a truncated escape, not a byte the lexer doesn't recognize at all) and diag.ml's convention is one code per distinct lexing situation. Deliberately NOT applied to a plain unterminated string: a string that runs off the end of the file with no dangling backslash (say, a quote-open let-binding with nothing after it and no closing quote) — rt does not bail there either, its scanning loop just stops when peek returns None and emits whatever was collected as the Str token. Reporting nothing in that case is intentional rt parity, not an oversight; pinned by the plain-unterminated-string-reports-nothing assertion in compiler/test/runner.ml. *) let unterminated_escape_code = Diag.lexing_prefix ^ "02" (* WO-E002 *) let directive_code = Diag.lexing_prefix ^ "03" (* WO-E003: #if/#else/#end misuse *) (* iteration 37, the raw text literal (backtick-delimited, verbatim content, no backslash escapes). Two codes, because the two shapes are genuinely different situations: WO-E004 — a raw literal that runs off the end of the file. Unlike a plain "..." string (silent, rt parity, see above), this one IS reported: multi-line is the raw literal's normal case, so a missing closing backtick would otherwise swallow every remaining line of the file with nothing to show for it. Reported at the OPENING backtick, which is the only position that helps -- EOF tells the reader nothing about which literal never closed. WO-E005 — a raw newline inside a "..." or '...' string. This used to be accepted silently: the string scanner's catch-all appended the newline like any other byte, so a forgotten closing quote ate the rest of the file with no diagnostic at all. Nothing in the repo ever relied on it (zero of the .wo sources span a line inside quotes) and the backtick literal is now the spelling for multi-line text, so the accident becomes an error. The scan stops at the newline WITHOUT consuming it, so the Newline token is still emitted and the statement terminates -- one diagnostic, and the next line parses normally instead of being swallowed. The rt-parity silence for a plain unterminated string with no newline is untouched. *) let unterminated_raw_code = Diag.lexing_prefix ^ "04" (* WO-E004 *) let newline_in_string_code = Diag.lexing_prefix ^ "05" (* WO-E005 *) (* haxe-parity Task 8: build flags. `woc -D name` fills this before any tokenize call; undefined flags are false. A module-level ref because the compiler is a single-shot process — tests that care set it explicitly and reset to empty. *) module StringSet = Set.Make (String) let defines : StringSet.t ref = ref StringSet.empty type lexer = { src : string; len : int; mutable pos : int; mutable line : int; mutable col : int; } let make src = { src; len = String.length src; pos = 0; line = 1; col = 1 } let peek lx = if lx.pos < lx.len then Some lx.src.[lx.pos] else None let peek_at lx n = let i = lx.pos + n in if i < lx.len then Some lx.src.[i] else None let advance lx = match peek lx with | None -> None | Some c -> lx.pos <- lx.pos + 1; if c = '\n' then begin lx.line <- lx.line + 1; lx.col <- 1 end else lx.col <- lx.col + 1; Some c let is_digit c = c >= '0' && c <= '9' let is_alpha c = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') let is_ident_start c = is_alpha c || c = '_' let is_ident_cont c = is_alpha c || is_digit c || c = '_' || c = '-' (* Consumes a run of identifier characters starting at the lexer's current position (caller has already confirmed is_ident_start on the character at that position) and returns the collected text. Mirrors rt's read_ident_chars, dash-continuation included (see module doc). *) let read_ident_chars lx = let start = lx.pos in let continue_ = ref true in while !continue_ do match peek lx with | Some c when is_ident_cont c -> ignore (advance lx) | _ -> continue_ := false done; String.sub lx.src start (lx.pos - start) (* The milestone-1 keyword set, exact per the Task 3 brief: type class interface fn let mut take return if else while for in true false, plus uppercase-only INSERT/SELECT. Deliberately absent: self, me, subscribe, receive, and lowercase insert/select — those fall through to the `_ -> None` case below and lex as plain Ident, matching rt and the CLAUDE.md gotcha this task exists to preserve. `use`/`pub` (haxe-parity Task 1, modules) added on top of that set. *) let keyword_kind = function | "type" -> Some Token.KwType | "class" -> Some Token.KwClass | "interface" -> Some Token.KwInterface | "fn" -> Some Token.KwFn | "let" -> Some Token.KwLet | "mut" -> Some Token.KwMut | "take" -> Some Token.KwTake | "return" -> Some Token.KwReturn | "if" -> Some Token.KwIf | "else" -> Some Token.KwElse | "while" -> Some Token.KwWhile | "for" -> Some Token.KwFor | "in" -> Some Token.KwIn | "true" -> Some Token.KwTrue | "false" -> Some Token.KwFalse | "use" -> Some Token.KwUse | "spawn" -> Some Token.KwSpawn | "using" -> Some Token.KwUsing | "pub" -> Some Token.KwPub | "break" -> Some Token.KwBreak | "continue" -> Some Token.KwContinue | "do" -> Some Token.KwDo | "const" -> Some Token.KwConst | "and" -> Some Token.KwAnd | "or" -> Some Token.KwOr | "not" -> Some Token.KwNot | "inline" -> Some Token.KwInline | "switch" -> Some Token.KwSwitch | "case" -> Some Token.KwCase | "default" -> Some Token.KwDefault | "typedef" -> Some Token.KwTypedef | "try" -> Some Token.KwTry | "catch" -> Some Token.KwCatch | "nil" -> Some Token.KwNil | "as" -> Some Token.KwAs | "INSERT" -> Some Token.KwInsert | "SELECT" -> Some Token.KwSelect | _ -> None (* haxe-parity Task 2: scans the raw source of one `${...}` interpolation body, starting right after the `{` (caller already consumed `$` and `{`). Returns that raw, unlexed text -- the parser re-tokenizes it as a full expression (parser.ml's own desugar-to-Concat step; this is a mechanical extraction only, no semantics). Tracks brace depth so a nested `{}` (a constructor literal inside an interpolation, `${Point{x:1}.x}`) doesn't end the scan early, and skips a nested string literal verbatim (honoring its own backslash escapes) so a quote or brace *inside* that nested string can't confuse either count. Runs off the end of the file the same silent way an unterminated outer string does -- the caller's own EOF handling picks up right after. *) let read_interp_expr lx = let buf = Buffer.create 16 in let depth = ref 0 in let continue_ = ref true in while !continue_ do match peek lx with | None -> continue_ := false | Some '}' when !depth = 0 -> ignore (advance lx); continue_ := false | Some ('{' as c) -> incr depth; Buffer.add_char buf c; ignore (advance lx) | Some ('}' as c) -> decr depth; Buffer.add_char buf c; ignore (advance lx) | Some (('"' | '\'') as q) -> Buffer.add_char buf q; ignore (advance lx); let scanning = ref true in while !scanning do match peek lx with | None -> scanning := false | Some c when c = q -> Buffer.add_char buf c; ignore (advance lx); scanning := false | Some '\\' -> ( Buffer.add_char buf '\\'; ignore (advance lx); match peek lx with | Some c -> Buffer.add_char buf c; ignore (advance lx) | None -> scanning := false) | Some c -> Buffer.add_char buf c; ignore (advance lx) done | Some c -> Buffer.add_char buf c; ignore (advance lx) done; Buffer.contents buf (* haxe-parity Task 8: the #if filter, run over the in-order token list at the end of tokenize. A `#if ` section is kept when the flag is defined AND every enclosing section is kept; `#else` flips the section; `#end` closes it. Nesting allowed; flag NAMES only (no expression language — the spec's limit); undefined flags are false. Misuse is WO-E003: a #if without a flag name, a second #else, a stray #else/#end, or a #if left open at end of file. Eof always survives so the parser still terminates after a reported error. *) let preprocess (collector : Diag.Collector.t) ~(file : string) (toks : Token.t list) : Token.t list = let err line col msg = Diag.Collector.add collector (Diag.error ~code:directive_code ~file ~line ~col ~message:msg ()) in (* frame: (emitting, seen_else, opening line, opening col) *) let stack : (bool * bool * int * int) list ref = ref [] in let emitting () = List.for_all (fun (e, _, _, _) -> e) !stack in let out = ref [] in let rec go = function | [] -> ( match !stack with | (_, _, l, c) :: _ -> err l c "#if left open — missing #end" | [] -> ()) | { Token.kind = Token.HashIf; line; col } :: rest -> ( match rest with | { Token.kind = Token.Ident flag; _ } :: rest2 -> stack := (StringSet.mem flag !defines, false, line, col) :: !stack; go rest2 | _ -> err line col "#if needs a flag name (`#if portable`)"; stack := (false, false, line, col) :: !stack; go rest) | { Token.kind = Token.HashElse; line; col } :: rest -> (match !stack with | (e, false, l, c) :: tl -> stack := (not e, true, l, c) :: tl | (_, true, _, _) :: _ -> err line col "second #else in one #if section" | [] -> err line col "#else outside any #if"); go rest | { Token.kind = Token.HashEnd; line; col } :: rest -> (match !stack with | _ :: tl -> stack := tl | [] -> err line col "#end outside any #if"); go rest | ({ Token.kind = Token.Eof; _ } as t) :: rest -> out := t :: !out; go rest | t :: rest -> if emitting () then out := t :: !out; go rest in go toks; List.rev !out (* ---- the raw literal's margin rule (iteration 37) ------------------- A render() body is written at its method's indentation, but that indentation is an artifact of the SOURCE, not of the markup -- nobody wants six leading spaces on every line of the served HTML. So the common margin is removed here, at lex time: the constant pool holds the dedented text, no downstream stage ever sees the source indentation, and the whole rule costs nothing at run time. The rule (Java's text blocks, which solved exactly this): - one newline immediately after the opening backtick is dropped, so the first markup line can start on its own line; - the smallest leading run of spaces/tabs across all non-blank lines is removed from every line (characters counted, tabs NOT expanded -- mixing them is the author's problem, and expanding would need a tab width the language does not have); - a whitespace-only final line (the usual case: the closing backtick sits on its own line) loses its whitespace but keeps its newline. A literal with no newline in it is left completely alone -- there is no margin to speak of, and silently eating the leading spaces of ` hi` would be a surprise, not a service. Holes do not disturb any of this. A line's indentation is by definition the run of whitespace at its start, and the only thing that can split a line across segments is a hole, which ends that run -- so an indentation run always lives whole inside one SText. The measuring pass replaces each hole with a single non-whitespace sentinel byte so that a line that is ` {{ x }}` correctly counts as indent 4 and as NON-blank. *) let is_indent_char c = c = ' ' || c = '\t' let segments_shadow (segs : Token.str_part list) : string = let b = Buffer.create 64 in List.iter (function | Token.SText s -> Buffer.add_string b s | Token.SExpr _ | Token.SEsc _ -> Buffer.add_char b '\001') segs; Buffer.contents b let min_indent (shadow : string) : int = let m = ref max_int in List.iter (fun line -> let n = String.length line in let i = ref 0 in while !i < n && is_indent_char line.[!i] do incr i done; (* a blank (or whitespace-only) line never sets the margin *) if !i < n && !i < !m then m := !i) (String.split_on_char '\n' shadow); if !m = max_int then 0 else !m let strip_margin (k : int) (segs : Token.str_part list) : Token.str_part list = if k = 0 then segs else begin let at_line_start = ref true in let one seg = match seg with | Token.SExpr _ | Token.SEsc _ -> at_line_start := false; seg | Token.SText s -> let n = String.length s in let b = Buffer.create n in let i = ref 0 in while !i < n do if !at_line_start then begin let dropped = ref 0 in while !dropped < k && !i < n && is_indent_char s.[!i] do incr dropped; incr i done; at_line_start := false end else begin let c = s.[!i] in Buffer.add_char b c; if c = '\n' then at_line_start := true; incr i end done; Token.SText (Buffer.contents b) in (* fold_left, not List.map: `one` carries state across segments and List.map's application order is unspecified. *) List.rev (List.fold_left (fun acc seg -> one seg :: acc) [] segs) end let drop_trailing_margin (segs : Token.str_part list) : Token.str_part list = match List.rev segs with | Token.SText s :: rest_rev -> let n = String.length s in let i = ref n in while !i > 0 && is_indent_char s.[!i - 1] do decr i done; (* only a run that directly follows a newline is a closing line *) if !i < n && !i > 0 && s.[!i - 1] = '\n' then List.rev (Token.SText (String.sub s 0 !i) :: rest_rev) else segs | _ -> segs let dedent (segs : Token.str_part list) : Token.str_part list = let shadow = segments_shadow segs in if not (String.contains shadow '\n') then segs else begin let segs = match segs with | Token.SText s :: rest when String.length s > 0 && s.[0] = '\n' -> Token.SText (String.sub s 1 (String.length s - 1)) :: rest | _ -> segs in let k = min_indent (segments_shadow segs) in drop_trailing_margin (strip_margin k segs) end let tokenize (collector : Diag.Collector.t) ~(file : string) (src : string) : Token.t list = let lx = make src in let out = ref [] in let emit kind line col = out := { Token.kind; line; col } :: !out in let last_is_newline () = match !out with | { Token.kind = Token.Newline; _ } :: _ -> true | _ -> false in (* A line ending in `..` continues on the next line — the ONE newline suppression in the language, so multi-line markup/text builds read as one expression (the shop template's ask; story 37 rides it). *) let last_is_dotdot () = match !out with | { Token.kind = Token.DotDot; _ } :: _ -> true | _ -> false in let report_unknown line col c = Diag.Collector.add collector (Diag.error ~code:unknown_char_code ~file ~line ~col ~message:(Printf.sprintf "unknown character '%c'" c) ()) in let report_unterminated_escape line col = Diag.Collector.add collector (Diag.error ~code:unterminated_escape_code ~file ~line ~col ~message:"unterminated string escape" ()) in let report_unterminated_raw line col = Diag.Collector.add collector (Diag.error ~code:unterminated_raw_code ~file ~line ~col ~message:"unterminated raw text literal" ()) in let report_newline_in_string line col = Diag.Collector.add collector (Diag.error ~code:newline_in_string_code ~file ~line ~col ~message: "newline in string literal (use a `...` raw text literal for \ multi-line text)" ()) in let running = ref true in while !running do match peek lx with | None -> running := false | Some c -> ( let line = lx.line and col = lx.col in if c = '-' && peek_at lx 1 = Some '-' then begin (* line comment: -- ... EOL (EOL itself is left for the next iteration to turn into its own Newline token). *) let scanning = ref true in while !scanning do match peek lx with | Some '\n' | None -> scanning := false | Some _ -> ignore (advance lx) done end else if c = '\n' then begin ignore (advance lx); if not (last_is_newline ()) && not (last_is_dotdot ()) then emit Token.Newline line col end else if c = ' ' || c = '\t' || c = '\r' then ignore (advance lx) else if c = '"' || c = '\'' then begin let quote = c in ignore (advance lx); let buf = Buffer.create 16 in (* haxe-parity Task 2: segments accumulate here only when at least one `${...}` is actually found (flush_text below); a plain string never touches `parts` at all, so it emits the exact same `Token.Str` it always did -- see the `match !parts` dispatch after the loop. *) let parts = ref [] in let flush_text () = parts := Token.SText (Buffer.contents buf) :: !parts; Buffer.clear buf in let scanning = ref true in while !scanning do match peek lx with | None -> (* Plain unterminated string (ran off the end of the file with no closing quote and no dangling backslash): rt does not bail here either, it just stops and emits whatever was collected. No diagnostic, deliberately — see unterminated_escape_code's doc comment above. *) scanning := false | Some c when c = quote -> ignore (advance lx); scanning := false | Some '$' when peek_at lx 1 = Some '{' -> (* Unescaped `${` -- `\$` never reaches here, it is fully consumed by the backslash branch below, one dispatch earlier, so this is always a genuine interpolation start, never an escaped `$` that happens to be followed by `{`. *) flush_text (); ignore (advance lx); (* '$' *) ignore (advance lx); (* '{' *) parts := Token.SExpr (read_interp_expr lx) :: !parts | Some '\\' -> ( (* Captured before advancing: this is the backslash's own position, so a dangling-escape diagnostic points at the `\` itself rather than wherever the scan happens to stop. *) let esc_line = lx.line and esc_col = lx.col in ignore (advance lx); match advance lx with | Some 'n' -> Buffer.add_char buf '\n' | Some 't' -> Buffer.add_char buf '\t' (* `\r` — added 2026-08-14: without it a program cannot write CRLF at all, and the driving workload's HTTP server needs it (its `index_of(buf, "\r\n\r\n")` was searching for a literal backslash-r, so it never found a header terminator). *) | Some 'r' -> Buffer.add_char buf '\r' | Some '0' -> Buffer.add_char buf '\000' | Some '\\' -> Buffer.add_char buf '\\' | Some '"' -> Buffer.add_char buf '"' | Some '\'' -> Buffer.add_char buf '\'' (* `\$` -- not one of the escapes above, so it falls into this catch-all exactly like any other unrecognized backslash sequence, producing a literal `$` that the `$` dispatch above never sees (it already advanced past it). *) | Some other -> Buffer.add_char buf other | None -> report_unterminated_escape esc_line esc_col; scanning := false) | Some '\n' -> (* WO-E005. Deliberately NOT consumed: the outer loop turns it into the Newline token that terminates the statement, so recovery is one bad line rather than the rest of the file. *) report_newline_in_string lx.line lx.col; scanning := false | Some other -> ignore (advance lx); Buffer.add_char buf other done; flush_text (); (match List.rev !parts with | [] -> emit (Token.Str "") line col | [ Token.SText s ] -> emit (Token.Str s) line col | segs -> emit (Token.InterpStr segs) line col) end else if c = '`' then begin (* iteration 37: the raw text literal. Everything up to the closing backtick is content -- newlines included, and with NO escape processing at all, which is the whole point: markup carries quotes and backslashes verbatim. A literal backtick (or a literal `{{`) is written by concatenating an ordinary "..." string with `..`; that door is one greppable operator, which beats inventing an escape character for the one form whose selling point is not having any. Two hole forms, and ONLY here -- inside "..." a `{{` is still two literal braces, so existing CSS/JS text is untouched: ${ expr } raw, exactly like a "..." string's hole {{ expr }} HTML-escaped (the parser wraps it in esc()) *) ignore (advance lx); let buf = Buffer.create 64 in let parts = ref [] in let flush_text () = parts := Token.SText (Buffer.contents buf) :: !parts; Buffer.clear buf in let scanning = ref true in while !scanning do match peek lx with | None -> report_unterminated_raw line col; scanning := false | Some '`' -> ignore (advance lx); scanning := false | Some '$' when peek_at lx 1 = Some '{' -> flush_text (); ignore (advance lx); ignore (advance lx); parts := Token.SExpr (read_interp_expr lx) :: !parts | Some '{' when peek_at lx 1 = Some '{' -> flush_text (); ignore (advance lx); ignore (advance lx); (* read_interp_expr stops at the first `}` at depth 0 and consumes it -- the second one closes this hole. Reusing it means brace depth and nested string literals are already handled, so `{{ Point{x:1}.x }}` scans correctly. *) let raw = read_interp_expr lx in (match peek lx with | Some '}' -> ignore (advance lx) | _ -> report_unterminated_raw line col); parts := Token.SEsc raw :: !parts | Some other -> ignore (advance lx); Buffer.add_char buf other done; flush_text (); (match dedent (List.rev !parts) with | [] -> emit (Token.Str "") line col | [ Token.SText s ] -> emit (Token.Str s) line col | segs -> emit (Token.InterpStr segs) line col) end else if is_digit c then begin (* iteration 19: one scanner for both numeric worlds. The integer run is scanned into a buffer as well as accumulated, because a fraction or an exponent turns the whole thing into a Float and OCaml's float_of_string wants the original text. A digit run stays an Int unless it is followed by: - '.' AND a digit -> `1.5`. The digit requirement is what keeps `0..10` a range (Dot Dot after Int 0) and leaves any future `1.method()` reachable; without it `0..10` would lex as Float 0. followed by `.10`. - 'e'/'E' with an optional sign AND a digit -> `2e10`. Checked before consuming, so `2eggs` is still Int 2 then Ident. *) (* `c` is PEEKED, not consumed — the loop below reads it. Adding it to the buffer here as well would count the first digit twice. *) (* iteration 36: hex (`0x`) and binary (`0b`) Int literals, and `_` digit separators in every integer form. The prefix commits only when the character AFTER it is a real digit of that base, so `0x` followed by anything else stays Int 0 + Ident — a parse error at its own position, no new lexer diagnostic. Accumulation uses OCaml's native int (63-bit): a full-width 64-bit literal like 0xFFFFFFFFFFFFFFFF is out of reach — all-ones is spelled -1. The float path below is untouched: neither prefix can reach it (a fraction/exponent needs the decimal branch), and `_` is consumed only between digits of an integer run. *) let is_hex_digit ch = is_digit ch || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') in let hex_val ch = if is_digit ch then Char.code ch - Char.code '0' else if ch >= 'a' && ch <= 'f' then Char.code ch - Char.code 'a' + 10 else Char.code ch - Char.code 'A' + 10 in let scan_prefixed base is_base_digit digit_val = (* consumes the peeked '0' and the prefix char, then the run *) ignore (advance lx); ignore (advance lx); let n = ref 0 in let scanning = ref true in while !scanning do match peek lx with | Some d when is_base_digit d -> n := (!n * base) + digit_val d; ignore (advance lx) | Some '_' when (match peek_at lx 1 with | Some d -> is_base_digit d | None -> false) -> ignore (advance lx) | _ -> scanning := false done; emit (Token.Int !n) line col in match (c, peek_at lx 1, peek_at lx 2) with | '0', Some ('x' | 'X'), Some d when is_hex_digit d -> scan_prefixed 16 is_hex_digit hex_val | '0', Some ('b' | 'B'), Some ('0' | '1') -> scan_prefixed 2 (fun ch -> ch = '0' || ch = '1') (fun ch -> Char.code ch - Char.code '0') | _ -> let buf = Buffer.create 16 in let n = ref 0 in let scanning = ref true in while !scanning do match peek lx with | Some d when is_digit d -> n := (!n * 10) + (Char.code d - Char.code '0'); Buffer.add_char buf d; ignore (advance lx) | Some '_' when (match peek_at lx 1 with | Some d -> is_digit d | None -> false) -> (* separator only BETWEEN digits: `1_` stops the run and the `_` lexes as its own ident, a parse error at its position *) ignore (advance lx) | _ -> scanning := false done; let is_float = ref false in (match (peek lx, peek_at lx 1) with | Some '.', Some d when is_digit d -> is_float := true; Buffer.add_char buf '.'; ignore (advance lx); let frac = ref true in while !frac do match peek lx with | Some d when is_digit d -> Buffer.add_char buf d; ignore (advance lx) | _ -> frac := false done | _ -> ()); (* exponent, on an integer run (`2e10`) or after a fraction (`1.5e-3`) *) (match (peek lx, peek_at lx 1, peek_at lx 2) with | Some ('e' | 'E'), Some d, _ when is_digit d -> is_float := true | Some ('e' | 'E'), Some ('+' | '-'), Some d when is_digit d -> is_float := true | _ -> ()); if !is_float then begin (match peek lx with | Some (('e' | 'E') as e) -> Buffer.add_char buf e; ignore (advance lx); (match peek lx with | Some (('+' | '-') as s) -> Buffer.add_char buf s; ignore (advance lx) | _ -> ()); let ex = ref true in while !ex do match peek lx with | Some d when is_digit d -> Buffer.add_char buf d; ignore (advance lx) | _ -> ex := false done | _ -> ()); (* float_of_string cannot fail here: the buffer is a well-formed decimal by construction. Overflow is not an error either — it yields infinity, which is a legitimate Float per IEEE quiet semantics (`1e400` is `inf`, not a compile error). *) emit (Token.Float (float_of_string (Buffer.contents buf))) line col end else emit (Token.Int !n) line col end else if c = '#' then begin (* haxe-parity Task 8: `#if` / `#else` / `#end` build-flag directives. Names only — anything else after '#' is WO-E003. *) ignore (advance lx); let name = match peek lx with | Some d when is_ident_start d -> read_ident_chars lx | _ -> "" in match name with | "if" -> emit Token.HashIf line col | "else" -> emit Token.HashElse line col | "end" -> emit Token.HashEnd line col | other -> Diag.Collector.add collector (Diag.error ~code:directive_code ~file ~line ~col ~message: (Printf.sprintf "unknown directive `#%s` — the build-flag directives are #if , #else, #end" other) ()) end else if is_ident_start c then begin let name = read_ident_chars lx in let kind = match keyword_kind name with Some k -> k | None -> Token.Ident name in emit kind line col end else match c with | '{' -> ignore (advance lx); emit Token.LBrace line col | '}' -> ignore (advance lx); emit Token.RBrace line col | '(' -> ignore (advance lx); emit Token.LParen line col | ')' -> ignore (advance lx); emit Token.RParen line col | '[' -> ignore (advance lx); emit Token.LBracket line col | ']' -> ignore (advance lx); emit Token.RBracket line col | ',' -> ignore (advance lx); emit Token.Comma line col | ';' -> ignore (advance lx); emit Token.Semicolon line col | ':' -> ignore (advance lx); emit Token.Colon line col | '.' -> ( ignore (advance lx); match peek lx with | Some '.' -> ignore (advance lx); emit Token.DotDot line col | _ -> emit Token.Dot line col) | '?' -> ignore (advance lx); emit Token.Question line col | '@' -> ignore (advance lx); emit Token.At line col | '|' -> ignore (advance lx); emit Token.Pipe line col | '-' -> ( ignore (advance lx); match peek lx with | Some '>' -> ignore (advance lx); emit Token.Arrow line col | Some '=' -> ignore (advance lx); emit Token.MinusEq line col | _ -> emit Token.Dash line col) | '+' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.PlusEq line col | _ -> emit Token.Plus line col) | '*' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.StarEq line col | _ -> emit Token.Star line col) | '/' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.SlashEq line col | _ -> emit Token.Slash line col) | '%' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.PercentEq line col | _ -> emit Token.Percent line col) (* iteration 36: the bitwise operators. `&` and `^` were unknown characters before this; `|` (Pipe, above) is reused in expression position by the parser. No `&=`/`^=`/`<<=`/`>>=` — bitwise compound assigns are out of scope per the story. *) | '&' -> ignore (advance lx); emit Token.Amp line col | '^' -> ignore (advance lx); emit Token.Caret line col | '=' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.EqEq line col | Some '>' -> ignore (advance lx); emit Token.FatArrow line col | _ -> emit Token.Eq line col) | '!' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.NotEq line col | _ -> report_unknown line col '!') | '<' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.LtEq line col | Some '<' -> ignore (advance lx); emit Token.Shl line col | _ -> emit Token.Lt line col) | '>' -> ( ignore (advance lx); match peek lx with | Some '=' -> ignore (advance lx); emit Token.GtEq line col | Some '>' -> ignore (advance lx); emit Token.Shr line col | _ -> emit Token.Gt line col) | other -> ignore (advance lx); report_unknown line col other) done; emit Token.Eof lx.line lx.col; preprocess collector ~file (List.rev !out)