From 3f386a9a4f1df0260dc47357ec5daa07941589ed Mon Sep 17 00:00:00 2001
From: Leonard Hecker <leonard@hecker.io>
Date: Mon, 26 Jan 2026 23:14:28 +0100
Subject: [PATCH 1/8] Syntax Highlighting V2

---
 .cargo/debug_refcell.toml                     |    3 +
 .vscode/launch.json                           |    2 +-
 Cargo.lock                                    |   57 +
 Cargo.toml                                    |    4 +-
 assets/highlighting-tests/COMMIT_EDITMSG      |   62 +
 assets/highlighting-tests/bash.sh             |   71 +
 assets/highlighting-tests/batch.bat           |   41 +
 assets/highlighting-tests/diff.diff           |  142 ++
 assets/highlighting-tests/git-rebase-todo     |   54 +
 assets/highlighting-tests/markdown.md         |   75 ++
 assets/highlighting-tests/powershell.ps1      |   78 ++
 assets/highlighting-tests/properties.conf     |   13 +
 assets/highlighting-tests/yaml.yml            |   43 +
 crates/edit/Cargo.toml                        |    2 +
 crates/edit/benches/lib.rs                    |  104 +-
 crates/edit/build/main.rs                     |   21 +
 crates/edit/src/bin/edit/apperr.rs            |    2 +
 crates/edit/src/bin/edit/documents.rs         |   43 +-
 crates/edit/src/bin/edit/draw_menubar.rs      |   16 +-
 crates/edit/src/bin/edit/draw_statusbar.rs    |   64 +-
 crates/edit/src/bin/edit/main.rs              |    8 +
 crates/edit/src/bin/edit/settings.rs          |  107 ++
 crates/edit/src/bin/edit/state.rs             |    6 +
 crates/edit/src/buffer/line_cache.rs          |  116 --
 crates/edit/src/buffer/mod.rs                 |   91 +-
 crates/edit/src/framebuffer.rs                |   22 +-
 crates/edit/src/helpers.rs                    |   11 -
 crates/edit/src/lib.rs                        |    2 +-
 crates/edit/src/lsh/cache.rs                  |   86 ++
 crates/edit/src/lsh/definitions.rs            |    4 +
 crates/edit/src/lsh/highlighter.rs            |  149 ++
 crates/edit/src/lsh/mod.rs                    |   33 +
 crates/edit/src/sys/unix.rs                   |    7 +-
 crates/lsh-bin/Cargo.toml                     |   14 +
 crates/lsh-bin/src/main.rs                    |  198 +++
 crates/lsh/Cargo.toml                         |   12 +
 crates/lsh/definitions/diff.lsh               |   12 +
 .../lsh/definitions/git_commit_messsage.lsh   |   25 +
 crates/lsh/definitions/git_ignore.lsh         |    8 +
 crates/lsh/definitions/git_rebase_todo.lsh    |   27 +
 crates/lsh/definitions/json.lsh               |   37 +
 crates/lsh/definitions/lsh.lsh                |   48 +
 crates/lsh/definitions/markdown.lsh           |  202 +++
 crates/lsh/definitions/powershell.lsh         |   62 +
 crates/lsh/definitions/properties.lsh         |   52 +
 crates/lsh/definitions/utility.lsh            |   21 +
 crates/lsh/definitions/yaml.lsh               |   78 ++
 crates/lsh/src/compiler/backend.rs            |  745 ++++++++++
 crates/lsh/src/compiler/frontend.rs           |  729 ++++++++++
 crates/lsh/src/compiler/generator.rs          |  313 +++++
 crates/lsh/src/compiler/helpers.rs            |   29 +
 crates/lsh/src/compiler/mod.rs                | 1197 +++++++++++++++++
 crates/lsh/src/compiler/optimizer.rs          |  210 +++
 crates/lsh/src/compiler/regex.rs              |  501 +++++++
 crates/lsh/src/compiler/tokenizer.rs          |  281 ++++
 crates/lsh/src/engine.rs                      |  427 ++++++
 crates/lsh/src/lib.rs                         |   57 +
 crates/{edit => stdext}/src/glob.rs           |    0
 crates/stdext/src/helpers.rs                  |   20 +
 crates/stdext/src/lib.rs                      |    4 +-
 crates/stdext/src/varint.rs                   |  142 ++
 i18n/edit.toml                                |   39 +
 62 files changed, 6878 insertions(+), 151 deletions(-)
 create mode 100644 .cargo/debug_refcell.toml
 create mode 100644 assets/highlighting-tests/COMMIT_EDITMSG
 create mode 100644 assets/highlighting-tests/bash.sh
 create mode 100644 assets/highlighting-tests/batch.bat
 create mode 100644 assets/highlighting-tests/diff.diff
 create mode 100644 assets/highlighting-tests/git-rebase-todo
 create mode 100644 assets/highlighting-tests/markdown.md
 create mode 100644 assets/highlighting-tests/powershell.ps1
 create mode 100644 assets/highlighting-tests/properties.conf
 create mode 100644 assets/highlighting-tests/yaml.yml
 create mode 100644 crates/edit/src/bin/edit/settings.rs
 delete mode 100644 crates/edit/src/buffer/line_cache.rs
 create mode 100644 crates/edit/src/lsh/cache.rs
 create mode 100644 crates/edit/src/lsh/definitions.rs
 create mode 100644 crates/edit/src/lsh/highlighter.rs
 create mode 100644 crates/edit/src/lsh/mod.rs
 create mode 100644 crates/lsh-bin/Cargo.toml
 create mode 100644 crates/lsh-bin/src/main.rs
 create mode 100644 crates/lsh/Cargo.toml
 create mode 100644 crates/lsh/definitions/diff.lsh
 create mode 100644 crates/lsh/definitions/git_commit_messsage.lsh
 create mode 100644 crates/lsh/definitions/git_ignore.lsh
 create mode 100644 crates/lsh/definitions/git_rebase_todo.lsh
 create mode 100644 crates/lsh/definitions/json.lsh
 create mode 100644 crates/lsh/definitions/lsh.lsh
 create mode 100644 crates/lsh/definitions/markdown.lsh
 create mode 100644 crates/lsh/definitions/powershell.lsh
 create mode 100644 crates/lsh/definitions/properties.lsh
 create mode 100644 crates/lsh/definitions/utility.lsh
 create mode 100644 crates/lsh/definitions/yaml.lsh
 create mode 100644 crates/lsh/src/compiler/backend.rs
 create mode 100644 crates/lsh/src/compiler/frontend.rs
 create mode 100644 crates/lsh/src/compiler/generator.rs
 create mode 100644 crates/lsh/src/compiler/helpers.rs
 create mode 100644 crates/lsh/src/compiler/mod.rs
 create mode 100644 crates/lsh/src/compiler/optimizer.rs
 create mode 100644 crates/lsh/src/compiler/regex.rs
 create mode 100644 crates/lsh/src/compiler/tokenizer.rs
 create mode 100644 crates/lsh/src/engine.rs
 create mode 100644 crates/lsh/src/lib.rs
 rename crates/{edit => stdext}/src/glob.rs (100%)
 create mode 100644 crates/stdext/src/varint.rs

diff --git a/.cargo/debug_refcell.toml b/.cargo/debug_refcell.toml
new file mode 100644
index 000000000000..98ec4965922b
--- /dev/null
+++ b/.cargo/debug_refcell.toml
@@ -0,0 +1,3 @@
+[unstable]
+build-std = ["std"]
+build-std-features = ["debug_refcell"]
diff --git a/.vscode/launch.json b/.vscode/launch.json
index 7142960c4f9d..aaed2864490b 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -10,7 +10,7 @@
             "program": "${workspaceFolder}/target/debug/edit",
             "cwd": "${workspaceFolder}",
             "args": [
-                "${workspaceFolder}/crates/edit/src/bin/edit/main.rs"
+                "${workspaceFolder}/assets/highlighting-tests/json.json"
             ],
         },
         {
diff --git a/Cargo.lock b/Cargo.lock
index 8375271b12fe..4f39ccae9dbc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -47,6 +47,38 @@ version = "1.0.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
 
+[[package]]
+name = "argh"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34ff18325c8a36b82f992e533ece1ec9f9a9db446bd1c14d4f936bac88fcd240"
+dependencies = [
+ "argh_derive",
+ "argh_shared",
+ "rust-fuzzy-search",
+]
+
+[[package]]
+name = "argh_derive"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb7b2b83a50d329d5d8ccc620f5c7064028828538bdf5646acd60dc1f767803"
+dependencies = [
+ "argh_shared",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "argh_shared"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a464143cc82dedcdc3928737445362466b7674b5db4e2eb8e869846d6d84f4f6"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.5.0"
@@ -226,6 +258,7 @@ version = "1.2.1"
 dependencies = [
  "criterion",
  "libc",
+ "lsh",
  "stdext",
  "toml-span",
  "windows-sys",
@@ -348,6 +381,24 @@ version = "0.4.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
 
+[[package]]
+name = "lsh"
+version = "0.0.0"
+dependencies = [
+ "regex-syntax",
+ "stdext",
+]
+
+[[package]]
+name = "lsh-bin"
+version = "0.0.0"
+dependencies = [
+ "anyhow",
+ "argh",
+ "lsh",
+ "stdext",
+]
+
 [[package]]
 name = "memchr"
 version = "2.7.6"
@@ -507,6 +558,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "rust-fuzzy-search"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a157657054ffe556d8858504af8a672a054a6e0bd9e8ee531059100c0fa11bb2"
+
 [[package]]
 name = "rustversion"
 version = "1.0.22"
diff --git a/Cargo.toml b/Cargo.toml
index fb53ab464f5f..99ef8715a6ac 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,7 +7,7 @@ resolver = "2"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/microsoft/edit"
-rust-version = "1.88"
+rust-version = "1.92"
 
 # We use `opt-level = "s"` as it significantly reduces binary size.
 # We could then use the `#[optimize(speed)]` attribute for spot optimizations.
@@ -28,5 +28,7 @@ lto = "thin"                # Similarly, speed up linking by a ton
 
 [workspace.dependencies]
 edit = { path = "./crates/edit" }
+lsh = { path = "./crates/lsh" }
+lsh-bin = { path = "./crates/lsh-bin" }
 stdext = { path = "./crates/stdext" }
 unicode-gen = { path = "./crates/unicode-gen" }
diff --git a/assets/highlighting-tests/COMMIT_EDITMSG b/assets/highlighting-tests/COMMIT_EDITMSG
new file mode 100644
index 000000000000..cfba35d08da9
--- /dev/null
+++ b/assets/highlighting-tests/COMMIT_EDITMSG
@@ -0,0 +1,62 @@
+lsh: flip legacy flag, add follow-up hook & diff test
+
+Refactors placeholder logic and introduces a lightweight follow-up hook.
+Adds diff highlighting test and removes outdated asset to streamline visuals.
+
+# Please enter the commit message for your changes. Lines starting
+# with '#' will be ignored, and an empty message aborts the commit.
+#
+# On branch foobar
+# Your branch is up to date with 'origin/foobar'.
+#
+# Changes to be committed:
+#	renamed:    foo.rs -> bar.rs
+#	modified:   src/lsh/definitions.rs
+#	deleted:    assets/old_logo.svg
+#	new file:   baz.rs
+#
+# ------------------------ >8 ------------------------
+# Do not modify or remove the line above.
+# Everything below it will be ignored.
+diff --git a/src/lsh/definitions.rs b/src/lsh/definitions.rs
+index a1b2c3d..d4c3b2a 100644
+--- a/src/lsh/definitions.rs
++++ b/src/lsh/definitions.rs
+@@ -12,9 +12,11 @@ // context line 12
+ // context line 13
+-// old placeholder logic A
+-const LEGACY_FLAG: bool = true;
+-// end legacy block
++// updated placeholder logic A
++const LEGACY_FLAG: bool = false; // flipped for test
++// added note: migration in progress
++// end legacy block
+ // context line 19
+ // context line 20
+@@ -42,7 +44,12 @@ // context line 42
+ // context line 43
+-    do_placeholder_action(alpha, beta);
++    // Split actions for clarity
++    do_placeholder_prepare(alpha);
++    do_placeholder_action(alpha, beta);
++    if (enable_extra()) {
++        do_placeholder_followup(beta);
++    }
+ // context line 50
+ // context line 51
+ // context line 52
+@@ -90,6 +97,15 @@ // context line 90
+ // context line 91
+ // context line 92
++/// Added lightweight fake helper
++fn enable_extra() -> bool {
++    // Pretend this consults a config value
++    true
++}
++
++// Temporary debug instrumentation (to be removed)
++const _DEBUG_HOOK: &str = "lsh:extra";
++
+ // context line 93
+ // context line 94
+ // context line 95
diff --git a/assets/highlighting-tests/bash.sh b/assets/highlighting-tests/bash.sh
new file mode 100644
index 000000000000..dfd5238724db
--- /dev/null
+++ b/assets/highlighting-tests/bash.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# This is a comment
+
+readonly VAR1="Hello"   # String literal
+VAR2=42                 # Integer literal
+VAR3=$((VAR2 + 8))      # Arithmetic expansion
+VAR4=$(echo "World")    # Command substitution
+
+function greet() {      # Function definition
+    local name="$1"     # Local variable, parameter expansion
+    echo "${VAR1}, $name! $VAR4"  # String, parameter expansion, variable
+}
+
+greet "User"            # Function call, string literal
+
+if [[ $VAR2 -gt 40 && $VAR3 -eq 50 ]]; then  # Conditional, test, operators
+    echo "Numbers are correct"   # String literal
+elif (( VAR2 < 40 )); then       # Arithmetic test
+    echo 'VAR2 is less than 40'  # Single-quoted string
+else
+    echo "Other case"
+fi
+
+for i in {1..3}; do     # Brace expansion, for loop
+    echo "Loop $i"      # String, variable
+done
+
+case "$VAR4" in         # Case statement
+    World) echo "It's World";;   # Pattern, string
+    *) echo "Unknown";;          # Wildcard
+esac
+
+arr=(one two three)     # Array
+echo "${arr[1]}"        # Array access
+
+declare -A assoc        # Associative array
+assoc[key]="value"
+echo "${assoc[key]}"
+
+# Here document
+cat <<EOF
+Multi-line
+string with $VAR1
+EOF
+
+# Here string
+grep H <<< "$VAR1"
+
+# Subshell
+(subshell_var=99; echo $subshell_var)
+
+# Redirection
+echo "Redirected" > /dev/null
+
+# Background job
+sleep 1 &
+
+# Arithmetic assignment
+let VAR2+=1
+
+# Process substitution
+diff <(echo foo) <(echo bar)
+
+# Command grouping
+{ echo "Group 1"; echo "Group 2"; }
+
+# Escaped characters
+echo "A quote: \" and a backslash: \\"
+
+# End of file
diff --git a/assets/highlighting-tests/batch.bat b/assets/highlighting-tests/batch.bat
new file mode 100644
index 000000000000..962ef66007af
--- /dev/null
+++ b/assets/highlighting-tests/batch.bat
@@ -0,0 +1,41 @@
+@echo off
+REM --- String, Variable, Label, Command, Operator, Number, Delimiter, Comment ---
+
+:: Label
+:Start
+
+:: Variable assignment and usage
+set "VAR1=Hello"
+set VAR2=World
+
+:: String with spaces and special characters
+set "STR=Batch ^& CMD!"
+
+:: Arithmetic operation (number, operator)
+set /a SUM=5+10
+
+:: IF statement (keyword, operator, string, variable)
+if "%VAR1%"=="Hello" (
+    echo %VAR1%, %VAR2%! %STR%
+) else (
+    echo Not matched!
+)
+
+:: FOR loop (keyword, variable, delimiter, string)
+for %%F in (*.bat) do (
+    echo Found file: %%F
+)
+
+:: CALL command (keyword, label)
+call :SubRoutine
+
+:: GOTO command (keyword, label)
+goto :End
+
+:: Subroutine with parameter
+:SubRoutine
+echo In subroutine with SUM=%SUM%
+goto :eof
+
+:End
+REM End of script
diff --git a/assets/highlighting-tests/diff.diff b/assets/highlighting-tests/diff.diff
new file mode 100644
index 000000000000..8d033f94bfcd
--- /dev/null
+++ b/assets/highlighting-tests/diff.diff
@@ -0,0 +1,142 @@
+diff --git a/src/lsh/definitions.rs b/src/lsh/definitions.rs
+index a1b2c3d..d4c3b2a 100644
+--- a/src/lsh/definitions.rs
++++ b/src/lsh/definitions.rs
+@@ -12,9 +12,11 @@ // context line 12
+ // context line 13
+-// old placeholder logic A
+-const LEGACY_FLAG: bool = true;
+-// end legacy block
++// updated placeholder logic A
++const LEGACY_FLAG: bool = false; // flipped for test
++// added note: migration in progress
++// end legacy block
+ // context line 19
+ // context line 20
+@@ -42,7 +44,12 @@ // context line 42
+ // context line 43
+-    do_placeholder_action(alpha, beta);
++    // Split actions for clarity
++    do_placeholder_prepare(alpha);
++    do_placeholder_action(alpha, beta);
++    if (enable_extra()) {
++        do_placeholder_followup(beta);
++    }
+ // context line 50
+ // context line 51
+ // context line 52
+@@ -90,6 +97,15 @@ // context line 90
+ // context line 91
+ // context line 92
++/// Added lightweight fake helper
++fn enable_extra() -> bool {
++    // Pretend this consults a config value
++    true
++}
++
++// Temporary debug instrumentation (to be removed)
++const _DEBUG_HOOK: &str = "lsh:extra";
++
+ // context line 93
+ // context line 94
+ // context line 95
+
+diff --git a/src/tui.rs b/src/tui.rs
+index 1122334..5566778 100644
+--- a/src/tui.rs
++++ b/src/tui.rs
+@@ -5,8 +5,13 @@ // context line 5
+ // context line 6
+ // context line 7
+-// previous rendering stub
+-render_placeholder(frame);
++// refined rendering sequence
++begin_frame(frame);
++render_header(frame);
++render_body(frame);
++render_footer(frame);
++end_frame(frame);
++
+ // context line 14
+ // context line 15
+ // context line 16
+
+diff --git a/README.md b/README.md
+index 9aa9aa9..1bb1bb1 100644
+--- a/README.md
++++ b/README.md
+@@ -1,6 +1,11 @@
+ # Project Title
+-Some brief description here.
++Some brief description here (updated).
++
++Experimental notes:
++- Added fake diff example
++- Demonstrates multi-hunk & multi-file patch
++
+ ## Usage
+ Run the binary as needed.
+
+diff --git a/Cargo.toml b/Cargo.toml
+index cafe123..fade321 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -8,7 +8,10 @@ edition = "2021"
+ # context
+ [features]
+-default = []
++default = ["experimental-preview"]
++experimental-preview = []
++
++# NOTE: Above feature is placeholder & fake
+
+ [dependencies]
+ # context
+@@ -20,6 +23,7 @@ anyhow = "1"
+ # context
+ # instrumentation (fake addition)
+ tracing = "0.1"
++tracing-subscriber = "0.3"
+
+diff --git a/src/feature/experimental.rs b/src/feature/experimental.rs
+new file mode 100644
+index 0000000..abc1234
+--- /dev/null
++++ b/src/feature/experimental.rs
+@@ -0,0 +1,24 @@
++//! Placeholder experimental module (fake content).
++//! This file exists only to demonstrate a multi-file diff.
++#![allow(dead_code)]
++
++pub struct ExperimentalToggle {
++    enabled: bool,
++}
++
++impl ExperimentalToggle {
++    pub fn new() -> Self {
++        Self { enabled: true }
++    }
++    pub fn enabled(&self) -> bool {
++        self.enabled
++    }
++}
++
++pub fn run_experimental_path() {
++    if ExperimentalToggle::new().enabled() {
++        // Fake behavior
++        eprintln!("(fake) running experimental path");
++    }
++}
++
+
+diff --git a/assets/old_logo.svg b/assets/old_logo.svg
+deleted file mode 100644
+index 55aa55a..0000000
+--- a/assets/old_logo.svg
++++ /dev/null
+@@ -1,5 +0,0 @@
+-<!-- Fake removed asset -->
+-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 10 10">
+-  <rect width="10" height="10" fill="#FF00FF"/>
+-  <text x="1" y="9" font-size="3">X</text>
+-</svg>
diff --git a/assets/highlighting-tests/git-rebase-todo b/assets/highlighting-tests/git-rebase-todo
new file mode 100644
index 000000000000..d051d1222924
--- /dev/null
+++ b/assets/highlighting-tests/git-rebase-todo
@@ -0,0 +1,54 @@
+pick 3f9d2ab # chore(ci): add initial pipeline
+pick e12c0ff # feat(io): introduce streaming reader
+reword 7ac14d0 feat(io): add zero-copy buffered writer
+edit 4d92b67 fix(io): handle partial UTF-8 sequences at buffer boundary
+pick 8bb3a21 docs: expand README performance section
+squash 1c02f55 docs: tidy wording in performance section
+pick 54ae91b feat(cli): add --color=auto detection
+fixup 0b77c3e feat(cli): typo in help text
+fixup -c 2f4e8ab feat(cli): adjust arg parsing error message
+pick 6d8e5f1 refactor(core): split large module into submodules
+break
+pick a41e9d3 test(core): add regression tests for issue #410
+exec cargo test --package edit --lib --quiet
+pick c3bb921 perf(render): SIMD accelerate line diffing
+drop 91d0f2e debug(render): temporary logging (to be dropped)
+label pre-merge-split
+pick d2f7ac4 feat(ui): add mouse wheel smooth scrolling
+pick f1a4bd9 feat(ui): add high-DPI awareness
+label post-ui
+update-ref refs/heads/feature/ui-stack-post
+reset pre-merge-split
+merge -C 5ab77e3 post-ui # Merge UI feature stack
+pick d8c4b91 build: bump version to 0.9.0
+
+# Rebase 3f9d2ab..d8c4b91 onto d8c4b91 (18 commands)
+#
+# Commands:
+# p, pick <commit> = use commit
+# r, reword <commit> = use commit, but edit the commit message
+# e, edit <commit> = use commit, but stop for amending
+# s, squash <commit> = use commit, but meld into previous commit
+# f, fixup [-C | -c] <commit> = like "squash" but keep only the previous
+#                    commit's log message, unless -C is used, in which case
+#                    keep only this commit's message; -c is same as -C but
+#                    opens the editor
+# x, exec <command> = run command (the rest of the line) using shell
+# b, break = stop here (continue rebase later with 'git rebase --continue')
+# d, drop <commit> = remove commit
+# l, label <label> = label current HEAD with a name
+# t, reset <label> = reset HEAD to a label
+# m, merge [-C <commit> | -c <commit>] <label> [# <oneline>]
+#         create a merge commit using the original merge commit's
+#         message (or the oneline, if no original merge commit was
+#         specified); use -c <commit> to reword the commit message
+# u, update-ref <ref> = track a placeholder for the <ref> to be updated
+#                       to this position in the new commits. The <ref> is
+#                       updated at the end of the rebase
+#
+# These lines can be re-ordered; they are executed from top to bottom.
+#
+# If you remove a line here THAT COMMIT WILL BE LOST.
+#
+# However, if you remove everything, the rebase will be aborted.
+#
diff --git a/assets/highlighting-tests/markdown.md b/assets/highlighting-tests/markdown.md
new file mode 100644
index 000000000000..c4845eb68014
--- /dev/null
+++ b/assets/highlighting-tests/markdown.md
@@ -0,0 +1,75 @@
+# H1
+
+## H2
+
+### H3
+
+#### H4
+
+##### H5
+
+###### H6
+
+regular
+*italic*
+_italic_
+**bold**
+__bold__
+***bold italic***
+**_bold italic_**
+__*bold italic*__
+~~strikethrough~~
+`inline code`
+`` `literal` ``
+\*not\* \_italic\_ # not a heading
+
+* Unordered item
+  - Nested item
+    * Third level
+* Task list:
+  * [ ] To do
+  * [x] Done
+  * [ ] *Mixed* **formatting** with `code`
+1. Ordered can start anywhere
+2. …like here (intentional)
+   1. Nested ordered
+   2. Multiple paragraphs within a list item:
+      Still the same item.
+
+> A single-level quote
+>
+> > A nested quote with **bold** and `code`
+>
+> * List in a quote
+> * [Link in quote](#links)
+
+Inline: [Example](https://example.com "Example Title")
+Reference: [Ref Link][ref] and [Another][another-ref]
+Relative: [This section](#tables)
+Footnote: [^note]
+[ref]: https://example.com
+[another-ref]: https://github.com
+[^note]: This is a footnote with **formatting** and a [link](https://github.com).
+
+Inline: ![Alt text](https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png "GitHub Mark")
+Reference: ![Logo][logo-ref]
+[logo-ref]: https://github.githubassets.com/images/modules/logos_page/GitHub-Logo.png "GitHub Logo"
+
+| Left        |   Center   | Right |
+| :---------- | :--------: | ----: |
+| *italic*    |   `code`   |   123 |
+| **bold**    | ~~strike~~ |  4.56 |
+| [link][ref] |   :tada:   | `end` |
+
+```bash
+# Shell
+echo "Hello, world" | tr a-z A-Z
+```
+
+```json
+{
+  "name": "gfm-kitchen-sink",
+  "private": true,
+  "scripts": { "test": "echo ok" }
+}
+```
diff --git a/assets/highlighting-tests/powershell.ps1 b/assets/highlighting-tests/powershell.ps1
new file mode 100644
index 000000000000..e2bb01f39c6c
--- /dev/null
+++ b/assets/highlighting-tests/powershell.ps1
@@ -0,0 +1,78 @@
+# Single-line comment
+
+<#
+Multi-line
+comment
+#>
+
+function Get-SampleData {
+    param(
+        [string]$Name = "World", # String literal, parameter
+        [int]$Count = 3
+    )
+
+    $array = @(1, 2, 3) # Array literal
+    $hashtable = @{ Key1 = 'Value1'; Key2 = 42 } # Hashtable literal
+
+    $nullVar = $null
+    $boolTrue = $true
+    $boolFalse = $false
+
+    $regexMatch = "abc123" -match '\d+' # Regex literal
+
+    for ($i = 0; $i -lt $Count; $i++) {
+        Write-Host "Hello, $Name! Iteration: $i" # Variable interpolation, string
+    }
+
+    if ($hashtable.Key2 -eq 42) {
+        Write-Output "Hashtable value is 42"
+    }
+    elseif ($hashtable.Key2 -gt 40) {
+        Write-Output "Hashtable value is greater than 40"
+    }
+    else {
+        Write-Output "Hashtable value is less than or equal to 40"
+    }
+
+    switch ($Name) {
+        "World" { Write-Host "Default name used." }
+        default { Write-Host "Custom name: $Name" }
+    }
+
+    try {
+        throw "An error occurred"
+    }
+    catch {
+        Write-Warning $_
+    }
+    finally {
+        Write-Verbose "Finally block executed"
+    }
+
+    $script:globalVar = 99 # Scope modifier
+
+    # Here-String
+    $hereString = @"
+This is a here-string.
+Name: $Name
+"@
+
+    return $hereString
+}
+
+# Command invocation, pipeline, splatting
+$paramSplat = @{
+    Name  = 'PowerShell'
+    Count = 2
+}
+Get-SampleData @paramSplat | Out-File -FilePath "./output.txt"
+
+# Type literal, member access, method call
+[System.DateTime]::Now.ToString("yyyy-MM-dd")
+
+# Subexpression
+Write-Host "2 + 2 = $($array[0] + $array[1])"
+
+# Command substitution
+$pwdPath = $(Get-Location).Path
+Write-Host "Current directory: $pwdPath"
diff --git a/assets/highlighting-tests/properties.conf b/assets/highlighting-tests/properties.conf
new file mode 100644
index 000000000000..3bd3b62693ba
--- /dev/null
+++ b/assets/highlighting-tests/properties.conf
@@ -0,0 +1,13 @@
+# General Settings
+[General]
+enabled = true
+debug = false
+log_level = info
+max_connections = 1000
+
+[SSL]
+enabled = true
+cert_file = /etc/ssl/certs/server.crt
+key_file = /etc/ssl/private/server.key
+protocols = TLSv1.2, TLSv1.3 # Supported protocols: "TLSv1.2" and "TLSv1.3"
+cipher_suite = "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256"
diff --git a/assets/highlighting-tests/yaml.yml b/assets/highlighting-tests/yaml.yml
new file mode 100644
index 000000000000..6dbf0abada91
--- /dev/null
+++ b/assets/highlighting-tests/yaml.yml
@@ -0,0 +1,43 @@
+# This is a comment
+---
+string: "Hello, world!"
+plain: plainValue
+multiline: |
+  This is a
+  true
+  multiline string.
+folded: >
+  This is a
+  true
+  folded string.
+number_int: 42
+number_float: 3.1415
+number_scientific: 1.23e45
+number_negative: -7
+boolean_true: true
+boolean_false: false
+null_value: null
+explicit_null: ~
+date: 2024-06-01
+timestamp: 2024-06-01T12:34:56Z
+confusable_string_number: 1.23e45 1.23e45 # This is a comment
+sequence:
+  - item1
+  - item2
+  - 3
+  - true
+mapping:
+  key1: value1
+  key2: value2
+nested:
+  - name: Alice
+    age: 30
+    married: false
+  - name: Bob
+    age: 25
+    married: true
+empty_sequence: [foo, 123, bar]
+empty_mapping: { foo: bar }
+literal_colon: "value:with:colons"
+literal_dash: "-not-a-sequence"
+special_chars: "Tab:\t Newline:\n Unicode:\u2713"
diff --git a/crates/edit/Cargo.toml b/crates/edit/Cargo.toml
index fcf8edeb79d1..3858b585cce7 100644
--- a/crates/edit/Cargo.toml
+++ b/crates/edit/Cargo.toml
@@ -19,6 +19,7 @@ harness = false
 debug-latency = []
 
 [dependencies]
+lsh.workspace = true
 stdext.workspace = true
 
 [target.'cfg(unix)'.dependencies]
@@ -26,6 +27,7 @@ libc = "0.2"
 
 [build-dependencies]
 stdext.workspace = true
+lsh.workspace = true
 # The default toml crate bundles its dependencies with bad compile times. Thanks.
 # Thankfully toml-span exists. FWIW the alternative is yaml-rust (without the 2 suffix).
 toml-span = { version = "0.6", default-features = false }
diff --git a/crates/edit/benches/lib.rs b/crates/edit/benches/lib.rs
index 56b92c067ee2..2caf7d5ddb2c 100644
--- a/crates/edit/benches/lib.rs
+++ b/crates/edit/benches/lib.rs
@@ -5,13 +5,16 @@
 
 use std::hint::black_box;
 use std::io::Cursor;
+use std::ops::Range;
+use std::path::Path;
 use std::{mem, vec};
 
 use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
 use edit::helpers::*;
 use edit::simd::MemsetSafe;
-use edit::{buffer, glob, hash, json, oklab, simd, unicode};
-use stdext::arena::{self, Arena, scratch_arena};
+use edit::{buffer, hash, json, lsh, oklab, simd, unicode};
+use stdext::arena::{Arena, scratch_arena};
+use stdext::{arena, glob, varint};
 
 struct EditingTracePatch<'a>(usize, usize, &'a str);
 
@@ -178,6 +181,34 @@ fn bench_json(c: &mut Criterion) {
     );
 }
 
+fn bench_lsh(c: &mut Criterion) {
+    let bytes = include_bytes!("../../../assets/highlighting-tests/markdown.md");
+    let bytes = &bytes[..];
+    let lang = lsh::LANGUAGES.iter().find(|lang| lang.id == "markdown").unwrap();
+    let highlighter = lsh::Highlighter::new(black_box(&bytes), lang);
+
+    c.benchmark_group("lsh").throughput(Throughput::Bytes(bytes.len() as u64)).bench_function(
+        "markdown",
+        |b| {
+            b.iter(|| {
+                let mut h = highlighter.clone();
+                loop {
+                    let scratch = scratch_arena(None);
+                    let res = h.parse_next_line(&scratch);
+                    if res.is_empty() {
+                        break;
+                    }
+                }
+            })
+        },
+    );
+
+    c.benchmark_group("lsh").bench_function("process_file_associations", |b| {
+        let path = Path::new("/some/long/path/to/file/foo.bar.foo.bar.foo.bar");
+        b.iter(|| lsh::process_file_associations(lsh::FILE_ASSOCIATIONS, black_box(path)))
+    });
+}
+
 fn bench_oklab(c: &mut Criterion) {
     c.benchmark_group("oklab")
         .bench_function("StraightRgba::as_oklab", |b| {
@@ -279,6 +310,73 @@ fn bench_unicode(c: &mut Criterion) {
         });
 }
 
+fn bench_varint(c: &mut Criterion) {
+    const BUFFER_SIZE: usize = MEBI;
+
+    let mut buffer = Vec::with_capacity(BUFFER_SIZE + 16);
+
+    // Knuth's MMIX LCG
+    let mut rng_state = 1442695040888963407u64;
+    let mut rng = || {
+        rng_state = rng_state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
+        rng_state as u32
+    };
+
+    // Bitmask with Rejection (as used by Apple)
+    let mut rng_state = 1442695040888963407u64;
+    let mut rng_range = |range: Range<u32>| {
+        let range_size = range.len() as u32;
+        let mask = range_size.next_power_of_two() - 1;
+        loop {
+            rng_state =
+                rng_state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
+            let value = rng_state as u32 & mask;
+            if value < range_size {
+                return range.start.wrapping_add(value);
+            }
+        }
+    };
+
+    loop {
+        // Generate values according to a non-uniform distribution.
+        // The distribution roughly corresponds to what LSH encounters.
+        let value = match rng() {
+            // ~35%: <=7 bits
+            0..1503238553 => rng_range(0..0x7F),
+            // ~40%: <=14 bits
+            1503238553..3221225472 => rng_range(0x80..0x3FFF),
+            // ~20%: <=21 bits
+            3221225472..4026531840 => rng_range(0x4000..0x1FFFFF),
+            // ~5%: u32::MAX
+            _ => (1 << 28) - 1,
+        };
+
+        buffer.extend(varint::encode(value));
+
+        if buffer.len() > BUFFER_SIZE {
+            break;
+        }
+    }
+
+    // As per the varint::decode() safety requirements, we need 8 bytes of padding.
+    // We pre-allocated `buffer` with extra capacity, so we technically fulfill that.
+    // _Technically_, however, we also make Rust unhappy, because it's uninitialized memory.
+    // It's just that I really really don't care about any such antics. It's memory.
+
+    c.benchmark_group("varint").bench_function("decode", |b| {
+        let mut off = 0;
+
+        b.iter(|| {
+            let (val, len) = unsafe { varint::decode(buffer.as_ptr().add(off)) };
+            black_box(val);
+            off += len;
+            if off >= buffer.len() {
+                off = 0;
+            }
+        });
+    });
+}
+
 fn bench(c: &mut Criterion) {
     arena::init(128 * MEBI).unwrap();
 
@@ -286,12 +384,14 @@ fn bench(c: &mut Criterion) {
     bench_glob(c);
     bench_hash(c);
     bench_json(c);
+    bench_lsh(c);
     bench_oklab(c);
     bench_simd_lines_fwd(c);
     bench_simd_memchr2(c);
     bench_simd_memset::<u32>(c);
     bench_simd_memset::<u8>(c);
     bench_unicode(c);
+    bench_varint(c);
 }
 
 criterion_group!(benches, bench);
diff --git a/crates/edit/build/main.rs b/crates/edit/build/main.rs
index 4acb58a87817..96ae2f201421 100644
--- a/crates/edit/build/main.rs
+++ b/crates/edit/build/main.rs
@@ -3,6 +3,8 @@
 
 #![allow(irrefutable_let_patterns)]
 
+use stdext::arena::scratch_arena;
+
 use crate::helpers::env_opt;
 
 mod helpers;
@@ -24,12 +26,31 @@ fn main() {
         _ => TargetOs::Unix,
     };
 
+    compile_lsh();
     compile_i18n();
     configure_icu(target_os);
     #[cfg(windows)]
     configure_windows_binary(target_os);
 }
 
+fn compile_lsh() {
+    let scratch = scratch_arena(None);
+
+    let lsh_path = lsh::compiler::builtin_definitions_path();
+    let out_dir = env_opt("OUT_DIR");
+    let out_path = format!("{out_dir}/lsh_definitions.rs");
+
+    let mut generator = lsh::compiler::Generator::new(&scratch);
+    match generator.read_directory(lsh_path).and_then(|_| generator.generate_rust()) {
+        Ok(c) => std::fs::write(out_path, c).unwrap(),
+        Err(err) => {
+            panic!("failed to compile lsh definitions: {err}");
+        }
+    };
+
+    println!("cargo::rerun-if-changed={}", lsh_path.display());
+}
+
 fn compile_i18n() {
     let i18n_path = "../../i18n/edit.toml";
 
diff --git a/crates/edit/src/bin/edit/apperr.rs b/crates/edit/src/bin/edit/apperr.rs
index baa8d071a8e5..f8aea68da7e5 100644
--- a/crates/edit/src/bin/edit/apperr.rs
+++ b/crates/edit/src/bin/edit/apperr.rs
@@ -7,6 +7,8 @@ use edit::{buffer, icu};
 
 #[derive(Debug)]
 pub enum Error {
+    SettingsInvalidJson,
+    SettingsInvalidValue,
     Io(io::Error),
     Icu(icu::Error),
 }
diff --git a/crates/edit/src/bin/edit/documents.rs b/crates/edit/src/bin/edit/documents.rs
index eabf798678f1..eb655f2117d8 100644
--- a/crates/edit/src/bin/edit/documents.rs
+++ b/crates/edit/src/bin/edit/documents.rs
@@ -9,9 +9,11 @@ use std::{fs, io};
 
 use edit::buffer::{RcTextBuffer, TextBuffer};
 use edit::helpers::{CoordType, Point};
+use edit::lsh::{FILE_ASSOCIATIONS, Language, process_file_associations};
 use edit::{path, sys};
 
 use crate::apperr;
+use crate::settings::Settings;
 use crate::state::DisplayablePathBuf;
 
 pub struct Document {
@@ -21,6 +23,7 @@ pub struct Document {
     pub filename: String,
     pub file_id: Option<sys::FileId>,
     pub new_file_counter: usize,
+    pub language_override: Option<Option<&'static Language>>,
 }
 
 impl Document {
@@ -63,15 +66,45 @@ impl Document {
     fn set_path(&mut self, path: PathBuf) {
         let filename = path.file_name().unwrap_or_default().to_string_lossy().into_owned();
         let dir = path.parent().map(ToOwned::to_owned).unwrap_or_default();
+
         self.filename = filename;
         self.dir = Some(DisplayablePathBuf::from_path(dir));
         self.path = Some(path);
-        self.update_file_mode();
+
+        self.buffer.borrow_mut().set_ruler(if self.filename == "COMMIT_EDITMSG" { 72 } else { 0 });
+        self.update_language();
+    }
+
+    pub fn auto_detect_language(&mut self) {
+        self.language_override = None;
+        self.update_language();
+    }
+
+    pub fn override_language(&mut self, lang: Option<&'static Language>) {
+        self.language_override = Some(lang);
+        self.update_language();
+    }
+
+    fn update_language(&mut self) {
+        self.buffer.borrow_mut().set_language(self.get_language());
     }
 
-    fn update_file_mode(&mut self) {
-        let mut tb = self.buffer.borrow_mut();
-        tb.set_ruler(if self.filename == "COMMIT_EDITMSG" { 72 } else { 0 });
+    fn get_language(&self) -> Option<&'static Language> {
+        if let Some(lang) = self.language_override {
+            return lang;
+        }
+
+        if let Some(path) = &self.path {
+            let settings = Settings::borrow();
+            if let Some(lang) = process_file_associations(&settings.file_associations, path) {
+                return Some(lang);
+            }
+            if let Some(lang) = process_file_associations(FILE_ASSOCIATIONS, path) {
+                return Some(lang);
+            }
+        }
+
+        None
     }
 }
 
@@ -123,6 +156,7 @@ impl DocumentManager {
             filename: Default::default(),
             file_id: None,
             new_file_counter: 0,
+            language_override: None,
         };
         self.gen_untitled_name(&mut doc);
 
@@ -183,6 +217,7 @@ impl DocumentManager {
             filename: Default::default(),
             file_id,
             new_file_counter: 0,
+            language_override: None,
         };
         doc.set_path(path);
 
diff --git a/crates/edit/src/bin/edit/draw_menubar.rs b/crates/edit/src/bin/edit/draw_menubar.rs
index 8f14f70c1ed0..5ccec4d180b7 100644
--- a/crates/edit/src/bin/edit/draw_menubar.rs
+++ b/crates/edit/src/bin/edit/draw_menubar.rs
@@ -7,6 +7,7 @@ use edit::tui::*;
 use stdext::arena_format;
 
 use crate::localization::*;
+use crate::settings::Settings;
 use crate::state::*;
 
 pub fn draw_menubar(ctx: &mut Context, state: &mut State) {
@@ -51,9 +52,18 @@ fn draw_menu_file(ctx: &mut Context, state: &mut State) {
         if ctx.menubar_menu_button(loc(LocId::FileSaveAs), 'A', vk::NULL) {
             state.wants_file_picker = StateFilePicker::SaveAs;
         }
-        if ctx.menubar_menu_button(loc(LocId::FileClose), 'C', kbmod::CTRL | vk::W) {
-            state.wants_close = true;
-        }
+    }
+    #[allow(irrefutable_let_patterns)]
+    if let path = Settings::borrow().path.as_path()
+        && ctx.menubar_menu_button(loc(LocId::FilePreferences), 'P', vk::NULL)
+        && let Err(err) = state.documents.add_file_path(path)
+    {
+        error_log_add(ctx, state, err);
+    }
+    if state.documents.active().is_some()
+        && ctx.menubar_menu_button(loc(LocId::FileClose), 'C', kbmod::CTRL | vk::W)
+    {
+        state.wants_close = true;
     }
     if ctx.menubar_menu_button(loc(LocId::FileExit), 'X', kbmod::CTRL | vk::Q) {
         state.wants_exit = true;
diff --git a/crates/edit/src/bin/edit/draw_statusbar.rs b/crates/edit/src/bin/edit/draw_statusbar.rs
index 935e158e176b..d812150922b7 100644
--- a/crates/edit/src/bin/edit/draw_statusbar.rs
+++ b/crates/edit/src/bin/edit/draw_statusbar.rs
@@ -6,6 +6,7 @@ use edit::fuzzy::score_fuzzy;
 use edit::helpers::*;
 use edit::icu;
 use edit::input::vk;
+use edit::lsh::LANGUAGES;
 use edit::tui::*;
 use stdext::arena::scratch_arena;
 use stdext::arena_format;
@@ -27,15 +28,21 @@ pub fn draw_statusbar(ctx: &mut Context, state: &mut State) {
 
         ctx.table_next_row();
 
-        if ctx.button("newline", if tb.is_crlf() { "CRLF" } else { "LF" }, ButtonStyle::default()) {
-            let is_crlf = tb.is_crlf();
-            tb.normalize_newlines(!is_crlf);
-        }
+        state.wants_language_picker |= ctx.button(
+            "language",
+            tb.language().map_or("Plain Text", |l| l.name),
+            ButtonStyle::default(),
+        );
         if state.wants_statusbar_focus {
             state.wants_statusbar_focus = false;
             ctx.steal_focus();
         }
 
+        if ctx.button("newline", if tb.is_crlf() { "CRLF" } else { "LF" }, ButtonStyle::default()) {
+            let is_crlf = tb.is_crlf();
+            tb.normalize_newlines(!is_crlf);
+        }
+
         state.wants_encoding_picker |=
             ctx.button("encoding", tb.encoding(), ButtonStyle::default());
         if state.wants_encoding_picker {
@@ -200,6 +207,55 @@ pub fn draw_statusbar(ctx: &mut Context, state: &mut State) {
     ctx.table_end();
 }
 
+pub fn draw_dialog_language_change(ctx: &mut Context, state: &mut State) {
+    let doc = state.documents.active_mut();
+    let mut done = doc.is_none();
+
+    ctx.modal_begin("language", loc(LocId::LanguageSelectMode));
+    if let Some(doc) = doc {
+        let width = (ctx.size().width - 20).max(10);
+        let height = (ctx.size().height - 10).max(10);
+
+        ctx.scrollarea_begin("scrollarea", Size { width, height });
+        ctx.attr_background_rgba(ctx.indexed_alpha(IndexedColor::Black, 1, 4));
+        ctx.inherit_focus();
+        {
+            ctx.list_begin("languages");
+            ctx.inherit_focus();
+
+            let auto_detect = doc.language_override.is_none();
+            let selected = if auto_detect { None } else { doc.buffer.borrow().language() };
+
+            if ctx.list_item(auto_detect, loc(LocId::LanguageAutoDetect))
+                == ListSelection::Activated
+            {
+                doc.auto_detect_language();
+                done = true;
+            }
+
+            if ctx.list_item(selected.is_none(), "Plain Text") == ListSelection::Activated {
+                doc.override_language(None);
+                done = true;
+            }
+
+            for lang in LANGUAGES {
+                if ctx.list_item(Some(lang) == selected, lang.name) == ListSelection::Activated {
+                    doc.override_language(Some(lang));
+                    done = true;
+                }
+            }
+            ctx.list_end();
+        }
+        ctx.scrollarea_end();
+    }
+    done |= ctx.modal_end();
+
+    if done {
+        state.wants_language_picker = false;
+        ctx.needs_rerender();
+    }
+}
+
 pub fn draw_dialog_encoding_change(ctx: &mut Context, state: &mut State) {
     let encoding = state.documents.active_mut().map_or("", |doc| doc.buffer.borrow().encoding());
     let reopen = state.wants_encoding_change == StateEncodingChange::Reopen;
diff --git a/crates/edit/src/bin/edit/main.rs b/crates/edit/src/bin/edit/main.rs
index a05756a009cd..eedd71fc96ff 100644
--- a/crates/edit/src/bin/edit/main.rs
+++ b/crates/edit/src/bin/edit/main.rs
@@ -10,6 +10,7 @@ mod draw_filepicker;
 mod draw_menubar;
 mod draw_statusbar;
 mod localization;
+mod settings;
 mod state;
 
 use std::borrow::Cow;
@@ -33,6 +34,8 @@ use state::*;
 use stdext::arena::{self, Arena, ArenaString, scratch_arena};
 use stdext::arena_format;
 
+use crate::settings::Settings;
+
 #[cfg(target_pointer_width = "32")]
 const SCRATCH_ARENA_CAPACITY: usize = 128 * MEBI;
 #[cfg(target_pointer_width = "64")]
@@ -73,6 +76,8 @@ fn run() -> apperr::Result<()> {
         return Ok(());
     }
 
+    Settings::reload()?;
+
     // This will reopen stdin if it's redirected (which may fail) and switch
     // the terminal to raw mode which prevents the user from pressing Ctrl+C.
     // `handle_args` may want to print a help message (must not fail),
@@ -325,6 +330,9 @@ fn draw(ctx: &mut Context, state: &mut State) {
     if state.wants_save {
         draw_handle_save(ctx, state);
     }
+    if state.wants_language_picker {
+        draw_dialog_language_change(ctx, state);
+    }
     if state.wants_encoding_change != StateEncodingChange::None {
         draw_dialog_encoding_change(ctx, state);
     }
diff --git a/crates/edit/src/bin/edit/settings.rs b/crates/edit/src/bin/edit/settings.rs
new file mode 100644
index 000000000000..cf375a29b485
--- /dev/null
+++ b/crates/edit/src/bin/edit/settings.rs
@@ -0,0 +1,107 @@
+use std::path::PathBuf;
+
+use edit::cell::{Ref, SemiRefCell};
+use edit::json;
+use edit::lsh::{LANGUAGES, Language};
+use stdext::arena::{read_to_string, scratch_arena};
+use stdext::arena_format;
+
+use crate::apperr;
+
+pub struct Settings {
+    pub path: PathBuf,
+    pub file_associations: Vec<(String, &'static Language)>,
+}
+
+struct SettingsCell(SemiRefCell<Settings>);
+unsafe impl Sync for SettingsCell {}
+static SETTINGS: SettingsCell = SettingsCell(SemiRefCell::new(Settings::new()));
+
+impl Settings {
+    const fn new() -> Self {
+        Settings { path: PathBuf::new(), file_associations: Vec::new() }
+    }
+
+    pub fn borrow() -> Ref<'static, Settings> {
+        SETTINGS.0.borrow()
+    }
+
+    pub fn reload() -> apperr::Result<()> {
+        let s = Self::load()?;
+        *SETTINGS.0.borrow_mut() = s;
+        Ok(())
+    }
+
+    fn load() -> apperr::Result<Self> {
+        let mut settings = Self::new();
+
+        settings.path = match settings_json_path() {
+            Some(p) => p,
+            None => return Ok(settings),
+        };
+
+        let scratch = scratch_arena(None);
+        let str = match read_to_string(&scratch, &settings.path) {
+            Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(settings),
+            Err(err) => return Err(err.into()),
+            Ok(str) => str,
+        };
+        let Ok(json) = json::parse(&scratch, &str) else {
+            return Err(apperr::Error::SettingsInvalidJson);
+        };
+        let Some(root) = json.as_object() else {
+            return Err(apperr::Error::SettingsInvalidValue);
+        };
+
+        if let Some(f) = root.get_object("fileAssociations") {
+            for &(mut key, ref value) in f.iter() {
+                if !key.contains('/') {
+                    key = arena_format!(&scratch, "**/{key}").leak();
+                }
+
+                let Some(id) = value.as_str() else {
+                    return Err(apperr::Error::SettingsInvalidValue);
+                };
+                let Some(language) = LANGUAGES.iter().find(|lang| lang.id == id) else {
+                    return Err(apperr::Error::SettingsInvalidValue);
+                };
+
+                settings.file_associations.push((key.to_string(), language));
+            }
+        }
+
+        Ok(settings)
+    }
+}
+
+fn settings_json_path() -> Option<PathBuf> {
+    let mut config_dir = config_dir()?;
+    config_dir.push("settings.json");
+    Some(config_dir)
+}
+
+fn config_dir() -> Option<PathBuf> {
+    fn var_path(key: &str) -> Option<PathBuf> {
+        std::env::var_os(key).map(PathBuf::from)
+    }
+
+    fn push(mut path: PathBuf, suffix: &str) -> PathBuf {
+        path.push(suffix);
+        path
+    }
+
+    #[cfg(target_os = "windows")]
+    {
+        var_path("APPDATA").map(|p| push(p, "Microsoft/Edit"))
+    }
+    #[cfg(any(target_os = "macos", target_os = "ios"))]
+    {
+        var_path("HOME").map(|p| push(p, "Library/Application Support/com.microsoft.edit"))
+    }
+    #[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios")))]
+    {
+        var_path("XDG_CONFIG_HOME")
+            .or_else(|| var_path("HOME").map(|p| push(p, ".config")))
+            .map(|p| push(p, "msedit"))
+    }
+}
diff --git a/crates/edit/src/bin/edit/state.rs b/crates/edit/src/bin/edit/state.rs
index c8d45bd8ca64..653efef06ecf 100644
--- a/crates/edit/src/bin/edit/state.rs
+++ b/crates/edit/src/bin/edit/state.rs
@@ -28,6 +28,8 @@ impl From<apperr::Error> for FormatApperr {
 impl std::fmt::Display for FormatApperr {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self.0 {
+            apperr::Error::SettingsInvalidJson => f.write_str("Settings JSON is malformed"),
+            apperr::Error::SettingsInvalidValue => f.write_str("Invalid settings value"),
             apperr::Error::Icu(icu::ICU_MISSING_ERROR) => f.write_str(loc(LocId::ErrorIcuMissing)),
             apperr::Error::Icu(ref err) => err.fmt(f),
             apperr::Error::Io(ref err) => err.fmt(f),
@@ -152,6 +154,8 @@ pub struct State {
     pub search_options: buffer::SearchOptions,
     pub search_success: bool,
 
+    pub wants_language_picker: bool,
+
     pub wants_encoding_picker: bool,
     pub wants_encoding_change: StateEncodingChange,
     pub encoding_picker_needle: String,
@@ -200,6 +204,8 @@ impl State {
             search_options: Default::default(),
             search_success: true,
 
+            wants_language_picker: false,
+
             wants_encoding_picker: false,
             encoding_picker_needle: Default::default(),
             encoding_picker_results: Default::default(),
diff --git a/crates/edit/src/buffer/line_cache.rs b/crates/edit/src/buffer/line_cache.rs
deleted file mode 100644
index af7cd59493f5..000000000000
--- a/crates/edit/src/buffer/line_cache.rs
+++ /dev/null
@@ -1,116 +0,0 @@
-use std::ops::Range;
-
-use crate::{document::ReadableDocument, simd::memchr2};
-
-/// Cache a line/offset pair every CACHE_EVERY lines to speed up line/offset calculations
-const CACHE_EVERY: usize = 1024 * 64;
-
-#[derive(Clone)]
-pub struct CachePoint {
-    pub index: usize,
-    pub line: usize,
-    // pub snapshot: ParserSnapshot
-}
-
-pub struct LineCache {
-    cache: Vec<CachePoint>,
-}
-
-impl LineCache {
-    pub fn new() -> Self {
-        Self { cache: vec![] }
-    }
-
-    pub fn from_document<T: ReadableDocument>(&mut self, document: &T) {
-        self.cache.clear();
-
-        let mut offset = 0;
-        let mut line = 0;
-        loop {
-            let text = document.read_forward(offset);
-            if text.is_empty() { return; }
-            
-            let mut off = 0;
-            loop {
-                off = memchr2(b'\n', b'\n', text, off);
-                if off == text.len() { break; }
-
-                if line % CACHE_EVERY == 0 {
-                    self.cache.push(CachePoint { index: offset+off, line });
-                }
-                line += 1;
-                off += 1;
-            }
-
-            offset += text.len();
-        }
-    }
-
-    /// Updates the cache after a deletion.
-    /// `range` is the deleted byte range, and `text` is the content that was deleted.
-    pub fn delete(&mut self, range: Range<usize>, text: &Vec<u8>) {
-        let mut newlines = 0;
-        for c in text {
-            if *c == b'\n' {
-                newlines += 1;
-            }
-        }
-
-        let mut beg_del = None;
-        let mut end_del = None;
-        for (i, point) in self.cache.iter_mut().enumerate() {
-            if point.index >= range.start {
-                if point.index < range.end {
-                    // cache point is within the deleted range
-                    if beg_del.is_none() { beg_del = Some(i); }
-                    end_del = Some(i + 1);
-                }
-                else {
-                    point.index -= text.len();
-                    point.line -= newlines;
-                }
-            }
-        }
-
-        if let (Some(beg), Some(end)) = (beg_del, end_del) {
-            self.cache.drain(beg..end);
-        }
-    }
-
-    /// Updates the cache after an insertion.
-    /// `offset` is where the insertion occurs, and `text` is the inserted content.
-    pub fn insert(&mut self, offset: usize, text: &[u8]) {
-        // Count how many newlines were inserted
-        let mut newlines = 0;
-        for c in text {
-            if *c == b'\n' {
-                newlines += 1;
-            }
-        }
-
-        let len = text.len();
-        for point in &mut self.cache {
-            if point.index > offset {
-                point.index += len;
-                point.line += newlines;
-            }
-        }
-
-        // TODO: This also needs to insert new cache points
-    }
-
-    /// Finds the nearest cached line-offset pair relative to a target line.
-    /// If `reverse` is false, it returns the closest *before* the target.
-    /// If `reverse` is true, it returns the closest *after or at* the target.
-    pub fn nearest_offset(&self, target_count: usize, reverse: bool) -> Option<CachePoint> {
-        match self.cache.binary_search_by_key(&target_count, |p| p.line) {
-            Ok(i) => Some(self.cache[i].clone()),
-            Err(i) => {
-                if i == 0 || i == self.cache.len() { None }  // target < lowest cache point || target > highest cache point
-                else {
-                    Some(self.cache[ if reverse {i} else {i-1} ].clone())
-                }
-            }
-        }
-    }
-}
diff --git a/crates/edit/src/buffer/mod.rs b/crates/edit/src/buffer/mod.rs
index 251fa401d141..9e9ef6600634 100644
--- a/crates/edit/src/buffer/mod.rs
+++ b/crates/edit/src/buffer/mod.rs
@@ -41,8 +41,10 @@ use stdext::{ReplaceRange as _, minmax, slice_as_uninit_mut, slice_copy_safe};
 use crate::cell::SemiRefCell;
 use crate::clipboard::Clipboard;
 use crate::document::{ReadableDocument, WriteableDocument};
-use crate::framebuffer::{Framebuffer, IndexedColor};
+use crate::framebuffer::{Attributes, Framebuffer, IndexedColor};
 use crate::helpers::*;
+use crate::lsh::cache::HighlighterCache;
+use crate::lsh::{HighlightKind, Highlighter, Language};
 use crate::oklab::StraightRgba;
 use crate::simd::memchr2;
 use crate::unicode::{self, Cursor, MeasurementConfig, Utf8Chars};
@@ -249,6 +251,7 @@ pub struct TextBuffer {
     selection: Option<TextBufferSelection>,
     selection_generation: u32,
     search: Option<UnsafeCell<ActiveSearch>>,
+    highlighter_cache: HighlighterCache,
 
     width: CoordType,
     margin_width: CoordType,
@@ -258,6 +261,7 @@ pub struct TextBuffer {
     tab_size: CoordType,
     indent_with_tabs: bool,
     line_highlight_enabled: bool,
+    language: Option<&'static Language>,
     ruler: CoordType,
     encoding: &'static str,
     newlines_are_crlf: bool,
@@ -297,6 +301,7 @@ impl TextBuffer {
             selection: None,
             selection_generation: 0,
             search: None,
+            highlighter_cache: HighlighterCache::new(),
 
             width: 0,
             margin_width: 0,
@@ -306,6 +311,7 @@ impl TextBuffer {
             tab_size: 4,
             indent_with_tabs: false,
             line_highlight_enabled: false,
+            language: None,
             ruler: 0,
             encoding: "UTF-8",
             newlines_are_crlf: cfg!(windows), // Windows users want CRLF
@@ -598,6 +604,15 @@ impl TextBuffer {
         self.line_highlight_enabled = enabled;
     }
 
+    pub fn language(&self) -> Option<&'static Language> {
+        self.language
+    }
+
+    pub fn set_language(&mut self, language: Option<&'static Language>) {
+        self.language = language;
+        self.highlighter_cache.invalidate_from(0);
+    }
+
     /// Sets a ruler column, e.g. 80.
     pub fn set_ruler(&mut self, column: CoordType) {
         self.ruler = column;
@@ -676,6 +691,7 @@ impl TextBuffer {
         self.set_selection(None);
         self.mark_as_clean();
         self.reflow();
+        self.highlighter_cache.invalidate_from(0);
     }
 
     /// Copies the contents of the buffer into a string.
@@ -1736,6 +1752,7 @@ impl TextBuffer {
         let text_width = width - self.margin_width;
         let mut visualizer_buf = [0xE2, 0x90, 0x80]; // U+2400 in UTF8
         let mut visual_pos_x_max = 0;
+        let mut highlighter = self.language.map(|l| Highlighter::new(&self.buffer, l));
 
         // Pick the cursor closer to the `origin.y`.
         let mut cursor = {
@@ -1974,6 +1991,75 @@ impl TextBuffer {
                     global_off += chunk.len();
                 }
 
+                if let Some(highlighter) = &mut highlighter {
+                    let highlights = self.highlighter_cache.parse_line(
+                        &scratch,
+                        highlighter,
+                        cursor_beg.logical_pos.y,
+                    );
+                    let mut highlights = highlights.iter();
+
+                    if let Some(first) = highlights.next() {
+                        let mut highlight_kind = first.kind;
+                        let mut highlight_beg =
+                            self.cursor_move_to_offset_internal(cursor_beg, first.start);
+
+                        for next in highlights {
+                            let kind = highlight_kind;
+                            highlight_kind = next.kind;
+
+                            let beg = highlight_beg.visual_pos;
+                            highlight_beg =
+                                self.cursor_move_to_offset_internal(highlight_beg, next.start);
+                            let end = highlight_beg.visual_pos;
+                            let target = Rect {
+                                left: destination.left + self.margin_width + beg.x - origin.x,
+                                top: destination.top + y,
+                                right: destination.left + self.margin_width + end.x - origin.x,
+                                bottom: destination.top + y + 1,
+                            };
+
+                            'block: {
+                                let color = match kind {
+                                    HighlightKind::Other => break 'block,
+
+                                    HighlightKind::Comment => IndexedColor::Green,
+                                    HighlightKind::Method => IndexedColor::BrightYellow,
+                                    HighlightKind::String => IndexedColor::BrightRed,
+                                    HighlightKind::Variable => IndexedColor::BrightCyan,
+
+                                    HighlightKind::ConstantLanguage => IndexedColor::BrightBlue,
+                                    HighlightKind::ConstantNumeric => IndexedColor::BrightGreen,
+                                    HighlightKind::KeywordControl => IndexedColor::BrightMagenta,
+                                    HighlightKind::KeywordOther => IndexedColor::BrightBlue,
+                                    HighlightKind::MarkupBold => break 'block,
+                                    HighlightKind::MarkupChanged => IndexedColor::BrightBlue,
+                                    HighlightKind::MarkupDeleted => IndexedColor::BrightRed,
+                                    HighlightKind::MarkupHeading => IndexedColor::BrightBlue,
+                                    HighlightKind::MarkupInserted => IndexedColor::BrightGreen,
+                                    HighlightKind::MarkupItalic => break 'block,
+                                    HighlightKind::MarkupLink => break 'block,
+                                    HighlightKind::MarkupList => IndexedColor::BrightBlue,
+                                    HighlightKind::MarkupStrikethrough => break 'block,
+                                    HighlightKind::MetaHeader => IndexedColor::BrightBlue,
+                                };
+                                fb.blend_fg(target, fb.indexed(color));
+                            }
+
+                            'block: {
+                                let attr = match kind {
+                                    HighlightKind::MarkupBold => Attributes::Bold,
+                                    HighlightKind::MarkupItalic => Attributes::Italic,
+                                    HighlightKind::MarkupLink => Attributes::Underlined,
+                                    HighlightKind::MarkupStrikethrough => Attributes::Strikethrough,
+                                    _ => break 'block,
+                                };
+                                fb.replace_attr(target, Attributes::All, attr);
+                            }
+                        }
+                    }
+                }
+
                 visual_pos_x_max = visual_pos_x_max.max(cursor_end.visual_pos.x);
             }
 
@@ -2601,6 +2687,7 @@ impl TextBuffer {
         }
 
         self.active_edit_off = cursor.offset;
+        self.highlighter_cache.invalidate_from(cursor.logical_pos.y);
 
         // If word-wrap is enabled, the visual layout of all logical lines affected by the write
         // may have changed. This includes even text before the insertion point up to the line
@@ -2852,6 +2939,8 @@ impl TextBuffer {
             return;
         }
 
+        self.highlighter_cache.invalidate_from(damage_start);
+
         if entry_buffer_generation.is_some() {
             self.recalc_after_content_changed();
         }
diff --git a/crates/edit/src/framebuffer.rs b/crates/edit/src/framebuffer.rs
index f2225e15eddb..4a54673f58f8 100644
--- a/crates/edit/src/framebuffer.rs
+++ b/crates/edit/src/framebuffer.rs
@@ -508,6 +508,13 @@ impl Framebuffer {
 
                 if last_attr != attr {
                     let diff = last_attr ^ attr;
+                    if diff.is(Attributes::Bold) {
+                        if attr.is(Attributes::Bold) {
+                            result.push_str("\x1b[1m");
+                        } else {
+                            result.push_str("\x1b[22m");
+                        }
+                    }
                     if diff.is(Attributes::Italic) {
                         if attr.is(Attributes::Italic) {
                             result.push_str("\x1b[3m");
@@ -522,6 +529,13 @@ impl Framebuffer {
                             result.push_str("\x1b[24m");
                         }
                     }
+                    if diff.is(Attributes::Strikethrough) {
+                        if attr.is(Attributes::Strikethrough) {
+                            result.push_str("\x1b[9m");
+                        } else {
+                            result.push_str("\x1b[29m");
+                        }
+                    }
                     last_attr = attr;
                 }
 
@@ -830,9 +844,11 @@ pub struct Attributes(u8);
 #[allow(non_upper_case_globals)]
 impl Attributes {
     pub const None: Self = Self(0);
-    pub const Italic: Self = Self(0b1);
-    pub const Underlined: Self = Self(0b10);
-    pub const All: Self = Self(0b11);
+    pub const Bold: Self = Self(1);
+    pub const Italic: Self = Self(2);
+    pub const Underlined: Self = Self(4);
+    pub const Strikethrough: Self = Self(8);
+    pub const All: Self = Self(16 - 1);
 
     pub const fn is(self, attr: Self) -> bool {
         (self.0 & attr.0) == attr.0
diff --git a/crates/edit/src/helpers.rs b/crates/edit/src/helpers.rs
index 756292b85756..f8539f8155ff 100644
--- a/crates/edit/src/helpers.rs
+++ b/crates/edit/src/helpers.rs
@@ -78,9 +78,6 @@ pub struct Size {
 }
 
 impl Size {
-    pub const MIN: Self = Self { width: 0, height: 0 };
-    pub const MAX: Self = Self { width: CoordType::MAX, height: CoordType::MAX };
-
     pub fn as_rect(&self) -> Rect {
         Rect { left: 0, top: 0, right: self.width, bottom: self.height }
     }
@@ -149,14 +146,6 @@ impl Rect {
     }
 }
 
-/// [`std::cmp::minmax`] is unstable, as per usual.
-pub fn minmax<T>(v1: T, v2: T) -> [T; 2]
-where
-    T: Ord,
-{
-    if v2 < v1 { [v2, v1] } else { [v1, v2] }
-}
-
 /// [`Read`] but with [`MaybeUninit<u8>`] buffers.
 pub fn file_read_uninit<T: Read>(file: &mut T, buf: &mut [MaybeUninit<u8>]) -> io::Result<usize> {
     unsafe {
diff --git a/crates/edit/src/lib.rs b/crates/edit/src/lib.rs
index 6dea3a7380bd..1ea8c11da642 100644
--- a/crates/edit/src/lib.rs
+++ b/crates/edit/src/lib.rs
@@ -24,12 +24,12 @@ pub mod clipboard;
 pub mod document;
 pub mod framebuffer;
 pub mod fuzzy;
-pub mod glob;
 pub mod hash;
 pub mod helpers;
 pub mod icu;
 pub mod input;
 pub mod json;
+pub mod lsh;
 pub mod oklab;
 pub mod path;
 pub mod simd;
diff --git a/crates/edit/src/lsh/cache.rs b/crates/edit/src/lsh/cache.rs
new file mode 100644
index 000000000000..026221a15296
--- /dev/null
+++ b/crates/edit/src/lsh/cache.rs
@@ -0,0 +1,86 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+use lsh::engine::Higlight;
+use stdext::arena::{Arena, scratch_arena};
+
+use crate::helpers::CoordType;
+use crate::lsh::{HighlightKind, Highlighter, HighlighterState};
+
+#[cfg(debug_assertions)]
+const INTERVAL: CoordType = 16;
+#[cfg(not(debug_assertions))]
+const INTERVAL: CoordType = 1024;
+
+#[derive(Default)]
+pub struct HighlighterCache {
+    checkpoints: Vec<HighlighterState>,
+}
+
+impl HighlighterCache {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Drop any cached states starting at (including) the given logical line.
+    pub fn invalidate_from(&mut self, line: CoordType) {
+        self.checkpoints.truncate(Self::ceil_line_to_offset(line));
+    }
+
+    /// Parse the given logical line. Returns the highlight spans.
+    pub fn parse_line<'a>(
+        &mut self,
+        arena: &'a Arena,
+        highlighter: &mut Highlighter,
+        line: CoordType,
+    ) -> Vec<Higlight<HighlightKind>, &'a Arena> {
+        // Do we need to random seek?
+        if line != highlighter.logical_pos_y() {
+            // If so, restore the nearest, preceeding checkpoint...
+            if !self.checkpoints.is_empty() {
+                let n = Self::floor_line_to_offset(line);
+                let n = n.min(self.checkpoints.len() - 1);
+                highlighter.restore(&self.checkpoints[n]);
+            } else {
+                // The assumption is that you pass in a default constructed highlighter,
+                // and this class handles random seeking for you. As such, there should
+                // never be a case where we don't have a checkpoint for line 0,
+                // but you have a highlighter for line >0.
+                debug_assert!(highlighter.logical_pos_y() == 0);
+            }
+
+            // ...and then seek in front of the requested line.
+            while highlighter.logical_pos_y() < line {
+                // There's a bit of waste here, because we just throw away the results,
+                // but that's better than duplicating the logic. The arena is very fast.
+                let scratch = scratch_arena(Some(arena));
+                _ = self.parse_line_impl(&scratch, highlighter);
+            }
+        }
+
+        self.parse_line_impl(arena, highlighter)
+    }
+
+    fn parse_line_impl<'a>(
+        &mut self,
+        arena: &'a Arena,
+        highlighter: &mut Highlighter,
+    ) -> Vec<Higlight<HighlightKind>, &'a Arena> {
+        // If we need to store a checkpoint for the start of the next line, do so now.
+        if Self::floor_line_to_offset(highlighter.logical_pos_y()) == self.checkpoints.len() {
+            self.checkpoints.push(highlighter.snapshot());
+        }
+
+        highlighter.parse_next_line(arena)
+    }
+
+    /// Since this line cache is super simplistic (no insertions, only append),
+    /// we can directly map from line numbers to offsets in the cache.
+    fn floor_line_to_offset(line: CoordType) -> usize {
+        (line / INTERVAL).try_into().unwrap_or(0)
+    }
+
+    fn ceil_line_to_offset(line: CoordType) -> usize {
+        ((line + INTERVAL - 1) / INTERVAL).try_into().unwrap_or(0)
+    }
+}
diff --git a/crates/edit/src/lsh/definitions.rs b/crates/edit/src/lsh/definitions.rs
new file mode 100644
index 000000000000..7c3cea5cfff6
--- /dev/null
+++ b/crates/edit/src/lsh/definitions.rs
@@ -0,0 +1,4 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+include!(concat!(env!("OUT_DIR"), "/lsh_definitions.rs"));
diff --git a/crates/edit/src/lsh/highlighter.rs b/crates/edit/src/lsh/highlighter.rs
new file mode 100644
index 000000000000..b9fb9fc31332
--- /dev/null
+++ b/crates/edit/src/lsh/highlighter.rs
@@ -0,0 +1,149 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+use lsh::engine::*;
+use stdext::arena::{Arena, scratch_arena};
+
+use crate::document::ReadableDocument;
+use crate::helpers::*;
+use crate::lsh::definitions::*;
+use crate::{simd, unicode};
+
+const MAX_LINE_LEN: usize = 32 * KIBI;
+
+#[derive(Clone)]
+pub struct Highlighter<'a> {
+    doc: &'a dyn ReadableDocument,
+    offset: usize,
+    logical_pos_y: CoordType,
+    engine: Engine<'static, 'static, 'static>,
+}
+
+#[derive(Clone)]
+pub struct HighlighterState {
+    offset: usize,
+    logical_pos_y: CoordType,
+    state: EngineState,
+}
+
+impl<'doc> Highlighter<'doc> {
+    pub fn new(doc: &'doc dyn ReadableDocument, language: &'static Language) -> Self {
+        Self {
+            doc,
+            offset: 0,
+            logical_pos_y: 0,
+            engine: Engine::new(&ASSEMBLY, &STRINGS, &CHARSETS, language.entrypoint),
+        }
+    }
+
+    pub fn logical_pos_y(&self) -> CoordType {
+        self.logical_pos_y
+    }
+
+    /// Create a restorable snapshot of the current highlighter state
+    /// so we can resume highlighting from this point later.
+    pub fn snapshot(&self) -> HighlighterState {
+        HighlighterState {
+            offset: self.offset,
+            logical_pos_y: self.logical_pos_y,
+            state: self.engine.snapshot(),
+        }
+    }
+
+    /// Restore the highlighter state from a previously captured snapshot.
+    pub fn restore(&mut self, snapshot: &HighlighterState) {
+        self.offset = snapshot.offset;
+        self.logical_pos_y = snapshot.logical_pos_y;
+        self.engine.restore(&snapshot.state);
+    }
+
+    pub fn parse_next_line<'a>(
+        &mut self,
+        arena: &'a Arena,
+    ) -> Vec<Higlight<HighlightKind>, &'a Arena> {
+        let scratch = scratch_arena(Some(arena));
+        let (line_beg, line) = self.read_next_line(&scratch);
+
+        // Empty lines can be somewhat common.
+        //
+        // If the line is too long, we don't highlight it.
+        // This is to prevent performance issues with very long lines.
+        if line.is_empty() || line.len() >= MAX_LINE_LEN {
+            return Vec::new_in(arena);
+        }
+
+        let line = unicode::strip_newline(line);
+        let mut res = self.engine.parse_next_line(arena, line);
+
+        // Adjust the range to account for the line offset.
+        for h in &mut res {
+            h.start = line_beg + h.start.min(line.len());
+        }
+
+        res
+    }
+
+    fn read_next_line<'a>(&mut self, arena: &'a Arena) -> (usize, &'a [u8])
+    where
+        'doc: 'a,
+    {
+        self.logical_pos_y += 1;
+
+        let line_beg = self.offset;
+        let mut chunk;
+        let mut line_buf;
+
+        // Try to read a chunk and see if it contains a newline.
+        // In that case we can skip concatenating chunks.
+        {
+            chunk = self.doc.read_forward(self.offset);
+            if chunk.is_empty() {
+                return (line_beg, chunk);
+            }
+
+            let (off, line) = simd::lines_fwd(chunk, 0, 0, 1);
+            self.offset += off;
+
+            if line == 1 {
+                return (line_beg, &chunk[..off]);
+            }
+
+            let next_chunk = self.doc.read_forward(self.offset);
+            if next_chunk.is_empty() {
+                return (line_beg, &chunk[..off]);
+            }
+
+            line_buf = Vec::new_in(arena);
+
+            // Ensure we don't overflow the heap size with a 1GB long line.
+            let end = off.min(MAX_LINE_LEN - line_buf.len());
+            let end = end.min(chunk.len());
+            line_buf.extend_from_slice(&chunk[..end]);
+
+            chunk = next_chunk;
+        }
+
+        // Concatenate chunks until we get a full line.
+        while line_buf.len() < MAX_LINE_LEN {
+            let (off, line) = simd::lines_fwd(chunk, 0, 0, 1);
+            self.offset += off;
+
+            // Ensure we don't overflow the heap size with a 1GB long line.
+            let end = off.min(MAX_LINE_LEN - line_buf.len());
+            let end = end.min(chunk.len());
+            line_buf.extend_from_slice(&chunk[..end]);
+
+            // Start of the next line found.
+            if line == 1 {
+                break;
+            }
+
+            chunk = self.doc.read_forward(self.offset);
+            if chunk.is_empty() {
+                break;
+            }
+        }
+
+        (line_beg, line_buf.leak())
+    }
+}
diff --git a/crates/edit/src/lsh/mod.rs b/crates/edit/src/lsh/mod.rs
new file mode 100644
index 000000000000..1b47a24aa4b8
--- /dev/null
+++ b/crates/edit/src/lsh/mod.rs
@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Microsoft Edit's adapter to LSH.
+
+pub mod cache;
+mod definitions;
+mod highlighter;
+
+use std::path::Path;
+
+pub use definitions::{FILE_ASSOCIATIONS, HighlightKind, LANGUAGES};
+pub use highlighter::*;
+pub use lsh::engine::Language;
+use stdext::glob::glob_match;
+
+pub fn process_file_associations<T>(
+    associations: &[(T, &'static Language)],
+    path: &Path,
+) -> Option<&'static Language>
+where
+    T: AsRef<[u8]>,
+{
+    let path = path.as_os_str().as_encoded_bytes();
+
+    for a in associations {
+        if glob_match(a.0.as_ref(), path) {
+            return Some(a.1);
+        }
+    }
+
+    None
+}
diff --git a/crates/edit/src/sys/unix.rs b/crates/edit/src/sys/unix.rs
index ba9d06fde64b..b0e275cfc558 100644
--- a/crates/edit/src/sys/unix.rs
+++ b/crates/edit/src/sys/unix.rs
@@ -6,7 +6,7 @@
 //! Read the `windows` module for reference.
 //! TODO: This reminds me that the sys API should probably be a trait.
 
-use std::ffi::{CStr, c_char, c_int, c_void};
+use std::ffi::{c_char, c_int, c_void};
 use std::fs::File;
 use std::mem::{self, ManuallyDrop, MaybeUninit};
 use std::os::fd::{AsRawFd as _, FromRawFd as _};
@@ -425,6 +425,7 @@ pub fn load_icu() -> io::Result<LibIcu> {
         Ok(LibIcu { libicuuc, libicui18n })
     }
 }
+
 /// ICU, by default, adds the major version as a suffix to each exported symbol.
 /// They also recommend to disable this for system-level installations (`runConfigureICU Linux --disable-renaming`),
 /// but I found that many (most?) Linux distributions don't do this for some reason.
@@ -459,7 +460,7 @@ pub fn icu_detect_renaming_suffix(arena: &Arena, handle: NonNull<c_void>) -> Are
         }
 
         // The library path is in `info.dli_fname`.
-        let path = match CStr::from_ptr(info.dli_fname).to_str() {
+        let path = match std::ffi::CStr::from_ptr(info.dli_fname).to_str() {
             Ok(name) => name,
             Err(_) => return res,
         };
@@ -502,7 +503,7 @@ where
     } else {
         // SAFETY: In this particular case we know that the string
         // is valid UTF-8, because it comes from icu.rs.
-        let name = unsafe { CStr::from_ptr(name) };
+        let name = unsafe { std::ffi::CStr::from_ptr(name) };
         let name = unsafe { name.to_str().unwrap_unchecked() };
 
         let mut res = ManuallyDrop::new(ArenaString::new_in(arena));
diff --git a/crates/lsh-bin/Cargo.toml b/crates/lsh-bin/Cargo.toml
new file mode 100644
index 000000000000..8601fe1960e4
--- /dev/null
+++ b/crates/lsh-bin/Cargo.toml
@@ -0,0 +1,14 @@
+[package]
+name = "lsh-bin"
+version = "0.0.0"
+
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+anyhow = "*"
+argh = "*"
+lsh.workspace = true
+stdext.workspace = true
diff --git a/crates/lsh-bin/src/main.rs b/crates/lsh-bin/src/main.rs
new file mode 100644
index 000000000000..f7f9ac5bbd0a
--- /dev/null
+++ b/crates/lsh-bin/src/main.rs
@@ -0,0 +1,198 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+use std::fs::File;
+use std::io::{BufRead, BufReader, IsTerminal, Write as _, stdout};
+use std::path::{Path, PathBuf};
+use std::process::exit;
+
+use anyhow::bail;
+use argh::FromArgs;
+use lsh::engine::Engine;
+use stdext::arena::scratch_arena;
+use stdext::glob::glob_match;
+
+#[derive(FromArgs, PartialEq, Debug)]
+#[argh(description = "Debug and test frontend for Leonard's Shitty Highlighter")]
+struct Command {
+    #[argh(subcommand)]
+    sub: SubCommands,
+}
+
+#[derive(FromArgs, PartialEq, Debug)]
+#[argh(subcommand)]
+enum SubCommands {
+    Compile(SubCommandOneCompile),
+    Assembly(SubCommandAssembly),
+    Render(SubCommandRender),
+}
+
+#[derive(FromArgs, PartialEq, Debug)]
+#[argh(subcommand, name = "compile", description = "Generate Rust code from .lsh files")]
+struct SubCommandOneCompile {
+    #[argh(positional, description = "source .lsh file or directory")]
+    lsh: PathBuf,
+}
+
+#[derive(FromArgs, PartialEq, Debug)]
+#[argh(subcommand, name = "assembly", description = "Generate assembly from .lsh files")]
+struct SubCommandAssembly {
+    #[argh(positional, description = "source .lsh file or directory")]
+    lsh: PathBuf,
+}
+
+#[derive(FromArgs, PartialEq, Debug)]
+#[argh(subcommand, name = "render", description = "Highlight text files")]
+struct SubCommandRender {
+    #[argh(positional, description = "source .lsh file or directory")]
+    lsh: PathBuf,
+    #[argh(positional, description = "source text file")]
+    input: PathBuf,
+}
+
+pub fn main() {
+    if let Err(e) = run() {
+        eprintln!("{e}");
+        exit(1);
+    }
+}
+
+fn run() -> anyhow::Result<()> {
+    stdext::arena::init(128 * 1024 * 1024).unwrap();
+
+    let command: Command = argh::from_env();
+    let scratch = scratch_arena(None);
+    let mut generator = lsh::compiler::Generator::new(&scratch);
+    let mut read_lsh = |path: &Path| {
+        if path.is_dir() { generator.read_directory(path) } else { generator.read_file(path) }
+    };
+
+    match &command.sub {
+        SubCommands::Compile(cmd) => {
+            read_lsh(&cmd.lsh)?;
+            let output = generator.generate_rust()?;
+            _ = stdout().write_all(output.as_bytes());
+        }
+        SubCommands::Assembly(cmd) => {
+            read_lsh(&cmd.lsh)?;
+            let vt = stdout().is_terminal();
+            let output = generator.generate_assembly(vt)?;
+            _ = stdout().write_all(output.as_bytes());
+        }
+        SubCommands::Render(cmd) => {
+            read_lsh(&cmd.lsh)?;
+            run_render(generator, &cmd.input)?;
+        }
+    }
+
+    Ok(())
+}
+
+fn run_render(generator: lsh::compiler::Generator, path: &Path) -> anyhow::Result<()> {
+    let assembly = generator.assemble()?;
+
+    let Some(entrypoint) = assembly.entrypoints.iter().find(|ep| {
+        ep.paths
+            .iter()
+            .any(|pattern| glob_match(pattern.as_bytes(), path.as_os_str().as_encoded_bytes()))
+    }) else {
+        bail!("No matching highlighting definition found");
+    };
+
+    let mut color_map = Vec::new();
+    let mut unknown_kinds = Vec::new();
+    for hk in &assembly.highlight_kinds {
+        let color = match hk.identifier {
+            "other" => "",
+
+            "comment" => "\x1b[32m",  // Green
+            "method" => "\x1b[93m",   // Bright Yellow
+            "string" => "\x1b[91m",   // Bright Red
+            "variable" => "\x1b[96m", // Bright Cyan
+
+            "constant.language" => "\x1b[94m",   // Bright Blue
+            "constant.numeric" => "\x1b[92m",    // Bright Green
+            "keyword.control" => "\x1b[95m",     // Bright Magenta
+            "keyword.other" => "\x1b[94m",       // Bright Blue
+            "markup.bold" => "\x1b[1m",          // Bold
+            "markup.changed" => "\x1b[94m",      // Bright Blue
+            "markup.deleted" => "\x1b[91m",      // Bright Red
+            "markup.heading" => "\x1b[94m",      // Bright Blue
+            "markup.inserted" => "\x1b[92m",     // Bright Green
+            "markup.italic" => "\x1b[3m",        // Italic
+            "markup.link" => "\x1b[4m",          // Underlined
+            "markup.list" => "\x1b[94m",         // Bright Blue
+            "markup.strikethrough" => "\x1b[9m", // Strikethrough
+            "meta.header" => "\x1b[94m",         // Bright Blue
+
+            _ => {
+                unknown_kinds.push(hk.identifier.to_string());
+                ""
+            }
+        };
+
+        if !color.is_empty() {
+            if color_map.len() <= hk.value as usize {
+                color_map.resize(hk.value as usize + 1, "");
+            }
+            color_map[hk.value as usize] = color;
+        }
+    }
+    if !unknown_kinds.is_empty() {
+        eprintln!("\x1b[33mWarning: Unknown highlight kinds:");
+        for kind in &unknown_kinds {
+            eprintln!("  - {}", kind);
+        }
+        eprintln!("\x1b[m");
+    }
+
+    // Convert Assembly data to static references by leaking memory
+    // This is fine for a CLI tool that runs once and exits
+    let charsets: Vec<[u16; 16]> = assembly
+        .charsets
+        .into_iter()
+        .map(|cs| {
+            let mut arr = [0u16; 16];
+            for lo in 0..16 {
+                let mut u = 0u16;
+                for hi in 0..16 {
+                    u |= (cs[hi * 16 + lo] as u16) << hi;
+                }
+                arr[lo] = u;
+            }
+            arr
+        })
+        .collect();
+
+    let mut engine = Engine::new(
+        &assembly.instructions,
+        &assembly.strings,
+        &charsets,
+        entrypoint.address as u32,
+    );
+
+    let reader = BufReader::new(File::open(path)?);
+    let mut stdout = stdout();
+
+    for line in reader.lines() {
+        let line = line?;
+        let scratch = scratch_arena(None);
+        let highlights = engine.parse_next_line::<u32>(&scratch, line.as_bytes());
+
+        for [curr, next] in highlights.array_windows::<2>() {
+            let start = curr.start;
+            let end = next.start;
+            let kind = curr.kind;
+            let text = &line[start..end];
+
+            if let Some(color) = color_map.get(kind as usize) {
+                write!(stdout, "{color}{text}\x1b[m")?;
+            } else {
+                stdout.write_all(text.as_bytes())?;
+            }
+        }
+        writeln!(stdout)?;
+    }
+
+    Ok(())
+}
diff --git a/crates/lsh/Cargo.toml b/crates/lsh/Cargo.toml
new file mode 100644
index 000000000000..9cb5e20d6747
--- /dev/null
+++ b/crates/lsh/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "lsh"
+version = "0.0.0"
+
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+regex-syntax = { version = "0.8", default-features = false }
+stdext.workspace = true
diff --git a/crates/lsh/definitions/diff.lsh b/crates/lsh/definitions/diff.lsh
new file mode 100644
index 000000000000..7b71c3dd2cc1
--- /dev/null
+++ b/crates/lsh/definitions/diff.lsh
@@ -0,0 +1,12 @@
+#[display_name = "Diff"]
+#[path = "**/*.diff"]
+#[path = "**/*.patch"]
+pub fn diff() {
+    if /(?:diff|---|\+\+\+).*/ {
+        yield meta.header;
+    } else if /-.*/ {
+        yield markup.deleted;
+    } else if /\+.*/ {
+        yield markup.inserted;
+    }
+}
diff --git a/crates/lsh/definitions/git_commit_messsage.lsh b/crates/lsh/definitions/git_commit_messsage.lsh
new file mode 100644
index 000000000000..d0896f5fd8d3
--- /dev/null
+++ b/crates/lsh/definitions/git_commit_messsage.lsh
@@ -0,0 +1,25 @@
+#[display_name = "Git Commit Message"]
+#[path = "**/COMMIT_EDITMSG"]
+#[path = "**/MERGE_MSG"]
+pub fn git_commit_message() {
+    if /#\s*/ {
+        yield comment;
+        if /(?:modified:|renamed:).*/ {
+            yield markup.changed;
+        } else if /deleted:.*/ {
+            yield markup.deleted;
+        } else if /new file:.*/ {
+            yield markup.inserted;
+        } else if /.*/ {
+        }
+    } else if /diff --git.*/ {
+        yield meta.header;
+        loop {
+            await input;
+            diff();
+            // diff() may not have eaten that line.
+            // It doesn't have to. That's our responsibility.
+            if /.*/ {}
+        }
+    }
+}
diff --git a/crates/lsh/definitions/git_ignore.lsh b/crates/lsh/definitions/git_ignore.lsh
new file mode 100644
index 000000000000..72da71fce16c
--- /dev/null
+++ b/crates/lsh/definitions/git_ignore.lsh
@@ -0,0 +1,8 @@
+#[display_name = "Ignore"]
+#[path = "**/.git-blame-ignore-revs"]
+#[path = "**/.gitignore"]
+pub fn git_ignore() {
+    if /#.*/ {
+        yield comment;
+    }
+}
diff --git a/crates/lsh/definitions/git_rebase_todo.lsh b/crates/lsh/definitions/git_rebase_todo.lsh
new file mode 100644
index 000000000000..ee6de0a4b9ae
--- /dev/null
+++ b/crates/lsh/definitions/git_rebase_todo.lsh
@@ -0,0 +1,27 @@
+#[display_name = "Git Rebase Message"]
+#[path = "**/git-rebase-todo"]
+#[path = "**/rebase-merge/done"]
+pub fn git_rebase_todo() {
+    if /#.*/ {
+        yield comment;
+    } else if /(?:break|exec|b|x)\b{end-half}/ {
+        yield keyword.control;
+        if /.*/ {
+            yield comment;
+        }
+    } else if /(?:drop|edit|fixup|pick|reword|squash|d|e|f|p|r|s)\b{end-half}/ {
+        yield keyword.control;
+
+        if /\s+/ {
+        }
+
+        if /\S+/ {
+            // Hash parameter
+            yield constant.numeric;
+
+            if /.*/ {
+                yield comment;
+            }
+        }
+    }
+}
diff --git a/crates/lsh/definitions/json.lsh b/crates/lsh/definitions/json.lsh
new file mode 100644
index 000000000000..b11af213e5b2
--- /dev/null
+++ b/crates/lsh/definitions/json.lsh
@@ -0,0 +1,37 @@
+#[display_name = "JSON"]
+#[path = "**/*.json"]
+#[path = "**/*.jsonc"]
+pub fn json() {
+    until /$/ {
+        yield other;
+
+        if /\/\/.*/ {
+            yield comment;
+        } else if /\/\*/ {
+            loop {
+                yield comment;
+                await input;
+                if /\*\// {
+                    yield comment;
+                    break;
+                }
+            }
+        } else if /"/ {
+            double_quote_string();
+        } else if /true|false|null/ {
+            if /\w+/ {
+                // Not a keyword after all.
+            } else {
+                yield constant.language;
+            }
+        } else if /-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/ {
+            if /\w+/ {
+                // Not a number after all.
+            } else {
+                yield constant.numeric;
+            }
+        } else if /\w+/ {
+            // Invalid token? Skip.
+        }
+    }
+}
diff --git a/crates/lsh/definitions/lsh.lsh b/crates/lsh/definitions/lsh.lsh
new file mode 100644
index 000000000000..7984a88b383f
--- /dev/null
+++ b/crates/lsh/definitions/lsh.lsh
@@ -0,0 +1,48 @@
+#[display_name = "LSH"]
+#[path = "**/*.lsh"]
+pub fn lsh() {
+    until /$/ {
+        yield other;
+
+        if /\/\/.*/ {
+            yield comment;
+        } else if /pub|fn|await/ {
+            if /\w+/ {
+                yield other;
+            } else {
+                yield keyword.other;
+            }
+        } else if /if|else|until|loop|break|continue/ {
+            if /\w+/ {
+                yield other;
+            } else {
+                yield keyword.control;
+            }
+        } else if /yield/ {
+            if /\w+/ {
+                yield other;
+            } else {
+                yield keyword.other;
+                if /\s+/ {
+                    // Gobble space to the argument
+                    yield other;
+                }
+                if /[\w.]+/ {
+                    // The yield argument
+                    yield markup.link;
+                }
+            }
+        } else if /\// {
+            until /$/ {
+                if /\\./ {
+                    // Skip escape char
+                } else if /\// {
+                    yield string;
+                    break;
+                }
+            }
+        } else if /"/ {
+            double_quote_string();
+        }
+    }
+}
diff --git a/crates/lsh/definitions/markdown.lsh b/crates/lsh/definitions/markdown.lsh
new file mode 100644
index 000000000000..de7ed73d6e60
--- /dev/null
+++ b/crates/lsh/definitions/markdown.lsh
@@ -0,0 +1,202 @@
+#[display_name = "Markdown"]
+#[path = "**/*.md"]
+pub fn markdown() {
+    // Gobble any leading whitespace on the line.
+    if /\s+/ {
+        yield other;
+    }
+
+    if /#+\s+.*/ {
+        yield markup.heading;
+    } else if />.*/ {
+        yield comment;
+    } else if /```/ {
+        // NOTE: These checks are sorted alphabetically.
+        if /(?i:diff)/ {
+            loop {
+                await input;
+                if /\s*```/ {
+                    return;
+                } else {
+                    diff();
+                    // diff() may not have eaten that line.
+                    // It doesn't have to. That's our responsibility.
+                    if /.*/ {}
+                }
+            }
+        } else if /(?i:json)/ {
+            loop {
+                await input;
+                if /\s*```/ {
+                    return;
+                } else {
+                    json();
+                    if /.*/ {}
+                }
+            }
+        } else if /(?i:yaml)/ {
+            loop {
+                await input;
+                if /\s*```/ {
+                    return;
+                } else {
+                    yaml();
+                    if /.*/ {}
+                }
+            }
+        } else if /(?i:pwsh|powershell)/ {
+            loop {
+                await input;
+                if /\s*```/ {
+                    return;
+                } else {
+                    powershell();
+                    if /.*/ {}
+                }
+            }
+        } else {
+            loop {
+                await input;
+                if /\s*```.*/ {
+                    return;
+                } else if /.*/ {
+                    // Gobble the rest of the line with no highlighting.
+                }
+            }
+        }
+    } else {
+        if /[\*-]\s+/ {
+            yield markup.list;
+        }
+
+        // The structure of inline emphasis, etc., matchers is
+        // awfully wrong, but parsing Markdown is also awful in general.
+        // In fact I think it's categorically impossible to implement like this,
+        // because it requires lookahead across potentially infinite lines:
+        // The Markdown spec describes its "parsing strategy" as "two phase"
+        // where the first phase segments the document into block structures.
+        // For instance, a `comment` may span multiple lines, _unless_
+        // there's a paragraph break inbetween, e.g. due to a ">" comment.
+        until /$/ {
+            yield other;
+
+            if /\[/ {
+                // Anything inside [...] is a link text and is colored like a string.
+                yield other;
+                if /[^\]]*/ {
+                    yield string;
+
+                    // Now look for the link target in parentheses.
+                    if /\]\(/ {
+                        yield other;
+                        if /[^\s\)]+/ {
+                            yield markup.link;
+                        }
+
+                        if /\s+/ {
+                        }
+
+                        // The link may have a title in quotes.
+                        yield other;
+                        if /"/ {
+                            if /[^"]+/ {
+                            }
+                            if /"/ {
+                                yield string;
+                            }
+                        }
+                    }
+
+                }
+            } else if /__/ {
+                if /_+/ {
+                    // "_____" = no hit
+                } else if /\s+/ {
+                    // "__ foo __" = no hit
+                } else {
+                    until /$/ {
+                        if /\\./ {
+                            // gobble escape char
+                        } else if /[^\s_]+__/ {
+                            yield markup.bold;
+                            break;
+                        } else if /__/ {
+                            break;
+                        }
+                    }
+                }
+            } else if /\*\*/ {
+                if /\s+/ {
+                    // "** foo **" = no hit
+                } else {
+                    until /$/ {
+                        if /\\./ {
+                            // gobble escape char
+                        } else if /[^\s*]+\*\*/ {
+                            yield markup.bold;
+                            break;
+                        } else if /\*\*/ {
+                            break;
+                        }
+                    }
+                }
+            } else if /_/ {
+                if /\s+/ {
+                    // "_ foo _" = no hit
+                } else {
+                    until /$/ {
+                        if /\\./ {
+                            // gobble escape char
+                        } else if /[^\s_]+_/ {
+                            yield markup.italic;
+                            break;
+                        } else if /_/ {
+                            break;
+                        }
+                    }
+                }
+            } else if /\*/ {
+                if /\s+/ {
+                    // "* foo *" = no hit
+                } else {
+                    until /$/ {
+                        if /\\./ {
+                            // gobble escape char
+                        } else if /[^\s*]+\*/ {
+                            yield markup.italic;
+                            break;
+                        } else if /\*/ {
+                            break;
+                        }
+                    }
+                }
+            } else if /~~/ {
+                if /\s+/ {
+                    // "~~ foo ~~" = no hit
+                } else {
+                    until /$/ {
+                        if /\\./ {
+                            // gobble escape char
+                        } else if /[^\s~]+~~/ {
+                            yield markup.strikethrough;
+                            break;
+                        } else if /~~/ {
+                            break;
+                        }
+                    }
+                }
+            } else if /```/ {
+                until /$/ { if /```/ { break; } }
+                yield string;
+            } else if /``/ {
+                until /$/ { if /``/ { break; } }
+                yield string;
+            } else if /`/ {
+                until /$/ { if /`/ { break; } }
+                yield string;
+            } else if /\\./ {
+                // Gobble escape char
+            }
+        }
+    }
+}
diff --git a/crates/lsh/definitions/powershell.lsh b/crates/lsh/definitions/powershell.lsh
new file mode 100644
index 000000000000..fefe96676fc4
--- /dev/null
+++ b/crates/lsh/definitions/powershell.lsh
@@ -0,0 +1,62 @@
+#[display_name = "PowerShell"]
+#[path = "**/*.ps1"]
+#[path = "**/*.psd1"]
+#[path = "**/*.psm1"]
+pub fn powershell() {
+    until /$/ {
+        yield other;
+
+        if /#.*/ {
+            yield comment;
+        } else if /<#/ {
+            loop {
+                yield comment;
+                if /#>/ { yield comment; break; }
+                await input;
+            }
+        } else if /'/ {
+            loop {
+                yield string;
+                if /\\./ {}
+                else if /'/ { yield string; break; }
+                await input;
+            }
+        } else if /@"/ {
+            loop {
+                yield string;
+                if /\\./ {}
+                else if /"@/ { yield string; break; }
+                await input;
+            }
+        } else if /"/ {
+            loop {
+                yield string;
+                if /\\./ {}
+                else if /"/ { yield string; break; }
+                await input;
+            }
+        } else if /function|param/ {
+            yield keyword.other;
+        } else if /elseif|else|if|for|switch|default|throw|try|catch|finally/ {
+            yield keyword.control;
+        } else if /-?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?/ {
+            if /[\w\-]+/ {
+                yield method;
+            } else {
+                yield constant.numeric;
+            }
+        } else if /\$false|\$true|\$null/ {
+            if /\w+/ {
+                yield variable;
+            } else {
+                yield constant.language;
+            }
+        } else if /[$@][\w\d?:]+/ {
+            yield variable;
+        } else if /[\w\d?:]+-[\w\d?:-]*/ {
+            yield method;
+        } else if /[\w\d?:]+/ {
+            // Gobble any other tokens that should not be highlighted
+        }
+    }
+}
diff --git a/crates/lsh/definitions/properties.lsh b/crates/lsh/definitions/properties.lsh
new file mode 100644
index 000000000000..cc770fffb2a1
--- /dev/null
+++ b/crates/lsh/definitions/properties.lsh
@@ -0,0 +1,52 @@
+#[display_name = "Properties"]
+// ini
+#[path = "**/*.ini"]
+// git
+#[path = "**/.config/git/config"]
+#[path = "**/.git/config"]
+#[path = "**/gitconfig"]
+#[path = "**/*.gitattributes"]
+#[path = "**/*.gitconfig"]
+#[path = "**/*.gitmodules"]
+// things that VS Code calls "properties" files
+#[path = "**/*.cfg"]
+#[path = "**/*.conf"]
+#[path = "**/*.directory"]
+#[path = "**/*.editorconfig"]
+#[path = "**/*.properties"]
+#[path = "**/*.repo"]
+// systemd
+#[path = "**/*.automount"]
+#[path = "**/*.dnssd"]
+#[path = "**/*.link"]
+#[path = "**/*.mount"]
+#[path = "**/*.netdev"]
+#[path = "**/*.network"]
+#[path = "**/*.nspawn"]
+#[path = "**/*.path"]
+#[path = "**/*.service"]
+#[path = "**/*.slice"]
+#[path = "**/*.socket"]
+#[path = "**/*.swap"]
+#[path = "**/*.systemd"]
+#[path = "**/*.target"]
+#[path = "**/*.timer"]
+pub fn properties() {
+    if /\[[^\[]+\]/ {
+        yield meta.header;
+    }
+
+    until /$/ {
+        yield other;
+
+        if /[;#].*/ {
+            yield comment;
+        } else if /([\w.-]+)\s*(=)\s*/ {
+            yield $1 as variable;
+        } else if /'/ {
+            single_quote_string();
+        } else if /"/ {
+            double_quote_string();
+        }
+    }
+}
diff --git a/crates/lsh/definitions/utility.lsh b/crates/lsh/definitions/utility.lsh
new file mode 100644
index 000000000000..d8f6be7ed76d
--- /dev/null
+++ b/crates/lsh/definitions/utility.lsh
@@ -0,0 +1,21 @@
+fn single_quote_string() {
+    until /$/ {
+        if /\\./ {
+            // Escape sequences
+        } else if /'/ {
+            yield string;
+            break;
+        }
+    }
+}
+
+fn double_quote_string() {
+    until /$/ {
+        if /\\./ {
+            // Escape sequences
+        } else if /"/ {
+            yield string;
+            break;
+        }
+    }
+}
diff --git a/crates/lsh/definitions/yaml.lsh b/crates/lsh/definitions/yaml.lsh
new file mode 100644
index 000000000000..a090447e3dd5
--- /dev/null
+++ b/crates/lsh/definitions/yaml.lsh
@@ -0,0 +1,78 @@
+#[display_name = "YAML"]
+#[path = "**/*.yaml"]
+#[path = "**/*.yml"]
+pub fn yaml() {
+    if /\s+/ {
+        // Leading whitespace
+    }
+
+    var indentation = off;
+
+    loop {
+        if /-\s+/ {
+            // List item
+        }
+        if /\w+:(?:\s+|$)/ {
+            // Key: (Value follows)
+        }
+
+        if /[|>][-+]?.*/ {
+            // If we see a multiline string inducer, capture content until indentation is back to before.
+            // Since we'll skip multiple lines, we need to use nested loops here.
+
+            loop {
+                // Wait for the next line in this multiline string.
+                await input;
+
+                // Skip leading whitespace
+                if /\s+/ {
+                }
+
+                if off <= indentation {
+                    // Once the new indentation is smaller, the multiline string has ended.
+                    break;
+                }
+
+                if /.*/ {
+                    yield string;
+                }
+            }
+        } else {
+            break;
+        }
+    }
+
+    yield other;
+
+    if /---/ {
+        yield other;
+    } else if /true|false|yes|no|null|~/ {
+        // If it looks like a keyword, check if it's actually a string (e.g. "foo: true false").
+        // TODO: Ideally this would not consume the last chunk of /\s+/ before a /# comment/.
+        if /\s*[^\s#]+[^#]*/ {
+            yield string;
+        } else {
+            yield constant.language;
+        }
+    } else if /-?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?/ {
+        // Same as with keywords above.
+        if /\s*[^\s#]+[^#]*/ {
+            yield string;
+        } else {
+            yield constant.numeric;
+        }
+    } else if /[^\s#]+[^#]*/ {
+        // Same as before: Gobble anything that looks like a string.
+        // TODO: This could be simplified & combined with the above if we stored the
+        // highlight kind into a variable and then yield the variable value itself.
+        yield string;
+    }
+
+    if /\s+/ {
+        yield other;
+    }
+
+    if /#.*/ {
+        yield comment;
+    }
+}
diff --git a/crates/lsh/src/compiler/backend.rs b/crates/lsh/src/compiler/backend.rs
new file mode 100644
index 000000000000..93ea8f583aee
--- /dev/null
+++ b/crates/lsh/src/compiler/backend.rs
@@ -0,0 +1,745 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Backend: liveness analysis, register allocation, and bytecode generation.
+//!
+//! ## Algorithm (Cornell CS 4120)
+//!
+//! Source: https://www.cs.cornell.edu/courses/cs4120/2022sp/notes/
+//!
+//! - Linearize IR nodes using DFS to assign instruction indices
+//! - Compute liveness using backward dataflow analysis
+//! - Compute live intervals from liveness sets
+//! - Linear scan register allocation
+//! - Generate bytecode with physical registers
+//!
+//! ## Why DFS ordering is essential
+//!
+//! Live intervals are `[min_index, max_index]` across all instructions where a vreg
+//! is live. BFS interleaves independent branches, making branch-local vregs appear
+//! to span the whole function.
+//!
+//! Example: `if /a/ { x = off } else if /b/ { y = off }` with BFS numbering:
+//! ```text
+//! [0: if /a/] [1: if /b/] [2: x=off] [3: y=off] [4: use x] [5: use y]
+//! ```
+//! Here `x` appears live [2,4] and `y` appears live [3,5], overlapping at [3,4].
+//!
+//! DFS numbers each branch contiguously: `[0: if /a/] [1: x=off] [2: use x] [3: if /b/] ...`
+//! Now `x` is live [1,2] and `y` is live [4,5] - no overlap, can share a register.
+//!
+//! ## Relocation system
+//!
+//! Jump targets aren't known during code generation (forward references). The backend
+//! emits placeholder zeros and records `Relocation` entries. After each function,
+//! `process_relocations()` patches the bytecode with resolved addresses.
+//!
+//! Two relocation types:
+//! - `ByName` - cross-function calls, resolved when the target function is compiled
+//! - `ByNode` - intra-function jumps to IR nodes not yet serialized
+//!
+//! ## Quirks
+//!
+//! - `IR.offset` starts as `usize::MAX` (unvisited). Codegen sets it to the bytecode
+//!   address. If we encounter a node with `offset != MAX`, it's a backward reference (loop)
+//!   We need to then emit a jump to the already-assigned address.
+//! - Physical registers have `physical = Some(...)`.
+//!   Liveness analysis ignores them (they're always "live").
+//!
+//! ## TODO
+//!
+//! - The liveness analysis is a much later addition so it doesn't fit into existing structures very well.
+//! - The linear scan allocator has spill logic but doesn't generate spill code.
+
+use std::collections::{HashMap, HashSet, VecDeque};
+
+use stdext::arena::scratch_arena;
+
+use super::*;
+
+#[derive(Debug, Clone, Copy)]
+enum Relocation<'a> {
+    ByName(usize, &'a str),
+    ByNode(usize, IRCell<'a>),
+}
+
+pub struct Backend<'a> {
+    assembly: Assembly<'a>,
+    relocations: Vec<Relocation<'a>>,
+    functions_seen: HashMap<&'a str, usize>,
+    charsets_seen: HashMap<*const Charset, usize>,
+    strings_seen: HashMap<*const str, usize>,
+}
+
+impl<'a> Backend<'a> {
+    pub fn new() -> Self {
+        Self {
+            assembly: Assembly {
+                instructions: Default::default(),
+                entrypoints: Default::default(),
+                charsets: Default::default(),
+                strings: Default::default(),
+                highlight_kinds: Default::default(),
+            },
+            relocations: Default::default(),
+            functions_seen: Default::default(),
+            charsets_seen: Default::default(),
+            strings_seen: Default::default(),
+        }
+    }
+
+    pub fn compile(mut self, compiler: &Compiler<'a>) -> CompileResult<Assembly<'a>> {
+        for function in &compiler.functions {
+            self.allocate_registers(function)?;
+
+            let entrypoint_offset = self.assembly.instructions.len();
+            self.functions_seen.insert(function.name, entrypoint_offset);
+            self.generate_code(function)?;
+            self.process_relocations();
+        }
+
+        if !self.relocations.is_empty() {
+            let names: String = self
+                .relocations
+                .iter()
+                .filter_map(|reloc| match reloc {
+                    Relocation::ByName(_, name) => Some(*name),
+                    Relocation::ByNode(_, _) => None,
+                })
+                .collect::<Vec<&str>>()
+                .join(", ");
+            return Err(CompileError {
+                path: String::new(),
+                line: 0,
+                column: 0,
+                message: if !names.is_empty() {
+                    format!("unresolved function call names: {names}")
+                } else {
+                    "unresolved IR nodes".to_string()
+                },
+            });
+        }
+
+        // Normally the runtime/engine would need to do bounds checks at all times to be safe,
+        // since there may be malformed bytecode (e.g. a bug in this compiler).
+        // We can fix that by padding the instruction stream with invalid opcodes at the end.
+        // This works as long as the runtime checks for valid opcodes. Even if the last valid
+        // opcode is chopped off (due to a bug above), the runtime can do an unchecked read
+        // of `MAX_ENCODED_SIZE` bytes without risking OOB access.
+        self.assembly.instructions.extend(std::iter::repeat_n(0xff, Instruction::MAX_ENCODED_SIZE));
+
+        self.assembly.entrypoints = compiler
+            .functions
+            .iter()
+            .filter(|f| f.public)
+            .map(|f| Entrypoint {
+                name: f.name.to_string(),
+                display_name: f.attributes.display_name.unwrap_or(f.name).to_string(),
+                paths: f.attributes.paths.iter().map(|s| s.to_string()).collect(),
+                address: f.body.borrow().offset,
+            })
+            .collect();
+        self.assembly.highlight_kinds = compiler.highlight_kinds.clone();
+
+        Ok(self.assembly)
+    }
+
+    /// Perform liveness analysis and register allocation for a function.
+    fn allocate_registers(&mut self, function: &Function<'a>) -> CompileResult<()> {
+        let mut analysis = LivenessAnalysis::new(function);
+        if analysis.is_empty() {
+            return Ok(());
+        }
+
+        analysis.compute_liveness();
+        let intervals = analysis.compute_intervals();
+        let allocation = self.linear_scan_allocation(intervals)?;
+        analysis.apply_allocation(&allocation);
+
+        Ok(())
+    }
+
+    /// Linear scan register allocation (Poletto-Sarkar).
+    ///
+    /// Processes intervals in order of start position, expiring old intervals
+    /// and allocating registers greedily. When out of registers, spills the
+    /// interval with the furthest end point.
+    fn linear_scan_allocation(
+        &mut self,
+        intervals: Vec<LiveInterval>,
+    ) -> CompileResult<HashMap<u32, Register>> {
+        let mut allocation: HashMap<u32, Register> = HashMap::new();
+
+        if intervals.is_empty() {
+            return Ok(allocation);
+        }
+
+        // Active intervals sorted by end position
+        let mut active: Vec<LiveInterval> = Vec::new();
+
+        // Available user registers
+        let mut available: Vec<Register> =
+            (Register::FIRST_USER_REG..Register::COUNT).rev().map(Register::from_usize).collect();
+
+        for interval in intervals {
+            // Expire intervals that ended before this one starts
+            active.retain(|active_interval| {
+                if active_interval.end < interval.start {
+                    if let Some(&reg) = allocation.get(&active_interval.vreg_id)
+                        && reg as usize >= Register::FIRST_USER_REG
+                    {
+                        available.push(reg);
+                    }
+                    false
+                } else {
+                    true
+                }
+            });
+
+            // Allocate or spill
+            if let Some(reg) = available.pop() {
+                allocation.insert(interval.vreg_id, reg);
+                active.push(interval);
+                active.sort_by_key(|i| i.end);
+            } else if let Some(last) = active.last() {
+                if last.end > interval.end {
+                    // Spill the longest-living active interval
+                    let spilled = active.pop().unwrap();
+                    if let Some(&reg) = allocation.get(&spilled.vreg_id) {
+                        allocation.remove(&spilled.vreg_id);
+                        allocation.insert(interval.vreg_id, reg);
+                        active.push(interval);
+                        active.sort_by_key(|i| i.end);
+                    }
+                }
+            } else {
+                // TODO: current interval gets spilled
+                return Err(CompileError {
+                    path: String::new(),
+                    line: 0,
+                    column: 0,
+                    message: "out of physical registers".to_string(),
+                });
+            }
+        }
+
+        Ok(allocation)
+    }
+
+    /// Generate bytecode for a function (assumes registers already allocated).
+    fn generate_code(&mut self, function: &Function<'a>) -> CompileResult<()> {
+        use Instruction::*;
+
+        let mut stack: VecDeque<IRCell<'a>> = VecDeque::new();
+        stack.push_back(function.body);
+
+        while let Some(ir_cell) = stack.pop_front() {
+            let mut ir = ir_cell.borrow_mut();
+
+            if ir.offset != usize::MAX {
+                // Already serialized
+                continue;
+            }
+
+            loop {
+                ir.offset = self.assembly.instructions.len();
+
+                match ir.instr {
+                    IRI::Noop => {}
+                    IRI::Mov { dst, src } => {
+                        let dst_reg = dst.borrow();
+                        let src_reg = src.borrow();
+                        let dst = dst_reg.physical.unwrap();
+                        let src = src_reg.physical.unwrap();
+                        self.push_instruction(Mov { dst, src });
+                    }
+                    IRI::MovImm { dst, imm } => {
+                        let dst_reg = dst.borrow();
+                        let dst = dst_reg.physical.unwrap();
+                        self.push_instruction(MovImm { dst, imm });
+                    }
+                    IRI::MovKind { dst, kind } => {
+                        let dst_reg = dst.borrow();
+                        let dst = dst_reg.physical.unwrap();
+                        self.push_instruction(MovImm { dst, imm: kind });
+                    }
+                    IRI::AddImm { dst, imm } => {
+                        let dst_reg = dst.borrow();
+                        let dst = dst_reg.physical.unwrap();
+                        self.push_instruction(AddImm { dst, imm });
+                    }
+                    IRI::If { condition, then } => {
+                        stack.push_back(then);
+
+                        match condition {
+                            Condition::Cmp { lhs, rhs, op } => {
+                                let lhs_phys = lhs.borrow().physical.unwrap();
+                                let rhs_phys = rhs.borrow().physical.unwrap();
+                                let tgt = self.dst_by_node(then) as u32;
+
+                                match op {
+                                    ComparisonOp::Eq => self.push_instruction(JumpEQ {
+                                        lhs: lhs_phys,
+                                        rhs: rhs_phys,
+                                        tgt,
+                                    }),
+                                    ComparisonOp::Ne => self.push_instruction(JumpNE {
+                                        lhs: lhs_phys,
+                                        rhs: rhs_phys,
+                                        tgt,
+                                    }),
+                                    ComparisonOp::Lt => self.push_instruction(JumpLT {
+                                        lhs: lhs_phys,
+                                        rhs: rhs_phys,
+                                        tgt,
+                                    }),
+                                    ComparisonOp::Gt => self.push_instruction(JumpGT {
+                                        lhs: lhs_phys,
+                                        rhs: rhs_phys,
+                                        tgt,
+                                    }),
+                                    ComparisonOp::Le => self.push_instruction(JumpLE {
+                                        lhs: lhs_phys,
+                                        rhs: rhs_phys,
+                                        tgt,
+                                    }),
+                                    ComparisonOp::Ge => self.push_instruction(JumpGE {
+                                        lhs: lhs_phys,
+                                        rhs: rhs_phys,
+                                        tgt,
+                                    }),
+                                }
+                            }
+                            Condition::EndOfLine => {
+                                let tgt = self.dst_by_node(then) as u32;
+                                self.push_instruction(JumpIfEndOfLine { tgt });
+                            }
+                            Condition::Charset(h) => {
+                                let idx = self.visit_charset(h) as u32;
+                                let tgt = self.dst_by_node(then) as u32;
+                                self.push_instruction(JumpIfMatchCharset { idx, tgt });
+                            }
+                            Condition::Prefix(s) => {
+                                let idx = self.visit_string(s) as u32;
+                                let tgt = self.dst_by_node(then) as u32;
+                                self.push_instruction(JumpIfMatchPrefix { idx, tgt });
+                            }
+                            Condition::PrefixInsensitive(s) => {
+                                let idx = self.visit_string(s) as u32;
+                                let tgt = self.dst_by_node(then) as u32;
+                                self.push_instruction(JumpIfMatchPrefixInsensitive { idx, tgt });
+                            }
+                        }
+                    }
+                    IRI::Call { name } => {
+                        let tgt = self.dst_by_name(name) as u32;
+                        self.push_instruction(Call { tgt });
+                    }
+                    IRI::Return => {
+                        self.push_instruction(Return);
+                    }
+                    IRI::Flush { kind } => {
+                        let kind = kind.borrow().physical.unwrap();
+                        self.push_instruction(FlushHighlight { kind });
+                    }
+                    IRI::AwaitInput => {
+                        self.push_instruction(AwaitInput);
+                    }
+                }
+
+                let Some(next) = ir.next else {
+                    break;
+                };
+
+                ir = next.borrow_mut();
+
+                if ir.offset != usize::MAX {
+                    self.push_instruction(MovImm {
+                        dst: Register::ProgramCounter,
+                        imm: ir.offset as u32,
+                    });
+                    break;
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn push_instruction(&mut self, instr: Instruction) {
+        if let Some(reloc) = self.relocations.last_mut() {
+            let offset = match reloc {
+                Relocation::ByName(off, _) => off,
+                Relocation::ByNode(off, _) => off,
+            };
+
+            if *offset == self.assembly.instructions.len()
+                && let Some(delta) = instr.address_offset()
+            {
+                *offset += delta;
+            }
+        }
+
+        let scratch = scratch_arena(None);
+        self.assembly.instructions.extend(instr.encode(&scratch));
+    }
+
+    fn visit_charset(&mut self, h: &'a Charset) -> usize {
+        *self.charsets_seen.entry(h as *const _).or_insert_with(|| {
+            let idx = self.assembly.charsets.len();
+            self.assembly.charsets.push(h);
+            idx
+        })
+    }
+
+    fn visit_string(&mut self, s: &'a str) -> usize {
+        *self.strings_seen.entry(s as *const _).or_insert_with(|| {
+            let idx = self.assembly.strings.len();
+            self.assembly.strings.push(s);
+            idx
+        })
+    }
+
+    fn dst_by_node(&mut self, ir: IRCell<'a>) -> usize {
+        let off = ir.borrow().offset;
+        if off != usize::MAX {
+            off
+        } else {
+            self.relocations.push(Relocation::ByNode(self.assembly.instructions.len(), ir));
+            0
+        }
+    }
+
+    fn dst_by_name(&mut self, name: &'a str) -> usize {
+        match self.functions_seen.get(name) {
+            Some(&dst) => dst,
+            None => {
+                self.relocations.push(Relocation::ByName(self.assembly.instructions.len(), name));
+                0
+            }
+        }
+    }
+
+    fn process_relocations(&mut self) {
+        self.relocations.retain_mut(|reloc| {
+            let (off, resolved) = match *reloc {
+                Relocation::ByName(off, name) => match self.functions_seen.get(name) {
+                    None => return true,
+                    Some(&resolved) => (off, resolved),
+                },
+                Relocation::ByNode(off, node) => match node.borrow().offset {
+                    usize::MAX => return true,
+                    resolved => (off, resolved),
+                },
+            };
+
+            let range = off..off + 4;
+            if let Some(target) = self.assembly.instructions.get_mut(range) {
+                target.copy_from_slice(&(resolved as u32).to_le_bytes());
+            } else {
+                panic!("Unexpected relocation target offset: {off}");
+            }
+
+            false
+        });
+    }
+}
+
+/// A live interval represents the range of instructions where a vreg is live.
+#[derive(Debug, Clone, Copy)]
+struct LiveInterval {
+    vreg_id: u32,
+    start: usize,
+    end: usize,
+}
+
+/// Encapsulates liveness analysis state for a single function.
+///
+/// Performs DFS linearization, builds the CFG, and computes liveness sets.
+/// The analysis owns all intermediate data structures, exposing only what's
+/// needed for register allocation.
+struct LivenessAnalysis<'a> {
+    /// IR nodes in DFS order (instruction indices correspond to positions here).
+    nodes: Vec<IRCell<'a>>,
+    /// CFG: successors[i] contains indices of nodes that can follow node i.
+    successors: Vec<Vec<usize>>,
+    /// Map from vreg ID to its IRRegCell (for applying allocation results).
+    vreg_cells: HashMap<u32, IRRegCell<'a>>,
+    /// Liveness sets: live_in[i] = vregs live at entry to instruction i.
+    live_in: Vec<HashSet<u32>>,
+    /// Liveness sets: live_out[i] = vregs live at exit from instruction i.
+    live_out: Vec<HashSet<u32>>,
+}
+
+impl<'a> LivenessAnalysis<'a> {
+    /// Create a new liveness analysis for a function.
+    ///
+    /// This linearizes the IR using DFS and builds the CFG. Call `compute_liveness()`
+    /// to fill in the liveness sets, then `compute_intervals()` to get live intervals.
+    fn new(function: &Function<'a>) -> Self {
+        let mut nodes = Vec::new();
+        let mut node_to_idx = HashMap::new();
+        let mut vreg_cells = HashMap::new();
+        let mut visited = HashSet::new();
+
+        // DFS is essential for correctness. See module docs for details.
+        Self::dfs(function.body, &mut nodes, &mut node_to_idx, &mut vreg_cells, &mut visited);
+
+        // Build successor relationships from the node_to_idx map
+        let successors = Self::build_successors(&nodes, &node_to_idx);
+
+        let n = nodes.len();
+        Self {
+            nodes,
+            successors,
+            vreg_cells,
+            live_in: vec![HashSet::new(); n],
+            live_out: vec![HashSet::new(); n],
+        }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.nodes.is_empty()
+    }
+
+    /// DFS traversal to linearize IR nodes.
+    ///
+    /// Visits "then" branches before "next" (fallthrough) to keep branch
+    /// instructions contiguous in the numbering.
+    fn dfs(
+        cell: IRCell<'a>,
+        nodes: &mut Vec<IRCell<'a>>,
+        node_to_idx: &mut HashMap<*const RefCell<IR<'a>>, usize>,
+        vreg_cells: &mut HashMap<u32, IRRegCell<'a>>,
+        visited: &mut HashSet<*const RefCell<IR<'a>>>,
+    ) {
+        if !visited.insert(cell as *const _) {
+            return;
+        }
+
+        let idx = nodes.len();
+        node_to_idx.insert(cell as *const _, idx);
+        nodes.push(cell);
+
+        let ir = cell.borrow();
+
+        match ir.instr {
+            IRI::Mov { dst, src } => {
+                if dst.borrow().physical.is_none() {
+                    vreg_cells.insert(dst.borrow().id, dst);
+                }
+                if src.borrow().physical.is_none() {
+                    vreg_cells.insert(src.borrow().id, src);
+                }
+            }
+            IRI::MovImm { dst, .. } => {
+                if dst.borrow().physical.is_none() {
+                    vreg_cells.insert(dst.borrow().id, dst);
+                }
+            }
+            IRI::MovKind { dst, .. } => {
+                if dst.borrow().physical.is_none() {
+                    vreg_cells.insert(dst.borrow().id, dst);
+                }
+            }
+            IRI::AddImm { dst, .. } => {
+                if dst.borrow().physical.is_none() {
+                    vreg_cells.insert(dst.borrow().id, dst);
+                }
+            }
+            IRI::If { condition: Condition::Cmp { lhs, rhs, .. }, .. } => {
+                if lhs.borrow().physical.is_none() {
+                    vreg_cells.insert(lhs.borrow().id, lhs);
+                }
+                if rhs.borrow().physical.is_none() {
+                    vreg_cells.insert(rhs.borrow().id, rhs);
+                }
+            }
+            IRI::Flush { kind, .. } => {
+                if kind.borrow().physical.is_none() {
+                    vreg_cells.insert(kind.borrow().id, kind);
+                }
+            }
+            _ => {}
+        }
+
+        // Visit "then" branch first (DFS into branches), then "next" (fallthrough).
+        if let IRI::If { then, .. } = ir.instr {
+            Self::dfs(then, nodes, node_to_idx, vreg_cells, visited);
+            let ir = cell.borrow();
+            if let Some(next) = ir.next {
+                Self::dfs(next, nodes, node_to_idx, vreg_cells, visited);
+            }
+        } else if let Some(next) = ir.next {
+            Self::dfs(next, nodes, node_to_idx, vreg_cells, visited);
+        }
+    }
+
+    /// Build CFG successor relationships from the linearized nodes.
+    fn build_successors(
+        nodes: &[IRCell<'a>],
+        node_to_idx: &HashMap<*const RefCell<IR<'a>>, usize>,
+    ) -> Vec<Vec<usize>> {
+        let mut successors = vec![Vec::new(); nodes.len()];
+        for (idx, cell) in nodes.iter().enumerate() {
+            let ir = cell.borrow();
+            if let Some(next) = ir.next
+                && let Some(&next_idx) = node_to_idx.get(&(next as *const _))
+            {
+                successors[idx].push(next_idx);
+            }
+            if let IRI::If { then, .. } = ir.instr
+                && let Some(&then_idx) = node_to_idx.get(&(then as *const _))
+            {
+                successors[idx].push(then_idx);
+            }
+        }
+        successors
+    }
+
+    /// Compute liveness using backward dataflow analysis (worklist algorithm).
+    ///
+    /// Dataflow equations:
+    /// - `in[n] = use[n] ∪ (out[n] - def[n])`
+    /// - `out[n] = ∪_{s ∈ succ(n)} in[s]`
+    fn compute_liveness(&mut self) {
+        let n = self.nodes.len();
+        if n == 0 {
+            return;
+        }
+
+        // Build predecessors from successors (needed for worklist propagation)
+        let mut predecessors: Vec<Vec<usize>> = vec![Vec::new(); n];
+        for (idx, succs) in self.successors.iter().enumerate() {
+            for &succ in succs {
+                predecessors[succ].push(idx);
+            }
+        }
+
+        // Worklist: start with all nodes
+        let mut worklist: VecDeque<usize> = (0..n).collect();
+        let mut in_worklist = vec![true; n];
+
+        while let Some(idx) = worklist.pop_front() {
+            in_worklist[idx] = false;
+
+            // out[n] = ∪_{s ∈ succ(n)} in[s]
+            let mut new_out = HashSet::new();
+            for &succ in &self.successors[idx] {
+                new_out.extend(self.live_in[succ].iter().copied());
+            }
+
+            // in[n] = use[n] ∪ (out[n] - def[n])
+            let (use_set, def_set) = Self::compute_use_def(&self.nodes[idx].borrow());
+            let mut new_in = use_set;
+            for &vreg in &new_out {
+                if !def_set.contains(&vreg) {
+                    new_in.insert(vreg);
+                }
+            }
+
+            // If in[n] changed, propagate to predecessors
+            if new_in != self.live_in[idx] {
+                self.live_in[idx] = new_in;
+                for &pred in &predecessors[idx] {
+                    if !in_worklist[pred] {
+                        worklist.push_back(pred);
+                        in_worklist[pred] = true;
+                    }
+                }
+            }
+
+            self.live_out[idx] = new_out;
+        }
+    }
+
+    /// Compute use and def sets for a single IR instruction.
+    fn compute_use_def(ir: &IR<'a>) -> (HashSet<u32>, HashSet<u32>) {
+        let mut use_set = HashSet::new();
+        let mut def_set = HashSet::new();
+
+        match ir.instr {
+            IRI::Mov { dst, src } => {
+                let dst_reg = dst.borrow();
+                if dst_reg.physical.is_none() {
+                    def_set.insert(dst_reg.id);
+                }
+                let src_reg = src.borrow();
+                if src_reg.physical.is_none() {
+                    use_set.insert(src_reg.id);
+                }
+            }
+            IRI::MovImm { dst, .. } => {
+                let dst_reg = dst.borrow();
+                if dst_reg.physical.is_none() {
+                    def_set.insert(dst_reg.id);
+                }
+            }
+            IRI::MovKind { dst, .. } => {
+                let dst_reg = dst.borrow();
+                if dst_reg.physical.is_none() {
+                    def_set.insert(dst_reg.id);
+                }
+            }
+            IRI::AddImm { dst, .. } => {
+                let dst_reg = dst.borrow();
+                if dst_reg.physical.is_none() {
+                    def_set.insert(dst_reg.id);
+                }
+            }
+            IRI::If { condition: Condition::Cmp { lhs, rhs, .. }, .. } => {
+                let lhs_reg = lhs.borrow();
+                if lhs_reg.physical.is_none() {
+                    use_set.insert(lhs_reg.id);
+                }
+                let rhs_reg = rhs.borrow();
+                if rhs_reg.physical.is_none() {
+                    use_set.insert(rhs_reg.id);
+                }
+            }
+            IRI::Flush { kind, .. } => {
+                let kind_reg = kind.borrow();
+                if kind_reg.physical.is_none() {
+                    use_set.insert(kind_reg.id);
+                }
+            }
+            _ => {}
+        }
+
+        (use_set, def_set)
+    }
+
+    /// Compute live intervals from liveness sets, sorted by start position.
+    fn compute_intervals(&self) -> Vec<LiveInterval> {
+        let mut vreg_ranges: HashMap<u32, (usize, usize)> = HashMap::new();
+
+        for idx in 0..self.nodes.len() {
+            for &vreg_id in self.live_in[idx].iter().chain(self.live_out[idx].iter()) {
+                vreg_ranges
+                    .entry(vreg_id)
+                    .and_modify(|(start, end)| {
+                        *start = (*start).min(idx);
+                        *end = (*end).max(idx);
+                    })
+                    .or_insert((idx, idx));
+            }
+        }
+
+        let mut intervals: Vec<LiveInterval> = vreg_ranges
+            .into_iter()
+            .map(|(vreg_id, (start, end))| LiveInterval { vreg_id, start, end })
+            .collect();
+
+        intervals.sort_by_key(|i| i.start);
+        intervals
+    }
+
+    /// Apply register allocation results to the IRReg cells.
+    fn apply_allocation(&self, allocation: &HashMap<u32, Register>) {
+        for (&vreg_id, &reg) in allocation {
+            if let Some(cell) = self.vreg_cells.get(&vreg_id) {
+                cell.borrow_mut().physical = Some(reg);
+            }
+        }
+    }
+}
diff --git a/crates/lsh/src/compiler/frontend.rs b/crates/lsh/src/compiler/frontend.rs
new file mode 100644
index 000000000000..6a9c64a662ae
--- /dev/null
+++ b/crates/lsh/src/compiler/frontend.rs
@@ -0,0 +1,729 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Parser: transforms DSL source into IR graph.
+//!
+//! ## Gotchas
+//!
+//! - Variables are in SSA-ish form: `var x = off; x = x + 1;` creates a new vreg for the
+//!   second `x`. But the variable *name* still maps to the new vreg, so later reads see it.
+//! - Physical registers (off, etc.) don't follow SSA. Reading them doesn't create a vreg;
+//!   the `parse_expression` function handles this by copying to a fresh vreg.
+//! - The `raise!` macro captures tokenizer position at call time. Don't advance() before raising.
+
+use std::collections::HashMap;
+use std::collections::hash_map::Entry;
+
+use super::tokenizer::*;
+use super::*;
+
+macro_rules! raise {
+    ($self:ident, $msg:literal) => {{
+        let path = $self.path.to_string();
+        let (line, column) = $self.tokenizer.position();
+        return Err(CompileError { path, line, column, message: $msg.to_string() })
+    }};
+    ($self:ident, $($arg:tt)*) => {{
+        let path = $self.path.to_string();
+        let (line, column) = $self.tokenizer.position();
+        return Err(CompileError { path, line, column, message: format!($($arg)*) })
+    }};
+}
+
+struct RegexSpan<'a> {
+    pub src: IRCell<'a>,
+    pub dst_good: IRCell<'a>,
+    pub dst_bad: IRCell<'a>,
+    pub capture_groups: Vec<(IRRegCell<'a>, IRRegCell<'a>), &'a Arena>,
+}
+
+enum LoopFlowControlKind {
+    Break,
+    Continue,
+}
+
+struct LoopFlowControl<'a> {
+    line: usize,
+    column: usize,
+    kind: LoopFlowControlKind,
+    ir: IRCell<'a>,
+}
+
+struct Context<'a> {
+    loop_start: Option<IRCell<'a>>,
+    loop_exit: Option<IRCell<'a>>,
+    capture_groups: Vec<(IRRegCell<'a>, IRRegCell<'a>), &'a Arena>,
+}
+
+pub struct Parser<'a, 'c, 'src> {
+    compiler: &'c mut Compiler<'a>,
+    path: &'src str,
+    tokenizer: Tokenizer<'src>,
+    current_token: Token<'src>,
+    context: Vec<Context<'a>, &'a Arena>,
+    variables: HashMap<&'src str, IRRegCell<'a>>,
+}
+
+impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
+    pub fn new(compiler: &'c mut Compiler<'a>, path: &'src str, src: &'src str) -> Self {
+        let context = Vec::new_in(compiler.arena);
+        Self {
+            compiler,
+            path,
+            tokenizer: Tokenizer::new(src),
+            current_token: Token::Eof,
+            context,
+            variables: Default::default(),
+        }
+    }
+
+    pub fn run(&mut self) -> CompileResult<()> {
+        self.advance();
+        while !matches!(self.current_token, Token::Eof) {
+            let f = self.parse_function()?;
+            self.compiler.functions.push(f);
+        }
+        Ok(())
+    }
+
+    fn parse_attributes(&mut self) -> CompileResult<FunctionAttributes<'a>> {
+        let mut attributes = FunctionAttributes::default();
+
+        while self.current_token == Token::Hash {
+            self.advance();
+            self.expect_token(Token::LeftBracket)?;
+
+            let key = match self.current_token {
+                Token::Identifier(k) => k,
+                _ => raise!(self, "expected attribute name"),
+            };
+            self.advance();
+
+            self.expect_token(Token::Equals)?;
+
+            let value = match self.current_token {
+                Token::String(s) => arena_clone_str(self.compiler.arena, s),
+                _ => raise!(self, "expected attribute value"),
+            };
+            self.advance();
+
+            self.expect_token(Token::RightBracket)?;
+
+            match key {
+                "display_name" => attributes.display_name = Some(value),
+                "path" => attributes.paths.push(value),
+                _ => raise!(self, "unknown attribute '{}'", key),
+            }
+        }
+
+        Ok(attributes)
+    }
+
+    fn parse_function(&mut self) -> CompileResult<Function<'a>> {
+        // Reset symbol table for new function
+        self.variables =
+            HashMap::from_iter([("off", self.compiler.get_reg(Register::InputOffset))]);
+
+        let attributes = self.parse_attributes()?;
+
+        let public = if matches!(self.current_token, Token::Pub) {
+            self.advance();
+            true
+        } else {
+            false
+        };
+
+        self.expect_token(Token::Fn)?;
+
+        let name = match self.current_token {
+            Token::Identifier(n) => arena_clone_str(self.compiler.arena, n),
+            _ => raise!(self, "expected function name"),
+        };
+        self.advance();
+
+        self.expect_token(Token::LeftParen)?;
+        self.expect_token(Token::RightParen)?;
+
+        let span = self.parse_block()?;
+
+        if let mut last = span.last.borrow_mut()
+            && last.wants_next()
+        {
+            last.set_next(self.compiler.alloc_iri(IRI::Return));
+        }
+
+        Ok(Function { name, attributes, body: span.first, public })
+    }
+
+    fn parse_block(&mut self) -> CompileResult<IRSpan<'a>> {
+        self.expect_token(Token::LeftBrace)?;
+
+        // TODO: a bit inoptimal to always allocate a noop node
+        let mut result: Option<IRSpan> = None;
+
+        while !matches!(self.current_token, Token::RightBrace | Token::Eof) {
+            let s = self.parse_statement()?;
+            if let Some(span) = &mut result {
+                span.last.borrow_mut().set_next(s.first);
+                span.last = s.last;
+            } else {
+                result = Some(s);
+            }
+        }
+
+        self.expect_token(Token::RightBrace)?;
+        Ok(match result {
+            Some(span) => span,
+            None => IRSpan::single(self.compiler.alloc_noop()),
+        })
+    }
+
+    fn parse_statement(&mut self) -> CompileResult<IRSpan<'a>> {
+        match self.current_token {
+            Token::Var => self.parse_var_declaration(),
+            Token::Loop => self.parse_loop(),
+            Token::Until => self.parse_until(),
+            Token::Break => self.parse_break(),
+            Token::Continue => self.parse_continue(),
+            Token::Return => self.parse_return(),
+            Token::If => self.parse_if(),
+            Token::Await => self.parse_await(),
+            Token::Yield => self.parse_yield(),
+            Token::Identifier(_) => self.parse_identifier(),
+            _ => raise!(self, "unexpected token: {:?}", self.current_token),
+        }
+    }
+
+    fn parse_loop(&mut self) -> CompileResult<IRSpan<'a>> {
+        self.expect_token(Token::Loop)?;
+
+        let loop_start = self.compiler.alloc_noop();
+        let loop_exit = self.compiler.alloc_noop();
+        self.parse_until_impl(loop_start, loop_start, loop_exit)
+    }
+
+    fn parse_until(&mut self) -> CompileResult<IRSpan<'a>> {
+        self.expect_token(Token::Until)?;
+
+        let re = self.parse_if_regex()?;
+
+        let loop_exit = self.compiler.alloc_noop();
+        re.dst_good.borrow_mut().set_next(loop_exit);
+        self.parse_until_impl(re.src, re.dst_bad, loop_exit)
+    }
+
+    fn parse_until_impl(
+        &mut self,
+        loop_start: IRCell<'a>,
+        loop_good: IRCell<'a>,
+        loop_exit: IRCell<'a>,
+    ) -> CompileResult<IRSpan<'a>> {
+        // First, save the current input offset.
+        // This is used to detect if the loop made any progress.
+        let saved_offset = self.compiler.alloc_vreg();
+        let first = self.compiler.alloc_iri(IRI::Mov {
+            dst: saved_offset,
+            src: self.compiler.get_reg(Register::InputOffset),
+        });
+
+        self.context.push(Context {
+            loop_start: Some(loop_start),
+            loop_exit: Some(loop_exit),
+            capture_groups: Vec::new_in(self.compiler.arena),
+        });
+        let block = self.parse_block()?;
+        self.context.pop();
+
+        // Force advance the input offset by 1 if it got stuck in the loop iteration.
+        //   if input_offset == saved_offset {
+        //       input_offset += 1;
+        //   }
+        let advance = self.compiler.alloc_ir(IR {
+            next: Some(first),
+            instr: IRI::AddImm { dst: self.compiler.get_reg(Register::InputOffset), imm: 1 },
+            offset: usize::MAX,
+        });
+        let advance_check = self.compiler.alloc_ir(IR {
+            next: Some(first),
+            instr: IRI::If {
+                condition: Condition::Cmp {
+                    lhs: self.compiler.get_reg(Register::InputOffset),
+                    rhs: saved_offset,
+                    op: ComparisonOp::Eq,
+                },
+                then: advance,
+            },
+            offset: usize::MAX,
+        });
+
+        // Skip any uninteresting characters before the next loop iteration.
+        //   if /.*?/ {}
+        let interesting = self.compiler.collect_interesting_charset(block.first);
+        let fast_skip = if interesting.covers_all() {
+            advance_check
+        } else {
+            let mut skip_charset = interesting.clone();
+            skip_charset.invert();
+            let skip_charset = self.compiler.intern_charset(&skip_charset);
+            self.compiler.alloc_ir(IR {
+                next: Some(advance_check),
+                instr: IRI::If { condition: Condition::Charset(skip_charset), then: advance_check },
+                offset: usize::MAX,
+            })
+        };
+
+        first.borrow_mut().set_next(loop_start);
+        loop_good.borrow_mut().set_next(block.first);
+
+        if let mut block_last = block.last.borrow_mut()
+            && block_last.wants_next()
+        {
+            block_last.set_next(fast_skip);
+        }
+
+        Ok(IRSpan { first, last: loop_exit })
+    }
+
+    fn parse_break(&mut self) -> CompileResult<IRSpan<'a>> {
+        self.expect_token(Token::Break)?;
+        self.expect_token(Token::Semicolon)?;
+
+        if let Some(exit) = self.context.last_mut().and_then(|ctx| ctx.loop_exit) {
+            let ir = self.compiler.alloc_noop();
+            ir.borrow_mut().set_next(exit);
+            Ok(IRSpan::single(ir))
+        } else {
+            raise!(self, "loop control statement outside of a loop")
+        }
+    }
+
+    fn parse_continue(&mut self) -> CompileResult<IRSpan<'a>> {
+        self.expect_token(Token::Continue)?;
+        self.expect_token(Token::Semicolon)?;
+
+        if let Some(start) = self.context.last_mut().and_then(|ctx| ctx.loop_start) {
+            let ir = self.compiler.alloc_noop();
+            ir.borrow_mut().set_next(start);
+            Ok(IRSpan::single(ir))
+        } else {
+            raise!(self, "loop control statement outside of a loop")
+        }
+    }
+
+    fn parse_return(&mut self) -> CompileResult<IRSpan<'a>> {
+        self.expect_token(Token::Return)?;
+        self.expect_token(Token::Semicolon)?;
+        Ok(IRSpan::single(self.compiler.alloc_iri(IRI::Return)))
+    }
+
+    fn parse_if(&mut self) -> CompileResult<IRSpan<'a>> {
+        self.expect_token(Token::If)?;
+
+        if matches!(self.current_token, Token::Regex(_)) {
+            self.parse_if_regex_chain()
+        } else {
+            self.parse_if_comparison()
+        }
+    }
+
+    fn parse_if_regex_chain(&mut self) -> CompileResult<IRSpan<'a>> {
+        let mut prev: Option<IRCell<'a>> = None;
+
+        // First, save the current input offset.
+        // This is used to restore the position on failed matches.
+        let save_reg = self.compiler.alloc_vreg();
+        let first = self.compiler.alloc_iri(IRI::Mov {
+            dst: save_reg,
+            src: self.compiler.get_reg(Register::InputOffset),
+        });
+
+        let last = self.compiler.alloc_noop();
+
+        loop {
+            let re = self.parse_if_regex()?;
+
+            // Push context with capture groups for the block
+            let (loop_start, loop_exit) = self
+                .context
+                .last()
+                .map(|ctx| (ctx.loop_start, ctx.loop_exit))
+                .unwrap_or((None, None));
+            self.context.push(Context { loop_start, loop_exit, capture_groups: re.capture_groups });
+            let bl = self.parse_block()?;
+            self.context.pop();
+
+            // Connect the previous else branch to form an "else if".
+            // If there's no previous one, we're in the first iteration,
+            // and so we connect it to the instruction that saves the position.
+            prev.unwrap_or(first).borrow_mut().set_next(re.src);
+
+            // Connect the if to the {}.
+            re.dst_good.borrow_mut().set_next(bl.first);
+            // Connect the end of the {} to the end of the if/else chain.
+            if let mut block_last = bl.last.borrow_mut()
+                && block_last.wants_next()
+            {
+                block_last.set_next(last);
+            }
+
+            // The "else" branch of the if needs to restore the position.
+            let dst_bad = self.compiler.alloc_iri(IRI::Mov {
+                dst: self.compiler.get_reg(Register::InputOffset),
+                src: save_reg,
+            });
+            re.dst_bad.borrow_mut().set_next(dst_bad);
+
+            // No else branch? dst_bad (= no hit) means we're done, so make that connection.
+            if !matches!(self.current_token, Token::Else) {
+                dst_bad.borrow_mut().set_next(last);
+                break;
+            }
+
+            // Gobble the "else" token.
+            self.advance();
+
+            // The else branch has a block? That's our dst_bad.
+            if !matches!(self.current_token, Token::If) {
+                let bl = self.parse_block()?;
+                dst_bad.borrow_mut().set_next(bl.first);
+                // Connect the end of the {} to the end of the if/else chain.
+                if let mut bl_last = bl.last.borrow_mut()
+                    && bl_last.wants_next()
+                {
+                    bl_last.set_next(last);
+                }
+                break;
+            }
+
+            // Otherwise, we expect an "if" in the next iteration to form an "else if".
+            self.expect_token(Token::If)?;
+            prev = Some(dst_bad);
+        }
+
+        Ok(IRSpan { first, last })
+    }
+
+    fn parse_if_comparison(&mut self) -> CompileResult<IRSpan<'a>> {
+        // Parse: if var1 OP var2 { block } where OP is ==, !=, <, >, <=, >=
+        let lhs_name = match self.current_token {
+            Token::Identifier(n) => n,
+            _ => raise!(self, "expected variable name"),
+        };
+        let lhs_vreg = self.get_variable(lhs_name)?;
+        self.advance();
+
+        let op = match self.current_token {
+            Token::EqualsEquals => ComparisonOp::Eq,
+            Token::NotEquals => ComparisonOp::Ne,
+            Token::LessThan => ComparisonOp::Lt,
+            Token::GreaterThan => ComparisonOp::Gt,
+            Token::LessThanEquals => ComparisonOp::Le,
+            Token::GreaterThanEquals => ComparisonOp::Ge,
+            _ => raise!(self, "expected comparison operator (==, !=, <, >, <=, >=)"),
+        };
+        self.advance();
+
+        let rhs_name = match self.current_token {
+            Token::Identifier(n) => n,
+            _ => raise!(self, "expected variable name"),
+        };
+        let rhs_vreg = self.get_variable(rhs_name)?;
+        self.advance();
+
+        let dst_good = self.compiler.alloc_noop();
+        let dst_bad = self.compiler.alloc_noop();
+        let cmp = self.compiler.alloc_ir(IR {
+            next: Some(dst_bad),
+            instr: IRI::If {
+                condition: Condition::Cmp { lhs: lhs_vreg, rhs: rhs_vreg, op },
+                then: dst_good,
+            },
+            offset: usize::MAX,
+        });
+
+        let bl = self.parse_block()?;
+        dst_good.borrow_mut().set_next(bl.first);
+
+        let end = self.compiler.alloc_noop();
+
+        // Handle optional else branch
+        if matches!(self.current_token, Token::Else) {
+            self.advance();
+
+            // Check for else if
+            if matches!(self.current_token, Token::If) {
+                let else_if_span = self.parse_if()?;
+                dst_bad.borrow_mut().set_next(else_if_span.first);
+                if let mut block_last = bl.last.borrow_mut()
+                    && block_last.wants_next()
+                {
+                    block_last.set_next(end);
+                }
+                if let mut else_if_last = else_if_span.last.borrow_mut()
+                    && else_if_last.wants_next()
+                {
+                    else_if_last.set_next(end);
+                }
+                Ok(IRSpan { first: cmp, last: end })
+            } else {
+                let else_bl = self.parse_block()?;
+                dst_bad.borrow_mut().set_next(else_bl.first);
+                if let mut block_last = bl.last.borrow_mut()
+                    && block_last.wants_next()
+                {
+                    block_last.set_next(end);
+                }
+                if let mut else_last = else_bl.last.borrow_mut()
+                    && else_last.wants_next()
+                {
+                    else_last.set_next(end);
+                }
+                Ok(IRSpan { first: cmp, last: end })
+            }
+        } else {
+            dst_bad.borrow_mut().set_next(end);
+            if let mut block_last = bl.last.borrow_mut()
+                && block_last.wants_next()
+            {
+                block_last.set_next(end);
+            }
+            Ok(IRSpan { first: cmp, last: end })
+        }
+    }
+
+    fn parse_if_regex(&mut self) -> CompileResult<RegexSpan<'a>> {
+        let pattern = match self.current_token {
+            Token::Regex(s) => s,
+            _ => raise!(self, "expected regex"),
+        };
+        let dst_good = self.compiler.alloc_noop();
+        let dst_bad = self.compiler.alloc_noop();
+        let mut capture_groups = Vec::new_in(self.compiler.arena);
+        let src = match regex::parse(self.compiler, pattern, dst_good, dst_bad, &mut capture_groups)
+        {
+            Ok(s) => s,
+            Err(e) => raise!(self, "{}", e),
+        };
+
+        self.advance();
+
+        Ok(RegexSpan { src, dst_good, dst_bad, capture_groups })
+    }
+
+    fn parse_await(&mut self) -> CompileResult<IRSpan<'a>> {
+        self.expect_token(Token::Await)?;
+
+        match self.current_token {
+            Token::Identifier("input") => self.advance(),
+            _ => raise!(self, "expected 'input' after await"),
+        }
+
+        self.expect_token(Token::Semicolon)?;
+
+        let ir = self.compiler.alloc_iri(IRI::AwaitInput);
+        Ok(IRSpan::single(ir))
+    }
+
+    fn parse_yield(&mut self) -> CompileResult<IRSpan<'a>> {
+        self.expect_token(Token::Yield)?;
+
+        // Check if this is a capture group reference: yield $n as color
+        if let Token::Submatch(capture_index) = self.current_token {
+            let capture_index = capture_index as usize - 1;
+            self.advance();
+
+            // Expect "as"
+            match self.current_token {
+                Token::Identifier("as") => self.advance(),
+                _ => raise!(self, "expected 'as' after capture group reference"),
+            }
+
+            let color = match self.current_token {
+                Token::Identifier(c) => c,
+                _ => raise!(self, "expected color name after 'as'"),
+            };
+            let kind = self.compiler.intern_highlight_kind(color).value;
+            self.advance();
+            self.expect_token(Token::Semicolon)?;
+
+            let (start_vreg, end_vreg) = match self.context.last() {
+                Some(ctx) if capture_index < ctx.capture_groups.len() => {
+                    ctx.capture_groups[capture_index]
+                }
+                Some(_) => raise!(
+                    self,
+                    "capture group ${} not found in current context",
+                    capture_index + 1
+                ),
+                None => raise!(self, "no regex context available for capture group reference"),
+            };
+
+            let kind_vreg = self.compiler.alloc_vreg();
+
+            let span = self
+                .compiler
+                .build_chain()
+                // Set HighlightStart to the start of the capture group
+                .append(IRI::Mov {
+                    dst: self.compiler.get_reg(Register::HighlightStart),
+                    src: start_vreg,
+                })
+                // Set HighlightKind to the color
+                .append(IRI::MovKind { dst: kind_vreg, kind })
+                // Set InputOffset to the end of the capture group and flush
+                .append(IRI::Mov {
+                    dst: self.compiler.get_reg(Register::InputOffset),
+                    src: end_vreg,
+                })
+                .append(IRI::Flush { kind: kind_vreg })
+                .build();
+            Ok(span)
+        } else {
+            // Normal yield: yield color;
+            let color = match self.current_token {
+                Token::Identifier(c) => c,
+                _ => raise!(self, "expected color name after yield"),
+            };
+            let kind = self.compiler.intern_highlight_kind(color).value;
+
+            self.advance();
+            self.expect_token(Token::Semicolon)?;
+
+            let vreg = self.compiler.alloc_vreg();
+            let span = self
+                .compiler
+                .build_chain()
+                .append(IRI::MovKind { dst: vreg, kind })
+                .append(IRI::Flush { kind: vreg })
+                .build();
+            Ok(span)
+        }
+    }
+
+    fn parse_var_declaration(&mut self) -> CompileResult<IRSpan<'a>> {
+        self.expect_token(Token::Var)?;
+
+        let name = match self.current_token {
+            Token::Identifier(n) => n,
+            _ => raise!(self, "expected variable name"),
+        };
+        self.advance();
+
+        self.expect_token(Token::Equals)?;
+        let (expr, vreg) = self.parse_expression()?;
+        self.expect_token(Token::Semicolon)?;
+
+        match self.variables.entry(name) {
+            Entry::Vacant(e) => _ = e.insert(vreg),
+            Entry::Occupied(_) => raise!(self, "variable '{}' already declared", name),
+        }
+
+        Ok(expr)
+    }
+
+    fn parse_identifier(&mut self) -> CompileResult<IRSpan<'a>> {
+        let name = match self.current_token {
+            Token::Identifier(n) => n,
+            _ => raise!(self, "expected identifier"),
+        };
+        self.advance();
+
+        match self.current_token {
+            // foo();
+            Token::LeftParen => {
+                self.advance();
+                self.expect_token(Token::RightParen)?;
+                self.expect_token(Token::Semicolon)?;
+                let name = self.compiler.strings.intern(self.compiler.arena, name);
+                Ok(IRSpan::single(self.compiler.alloc_iri(IRI::Call { name })))
+            }
+            // foo = expr;
+            Token::Equals => {
+                self.advance();
+                let (expr, vreg) = self.parse_expression()?;
+                self.expect_token(Token::Semicolon)?;
+                self.variables.insert(name, vreg);
+                Ok(expr)
+            }
+            // foo += expr;
+            Token::PlusEquals => {
+                let lhs_vreg = self.get_variable(name)?;
+                self.advance();
+
+                let Token::Integer(val) = self.current_token else {
+                    raise!(self, "expected integer literal after '+='");
+                };
+                self.advance();
+
+                self.expect_token(Token::Semicolon)?;
+
+                let ir = self.compiler.alloc_iri(IRI::AddImm { dst: lhs_vreg, imm: val });
+                self.variables.insert(name, lhs_vreg);
+                Ok(IRSpan::single(ir))
+            }
+            _ => raise!(self, "expected '(', or '=', '+=' after identifier"),
+        }
+    }
+
+    fn parse_expression(&mut self) -> CompileResult<(IRSpan<'a>, IRRegCell<'a>)> {
+        let (lhs_span, lhs_vreg) = match self.current_token {
+            Token::Integer(val) => {
+                let vreg = self.compiler.alloc_vreg();
+                let ir = self.compiler.alloc_iri(IRI::MovImm { dst: vreg, imm: val });
+                self.advance();
+                (IRSpan::single(ir), vreg)
+            }
+            Token::Identifier(name) => {
+                let vreg = self.get_variable(name)?;
+                self.advance();
+                (IRSpan::single(self.compiler.alloc_noop()), vreg)
+            }
+            _ => raise!(self, "expected integer or identifier in expression"),
+        };
+
+        // Check for binary operator
+        if matches!(self.current_token, Token::Plus) {
+            self.advance();
+
+            // Parse right-hand side - only integer literals supported
+            match self.current_token {
+                Token::Integer(val) => {
+                    self.advance();
+                    let add_ir = self.compiler.alloc_iri(IRI::AddImm { dst: lhs_vreg, imm: val });
+                    lhs_span.last.borrow_mut().set_next(add_ir);
+                    Ok((IRSpan { first: lhs_span.first, last: add_ir }, lhs_vreg))
+                }
+                _ => raise!(self, "expected integer literal after '+'"),
+            }
+        } else if lhs_vreg.borrow().physical.is_some() {
+            // For expressions of type `var virtual = phyiscal;`, we need to ensure
+            // that we actually copy the physical register into a new virtual one.
+            // The remaining code assumes single assignment form, while physical registers are permanent.
+            let dst = self.compiler.alloc_vreg();
+            let node = self.compiler.alloc_iri(IRI::Mov { dst, src: lhs_vreg });
+            Ok((IRSpan::single(node), dst))
+        } else {
+            Ok((lhs_span, lhs_vreg))
+        }
+    }
+
+    fn get_variable(&self, name: &str) -> CompileResult<IRRegCell<'a>> {
+        match self.variables.get(name) {
+            Some(&reg) => Ok(reg),
+            None => raise!(self, "undefined variable '{}'", name),
+        }
+    }
+
+    fn expect_token(&mut self, expected: Token) -> CompileResult<()> {
+        if std::mem::discriminant(&self.current_token) == std::mem::discriminant(&expected) {
+            self.advance();
+            Ok(())
+        } else {
+            raise!(self, "expected {:?}, found {:?}", expected, self.current_token)
+        }
+    }
+
+    fn advance(&mut self) {
+        self.current_token = self.tokenizer.next();
+    }
+}
diff --git a/crates/lsh/src/compiler/generator.rs b/crates/lsh/src/compiler/generator.rs
new file mode 100644
index 000000000000..5ca94d948646
--- /dev/null
+++ b/crates/lsh/src/compiler/generator.rs
@@ -0,0 +1,313 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Code generators for different output formats.
+//!
+//! ## TODO
+//!
+//! - The label lookup (with a `HashMap`) is just plain dumb.
+//!   The backend should emit metadata, I think.
+
+use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::fmt::format;
+use std::fs::read_dir;
+use std::io;
+use std::path::PathBuf;
+
+use stdext::arena::scratch_arena;
+
+use super::*;
+
+pub struct Generator<'a> {
+    compiler: Compiler<'a>,
+}
+
+impl<'a> Generator<'a> {
+    pub fn new(arena: &'a Arena) -> Self {
+        Self { compiler: Compiler::new(arena) }
+    }
+
+    pub fn read_file(&mut self, path: &Path) -> CompileResult<()> {
+        let path_str = path.display().to_string();
+        match std::fs::read_to_string(path) {
+            Ok(src) => self.compiler.parse(&path_str, &src),
+            Err(e) => {
+                Err(CompileError { path: path_str, line: 0, column: 0, message: e.to_string() })
+            }
+        }
+    }
+
+    pub fn read_directory(&mut self, path: &Path) -> CompileResult<()> {
+        let files = Self::read_dir_to_vec(path).map_err(|e| CompileError {
+            path: path.display().to_string(),
+            line: 0,
+            column: 0,
+            message: e.to_string(),
+        })?;
+
+        for path in files {
+            if path.extension() == Some(OsStr::new("lsh")) {
+                self.read_file(&path)?;
+            }
+        }
+        Ok(())
+    }
+
+    fn read_dir_to_vec(path: &Path) -> io::Result<Vec<PathBuf>> {
+        let mut paths = Vec::new();
+
+        for entry in read_dir(path)? {
+            let entry = entry?;
+            if entry.metadata().is_ok_and(|f| f.is_file())
+                && entry.file_name().as_encoded_bytes().ends_with(b".lsh")
+            {
+                paths.push(entry.path());
+            }
+        }
+
+        paths.sort_unstable();
+        Ok(paths)
+    }
+
+    pub fn assemble(mut self) -> CompileResult<Assembly<'a>> {
+        self.compiler.assemble()
+    }
+
+    pub fn generate_assembly(mut self, vt: bool) -> CompileResult<String> {
+        let mut output = String::new();
+        let assembly = self.compiler.assemble()?;
+        let line_num_width = assembly.instructions.len().checked_ilog10().unwrap_or(0) as usize + 1;
+        let mnemonic_config = if vt {
+            MnemonicFormattingConfig {
+                instruction_prefix: "\x1b[33m", // yellow
+                instruction_suffix: "\x1b[39m", // default
+
+                register_prefix: "\x1b[32m", // green
+                register_suffix: "\x1b[39m",
+
+                address_prefix: "\x1b[90m", // bright black
+                address_suffix: "\x1b[39m",
+
+                numeric_prefix: "\x1b[36m", // cyan
+                numeric_suffix: "\x1b[39m",
+            }
+        } else {
+            Default::default()
+        };
+        let label_prefix = if vt { "\x1b[4;94m" } else { "" }; // underlined & bright blue
+        let label_suffix = if vt { "\x1b[m" } else { "" };
+        let line_prefix = if vt { "\x1b[90m" } else { "" }; // bright black
+        let comment_prefix = if vt { "\x1b[32m" } else { "" }; // green
+        let comment_suffix = if vt { "\x1b[39m" } else { "" };
+
+        // TODO: This is kind of stupid. There should be per-instruction annotations.
+        let labels: HashMap<usize, &str> =
+            assembly.entrypoints.iter().map(|ep| (ep.address, ep.name.as_str())).collect();
+
+        let mut off = 0;
+        while off < assembly.instructions.len() {
+            if let Some(label) = labels.get(&off) {
+                if off != 0 {
+                    output.push('\n');
+                }
+                _ = writeln!(output, "{label_prefix}{}:{label_suffix}", label);
+            }
+
+            let (instr, len) = Instruction::decode(&assembly.instructions[off..]);
+            let mnemonic_width = match instr {
+                Some(Instruction::JumpIfMatchCharset { .. })
+                | Some(Instruction::JumpIfMatchPrefix { .. })
+                | Some(Instruction::JumpIfMatchPrefixInsensitive { .. }) => {
+                    if vt {
+                        60
+                    } else {
+                        30
+                    }
+                }
+                _ => 0,
+            };
+
+            let scratch = scratch_arena(None);
+            if let Some(instr) = instr {
+                _ = write!(
+                    output,
+                    "{line_prefix}{off:>line_num_width$}:  {mnemonic:mnemonic_width$}",
+                    mnemonic = instr.mnemonic(&scratch, &mnemonic_config),
+                );
+            } else {
+                _ = write!(
+                    output,
+                    "{line_prefix}{off:>line_num_width$}:  {_n}{opcode:#02x}{n_}",
+                    opcode = assembly.instructions[off],
+                    _n = mnemonic_config.numeric_prefix,
+                    n_ = mnemonic_config.numeric_suffix,
+                );
+            }
+
+            match instr {
+                Some(Instruction::JumpIfMatchCharset { idx, .. }) => {
+                    _ = write!(
+                        output,
+                        " {comment_prefix}// {:?}{comment_suffix}",
+                        assembly.charsets[idx as usize]
+                    )
+                }
+                Some(Instruction::JumpIfMatchPrefix { idx, .. })
+                | Some(Instruction::JumpIfMatchPrefixInsensitive { idx, .. }) => {
+                    _ = write!(
+                        output,
+                        " {comment_prefix}// {:?}{comment_suffix}",
+                        assembly.strings[idx as usize]
+                    )
+                }
+                _ => {}
+            }
+
+            output.push('\n');
+            off += len;
+        }
+
+        Ok(output)
+    }
+
+    pub fn generate_rust(mut self) -> CompileResult<String> {
+        let assembly = self.compiler.assemble()?;
+
+        let mut output = String::new();
+        output.push_str("// This file is auto-generated. Do not edit it manually.\n\n");
+        output.push_str("use lsh::engine::Language;\n\n");
+
+        output.push_str("#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub enum HighlightKind {\n");
+        let members: Vec<_> = assembly
+            .highlight_kinds
+            .iter()
+            .map(|hk| (hk, format!("{} = {},", hk.fmt_camelcase(), hk.value)))
+            .collect();
+        let width = members.iter().map(|s| s.1.len()).max().unwrap_or(0);
+        for (hk, member) in members {
+            _ = writeln!(output, "    {member:<width$} // {}", hk.identifier);
+        }
+        output.push_str("}\n");
+
+        if let Some(last) = assembly.highlight_kinds.last() {
+            _ = write!(
+                output,
+                "
+impl TryFrom<u32> for HighlightKind {{
+    type Error = ();
+
+    #[inline]
+    fn try_from(value: u32) -> Result<Self, Self::Error> {{
+        if value <= Self::{} as u32 {{
+            Ok(unsafe {{ std::mem::transmute::<u8, Self>(value as u8) }})
+        }} else {{
+            Err(())
+        }}
+    }}
+}}
+",
+                last.fmt_camelcase()
+            );
+        }
+
+        output.push_str("/*\n");
+        output.push_str(&self.compiler.as_mermaid());
+        output.push_str("*/\n");
+
+        output.push_str("\n#[rustfmt::skip] pub const LANGUAGES: &[Language] = &[\n");
+        for ep in &assembly.entrypoints {
+            _ = writeln!(
+                output,
+                "    Language {{ id: {:?}, name: {:?}, entrypoint: {} }},",
+                ep.name, ep.display_name, ep.address
+            );
+        }
+        output.push_str("];\n");
+
+        output.push_str(
+            "\n#[rustfmt::skip] pub const FILE_ASSOCIATIONS: &[(&str, &Language)] = &[\n",
+        );
+        for (idx, ep) in assembly.entrypoints.iter().enumerate() {
+            for path in &ep.paths {
+                _ = writeln!(output, "    ({path:?}, &LANGUAGES[{idx}]),");
+            }
+        }
+        output.push_str("];\n");
+
+        _ = writeln!(
+            output,
+            "\n#[rustfmt::skip] pub const ASSEMBLY: [u8; {len}] = [",
+            len = assembly.instructions.len(),
+        );
+        let line_num_width = assembly.instructions.len().checked_ilog10().unwrap_or(0) as usize + 1;
+
+        // TODO: This is kind of stupid. There should be per-instruction annotations.
+        let labels: HashMap<usize, &str> =
+            assembly.entrypoints.iter().map(|ep| (ep.address, ep.name.as_str())).collect();
+
+        let mut off = 0;
+        while off < assembly.instructions.len() {
+            if let Some(label) = labels.get(&off) {
+                if off != 0 {
+                    output.push('\n');
+                }
+                _ = writeln!(output, "    // {}:", label);
+            }
+
+            output.push_str("    ");
+
+            let (instr, len) = Instruction::decode(&assembly.instructions[off..]);
+            let scratch = scratch_arena(None);
+            for i in 0..len {
+                _ = write!(output, "0x{:02x}, ", assembly.instructions[off + i]);
+            }
+
+            if let Some(instr) = instr {
+                _ = writeln!(
+                    output,
+                    "{:<padding_width$}// {off:>line_num_width$}:  {mnemonic}",
+                    "",
+                    padding_width = 9usize.saturating_sub(len) * 6,
+                    mnemonic = instr.mnemonic(&scratch, &Default::default())
+                );
+            }
+
+            off += len;
+        }
+        output.push_str("];\n");
+
+        _ = writeln!(
+            output,
+            "\n#[rustfmt::skip] pub const CHARSETS: [[u16; 16]; {len}] = [",
+            len = assembly.charsets.len(),
+        );
+        for cs in assembly.charsets {
+            output.push_str("    [");
+            for lo in 0..16 {
+                if lo > 0 {
+                    output.push_str(", ");
+                }
+                let mut u = 0u16;
+                for hi in 0..16 {
+                    u |= (cs[hi * 16 + lo] as u16) << hi;
+                }
+                _ = write!(output, "0x{u:04x}");
+            }
+            output.push_str("],\n");
+        }
+        output.push_str("];\n");
+
+        _ = writeln!(
+            output,
+            "\n#[rustfmt::skip] pub const STRINGS: [&str; {len}] = [",
+            len = assembly.strings.len(),
+        );
+        for s in assembly.strings {
+            _ = writeln!(output, "    {s:?},");
+        }
+        output.push_str("];\n");
+
+        Ok(output)
+    }
+}
diff --git a/crates/lsh/src/compiler/helpers.rs b/crates/lsh/src/compiler/helpers.rs
new file mode 100644
index 000000000000..eb1cc319295e
--- /dev/null
+++ b/crates/lsh/src/compiler/helpers.rs
@@ -0,0 +1,29 @@
+use std::cell::RefCell;
+use std::collections::HashSet;
+use std::fmt;
+use std::fmt::Write as _;
+use std::ops::{Index, IndexMut};
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
+
+use stdext::arena::*;
+
+pub fn arena_clone_str<'a>(arena: &'a Arena, s: &str) -> &'a str {
+    ArenaString::from_str(arena, s).leak()
+}
+
+pub trait Intern<'a, T: ?Sized> {
+    fn intern(&mut self, arena: &'a Arena, item: &T) -> &'a T;
+}
+
+impl<'a> Intern<'a, str> for Vec<&'a str> {
+    fn intern(&mut self, arena: &'a Arena, value: &str) -> &'a str {
+        if let Some(&s) = self.iter().find(|&&v| v == value) {
+            s
+        } else {
+            let s = arena_clone_str(arena, value);
+            self.push(s);
+            s
+        }
+    }
+}
diff --git a/crates/lsh/src/compiler/mod.rs b/crates/lsh/src/compiler/mod.rs
new file mode 100644
index 000000000000..9a54624e56c7
--- /dev/null
+++ b/crates/lsh/src/compiler/mod.rs
@@ -0,0 +1,1197 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Compiler core: IR definitions, instruction encoding, and shared infrastructure.
+//!
+//! ## IR representation
+//!
+//! The IR is a graph of `RefCell<IR>` nodes linked via `next` and `then` pointers.
+//! This isn't a traditional SSA or basic block representation - it's more like a
+//! flowchart where each node is one instruction. The `offset` field is set during
+//! codegen to the final bytecode address.
+//!
+//! ## Register architecture
+//!
+//! The `IRReg.physical` field starts as `None` for vregs and gets filled in by regalloc.
+//! For physical registers, it's set during `Compiler::new()`.
+//!
+//! ## Charset representation
+//!
+//! `Charset` is a 256-bit bitmap (`[bool; 256]`). The engine uses a transposed `[u16; 16]`
+//! layout for SIMD reasons - see `generator.rs` where the conversion happens.
+//!
+//! ## Instruction encoding
+//!
+//! Variable-length encoding, 1-9 bytes per instruction. See `Instruction::encode/decode`.
+//! The `address_offset()` method returns where within an instruction the jump target lives,
+//! used by the backend's relocation system.
+//!
+//! ## Quirks
+//!
+//! - `TreeVisitor` does BFS, which is fine for iteration but *not* for linearization.
+//!   The backend has its own DFS traversal for liveness analysis.
+
+mod backend;
+mod frontend;
+mod generator;
+mod helpers;
+mod optimizer;
+mod regex;
+mod tokenizer;
+
+use std::cell::{Cell, RefCell};
+use std::collections::{HashSet, VecDeque};
+use std::fmt::Write as _;
+use std::marker::PhantomData;
+use std::mem::{MaybeUninit, transmute, zeroed};
+use std::ops::{Index, IndexMut};
+use std::path::Path;
+use std::{fmt, ptr};
+
+use stdext::arena::{Arena, ArenaString};
+
+use self::frontend::*;
+pub use self::generator::Generator;
+use crate::compiler::helpers::{Intern, arena_clone_str};
+
+pub fn builtin_definitions_path() -> &'static Path {
+    #[cfg(windows)]
+    let path = concat!(env!("CARGO_MANIFEST_DIR"), "\\definitions");
+    #[cfg(not(windows))]
+    let path = concat!(env!("CARGO_MANIFEST_DIR"), "/definitions");
+    Path::new(path)
+}
+
+pub type CompileResult<T> = Result<T, CompileError>;
+
+#[derive(Debug)]
+pub struct CompileError {
+    pub path: String,
+    pub line: usize,
+    pub column: usize,
+    pub message: String,
+}
+
+impl std::error::Error for CompileError {}
+
+impl fmt::Display for CompileError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "error at {}:{}:{}: {}", self.path, self.line, self.column, self.message)
+    }
+}
+
+pub struct Compiler<'a> {
+    arena: &'a Arena,
+    physical_registers: [IRRegCell<'a>; Register::COUNT],
+    functions: Vec<Function<'a>>,
+    charsets: Vec<&'a Charset>,
+    strings: Vec<&'a str>,
+    highlight_kinds: Vec<HighlightKind<'a>>,
+    next_vreg_id: Cell<u32>,
+}
+
+impl<'a> Compiler<'a> {
+    pub fn new(arena: &'a Arena) -> Self {
+        #[allow(invalid_value)]
+        let mut s = Self {
+            arena,
+            physical_registers: unsafe { zeroed() },
+            functions: Default::default(),
+            charsets: Default::default(),
+            strings: Default::default(),
+            highlight_kinds: vec![HighlightKind { identifier: "other", value: 0 }],
+            next_vreg_id: Cell::new(0),
+        };
+
+        for i in 0..Register::COUNT {
+            let reg = s.alloc_vreg();
+            reg.borrow_mut().physical = Some(Register::from_usize(i));
+            s.physical_registers[i] = reg;
+        }
+
+        s
+    }
+
+    pub fn parse<'src>(&mut self, path: &'src str, src: &'src str) -> CompileResult<()> {
+        let mut parser = Parser::new(self, path, src);
+        parser.run()?;
+        Ok(())
+    }
+
+    pub fn assemble(&mut self) -> CompileResult<Assembly<'a>> {
+        optimizer::optimize(self);
+        backend::Backend::new().compile(self)
+    }
+
+    fn alloc_ir(&self, ir: IR<'a>) -> IRCell<'a> {
+        self.arena.alloc_uninit().write(RefCell::new(ir))
+    }
+
+    fn alloc_iri(&self, instr: IRI<'a>) -> IRCell<'a> {
+        self.arena.alloc_uninit().write(RefCell::new(IR { next: None, instr, offset: usize::MAX }))
+    }
+
+    fn alloc_noop(&self) -> IRCell<'a> {
+        self.alloc_iri(IRI::Noop)
+    }
+
+    fn build_chain<'s>(&'s self) -> IRChainBuilder<'a, 's> {
+        IRChainBuilder { compiler: self, span: None }
+    }
+
+    fn get_reg(&self, reg: Register) -> IRRegCell<'a> {
+        self.physical_registers[reg as usize]
+    }
+
+    fn alloc_vreg(&self) -> IRRegCell<'a> {
+        let id = self.next_vreg_id.get();
+        self.next_vreg_id.set(id + 1);
+        self.arena.alloc_uninit().write(RefCell::new(IRReg::new(id)))
+    }
+
+    fn intern_charset(&mut self, charset: &Charset) -> &'a Charset {
+        self.charsets.intern(self.arena, charset)
+    }
+
+    fn intern_string(&mut self, s: &str) -> &'a str {
+        self.strings.intern(self.arena, s)
+    }
+
+    fn intern_highlight_kind(&mut self, identifier: &str) -> &HighlightKind<'a> {
+        let idx = match self.highlight_kinds.binary_search_by(|hk| hk.identifier.cmp(identifier)) {
+            Ok(idx) => idx,
+            Err(idx) => {
+                let identifier = arena_clone_str(self.arena, identifier);
+                let value = self.highlight_kinds.len() as u32;
+                self.highlight_kinds.insert(idx, HighlightKind { identifier, value });
+                idx
+            }
+        };
+        &self.highlight_kinds[idx]
+    }
+
+    fn visit_nodes_from(&self, root: IRCell<'a>) -> TreeVisitor<'a> {
+        let mut stack = VecDeque::new();
+        stack.push_back(root);
+        TreeVisitor { current: None, stack, visited: Default::default() }
+    }
+
+    /// Collect all "interesting" characters from conditions in a loop body.
+    /// Returns a charset where true = interesting character that should be checked.
+    fn collect_interesting_charset(&self, loop_body: IRCell<'a>) -> Charset {
+        let mut charset = Charset::no();
+        let mut visited = HashSet::new();
+
+        for node in self.visit_nodes_from(loop_body) {
+            let node_ptr = node as *const _;
+            if !visited.insert(node_ptr) {
+                continue;
+            }
+
+            if let IRI::If { condition, .. } = node.borrow().instr {
+                match condition {
+                    Condition::Cmp { .. } => {}
+                    Condition::EndOfLine => {}
+                    Condition::Charset(cs) => {
+                        // Merge this charset
+                        charset.merge(cs);
+                    }
+                    Condition::Prefix(s) | Condition::PrefixInsensitive(s) => {
+                        // First character of the prefix is interesting
+                        if let Some(&b) = s.as_bytes().first() {
+                            charset.set(b, true);
+                            if matches!(condition, Condition::PrefixInsensitive(_)) {
+                                charset.set(b.to_ascii_uppercase(), true);
+                                charset.set(b.to_ascii_lowercase(), true);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        charset
+    }
+
+    pub fn as_mermaid(&self) -> String {
+        let mut output = String::new();
+
+        // ---
+        // config:
+        // layout: elk
+        // elk:
+        //   considerModelOrder: NONE
+        // ---
+        output.push_str("flowchart TB\n");
+
+        for func in &self.functions {
+            _ = writeln!(output, "  subgraph {}", func.name);
+            _ = writeln!(output, "    direction TB");
+            _ = writeln!(
+                output,
+                "    {}_start@{{shape: start}} --> {}",
+                func.name,
+                func.body.borrow()
+            );
+
+            let mut visited = HashSet::new();
+            let mut to_visit = vec![func.body];
+
+            while let Some(node_cell) = to_visit.pop() {
+                if !visited.insert(node_cell.as_ptr()) {
+                    continue;
+                }
+
+                let node = node_cell.borrow();
+                let offset = node.offset;
+                _ = write!(output, "    {}", node);
+
+                match node.instr {
+                    IRI::Noop => {
+                        _ = write!(output, "[{offset}: noop]");
+                    }
+                    IRI::Mov { dst, src } => {
+                        let src = src.borrow();
+                        let dst = dst.borrow();
+                        _ = write!(output, "[\"{offset}: {dst:?} = {src:?}\"]");
+                    }
+                    IRI::MovImm { dst, imm } => {
+                        let dst = dst.borrow();
+                        _ = write!(output, "[\"{offset}: {dst:?} = {imm}\"]");
+                    }
+                    IRI::MovKind { dst, kind } => {
+                        let dst = dst.borrow();
+                        let kind = self
+                            .highlight_kinds
+                            .iter()
+                            .find(|hk| hk.value == kind)
+                            .map_or("???", |hk| hk.identifier);
+                        _ = write!(output, "[\"{offset}: {dst:?} = {kind}\"]");
+                    }
+                    IRI::AddImm { dst, imm } => {
+                        let dst = dst.borrow();
+                        _ = write!(output, "[\"{offset}: {dst:?} += {imm}\"]");
+                    }
+                    IRI::If { condition, then } => {
+                        _ = write!(output, "{{\"{offset}: ");
+                        _ = match condition {
+                            Condition::Cmp { lhs, rhs, op } => {
+                                let op_str = match op {
+                                    ComparisonOp::Eq => "==",
+                                    ComparisonOp::Ne => "!=",
+                                    ComparisonOp::Lt => "<",
+                                    ComparisonOp::Gt => ">",
+                                    ComparisonOp::Le => "<=",
+                                    ComparisonOp::Ge => ">=",
+                                };
+                                write!(output, "{lhs:?} {op_str} {rhs:?}")
+                            }
+                            Condition::EndOfLine => write!(output, "eol"),
+                            Condition::Charset(cs) => write!(output, "charset: {cs:?}"),
+                            Condition::Prefix(s) => write!(output, "match: {s}"),
+                            Condition::PrefixInsensitive(s) => write!(output, "imatch: {s}"),
+                        };
+                        _ = writeln!(output, "\"}}");
+                        _ = writeln!(output, "    {} -->|yes| {}", node, then.borrow());
+                        to_visit.push(then);
+                    }
+                    IRI::Call { name } => {
+                        _ = write!(output, "[\"{offset}: call {name}\"]");
+                    }
+                    IRI::Return => {
+                        _ = write!(output, "[{offset}: return]");
+                    }
+                    IRI::Flush { kind } => {
+                        _ = write!(output, "[{offset}: flush]");
+                    }
+                    IRI::AwaitInput => {
+                        _ = write!(output, "[{offset}: await input]");
+                    }
+                }
+
+                match node.instr {
+                    IRI::If { .. } => {
+                        if let Some(next) = node.next {
+                            _ = writeln!(output, "    {} -->|no| {}", node, next.borrow());
+                        }
+                    }
+                    _ => {
+                        if let Some(next) = node.next {
+                            _ = writeln!(output, " --> {}", next.borrow());
+                        } else {
+                            _ = writeln!(output);
+                        }
+                    }
+                }
+
+                if let Some(next) = node.next {
+                    to_visit.push(next);
+                }
+            }
+
+            _ = writeln!(output, "  end");
+        }
+
+        output
+    }
+}
+
+struct TreeVisitor<'a> {
+    current: Option<IRCell<'a>>,
+    stack: VecDeque<IRCell<'a>>,
+    visited: HashSet<*const RefCell<IR<'a>>>,
+}
+
+impl<'a> Iterator for TreeVisitor<'a> {
+    type Item = IRCell<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(cell) = self.current.take() {
+            {
+                let ir = cell.borrow();
+                if let IRI::If { then, .. } = ir.instr {
+                    self.stack.push_back(then);
+                }
+                if let Some(next) = ir.next {
+                    self.stack.push_back(next);
+                }
+            }
+        }
+
+        while let Some(cell) = self.stack.pop_front() {
+            if self.visited.insert(cell) {
+                self.current = Some(cell);
+                return self.current;
+            }
+        }
+
+        None
+    }
+}
+
+#[derive(Clone)]
+pub struct HighlightKind<'a> {
+    pub identifier: &'a str,
+    pub value: u32,
+}
+
+impl<'a> HighlightKind<'a> {
+    pub fn fmt_camelcase(&self) -> HighlightKindCamelcaseFormatter<'a> {
+        HighlightKindCamelcaseFormatter { identifier: self.identifier }
+    }
+}
+
+pub struct HighlightKindCamelcaseFormatter<'a> {
+    identifier: &'a str,
+}
+
+impl<'a> fmt::Display for HighlightKindCamelcaseFormatter<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut capitalize_next = true;
+        for c in self.identifier.chars() {
+            if c == '.' {
+                capitalize_next = true;
+            } else if capitalize_next {
+                capitalize_next = false;
+                f.write_char(c.to_ascii_uppercase())?;
+            } else {
+                f.write_char(c)?;
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+struct FunctionAttributes<'a> {
+    display_name: Option<&'a str>,
+    paths: Vec<&'a str>,
+}
+
+#[derive(Debug, Clone)]
+struct Function<'a> {
+    name: &'a str,
+    attributes: FunctionAttributes<'a>,
+    body: IRCell<'a>,
+    public: bool,
+}
+
+// To be honest, I don't think this qualifies as an "intermediate representation",
+// if we compare this to popular compilers. But whatever. It's still intermediate to us.
+#[derive(Debug)]
+struct IR<'a> {
+    next: Option<IRCell<'a>>,
+    instr: IRI<'a>,
+    offset: usize,
+}
+
+impl<'a> fmt::Display for IR<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "i{self:p}")
+    }
+}
+
+type IRCell<'a> = &'a RefCell<IR<'a>>;
+
+// IRI = Immediate Representation Instruction
+#[derive(Debug, Clone, Copy)]
+enum IRI<'a> {
+    Noop,
+    Mov { dst: IRRegCell<'a>, src: IRRegCell<'a> },
+    MovImm { dst: IRRegCell<'a>, imm: u32 },
+    MovKind { dst: IRRegCell<'a>, kind: u32 },
+    AddImm { dst: IRRegCell<'a>, imm: u32 },
+    If { condition: Condition<'a>, then: IRCell<'a> },
+    Call { name: &'a str },
+    Return,
+    Flush { kind: IRRegCell<'a> },
+    AwaitInput,
+}
+
+#[derive(Default)]
+pub struct IRReg {
+    id: u32,
+    physical: Option<Register>,
+}
+
+pub type IRRegCell<'a> = &'a RefCell<IRReg>;
+
+impl IRReg {
+    fn new(id: u32) -> Self {
+        IRReg { id, physical: None }
+    }
+}
+
+impl fmt::Debug for IRReg {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if let Some(p) = self.physical
+            && self.id < Register::COUNT as u32
+        {
+            write!(f, "{}", p.mnemonic())
+        } else {
+            write!(f, "v{}", self.id)
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+struct IRSpan<'a> {
+    pub first: IRCell<'a>,
+    pub last: IRCell<'a>,
+}
+
+impl<'a> IRSpan<'a> {
+    pub fn single(node: IRCell<'a>) -> Self {
+        Self { first: node, last: node }
+    }
+}
+
+struct IRChainBuilder<'a, 's> {
+    compiler: &'s Compiler<'a>,
+    span: Option<IRSpan<'a>>,
+}
+
+impl<'a, 's> IRChainBuilder<'a, 's> {
+    fn append(&mut self, instr: IRI<'a>) -> &mut Self {
+        let node = self.compiler.alloc_iri(instr);
+        if let Some(span) = &mut self.span {
+            span.last.borrow_mut().set_next(node);
+            span.last = node;
+        } else {
+            self.span = Some(IRSpan::single(node));
+        }
+        self
+    }
+
+    fn build(&self) -> IRSpan<'a> {
+        self.span.unwrap_or_else(|| IRSpan::single(self.compiler.alloc_noop()))
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ComparisonOp {
+    Eq,
+    Ne,
+    Lt,
+    Gt,
+    Le,
+    Ge,
+}
+
+#[derive(Debug, Clone, Copy)]
+enum Condition<'a> {
+    Cmp { lhs: IRRegCell<'a>, rhs: IRRegCell<'a>, op: ComparisonOp },
+    EndOfLine,
+    Charset(&'a Charset),
+    Prefix(&'a str),
+    PrefixInsensitive(&'a str),
+}
+
+impl<'a> IR<'a> {
+    fn wants_next(&self) -> bool {
+        self.next.is_none() && !matches!(self.instr, IRI::Return)
+    }
+
+    fn set_next(&mut self, n: IRCell<'a>) {
+        debug_assert!(self.wants_next());
+        self.next = Some(n);
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct Charset {
+    bits: [bool; 256],
+}
+
+impl Charset {
+    const fn no() -> Self {
+        Charset { bits: [false; 256] }
+    }
+
+    const fn yes() -> Self {
+        Charset { bits: [true; 256] }
+    }
+
+    fn fill(&mut self, value: bool) {
+        self.bits.fill(value);
+    }
+
+    fn invert(&mut self) {
+        for b in &mut self.bits {
+            *b = !*b;
+        }
+    }
+
+    fn set(&mut self, index: u8, value: bool) {
+        self.bits[index as usize] = value;
+    }
+
+    fn merge(&mut self, other: &Charset) {
+        for (a, b) in self.bits.iter_mut().zip(other.bits.iter()) {
+            *a |= *b;
+        }
+    }
+
+    fn covers_all(&self) -> bool {
+        self.bits.iter().all(|&b| b)
+    }
+
+    fn covers_char(&self, b: u8) -> bool {
+        self.bits[b as usize]
+    }
+
+    fn covers_char_insensitive(&self, b: u8) -> bool {
+        self.bits[b.to_ascii_uppercase() as usize] && self.bits[b.to_ascii_lowercase() as usize]
+    }
+
+    fn covers_str(&self, s: &str) -> bool {
+        s.as_bytes().iter().all(|&b| self.bits[b as usize])
+    }
+
+    fn covers_str_insensitive(&self, s: &str) -> bool {
+        s.as_bytes().iter().all(|&b| {
+            self.bits[b.to_ascii_uppercase() as usize] && self.bits[b.to_ascii_lowercase() as usize]
+        })
+    }
+
+    fn is_superset(&self, other: &Charset) -> bool {
+        for (&s, &o) in self.bits.iter().zip(other.bits.iter()) {
+            if s && !o {
+                return false;
+            }
+        }
+        true
+    }
+
+    fn is_strict_superset(&self, other: &Charset) -> bool {
+        let mut has_extra = false;
+        for (&s, &o) in self.bits.iter().zip(other.bits.iter()) {
+            if !s && o {
+                return false;
+            }
+            has_extra |= s && !o;
+        }
+        has_extra
+    }
+}
+
+impl Default for Charset {
+    fn default() -> Self {
+        Self::no()
+    }
+}
+
+impl fmt::Debug for Charset {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let show_char = |f: &mut fmt::Formatter<'_>, b: usize| {
+            let b = b as u8;
+            if b == b'"' {
+                write!(f, "&quot;")
+            } else if b.is_ascii_graphic() {
+                let b = b as char;
+                write!(f, "{b}")
+            } else {
+                write!(f, "0x{b:02X}")
+            }
+        };
+
+        let mut beg = 0;
+        let mut first = true;
+
+        write!(f, "[")?;
+
+        while beg < 256 {
+            while beg < 256 && !self.bits[beg] {
+                beg += 1;
+            }
+            if beg >= 256 {
+                break;
+            }
+
+            let mut end = beg;
+            while end < 256 && self.bits[end] {
+                end += 1;
+            }
+
+            if !first {
+                write!(f, ", ")?;
+            }
+            show_char(f, beg)?;
+            if end - beg > 1 {
+                write!(f, "-")?;
+                show_char(f, end - 1)?;
+            }
+
+            beg = end;
+            first = false;
+        }
+
+        write!(f, "]")
+    }
+}
+
+impl<I> Index<I> for Charset
+where
+    [bool]: Index<I>,
+{
+    type Output = <[bool] as Index<I>>::Output;
+
+    #[inline]
+    fn index(&self, index: I) -> &Self::Output {
+        self.bits.index(index)
+    }
+}
+
+impl<I> IndexMut<I> for Charset
+where
+    [bool]: IndexMut<I>,
+{
+    #[inline]
+    fn index_mut(&mut self, index: I) -> &mut Self::Output {
+        self.bits.index_mut(index)
+    }
+}
+
+impl<'a> Intern<'a, Charset> for Vec<&'a Charset> {
+    fn intern(&mut self, arena: &'a Arena, value: &Charset) -> &'a Charset {
+        if let Some(&s) = self.iter().find(|&&v| v == value) {
+            s
+        } else {
+            let s = arena.alloc_uninit().write(value.clone());
+            self.push(s);
+            s
+        }
+    }
+}
+
+pub struct Assembly<'a> {
+    pub instructions: Vec<u8>,
+    pub entrypoints: Vec<Entrypoint>,
+    pub charsets: Vec<&'a Charset>,
+    pub strings: Vec<&'a str>,
+    pub highlight_kinds: Vec<HighlightKind<'a>>,
+}
+
+pub struct Entrypoint {
+    pub name: String,
+    pub display_name: String,
+    pub paths: Vec<String>,
+    pub address: usize,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Register {
+    ProgramCounter,
+    InputOffset,
+    HighlightStart,
+    X3,
+    X4,
+    X5,
+    X6,
+    X7,
+    X8,
+    X9,
+    X10,
+    X11,
+    X12,
+    X13,
+    X14,
+    X15,
+}
+
+impl Register {
+    const FIRST_USER_REG: usize = 3; // aka x3
+    const COUNT: usize = 16;
+
+    fn from_usize(value: usize) -> Self {
+        debug_assert!(value < Self::COUNT);
+        unsafe { transmute::<u8, Register>(value as u8) }
+    }
+
+    fn mnemonic(&self) -> &'static str {
+        match self {
+            Register::ProgramCounter => "pc",
+            Register::InputOffset => "off",
+            Register::HighlightStart => "hs",
+            Register::X3 => "x3",
+            Register::X4 => "x4",
+            Register::X5 => "x5",
+            Register::X6 => "x6",
+            Register::X7 => "x7",
+            Register::X8 => "x8",
+            Register::X9 => "x9",
+            Register::X10 => "x10",
+            Register::X11 => "x11",
+            Register::X12 => "x12",
+            Register::X13 => "x13",
+            Register::X14 => "x14",
+            Register::X15 => "x15",
+        }
+    }
+}
+
+impl fmt::Display for Register {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.mnemonic())
+    }
+}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy)]
+pub struct Registers {
+    pub pc: u32,  // x0 = ProgramCounter
+    pub off: u32, // x1 = InputOffset
+    pub hs: u32,  // x2 = HighlightStart
+    pub x3: u32,
+    pub x4: u32,
+    pub x5: u32,
+    pub x6: u32,
+    pub x7: u32,
+    pub x8: u32,
+    pub x9: u32,
+    pub x10: u32,
+    pub x11: u32,
+    pub x12: u32,
+    pub x13: u32,
+    pub x14: u32,
+    pub x15: u32,
+}
+
+impl Registers {
+    const COUNT: usize = 16;
+
+    #[inline(always)]
+    pub fn get(&self, reg: usize) -> u32 {
+        debug_assert!(reg < Self::COUNT);
+        unsafe { (self as *const _ as *const u32).add(reg).read() }
+    }
+
+    #[inline(always)]
+    pub fn set(&mut self, reg: usize, val: u32) {
+        debug_assert!(reg < Self::COUNT);
+        unsafe { (self as *mut _ as *mut u32).add(reg).write(val) }
+    }
+}
+
+struct RegisterAllocator(u16);
+
+impl RegisterAllocator {
+    fn new() -> Self {
+        RegisterAllocator((1 << Register::FIRST_USER_REG) - 1)
+    }
+
+    fn alloc(&mut self) -> Option<Register> {
+        let available = !self.0;
+        if available == 0 {
+            return None;
+        }
+
+        let idx = available.trailing_zeros() as usize;
+        self.0 |= 1 << idx;
+        Some(Register::from_usize(idx))
+    }
+
+    fn dealloc(&mut self, reg: Register) {
+        let idx = reg as usize;
+        if idx >= Register::FIRST_USER_REG {
+            self.0 &= !(1 << idx);
+        }
+    }
+}
+
+impl Default for RegisterAllocator {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+pub struct AnnotatedInstruction<'a> {
+    pub instr: Instruction,
+    pub label: &'a str,
+}
+
+#[allow(dead_code)]
+#[repr(u8)]
+#[derive(Debug, Clone, Copy)]
+pub enum Instruction {
+    // NOTE: This allows for jumps by manipulating Register::ProgramCounter.
+    Mov { dst: Register, src: Register },
+    Add { dst: Register, src: Register },
+    Sub { dst: Register, src: Register },
+    MovImm { dst: Register, imm: u32 },
+    AddImm { dst: Register, imm: u32 },
+    SubImm { dst: Register, imm: u32 },
+
+    Call { tgt: u32 },
+    Return,
+
+    JumpEQ { lhs: Register, rhs: Register, tgt: u32 }, // ==
+    JumpNE { lhs: Register, rhs: Register, tgt: u32 }, // !=
+    JumpLT { lhs: Register, rhs: Register, tgt: u32 }, // <
+    JumpLE { lhs: Register, rhs: Register, tgt: u32 }, // <=
+    JumpGT { lhs: Register, rhs: Register, tgt: u32 }, // >
+    JumpGE { lhs: Register, rhs: Register, tgt: u32 }, // >=
+
+    // Jumps to `tgt` if we're at the end of the line.
+    JumpIfEndOfLine { tgt: u32 },
+
+    // Jumps to `tgt` if the test succeeds.
+    // `idx` specifies the charset/string to use.
+    JumpIfMatchCharset { idx: u32, tgt: u32 },
+    JumpIfMatchPrefix { idx: u32, tgt: u32 },
+    JumpIfMatchPrefixInsensitive { idx: u32, tgt: u32 },
+
+    // Flushes the current HighlightKind to the output.
+    FlushHighlight { kind: Register },
+
+    // Awaits more input to be available.
+    AwaitInput,
+}
+
+impl Instruction {
+    // JumpIfMatchCharset, etc., are 1 byte opcode + two u32 parameters.
+    pub const MAX_ENCODED_SIZE: usize = 9;
+
+    pub fn address_offset(&self) -> Option<usize> {
+        match *self {
+            Instruction::MovImm { .. }
+            | Instruction::AddImm { .. }
+            | Instruction::SubImm { .. } => Some(2), // opcode + dst
+
+            Instruction::Call { .. } => Some(1), // opcode
+
+            Instruction::JumpEQ { .. }
+            | Instruction::JumpNE { .. }
+            | Instruction::JumpLT { .. }
+            | Instruction::JumpLE { .. }
+            | Instruction::JumpGT { .. }
+            | Instruction::JumpGE { .. } => Some(2), // opcode + lhs/rhs pair
+
+            Instruction::JumpIfEndOfLine { .. } => Some(1), // opcode
+
+            Instruction::JumpIfMatchCharset { .. }
+            | Instruction::JumpIfMatchPrefix { .. }
+            | Instruction::JumpIfMatchPrefixInsensitive { .. } => Some(5), // opcode + idx
+
+            _ => None,
+        }
+    }
+
+    #[allow(clippy::identity_op)]
+    pub fn encode<'a>(&self, arena: &'a Arena) -> Vec<u8, &'a Arena> {
+        fn enc_reg_pair(lo: Register, hi: Register) -> u8 {
+            ((hi as u8) << 4) | (lo as u8)
+        }
+
+        fn enc_reg_single(lo: Register) -> u8 {
+            lo as u8
+        }
+
+        fn enc_u16(val: u16) -> [u8; 2] {
+            val.to_le_bytes()
+        }
+
+        fn enc_u32(val: u32) -> [u8; 4] {
+            val.to_le_bytes()
+        }
+
+        let mut bytes = Vec::with_capacity_in(16, arena);
+        #[allow(clippy::missing_transmute_annotations)]
+        bytes.push(unsafe { std::mem::transmute(std::mem::discriminant(self)) });
+
+        match *self {
+            Instruction::Mov { dst, src }
+            | Instruction::Add { dst, src }
+            | Instruction::Sub { dst, src } => {
+                bytes.push(enc_reg_pair(dst, src));
+            }
+            Instruction::MovImm { dst, imm }
+            | Instruction::AddImm { dst, imm }
+            | Instruction::SubImm { dst, imm } => {
+                bytes.push(enc_reg_single(dst));
+                bytes.extend_from_slice(&enc_u32(imm));
+            }
+
+            Instruction::Call { tgt } => {
+                bytes.extend_from_slice(&enc_u32(tgt));
+            }
+            Instruction::Return => {}
+
+            Instruction::JumpEQ { lhs, rhs, tgt }
+            | Instruction::JumpNE { lhs, rhs, tgt }
+            | Instruction::JumpLT { lhs, rhs, tgt }
+            | Instruction::JumpLE { lhs, rhs, tgt }
+            | Instruction::JumpGT { lhs, rhs, tgt }
+            | Instruction::JumpGE { lhs, rhs, tgt } => {
+                bytes.push(enc_reg_pair(lhs, rhs));
+                bytes.extend_from_slice(&enc_u32(tgt));
+            }
+
+            Instruction::JumpIfEndOfLine { tgt } => {
+                bytes.extend_from_slice(&enc_u32(tgt));
+            }
+            Instruction::JumpIfMatchCharset { idx, tgt }
+            | Instruction::JumpIfMatchPrefix { idx, tgt }
+            | Instruction::JumpIfMatchPrefixInsensitive { idx, tgt } => {
+                bytes.extend_from_slice(&enc_u32(idx));
+                bytes.extend_from_slice(&enc_u32(tgt));
+            }
+
+            Instruction::FlushHighlight { kind } => {
+                bytes.push(enc_reg_single(kind));
+            }
+            Instruction::AwaitInput => {}
+        }
+
+        bytes
+    }
+
+    pub fn decode(bytes: &[u8]) -> (Option<Self>, usize) {
+        fn dec_reg_pair(b: u8) -> (Register, Register) {
+            let hi = Register::from_usize((b >> 4) as usize);
+            let lo = Register::from_usize((b & 0xF) as usize);
+            (lo, hi)
+        }
+
+        fn dec_reg_single(b: u8) -> Register {
+            Register::from_usize(b as usize)
+        }
+
+        fn dec_u32(bytes: &[u8]) -> u32 {
+            u32::from_le_bytes(bytes[..4].try_into().unwrap())
+        }
+
+        let opcode = bytes[0];
+        match opcode {
+            0 => {
+                let (dst, src) = dec_reg_pair(bytes[1]);
+                (Some(Instruction::Mov { dst, src }), 2)
+            }
+            1 => {
+                let (dst, src) = dec_reg_pair(bytes[1]);
+                (Some(Instruction::Add { dst, src }), 2)
+            }
+            2 => {
+                let (dst, src) = dec_reg_pair(bytes[1]);
+                (Some(Instruction::Sub { dst, src }), 2)
+            }
+            3 => {
+                let dst = dec_reg_single(bytes[1]);
+                let imm = dec_u32(&bytes[2..]);
+                (Some(Instruction::MovImm { dst, imm }), 6)
+            }
+            4 => {
+                let dst = dec_reg_single(bytes[1]);
+                let imm = dec_u32(&bytes[2..]);
+                (Some(Instruction::AddImm { dst, imm }), 6)
+            }
+            5 => {
+                let dst = dec_reg_single(bytes[1]);
+                let imm = dec_u32(&bytes[2..]);
+                (Some(Instruction::SubImm { dst, imm }), 6)
+            }
+            6 => (Some(Instruction::Call { tgt: dec_u32(&bytes[1..]) }), 5),
+            7 => (Some(Instruction::Return), 1),
+            8 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpEQ { lhs, rhs, tgt }), 6)
+            }
+            9 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpNE { lhs, rhs, tgt }), 6)
+            }
+            10 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpLT { lhs, rhs, tgt }), 6)
+            }
+            11 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpLE { lhs, rhs, tgt }), 6)
+            }
+            12 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpGT { lhs, rhs, tgt }), 6)
+            }
+            13 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpGE { lhs, rhs, tgt }), 6)
+            }
+            14 => (Some(Instruction::JumpIfEndOfLine { tgt: dec_u32(&bytes[1..]) }), 5),
+            15 => {
+                let idx = dec_u32(&bytes[1..]);
+                let tgt = dec_u32(&bytes[5..]);
+                (Some(Instruction::JumpIfMatchCharset { idx, tgt }), 9)
+            }
+            16 => {
+                let idx = dec_u32(&bytes[1..]);
+                let tgt = dec_u32(&bytes[5..]);
+                (Some(Instruction::JumpIfMatchPrefix { idx, tgt }), 9)
+            }
+            17 => {
+                let idx = dec_u32(&bytes[1..]);
+                let tgt = dec_u32(&bytes[5..]);
+                (Some(Instruction::JumpIfMatchPrefixInsensitive { idx, tgt }), 9)
+            }
+            18 => {
+                let kind = dec_reg_single(bytes[1]);
+                (Some(Instruction::FlushHighlight { kind }), 2)
+            }
+            19 => (Some(Instruction::AwaitInput), 1),
+            _ => (None, 1),
+        }
+    }
+
+    pub fn mnemonic<'a>(
+        &self,
+        arena: &'a Arena,
+        config: &MnemonicFormattingConfig,
+    ) -> ArenaString<'a> {
+        let mut str = ArenaString::new_in(arena);
+        let _i = config.instruction_prefix;
+        let i_ = config.instruction_suffix;
+        let _r = config.register_prefix;
+        let r_ = config.register_suffix;
+        let _a = config.address_prefix;
+        let a_ = config.address_suffix;
+        let _n = config.numeric_prefix;
+        let n_ = config.numeric_suffix;
+
+        match *self {
+            Instruction::Mov { dst, src } => {
+                _ = write!(str, "{_i}mov{i_}    {_r}{dst}{r_}, {_r}{src}{r_}");
+            }
+            Instruction::Add { dst, src } => {
+                _ = write!(str, "{_i}add{i_}    {_r}{dst}{r_}, {_r}{src}{r_}");
+            }
+            Instruction::Sub { dst, src } => {
+                _ = write!(str, "{_i}sub{i_}    {_r}{dst}{r_}, {_r}{src}{r_}");
+            }
+            Instruction::MovImm { dst, imm } => {
+                if dst == Register::ProgramCounter {
+                    _ = write!(str, "{_i}movi{i_}   {_r}{dst}{r_}, {_a}{imm}{a_}");
+                } else {
+                    _ = write!(str, "{_i}movi{i_}   {_r}{dst}{r_}, {_n}{imm}{n_}");
+                }
+            }
+            Instruction::AddImm { dst, imm } => {
+                _ = write!(str, "{_i}addi{i_}   {_r}{dst}{r_}, {_n}{imm}{n_}");
+            }
+            Instruction::SubImm { dst, imm } => {
+                _ = write!(str, "{_i}subi{i_}   {_r}{dst}{r_}, {_n}{imm}{n_}");
+            }
+
+            Instruction::Call { tgt } => {
+                _ = write!(str, "{_i}call{i_}   {_a}{tgt}{a_}");
+            }
+            Instruction::Return => {
+                _ = write!(str, "{_i}ret{i_}");
+            }
+
+            Instruction::JumpEQ { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jeq{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpNE { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jne{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpLT { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jlt{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpLE { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jle{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpGT { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jgt{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpGE { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jge{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+
+            Instruction::JumpIfEndOfLine { tgt } => {
+                _ = write!(str, "{_i}jeol{i_}   {_a}{tgt}{a_}");
+            }
+            Instruction::JumpIfMatchCharset { idx, tgt } => {
+                _ = write!(str, "{_i}jc{i_}     {_n}{idx}{n_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpIfMatchPrefix { idx, tgt } => {
+                _ = write!(str, "{_i}jp{i_}     {_n}{idx}{n_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpIfMatchPrefixInsensitive { idx, tgt } => {
+                _ = write!(str, "{_i}jpi{i_}    {_n}{idx}{n_}, {_a}{tgt}{a_}");
+            }
+
+            Instruction::FlushHighlight { kind } => {
+                _ = write!(str, "{_i}flush{i_}  {_r}{kind}{r_}");
+            }
+            Instruction::AwaitInput => {
+                _ = write!(str, "{_i}await{i_}");
+            }
+        }
+
+        str
+    }
+}
+
+#[derive(Default)]
+pub struct MnemonicFormattingConfig<'a> {
+    // Color used for highlighting the instruction.
+    pub instruction_prefix: &'a str,
+    pub instruction_suffix: &'a str,
+
+    // Color used for highlighting a register name.
+    pub register_prefix: &'a str,
+    pub register_suffix: &'a str,
+
+    // Color used for highlighting an immediate value.
+    pub address_prefix: &'a str,
+    pub address_suffix: &'a str,
+
+    // Color used for highlighting an immediate value.
+    pub numeric_prefix: &'a str,
+    pub numeric_suffix: &'a str,
+}
diff --git a/crates/lsh/src/compiler/optimizer.rs b/crates/lsh/src/compiler/optimizer.rs
new file mode 100644
index 000000000000..01f0fa8c222d
--- /dev/null
+++ b/crates/lsh/src/compiler/optimizer.rs
@@ -0,0 +1,210 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! IR optimization passes, run before register allocation.
+//!
+//! ## Quirks
+//!
+//! - Pass order matters
+//!
+//! ## TODO
+//!
+//! - Could do copy propagation: if `vreg1 = vreg2`, replace all uses of vreg1 with vreg2.
+//! - Could merge consecutive `Add { off, off, 1 }` instructions.
+//! - Could eliminate unreachable code after `Return`.
+
+use std::cell::RefMut;
+use std::iter::repeat_n;
+use std::ptr;
+
+use stdext::arena::scratch_arena;
+
+use super::*;
+
+pub fn optimize<'a>(compiler: &mut Compiler<'a>) {
+    // Remove noops first, such that analyzing instruction chains becomes easier for the other passes.
+    optimize_noop(compiler);
+    optimize_redundant_offset_backup_restore(compiler);
+    optimize_highlight_kind_values(compiler);
+}
+
+/// Removes no-op instructions from the IR.
+fn optimize_noop<'a>(compiler: &mut Compiler<'a>) {
+    // Remove noops from the function entrypoint (the trunk of the tree).
+    for function in &mut compiler.functions {
+        while let body = function.body.borrow()
+            && let Some(next) = body.next
+            && matches!(body.instr, IRI::Noop)
+        {
+            function.body = next;
+        }
+    }
+
+    for function in &compiler.functions {
+        for current_cell in compiler.visit_nodes_from(function.body) {
+            // First, filter down to nodes that are not no-ops.
+            if let mut current = current_cell.borrow_mut()
+                && !matches!(current.instr, IRI::Noop)
+            {
+                // `IRI::If` nodes have an additional "next" pointer.
+                let current = &mut *current;
+                let mut nexts = [
+                    current.next.as_mut(),
+                    match &mut current.instr {
+                        IRI::If { then, .. } => Some(then),
+                        _ => None,
+                    },
+                ];
+
+                // Now, "pop_front" no-ops from the next pointer, until it
+                // points to a real op (or None, but that shouldn't happen).
+                for next_ref in nexts.into_iter().flatten() {
+                    while !ptr::eq(*next_ref, current_cell)
+                        && let next = next_ref.borrow()
+                        && matches!(next.instr, IRI::Noop)
+                        && let Some(skip_next) = next.next
+                    {
+                        *next_ref = skip_next;
+                    }
+                }
+            }
+        }
+    }
+}
+
+// Conditions in the VM advance the offset only if they match. The frontend doesn't
+// care about this and emits pointless backup/restore instructions for the offset.
+// This code is responsible for turning this chain of `.next` pointers:
+//   IRI::Add { off -> backup }
+//   IRI::If { .. }
+//   IRI::Add { backup -> off }
+//   IRI::If { .. }
+//   IRI::Add { backup -> off }
+//   IRI::If { .. }
+//   IRI::Add { backup -> off }
+// into this:
+//   IRI::Add { off -> backup }
+//   IRI::If { .. }
+//   IRI::If { .. }
+//   IRI::If { .. }
+fn optimize_redundant_offset_backup_restore<'a>(compiler: &mut Compiler<'a>) {
+    let off_reg = compiler.get_reg(Register::InputOffset);
+
+    // Remove pointless offset restore chains.
+    for function in &compiler.functions {
+        for current_cell in compiler.visit_nodes_from(function.body) {
+            // First, filter down to nodes that assign the `off` to a virtual register.
+            if let mut save = current_cell.borrow_mut()
+                && let IRI::Mov { dst: backup_reg, src } = save.instr
+                && ptr::eq(src, off_reg)
+                && backup_reg.borrow().physical.is_none()
+            {
+                let mut next_cond = save.next;
+
+                // Next optimize an entire chain of `if` conditions that pointlessly restore `off`.
+                while let Some(cond) = next_cond
+                    && let mut cond = cond.borrow_mut()
+                    && matches!(cond.instr, IRI::If { .. })
+                    && let Some(restore) = cond.next
+                    && let mut restore = restore.borrow_mut()
+                    && matches!(restore.instr, IRI::Mov { dst, src } if ptr::eq(dst, off_reg) && ptr::eq(src, backup_reg))
+                {
+                    cond.next = restore.next;
+                    next_cond = restore.next;
+                }
+            }
+        }
+    }
+
+    // Remove pointless offset backups.
+    // A backup is pointless if the destination vreg is never read.
+    for function in &compiler.functions {
+        // First, collect all vregs that are read anywhere in the function.
+        let mut used_vregs = HashSet::new();
+        for current_cell in compiler.visit_nodes_from(function.body) {
+            let current = current_cell.borrow();
+            match current.instr {
+                IRI::Mov { src, .. } => {
+                    let id = src.borrow().id;
+                    used_vregs.insert(id);
+                }
+                IRI::If { condition: Condition::Cmp { lhs, rhs, .. }, .. } => {
+                    used_vregs.insert(lhs.borrow().id);
+                    used_vregs.insert(rhs.borrow().id);
+                }
+                _ => {}
+            }
+        }
+
+        // Now remove dead stores (assignments to vregs that are never read).
+        for current_cell in compiler.visit_nodes_from(function.body) {
+            // First, filter down to nodes that assign the `off` to a virtual register.
+            if let mut cell = current_cell.borrow_mut()
+                && let IRI::Mov { dst, src } = cell.instr
+                && let src = src.borrow()
+                && let dst = dst.borrow()
+                // TODO: Technically we could also optimize vreg --> vreg assignments, but for that we
+                // need to be able to call `count_register_uses` multiple times, so that the count is
+                // accurate after removing an assignment. Physical registers don't care about that.
+                && src.physical.is_some()
+                // We can't optimize physical register --> physical register assignments.
+                && dst.physical.is_none()
+                && !used_vregs.contains(&dst.id)
+            {
+                cell.instr = IRI::Noop;
+            }
+        }
+    }
+    optimize_noop(compiler);
+}
+
+/// This isn't an optimization for the VM, it's one for my autistic side.
+/// I like it if the identifiers are sorted and the values contiguous.
+fn optimize_highlight_kind_values<'a>(compiler: &mut Compiler<'a>) {
+    let scratch = scratch_arena(None);
+    let mut mapping = Vec::new_in(&*scratch);
+
+    compiler.highlight_kinds.sort_unstable_by(|a, b| {
+        let a = a.identifier;
+        let b = b.identifier;
+
+        // Global identifiers without a dot come first.
+        let nested_a = a.contains('.');
+        let nested_b = b.contains('.');
+        let cmp = nested_a.cmp(&nested_b);
+        if cmp != std::cmp::Ordering::Equal {
+            return cmp;
+        }
+
+        // Among globals, "other" comes first. Due to the above,
+        // `nested_a == false` implies `nested_b == false`.
+        if !nested_a {
+            if a == "other" {
+                return std::cmp::Ordering::Less;
+            }
+            if b == "other" {
+                return std::cmp::Ordering::Greater;
+            }
+        }
+
+        // Otherwise, sort by dot-separated components.
+        a.split('.').cmp(b.split('.'))
+    });
+
+    mapping.resize(compiler.highlight_kinds.len(), u32::MAX);
+    for (idx, hk) in compiler.highlight_kinds.iter_mut().enumerate() {
+        let idx = idx as u32;
+        mapping[hk.value as usize] = idx;
+        hk.value = idx;
+    }
+
+    for function in &compiler.functions {
+        for current in compiler.visit_nodes_from(function.body) {
+            let mut current = current.borrow_mut();
+
+            if let IRI::MovKind { dst, kind } = &mut current.instr {
+                *kind = mapping[*kind as usize];
+            }
+        }
+    }
+}
diff --git a/crates/lsh/src/compiler/regex.rs b/crates/lsh/src/compiler/regex.rs
new file mode 100644
index 000000000000..b6b91a30f012
--- /dev/null
+++ b/crates/lsh/src/compiler/regex.rs
@@ -0,0 +1,501 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Regex-to-IR compiler using `regex_syntax` as the parser.
+//!
+//! ## Supported patterns
+//!
+//! - Literals: `foo`, `bar`
+//! - Character classes: `[a-z]`, `[^0-9]`, `.` (any byte), `\w`, `\d`, etc.
+//! - Quantifiers: `?`, `+`, `*` (greedy only)
+//! - Alternation: `a|b|c`
+//! - Concatenation: `abc`
+//! - Capture groups: `(...)` (for `yield $1 as color`)
+//! - Word boundary: `\b` (ASCII only, via `Look::WordEndHalfAscii`)
+//! - End of line: `$`
+//!
+//! ## Gotchas
+//!
+//! - Single-character classes like `[eE]` get expanded into a chain of `Prefix` conditions
+//!   rather than using `Charset`. This is because prefix matching advances `off` by exactly
+//!   the matched length, while charset matching is greedy (matches as many as possible).
+//! - For patterns like `[a-z]+|\w+`, the compiler tries to chain alternatives so that if
+//!   the more specific pattern fails partway through, it falls back to the more general one.
+//!   This is the `try_append_as_fallback` logic - it's complex and may have edge cases.
+//! - If a charset includes `\w` (word characters), the compiler also sets bits for UTF-8
+//!   continuation byte starters (0xC2-0xF4). This lets `\w+` consume multibyte characters
+//!   even though we operate on bytes. It's not Unicode-correct but works for identifiers.
+//! - The `parse()` function modifies `dst_good`/`dst_bad` nodes' `.next` pointers.
+//!   Don't pass nodes that are already wired into the IR graph.
+//!
+//! ## TODO
+//!
+//! - Returning `Err(String)` but the caller wraps it in `CompileError`.
+//!   Should just return `CompileError` directly.
+//! - The rest of the fucking owl.
+
+use std::ops::RangeInclusive;
+use std::ptr;
+
+use regex_syntax::hir::{Class, ClassBytes, ClassBytesRange, Hir, HirKind, Look};
+
+use super::*;
+
+pub fn parse<'a>(
+    compiler: &mut Compiler<'a>,
+    pattern: &str,
+    dst_good: IRCell<'a>,
+    dst_bad: IRCell<'a>,
+    capture_groups: &mut Vec<(IRRegCell<'a>, IRRegCell<'a>), &'a Arena>,
+) -> Result<IRCell<'a>, String> {
+    let hir = match regex_syntax::ParserBuilder::new()
+        .utf8(false)
+        .unicode(false)
+        .dot_matches_new_line(true)
+        .build()
+        .parse(pattern)
+    {
+        Ok(hir) => hir,
+        Err(e) => return Err(format!("{e}")),
+    };
+
+    let src = transform(compiler, dst_good, &hir, capture_groups)?;
+
+    // Connect all unset .next pointers to dst_bad.
+    for node in compiler.visit_nodes_from(src) {
+        if !ptr::eq(node, dst_good)
+            && !ptr::eq(node, dst_bad)
+            && let mut node = node.borrow_mut()
+            && node.wants_next()
+        {
+            node.set_next(dst_bad);
+        }
+    }
+
+    Ok(src)
+}
+
+fn transform<'a>(
+    compiler: &mut Compiler<'a>,
+    dst: IRCell<'a>,
+    hir: &Hir,
+    capture_groups: &mut Vec<(IRRegCell<'a>, IRRegCell<'a>), &'a Arena>,
+) -> Result<IRCell<'a>, String> {
+    fn is_any_class(class: &ClassBytes) -> bool {
+        class.ranges() == [ClassBytesRange::new(0, 255)]
+    }
+
+    match hir.kind() {
+        HirKind::Empty => Ok(dst),
+        HirKind::Literal(lit) => transform_literal(compiler, dst, &lit.0),
+        HirKind::Class(Class::Bytes(class)) if is_any_class(class) => transform_any(compiler, dst),
+        HirKind::Class(Class::Bytes(class)) => transform_class(compiler, dst, class),
+        HirKind::Class(Class::Unicode(class)) => {
+            transform_class(compiler, dst, &class.to_byte_class().unwrap())
+        }
+        HirKind::Look(Look::End) => {
+            Ok(compiler.alloc_iri(IRI::If { condition: Condition::EndOfLine, then: dst }))
+        }
+        HirKind::Look(Look::WordEndHalfAscii) => {
+            // If the test hits, then we failed the word boundary, so go to dst_bad.
+            // We don't have dst_bad here, so allocate a noop node, which later gets
+            // picked up with its unset .next pointer, and set to dst_bad automatically.
+            let bad = compiler.alloc_noop();
+            let c = compiler.intern_charset(&ASCII_WORD_CHARSET);
+            Ok(compiler.alloc_ir(IR {
+                next: Some(dst),
+                instr: IRI::If { condition: Condition::Charset(c), then: bad },
+                offset: usize::MAX,
+            }))
+        }
+        HirKind::Repetition(rep) => match (rep.min, rep.max, rep.greedy, rep.sub.kind()) {
+            (0, None, true, HirKind::Class(Class::Bytes(class))) => {
+                if is_any_class(class) {
+                    transform_any_star(compiler, dst)
+                } else {
+                    let src = transform_class_plus(compiler, dst, class)?;
+                    transform_option(src, dst)
+                }
+            }
+            (0, Some(1), true, _) => {
+                let src = transform(compiler, dst, &rep.sub, capture_groups)?;
+                transform_option(src, dst)
+            }
+            (1, None, true, HirKind::Literal(lit)) => transform_literal_plus(compiler, dst, lit),
+            (1, None, true, HirKind::Class(Class::Bytes(class))) => {
+                transform_class_plus(compiler, dst, class)
+            }
+            _ => panic!("Unsupported HIR: {hir:?}"),
+        },
+        HirKind::Concat(hirs) => transform_concat(compiler, dst, hirs, capture_groups),
+        HirKind::Alternation(hirs) => transform_alt(compiler, dst, hirs, capture_groups),
+        HirKind::Capture(capture) => {
+            // Save the current input offset before matching the capture group
+            let start_vreg = compiler.alloc_vreg();
+            let save_start = compiler.alloc_iri(IRI::Mov {
+                dst: start_vreg,
+                src: compiler.get_reg(Register::InputOffset),
+            });
+
+            // Transform the sub-expression
+            let end_marker = compiler.alloc_noop();
+            save_start.borrow_mut().set_next(transform(
+                compiler,
+                end_marker,
+                &capture.sub,
+                capture_groups,
+            )?);
+
+            // Save the end offset after matching
+            let end_vreg = compiler.alloc_vreg();
+            let save_end = compiler.alloc_iri(IRI::Mov {
+                dst: end_vreg,
+                src: compiler.get_reg(Register::InputOffset),
+            });
+            end_marker.borrow_mut().set_next(save_end);
+            save_end.borrow_mut().set_next(dst);
+
+            // Store the capture group (index is 1-based in regex, 0-based in our vec)
+            let vec_index = (capture.index as usize).saturating_sub(1);
+            // Ensure the vector is large enough
+            while capture_groups.len() <= vec_index {
+                let dummy_start = compiler.alloc_vreg();
+                let dummy_end = compiler.alloc_vreg();
+                capture_groups.push((dummy_start, dummy_end));
+            }
+            capture_groups[vec_index] = (start_vreg, end_vreg);
+
+            Ok(save_start)
+        }
+        _ => panic!("Unsupported HIR: {hir:?}"),
+    }
+}
+
+// string
+fn transform_literal<'a>(
+    compiler: &mut Compiler<'a>,
+    dst: IRCell<'a>,
+    lit: &[u8],
+) -> Result<IRCell<'a>, String> {
+    let s = String::from_utf8(lit.to_vec()).unwrap();
+    let s = compiler.intern_string(&s);
+    Ok(compiler.alloc_iri(IRI::If { condition: Condition::Prefix(s), then: dst }))
+}
+
+// a+
+fn transform_literal_plus<'a>(
+    compiler: &mut Compiler<'a>,
+    dst: IRCell<'a>,
+    lit: &regex_syntax::hir::Literal,
+) -> Result<IRCell<'a>, String> {
+    assert!(lit.0.len() == 1);
+    let mut c = Charset::default();
+    c[lit.0[0] as usize] = true;
+    let c = compiler.intern_charset(&c);
+    Ok(compiler.alloc_iri(IRI::If { condition: Condition::Charset(c), then: dst }))
+}
+
+// [a-z]+
+fn transform_class_plus<'a>(
+    compiler: &mut Compiler<'a>,
+    dst: IRCell<'a>,
+    class: &ClassBytes,
+) -> Result<IRCell<'a>, String> {
+    let c = class_to_charset(class);
+    let c = compiler.intern_charset(&c);
+    Ok(compiler.alloc_iri(IRI::If { condition: Condition::Charset(c), then: dst }))
+}
+
+// [eE]
+fn transform_class<'a>(
+    compiler: &mut Compiler<'a>,
+    dst: IRCell<'a>,
+    class: &ClassBytes,
+) -> Result<IRCell<'a>, String> {
+    let mut charset = class_to_charset(class);
+    let mut first: Option<IRCell<'a>> = None;
+    let mut last: Option<IRCell<'a>> = None;
+
+    for i in 0..256 {
+        if !charset[i] {
+            continue;
+        }
+
+        if i >= 128 {
+            panic!("Invalid non-ASCII class character {i}");
+        }
+
+        let ch = i as u8;
+        let mut str = String::new();
+        str.push(ch as char);
+
+        let upper = ch.to_ascii_lowercase() as usize;
+        // NOTE: Uppercase chars have a lower numeric value than lowercase chars.
+        // As such, we need to test for `is_ascii_uppercase`.
+        let insensitive = ch.is_ascii_uppercase() && charset[upper];
+
+        if insensitive {
+            charset[upper] = false;
+            str.make_ascii_lowercase();
+        }
+
+        let str = compiler.intern_string(&str);
+        let condition =
+            if insensitive { Condition::PrefixInsensitive(str) } else { Condition::Prefix(str) };
+
+        let node = compiler.alloc_iri(IRI::If { condition, then: dst });
+        if first.is_none() {
+            first = Some(node);
+        }
+        if let Some(last) = &last {
+            last.borrow_mut().set_next(node);
+        }
+        last = Some(node);
+    }
+
+    Ok(first.unwrap())
+}
+
+// .?
+fn transform_option<'a>(src: IRCell<'a>, dst: IRCell<'a>) -> Result<IRCell<'a>, String> {
+    let mut n = src.borrow_mut();
+
+    while let Some(next) = n.next {
+        n = next.borrow_mut();
+    }
+
+    n.set_next(dst);
+    Ok(src)
+}
+
+// .*
+fn transform_any_star<'a>(
+    compiler: &mut Compiler<'a>,
+    dst: IRCell<'a>,
+) -> Result<IRCell<'a>, String> {
+    Ok(compiler.alloc_ir(IR {
+        next: Some(dst),
+        instr: IRI::MovImm { dst: compiler.get_reg(Register::InputOffset), imm: u32::MAX },
+        offset: usize::MAX,
+    }))
+}
+
+// .
+fn transform_any<'a>(compiler: &mut Compiler<'a>, dst: IRCell<'a>) -> Result<IRCell<'a>, String> {
+    Ok(compiler.alloc_ir(IR {
+        next: Some(dst),
+        instr: IRI::AddImm { dst: compiler.get_reg(Register::InputOffset), imm: 1 },
+        offset: usize::MAX,
+    }))
+}
+
+// (a)(b)
+fn transform_concat<'a>(
+    compiler: &mut Compiler<'a>,
+    dst: IRCell<'a>,
+    hirs: &[Hir],
+    capture_groups: &mut Vec<(IRRegCell<'a>, IRRegCell<'a>), &'a Arena>,
+) -> Result<IRCell<'a>, String> {
+    fn is_lowercase_literal(hir: &Hir) -> Option<u8> {
+        if let HirKind::Class(Class::Bytes(class)) = hir.kind()
+            && let ranges = class.ranges()
+            && ranges.len() == 2
+            && ranges[0].len() == 1
+            && ranges[1].len() == 1
+            && let lower_a = ranges[0].start().to_ascii_lowercase()
+            && let lower_b = ranges[1].start().to_ascii_lowercase()
+            && lower_a == lower_b
+        {
+            Some(lower_a)
+        } else {
+            None
+        }
+    }
+
+    let mut it = hirs.iter().peekable();
+    let mut first: Option<IRCell<'a>> = None;
+    let mut last: Option<IRCell<'a>> = None;
+
+    while let Some(hir) = it.next() {
+        // Transform [aA][bB][cC] into PrefixInsensitive("abc").
+        let prefix_insensitive = is_lowercase_literal(hir).map(|ch| {
+            let mut str = String::new();
+            str.push(ch as char);
+
+            while let Some(next_hir) = it.peek() {
+                if let Some(next_ch) = is_lowercase_literal(next_hir) {
+                    str.push(next_ch as char);
+                    it.next();
+                } else {
+                    break;
+                }
+            }
+
+            str.make_ascii_lowercase();
+            str
+        });
+
+        let dst = compiler.alloc_noop();
+        let src = if let Some(str) = prefix_insensitive {
+            let str = compiler.intern_string(&str);
+            compiler.alloc_iri(IRI::If { condition: Condition::PrefixInsensitive(str), then: dst })
+        } else {
+            transform(compiler, dst, hir, capture_groups)?
+        };
+        if first.is_none() {
+            first = Some(src);
+        }
+        if let Some(last) = &last {
+            last.borrow_mut().set_next(src);
+        }
+        last = Some(dst);
+    }
+
+    if let Some(last) = &last {
+        last.borrow_mut().set_next(dst);
+    }
+
+    Ok(first.unwrap())
+}
+
+// (a|b)
+fn transform_alt<'a>(
+    compiler: &mut Compiler<'a>,
+    dst: IRCell<'a>,
+    hirs: &[Hir],
+    capture_groups: &mut Vec<(IRRegCell<'a>, IRRegCell<'a>), &'a Arena>,
+) -> Result<IRCell<'a>, String> {
+    let mut first: Option<IRCell<'a>> = None;
+    let mut last: Option<IRCell<'a>> = None;
+
+    for hir in hirs {
+        let node = transform(compiler, dst, hir, capture_groups)?;
+        if first.is_none() {
+            first = Some(node);
+        }
+        if let Some(last) = &last {
+            // Try to merge this alternative as a fallback of previous alternatives
+            // if their charsets are compatible (superset relationship)
+            if !try_append_as_fallback(last, node) {
+                last.borrow_mut().set_next(node);
+            }
+        }
+        last = Some(node);
+    }
+
+    Ok(first.unwrap())
+}
+
+// Try to append `fallback` as a fallback to nodes in `tree` where the charset
+// of `fallback` is a superset of the charset in `tree`. This allows patterns like
+// `[a-z]+foo|\w+bar` to match `foobar` by trying the more specific pattern first,
+// and falling back to the more general pattern on failure.
+fn try_append_as_fallback<'a>(tree: &IRCell<'a>, fallback: IRCell<'a>) -> bool {
+    let fallback_charset = get_initial_charset(fallback);
+    if fallback_charset.is_none() {
+        return false;
+    }
+
+    append_fallback_recursive(tree, fallback, fallback_charset.unwrap())
+}
+
+fn append_fallback_recursive<'a>(
+    node: &IRCell<'a>,
+    fallback: IRCell<'a>,
+    fallback_charset: &Charset,
+) -> bool {
+    let mut node_ref = node.borrow_mut();
+
+    if let IRI::If { condition: Condition::Charset(charset), then } = &node_ref.instr {
+        // If the fallback charset is a superset of this node's charset,
+        // we can append the fallback to this node's failure path
+        if is_superset(fallback_charset, charset) {
+            // Recursively try to append to the success path first
+            let appended_to_then = append_fallback_recursive(then, fallback, fallback_charset);
+
+            // If not appended to success path, append to failure path (next)
+            if !appended_to_then {
+                if let Some(next) = node_ref.next {
+                    // Try to append recursively to the next node
+                    if !append_fallback_recursive(&next, fallback, fallback_charset) {
+                        // If we couldn't append recursively, this might be the right place
+                        // But we already have a next pointer, so we need to be careful
+                        // For now, don't override existing next pointers in this branch
+                    }
+                } else {
+                    // This node has no next pointer yet, so we can set it to the fallback
+                    node_ref.next = Some(fallback);
+                    return true;
+                }
+            }
+            return appended_to_then;
+        }
+    }
+
+    // Try recursively on the next node
+    if let Some(next) = node_ref.next {
+        drop(node_ref); // Release the borrow before recursing
+        return append_fallback_recursive(&next, fallback, fallback_charset);
+    }
+
+    false
+}
+
+// Get the initial charset that a regex tree matches
+fn get_initial_charset<'a>(node: IRCell<'a>) -> Option<&'a Charset> {
+    let node_ref = node.borrow();
+    match &node_ref.instr {
+        IRI::If { condition: Condition::Charset(charset), .. } => Some(*charset),
+        _ => None,
+    }
+}
+
+// Check if `superset` is a superset of `subset`
+fn is_superset(superset: &Charset, subset: &Charset) -> bool {
+    for i in 0..256 {
+        if subset[i] && !superset[i] {
+            return false;
+        }
+    }
+    true
+}
+
+// [a-z] -> 256-ary LUT
+fn class_to_charset(class: &ClassBytes) -> Charset {
+    let mut charset = Charset::default();
+
+    for r in class.iter() {
+        charset[r.start() as usize..=r.end() as usize].fill(true);
+    }
+
+    // If the class includes \w, we also set any non-ASCII UTF8 starters.
+    // That's not how Unicode works, but it simplifies the implementation.
+    if [(b'0', b'9'), (b'A', b'Z'), (b'_', b'_'), (b'a', b'z')]
+        .iter()
+        .all(|&(beg, end)| charset[beg as usize..=end as usize].iter().all(|&b| b))
+    {
+        charset[0xC2..=0xF4].fill(true);
+    }
+
+    charset
+}
+
+const ASCII_WORD_CHARSET: Charset = {
+    const fn fill(bits: &mut [bool; 256], range: RangeInclusive<u8>) {
+        let beg = *range.start() as usize;
+        let end = *range.end() as usize;
+        let mut i = beg;
+        while i <= end {
+            bits[i] = true;
+            i += 1;
+        }
+    }
+
+    let mut charset = Charset::no();
+    fill(&mut charset.bits, b'0'..=b'9');
+    fill(&mut charset.bits, b'A'..=b'Z');
+    fill(&mut charset.bits, b'_'..=b'_');
+    fill(&mut charset.bits, b'a'..=b'z');
+    fill(&mut charset.bits, 0xC2..=0xF4);
+    charset
+};
diff --git a/crates/lsh/src/compiler/tokenizer.rs b/crates/lsh/src/compiler/tokenizer.rs
new file mode 100644
index 000000000000..97996c9e19af
--- /dev/null
+++ b/crates/lsh/src/compiler/tokenizer.rs
@@ -0,0 +1,281 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Lexer for the LSH DSL.
+//!
+//! ## Quirks
+//!
+//! - Regex and string tokens include escape sequences verbatim. The frontend/regex compiler
+//!   handles unescaping. This means `/\/foo/` tokenizes as `Regex("\\/foo")`.
+//! - `position()` computes line/column by scanning from the start.
+//!   Fine for error reporting, but don't call it in hot paths.
+//! - Identifiers can contain dots (e.g., `string.quoted`), which is unusual.
+//!   This is for hierarchical highlight kind names. Affects `is_ident_continuation`.
+
+use std::iter::Peekable;
+use std::str::Chars;
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum Token<'a> {
+    // Literals
+    Identifier(&'a str),
+    Integer(u32),
+    String(&'a str),
+    Regex(&'a str),
+    Submatch(u32),
+
+    // Keywords
+    Pub,
+    Fn,
+    Var,
+    If,
+    Else,
+    Loop,
+    Until,
+    Break,
+    Continue,
+    Return,
+    Await,
+    Yield,
+
+    // Operators
+    Equals,
+    EqualsEquals,
+    NotEquals,
+    LessThan,
+    LessThanEquals,
+    GreaterThan,
+    GreaterThanEquals,
+    Plus,
+    PlusEquals,
+
+    // Punctuation
+    LeftBrace,
+    RightBrace,
+    LeftParen,
+    RightParen,
+    Semicolon,
+    LeftBracket,
+    RightBracket,
+    Hash,
+    Comma,
+
+    // End of file
+    Eof,
+
+    // Error
+    Error(String),
+}
+
+pub struct Tokenizer<'a> {
+    input: &'a str,
+    chars: Peekable<Chars<'a>>,
+    current_pos: usize,
+    start_pos: usize,
+}
+
+impl<'a> Tokenizer<'a> {
+    pub fn new(input: &'a str) -> Self {
+        Self { input, chars: input.chars().peekable(), current_pos: 0, start_pos: 0 }
+    }
+
+    pub fn position(&self) -> (usize, usize) {
+        let mut line = 1;
+        let mut column = 1;
+
+        for ch in self.input[..self.start_pos].chars() {
+            column += 1;
+            if ch == '\n' {
+                line += 1;
+                column = 1;
+            }
+        }
+
+        (line, column)
+    }
+
+    pub fn next(&mut self) -> Token<'a> {
+        self.skip_whitespace();
+
+        self.start_pos = self.current_pos;
+
+        match self.advance() {
+            None => Token::Eof,
+            Some(ch) => match ch {
+                '{' => Token::LeftBrace,
+                '}' => Token::RightBrace,
+                '(' => Token::LeftParen,
+                ')' => Token::RightParen,
+                '[' => Token::LeftBracket,
+                ']' => Token::RightBracket,
+                '#' => Token::Hash,
+                ',' => Token::Comma,
+                '$' => {
+                    self.start_pos += 1; // Skip the '$'
+                    match self.read_integer(ch) {
+                        Token::Integer(idx) => Token::Submatch(idx),
+                        other => other,
+                    }
+                }
+                ';' => Token::Semicolon,
+                '"' => self.read_string(),
+                '/' => self.read_regex(),
+                '=' => {
+                    if self.peek() == Some('=') {
+                        self.advance();
+                        Token::EqualsEquals
+                    } else {
+                        Token::Equals
+                    }
+                }
+                '!' => {
+                    if self.peek() == Some('=') {
+                        self.advance();
+                        Token::NotEquals
+                    } else {
+                        Token::Error("Unexpected character: '!'".to_string())
+                    }
+                }
+                '<' => {
+                    if self.peek() == Some('=') {
+                        self.advance();
+                        Token::LessThanEquals
+                    } else {
+                        Token::LessThan
+                    }
+                }
+                '>' => {
+                    if self.peek() == Some('=') {
+                        self.advance();
+                        Token::GreaterThanEquals
+                    } else {
+                        Token::GreaterThan
+                    }
+                }
+                '+' => {
+                    if self.peek() == Some('=') {
+                        self.advance();
+                        Token::PlusEquals
+                    } else {
+                        Token::Plus
+                    }
+                }
+                '0'..='9' => self.read_integer(ch),
+                c => self.read_identifier_or_keyword(c),
+            },
+        }
+    }
+
+    fn advance(&mut self) -> Option<char> {
+        if let Some(ch) = self.chars.next() {
+            self.current_pos += ch.len_utf8();
+            Some(ch)
+        } else {
+            None
+        }
+    }
+
+    fn peek(&mut self) -> Option<char> {
+        self.chars.peek().copied()
+    }
+
+    fn skip_whitespace(&mut self) {
+        loop {
+            // Skip whitespace. advance() has been inlined with ASCII assumptions.
+            while let Some(ch) = self.peek()
+                && ch.is_ascii_whitespace()
+            {
+                self.chars.next();
+                self.current_pos += 1;
+            }
+
+            // Check for "//" comments. Skip the line if found.
+            if !self.input.get(self.current_pos..).is_some_and(|s| s.starts_with("//")) {
+                break;
+            }
+            for ch in &mut self.chars {
+                self.current_pos += 1;
+                if ch == '\n' {
+                    break;
+                }
+            }
+        }
+    }
+
+    fn read_string(&mut self) -> Token<'a> {
+        let beg = self.current_pos;
+        let mut last_char = '\0';
+
+        while let Some(ch) = self.advance()
+            && (ch != '"' || last_char == '\\')
+        {
+            last_char = ch;
+        }
+
+        let end = self.current_pos - 1;
+        Token::String(&self.input[beg..end])
+    }
+
+    fn read_regex(&mut self) -> Token<'a> {
+        let beg = self.current_pos;
+        let mut last_char = '\0';
+
+        while let Some(ch) = self.advance()
+            && (ch != '/' || last_char == '\\')
+        {
+            last_char = ch;
+        }
+
+        let end = self.current_pos - 1;
+        Token::Regex(&self.input[beg..end])
+    }
+
+    fn read_integer(&mut self, first: char) -> Token<'a> {
+        while let Some(ch) = self.peek()
+            && ch.is_ascii_digit()
+        {
+            self.advance();
+        }
+
+        let s = &self.input[self.start_pos..self.current_pos];
+        match s.parse::<u32>() {
+            Ok(val) => Token::Integer(val),
+            Err(_) => Token::Error(format!("Invalid integer: {s}")),
+        }
+    }
+
+    fn read_identifier_or_keyword(&mut self, ch: char) -> Token<'a> {
+        fn is_ident_start(ch: char) -> bool {
+            ch.is_ascii_alphabetic() || ch == '_'
+        }
+        fn is_ident_continuation(ch: char) -> bool {
+            ch.is_ascii_alphanumeric() || ch == '_' || ch == '.'
+        }
+
+        if !is_ident_start(ch) {
+            return Token::Error(format!("Unexpected character: '{ch}'"));
+        }
+
+        while let Some(ch) = self.peek()
+            && is_ident_continuation(ch)
+        {
+            self.advance();
+        }
+
+        match self.input.get(self.start_pos..self.current_pos).unwrap_or("") {
+            "pub" => Token::Pub,
+            "fn" => Token::Fn,
+            "var" => Token::Var,
+            "if" => Token::If,
+            "else" => Token::Else,
+            "loop" => Token::Loop,
+            "until" => Token::Until,
+            "break" => Token::Break,
+            "continue" => Token::Continue,
+            "return" => Token::Return,
+            "await" => Token::Await,
+            "yield" => Token::Yield,
+            ident => Token::Identifier(ident),
+        }
+    }
+}
diff --git a/crates/lsh/src/engine.rs b/crates/lsh/src/engine.rs
new file mode 100644
index 000000000000..dfceca46782f
--- /dev/null
+++ b/crates/lsh/src/engine.rs
@@ -0,0 +1,427 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! LSH bytecode interpreter.
+//!
+//! ## Performance notes
+//!
+//! - The main loop is "unsafe". Profile before "cleaning" it up.
+//! - `charset_gobble`, `inlined_mem(i)cmp` are hot paths.
+//!
+//! ## Gotchas
+//!
+//! - `Return` with empty stack resets the VM to entrypoint and clears registers.
+//!   This is how the DSL returns to the "idle" state between tokens.
+//! - `AwaitInput` only breaks the loop if `off >= line.len()`. If not at EOL, it's a no-op.
+//!   This allows the DSL to say "wait for more input OR continue if there is some".
+//! - The result always has a sentinel span at `line.len()`. Consumers can rely on this.
+
+use std::fmt::Debug;
+use std::mem;
+use std::path::Path;
+
+use stdext::arena::{Arena, scratch_arena};
+
+use crate::compiler::Registers;
+
+/// A compiled language definition with its bytecode entrypoint.
+pub struct Language {
+    /// Unique identifier (e.g., "rust", "markdown").
+    pub id: &'static str,
+    /// Human-readable display name.
+    pub name: &'static str,
+    /// Bytecode address where execution begins for this language.
+    pub entrypoint: u32,
+}
+
+impl PartialEq for &'static Language {
+    fn eq(&self, other: &Self) -> bool {
+        std::ptr::eq(*self, *other)
+    }
+}
+
+/// A highlight span indicating that text from `start` to the next span has the given `kind`.
+///
+/// Spans are half-open: `[start, next.start)`. The final span in a line extends to EOL.
+#[derive(Clone, PartialEq, Eq)]
+pub struct Higlight<T> {
+    /// Byte offset where this highlight begins.
+    pub start: usize,
+    /// The token/highlight type (e.g., keyword, string, comment).
+    pub kind: T,
+}
+
+impl<T: Debug> Debug for Higlight<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "({}, {:?})", self.start, self.kind)
+    }
+}
+
+/// The bytecode interpreter for syntax highlighting.
+#[derive(Clone)]
+pub struct Engine<'pa, 'ps, 'pc> {
+    assembly: &'pa [u8],
+    strings: &'ps [&'ps str],
+    charsets: &'pc [[u16; 16]],
+    entrypoint: u32,
+    stack: Vec<u32>,
+    registers: Registers,
+}
+
+/// Snapshot of engine state for incremental re-highlighting.
+///
+/// Save with [`Engine::snapshot`], restore with [`Engine::restore`].
+/// This allows re-highlighting from a known state when only part of a file changes.
+#[derive(Clone)]
+pub struct EngineState {
+    stack: Vec<u32>,
+    registers: Registers,
+}
+
+impl<'pa, 'ps, 'pc> Engine<'pa, 'ps, 'pc> {
+    pub fn new(
+        assembly: &'pa [u8],
+        strings: &'ps [&'ps str],
+        charsets: &'pc [[u16; 16]],
+        entrypoint: u32,
+    ) -> Self {
+        Engine {
+            assembly,
+            strings,
+            charsets,
+            entrypoint,
+            stack: Default::default(),
+            registers: Registers { pc: entrypoint, ..Default::default() },
+        }
+    }
+
+    pub fn snapshot(&self) -> EngineState {
+        EngineState { stack: self.stack.clone(), registers: self.registers }
+    }
+
+    pub fn restore(&mut self, state: &EngineState) {
+        self.stack = state.stack.clone();
+        self.registers = state.registers;
+    }
+
+    /// Parse a single line and return highlight spans.
+    ///
+    /// Executes bytecode until the line is fully consumed or a `Return` resets the VM.
+    /// The returned spans partition the line into highlighted regions.
+    ///
+    /// # Arguments
+    /// * `arena` - Allocator for the result vector
+    /// * `line` - The line bytes to highlight (without trailing newline)
+    ///
+    /// # Returns
+    /// A vector of [`Higlight`] spans. Always contains at least two spans:
+    /// one at offset 0 and one at `line.len()` as a sentinel.
+    pub fn parse_next_line<'a, T: PartialEq + TryFrom<u32>>(
+        &mut self,
+        arena: &'a Arena,
+        line: &[u8],
+    ) -> Vec<Higlight<T>, &'a Arena> {
+        let mut res: Vec<Higlight<T>, &Arena> = Vec::new_in(arena);
+        let mut reset = false;
+
+        self.registers.off = 0;
+        self.registers.hs = 0;
+
+        // By default, any line starts with HighlightKind::Other.
+        // If the DSL yields anything, this will be overwritten.
+        res.push(Higlight { start: 0, kind: unsafe { mem::zeroed() } });
+
+        loop {
+            let pc = self.registers.pc as usize;
+            let op = self.assembly[pc];
+            self.registers.pc += 1;
+
+            match op {
+                0 => {
+                    // Mov { dst: Register, src: Register }
+                    let (dst, src) = self.read_reg_pair();
+                    let s = self.registers.get(src);
+                    self.registers.set(dst, s);
+                }
+                1 => {
+                    // Add { dst: Register, src: Register }
+                    let (dst, src) = self.read_reg_pair();
+                    let d = self.registers.get(dst);
+                    let s = self.registers.get(src);
+                    self.registers.set(dst, d.saturating_add(s));
+                }
+                2 => {
+                    // Sub { dst: Register, src: Register }
+                    let (dst, src) = self.read_reg_pair();
+                    let d = self.registers.get(dst);
+                    let s = self.registers.get(src);
+                    self.registers.set(dst, d.saturating_sub(s));
+                }
+                3 => {
+                    // MovImm { dst: Register, imm: u32 }
+                    let (dst, _) = self.read_reg_pair();
+                    let imm = self.read::<u32>();
+                    self.registers.set(dst, imm);
+                }
+                4 => {
+                    // AddImm { dst: Register, imm: u32 }
+                    let (dst, _) = self.read_reg_pair();
+                    let imm = self.read::<u32>();
+                    let d = self.registers.get(dst);
+                    self.registers.set(dst, d.saturating_add(imm));
+                }
+                5 => {
+                    // SubImm { dst: Register, imm: u32 }
+                    let (dst, _) = self.read_reg_pair();
+                    let imm = self.read::<u32>();
+                    let d = self.registers.get(dst);
+                    self.registers.set(dst, d.saturating_sub(imm));
+                }
+
+                6 => {
+                    // Call { tgt: u32 }
+                    let tgt = self.read::<u32>();
+                    self.stack.push(self.registers.pc);
+                    self.registers.pc = tgt;
+                }
+                7 => {
+                    // Return
+                    if let Some(pc) = self.stack.pop() {
+                        self.registers.pc = pc;
+                    } else {
+                        reset = true;
+                        break;
+                    }
+                }
+
+                8 => {
+                    // JumpEQ { lhs: Register, rhs: Register, tgt: u32 }
+                    let (dst, src) = self.read_reg_pair();
+                    let tgt = self.read::<u32>();
+                    if self.registers.get(dst) == self.registers.get(src) {
+                        self.registers.pc = tgt;
+                    }
+                }
+                9 => {
+                    // JumpNE { lhs: Register, rhs: Register, tgt: u32 }
+                    let (dst, src) = self.read_reg_pair();
+                    let tgt = self.read::<u32>();
+                    if self.registers.get(dst) != self.registers.get(src) {
+                        self.registers.pc = tgt;
+                    }
+                }
+                10 => {
+                    // JumpLT { lhs: Register, rhs: Register, tgt: u32 }
+                    let (dst, src) = self.read_reg_pair();
+                    let tgt = self.read::<u32>();
+                    if self.registers.get(dst) < self.registers.get(src) {
+                        self.registers.pc = tgt;
+                    }
+                }
+                11 => {
+                    // JumpLE { lhs: Register, rhs: Register, tgt: u32 }
+                    let (dst, src) = self.read_reg_pair();
+                    let tgt = self.read::<u32>();
+                    if self.registers.get(dst) <= self.registers.get(src) {
+                        self.registers.pc = tgt;
+                    }
+                }
+                12 => {
+                    // JumpGT { lhs: Register, rhs: Register, tgt: u32 }
+                    let (dst, src) = self.read_reg_pair();
+                    let tgt = self.read::<u32>();
+                    if self.registers.get(dst) > self.registers.get(src) {
+                        self.registers.pc = tgt;
+                    }
+                }
+                13 => {
+                    // JumpGE { lhs: Register, rhs: Register, tgt: u32 }
+                    let (dst, src) = self.read_reg_pair();
+                    let tgt = self.read::<u32>();
+                    if self.registers.get(dst) >= self.registers.get(src) {
+                        self.registers.pc = tgt;
+                    }
+                }
+
+                14 => {
+                    // JumpIfEndOfLine { tgt: u32 }
+                    let tgt = self.read::<u32>();
+                    if (self.registers.off as usize) >= line.len() {
+                        self.registers.pc = tgt;
+                    }
+                }
+
+                15 => {
+                    // JumpIfMatchCharset { idx: u32, tgt: u32 }
+                    let idx = self.read::<u32>();
+                    let tgt = self.read::<u32>();
+                    let off = self.registers.off as usize;
+                    let cs = &self.charsets[idx as usize];
+
+                    if let Some(off) = Self::charset_gobble(line, off, cs) {
+                        self.registers.off = off as u32;
+                        self.registers.pc = tgt;
+                    }
+                }
+                16 => {
+                    // JumpIfMatchPrefix { idx: u32, tgt: u32 }
+                    let idx = self.read::<u32>();
+                    let tgt = self.read::<u32>();
+                    let off = self.registers.off as usize;
+                    let str = self.strings[idx as usize].as_bytes();
+
+                    if Self::inlined_memcmp(line, off, str) {
+                        self.registers.off = (off + str.len()) as u32;
+                        self.registers.pc = tgt;
+                    }
+                }
+                17 => {
+                    // JumpIfMatchPrefixInsensitive { idx: u32, tgt: u32 }
+                    let idx = self.read::<u32>();
+                    let tgt = self.read::<u32>();
+                    let off = self.registers.off as usize;
+                    let str = self.strings[idx as usize].as_bytes();
+
+                    if Self::inlined_memicmp(line, off, str) {
+                        self.registers.off = (off + str.len()) as u32;
+                        self.registers.pc = tgt;
+                    }
+                }
+
+                18 => {
+                    // FlushHighlight
+                    let (kind, _) = self.read_reg_pair();
+                    let kind = self.registers.get(kind);
+                    let kind = unsafe { kind.try_into().unwrap_unchecked() };
+                    let start = (self.registers.hs as usize).min(line.len());
+
+                    if let Some(last) = res.last_mut()
+                        && (last.start == start || last.kind == kind)
+                    {
+                        last.kind = kind;
+                    } else {
+                        res.push(Higlight { start, kind });
+                    }
+
+                    self.registers.hs = self.registers.off;
+                }
+                19 => {
+                    // AwaitInput
+                    let off = self.registers.off as usize;
+                    if off >= line.len() {
+                        break;
+                    }
+                }
+
+                _ => unreachable!(),
+            }
+        }
+
+        // Ensure that there's a past-the-end highlight.
+        if res.last().is_none_or(|last| last.start < line.len()) {
+            res.push(Higlight { start: line.len(), kind: unsafe { mem::zeroed() } });
+        }
+
+        if reset {
+            self.registers = unsafe { mem::zeroed() };
+            self.registers.pc = self.entrypoint;
+        }
+
+        res
+    }
+
+    // TODO: http://0x80.pl/notesen/2018-10-18-simd-byte-lookup.html#alternative-implementation
+    #[inline]
+    fn charset_gobble(haystack: &[u8], mut off: usize, cs: &[u16; 16]) -> Option<usize> {
+        if off >= haystack.len() || !Self::in_set(cs, haystack[off]) {
+            return None;
+        }
+
+        while {
+            off += 1;
+            off < haystack.len() && Self::in_set(cs, haystack[off])
+        } {}
+
+        Some(off)
+    }
+
+    /// A mini-memcmp implementation for short needles.
+    /// Compares the `haystack` at `off` with the `needle`.
+    #[inline]
+    fn inlined_memcmp(haystack: &[u8], off: usize, needle: &[u8]) -> bool {
+        unsafe {
+            if off >= haystack.len() || haystack.len() - off < needle.len() {
+                return false;
+            }
+
+            let a = haystack.as_ptr().add(off);
+            let b = needle.as_ptr();
+            let mut i = 0;
+
+            while i < needle.len() {
+                let a = *a.add(i);
+                let b = *b.add(i);
+                i += 1;
+                if a != b {
+                    return false;
+                }
+            }
+
+            true
+        }
+    }
+
+    /// Like `inlined_memcmp`, but case-insensitive.
+    #[inline]
+    fn inlined_memicmp(haystack: &[u8], off: usize, needle: &[u8]) -> bool {
+        unsafe {
+            if off >= haystack.len() || haystack.len() - off < needle.len() {
+                return false;
+            }
+
+            let a = haystack.as_ptr().add(off);
+            let b = needle.as_ptr();
+            let mut i = 0;
+
+            while i < needle.len() {
+                // str in PrefixInsensitive(str) is expected to be lowercase, printable ASCII.
+                let a = a.add(i).read().to_ascii_lowercase();
+                let b = b.add(i).read();
+                i += 1;
+                if a != b {
+                    return false;
+                }
+            }
+
+            true
+        }
+    }
+
+    #[inline]
+    fn in_set(bitmap: &[u16; 16], byte: u8) -> bool {
+        let lo_nibble = byte & 0xf;
+        let hi_nibble = byte >> 4;
+
+        let bitset = bitmap[lo_nibble as usize];
+        let bitmask = 1u16 << hi_nibble;
+
+        (bitset & bitmask) != 0
+    }
+
+    #[inline]
+    fn read<T: Copy>(&mut self) -> T {
+        let pc = self.registers.pc as usize;
+        self.registers.pc += mem::size_of::<T>() as u32;
+        // The LSH backend emits extra 0xff instructions at the end, to allow
+        // us to blindly read any instruction and only later check if it was valid.
+        unsafe { (self.assembly.as_ptr().add(pc) as *const T).read_unaligned() }
+    }
+
+    #[inline]
+    fn read_reg_pair(&mut self) -> (usize, usize) {
+        let reg_pair = self.read::<u8>();
+        let dst = (reg_pair & 0xf) as usize;
+        let src = (reg_pair >> 4) as usize;
+        (dst, src)
+    }
+}
diff --git a/crates/lsh/src/lib.rs b/crates/lsh/src/lib.rs
new file mode 100644
index 000000000000..e31254fb56f6
--- /dev/null
+++ b/crates/lsh/src/lib.rs
@@ -0,0 +1,57 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Welcome to Leonard's Syntax Highlighter (LSH), otherwise known as
+//! Leonard's Shitty Highlighter, which is really what it is.
+//!
+//! ## Architecture
+//!
+//!   DSL source
+//! → `frontend` (parse) → IR
+//! → `optimizer` → IR
+//! → `backend` (regalloc + codegen) → bytecode
+//! → `engine` (execute)
+//!
+//! The IR is a graph of `RefCell<IR>` nodes. Each node has a `.next` pointer and `If` nodes
+//! in particular have a `.then` pointer. This makes CFG manipulation trivial but means you can't
+//! iterate in program order without linearization (see `backend::LivenessAnalysis`).
+//!
+//! ## Register semantics
+//!
+//! - `off` advances only on successful regex matches. Failed matches leave it alone.
+//!   This is why the frontend emits backup/restore pairs around regex chains.
+//! - `hs` (highlight start) is used to track the input offset of the last yield statement,
+//!   which permits the engine (runtime) to highlight everything inbetween with the next yield's highlight kind.
+//!
+//! ## Charset encoding
+//!
+//! Charsets are 256-bit bitmaps stored as `[u16; 16]`. The encoding is transposed for SIMD:
+//! `bitmap[lo_nibble] & (1 << hi_nibble)` tests if byte `(hi_nibble << 4) | lo_nibble` is set.
+//! See `in_set()`. This layout allows parallel lookup of multiple bytes using pshufb.
+//!
+//! ## Gotchas
+//!
+//! - Physical VS virtual registers:
+//!   `IRReg.physical` being `Some` means it's pre-colored (e.g., `off` register).
+//!   The backend must preserve these assignments.
+//! - Semi-SSA:
+//!   The frontend emits IR where each vreg is written once,
+//!   but physical registers like `off` are mutated repeatedly.
+//!   The optimizer relies on this.
+//!
+//! ## TODO
+//!
+//! - Most importantly, the architecture is A GODDAMN MESS.
+//! - The regex compiler (`regex.rs`) panics on unsupported patterns instead of returning errors.
+//!   Should bubble up `CompileError` instead.
+//! - No support for spill code generation. If you run out of user registers, you're dead.
+//!   The current definition files don't hit this, but complex patterns could.
+//! - No include statements, and all functions across all files share a single namespace.
+
+#![feature(allocator_api)]
+#![allow(irrefutable_let_patterns, unused, clippy::upper_case_acronyms)]
+
+pub mod compiler;
+pub mod engine;
+
+pub use crate::compiler::CompileResult;
diff --git a/crates/edit/src/glob.rs b/crates/stdext/src/glob.rs
similarity index 100%
rename from crates/edit/src/glob.rs
rename to crates/stdext/src/glob.rs
diff --git a/crates/stdext/src/helpers.rs b/crates/stdext/src/helpers.rs
index 556206ae6ed7..fbf3127945d0 100644
--- a/crates/stdext/src/helpers.rs
+++ b/crates/stdext/src/helpers.rs
@@ -40,6 +40,26 @@ impl fmt::Display for MetricFormatter<usize> {
 #[cold]
 pub const fn cold_path() {}
 
+#[inline(always)]
+pub const fn likely(b: bool) -> bool {
+    if b {
+        true
+    } else {
+        cold_path();
+        false
+    }
+}
+
+#[inline(always)]
+pub const fn unlikely(b: bool) -> bool {
+    if b {
+        cold_path();
+        true
+    } else {
+        false
+    }
+}
+
 /// [`std::cmp::minmax`] is unstable, as per usual.
 pub fn minmax<T>(v1: T, v2: T) -> [T; 2]
 where
diff --git a/crates/stdext/src/lib.rs b/crates/stdext/src/lib.rs
index d7226d7e19c2..57eec5497be5 100644
--- a/crates/stdext/src/lib.rs
+++ b/crates/stdext/src/lib.rs
@@ -6,7 +6,9 @@
 #![feature(allocator_api)]
 
 pub mod arena;
+pub mod glob;
+mod helpers;
 pub mod sys;
+pub mod varint;
 
-mod helpers;
 pub use helpers::*;
diff --git a/crates/stdext/src/varint.rs b/crates/stdext/src/varint.rs
new file mode 100644
index 000000000000..64a40c037a6b
--- /dev/null
+++ b/crates/stdext/src/varint.rs
@@ -0,0 +1,142 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Variable-length `u32` encoding and decoding, with efficient storage of `u32::MAX`.
+//! `u32::MAX` is a common value in Microsoft Edit's syntax highlighter bytecode.
+//!
+//! # Format
+//!
+//! ```text
+//!       0-127        ( 7 bits): xxxxxxx0
+//!     128-16383      (14 bits): xxxxxx01 yyyyyyyx
+//!   16384-2097151    (21 bits): xxxxx011 yyyyyyxx zzzzzzzy
+//! 2097152-268435455  (28 bits): xxxx0111 yyyyyxxx zzzzzzyy wwwwwwwz
+//!         4294967295 (32 bits): ....1111
+//! ```
+//!
+//! The least significant bits indicate the length, in a format identical to UTF-8. The remaining bits store
+//! the value, in little-endian order. Little endian was chosen, as most architectures today use that.
+//!
+//! On x86, `tzcnt` (= `trailing_ones()` = what we need) has the benefit that its encoding is identical to `rep bsf`.
+//! Older CPUs without BMI1 will ignore the `rep` prefix and use `bsf`, while modern CPUs will use the faster `tzcnt`.
+//! So not just can we drop the need for `bswap` on x86, but we also speed up the bit count calculation.
+//! This makes this encoding faster than LEB128, Google Varint, and others.
+
+pub fn encode(val: u32) -> Vec<u8> {
+    let mut result = Vec::with_capacity(5);
+    let shift = match val {
+        0..0x80 => 0,
+        0x80..0x4000 => 1,
+        0x4000..0x200000 => 2,
+        0x200000..0x10000000 => 3,
+        _ => {
+            result.push(0xff);
+            return result;
+        }
+    };
+    let marker = (1u32 << shift) - 1;
+    let encoded = (val << (shift + 1)) | marker;
+    let bytes = encoded.to_le_bytes();
+    result.extend_from_slice(&bytes[..=shift]);
+    result
+}
+
+/// # Safety
+///
+/// The caller must ensure that `data..data+4` is valid memory.
+/// It doesn't need to be a valid value, but it must be readable.
+pub unsafe fn decode(data: *const u8) -> (u32, usize) {
+    // For inputs such as:
+    //   [0xff, 0xff, 0xff, 0xff]
+    // the shifts below will shift by more than 31 digits, which Rust considers undefined behavior.
+    // *We explicitly want UB here*.
+    //
+    // If we write an if condition here (like this one), LLVM will turn that into a proper branch. Since our inputs
+    // are relatively random, that branch will mispredict, hurting performance. The if condition at the end
+    // gets turned into conditional moves (good!), but that only works because it comes after the shifts.
+    // Unfortunately, there's no way to ask Rust for "platform-defined behavior" (`unchecked_shl/shr` is not it).
+    #[cfg(debug_assertions)]
+    unsafe {
+        if (*data & 0x0f) == 0x0f {
+            return (u32::MAX, 1);
+        }
+    }
+
+    unsafe {
+        // Read the following 4 bytes in a single u32 load. We need to swap to big-endian to move the lead
+        // 0/10/110/1110/1111 bits to the MSB. This then allows us to do a single, quick `leading_ones` call.
+        let val = u32::from_le((data as *const u32).read_unaligned());
+        let ones = val.trailing_ones();
+
+        let mut len = ones as usize + 1;
+        let mut res = 'bextr: {
+            // Give LLVM a helping hand for x86 CPUs with BMI1. It's not smart enough to figure out that `bextr` can
+            // be used here. To be fair, it's not faster, so maybe that's why. It is _a lot_ more compact, however.
+            #[cfg(target_feature = "bmi1")]
+            break 'bextr std::arch::x86_64::_bextr_u32(val, len as u32, (7 * len) as u32);
+
+            // This is where you'd put more architecture-specific optimizations.
+            // In fact this is where I'd put my ARM optimizations, but it doesn't have anything like `bextr`. :(
+
+            let mut res = val;
+            // Shift out the bytes we read but don't need.
+            res <<= 32 - 8 * len;
+            // Shift back down and remove the trailing 0/10/110/1110/1111 length bits.
+            res >>= 32 - 7 * len;
+            break 'bextr res;
+        };
+
+        // If the lead byte indicates >28 bits, assume `u32::MAX`.
+        // This doubles as a simple form of error correction.
+        if len > 4 {
+            res = u32::MAX;
+            len = 1;
+        }
+
+        (res, len)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_encode_decode_roundtrip() {
+        // Test various boundary values
+        let test_values = [
+            0u32,
+            1,
+            123,
+            127, // Max 1 byte
+            128, // Min 2 bytes
+            1234,
+            16383,     // Max 2 bytes
+            16384,     // Min 3 bytes
+            2097151,   // Max 3 bytes
+            2097152,   // Min 4 bytes
+            268435455, // Max 4 bytes
+            u32::MAX,  // Special case
+        ];
+
+        for &val in &test_values {
+            let encoded = encode(val);
+            println!("Value {} encoded as: {:02X?}", val, encoded);
+            let (decoded, len) = unsafe { decode(encoded.as_ptr()) };
+            println!("  Decoded as: {} with length {}", decoded, len);
+            assert_eq!(decoded, val, "Failed roundtrip for value {}", val);
+            assert_eq!(len, encoded.len(), "Length mismatch for value {}", val);
+        }
+    }
+
+    #[test]
+    fn test_specific_encodings() {
+        // Test specific byte patterns
+        unsafe {
+            assert_eq!((0, 1), decode([0, 0xbb, 0xcc, 0xdd].as_ptr()));
+            assert_eq!((123, 1), decode([0xf6, 0xbb, 0xcc, 0xdd].as_ptr()));
+            assert_eq!((1234, 2), decode([0x49, 0x13, 0xcc, 0xdd].as_ptr()));
+            assert_eq!((u32::MAX, 1), decode([0xff, 0xbb, 0xcc, 0xdd].as_ptr()));
+        }
+    }
+}
diff --git a/i18n/edit.toml b/i18n/edit.toml
index 01248b1577ad..5c992d32ebc4 100644
--- a/i18n/edit.toml
+++ b/i18n/edit.toml
@@ -376,6 +376,19 @@ vi = "Lưu thành…"
 zh_hans = "另存为…"
 zh_hant = "另存新檔…"
 
+[FilePreferences]
+en = "Preferences"
+de = "Einstellungen"
+es = "Configuración"
+fr = "Paramètres"
+it = "Impostazioni"
+ja = "設定"
+ko = "설정"
+pt_br = "Configurações"
+ru = "Параметры"
+zh_hans = "设置"
+zh_hant = "設定"
+
 [FileClose]
 en = "Close File"
 ar = "إغلاق الملف"
@@ -1661,6 +1674,32 @@ vi = "Đóng"
 zh_hans = "关闭"
 zh_hant = "關閉"
 
+[LanguageSelectMode]
+en = "Select Language Mode"
+de = "Sprachmodus auswählen"
+es = "Seleccionar modo de lenguaje"
+fr = "Sélectionner le mode du langage"
+it = "Seleziona modalità del linguaggio"
+ja = "言語モードの選択"
+ko = "언어 모드 선택"
+pt_br = "Selecionar modo de linguagem"
+ru = "Выбрать режим языка"
+zh_hans = "选择语言模式"
+zh_hant = "選擇語言模式"
+
+[LanguageAutoDetect]
+en = "Auto Detect"
+de = "Automatisch erkennen"
+es = "Detección automática"
+fr = "Détection automatique"
+it = "Rilevamento automatico"
+ja = "自動検出"
+ko = "자동 감지"
+pt_br = "Detectar automaticamente"
+ru = "Определить автоматически"
+zh_hans = "自动检测"
+zh_hant = "自動偵測"
+
 [EncodingReopen]
 en = "Reopen with encoding…"
 ar = "إعادة فتح مع الترميز…"

From 9e927e2cb959b2a7a7ae6aa3a47c4cadb34a8106 Mon Sep 17 00:00:00 2001
From: Leonard Hecker <leonard@hecker.io>
Date: Mon, 26 Jan 2026 23:19:47 +0100
Subject: [PATCH 2/8] Fix the latest clippy warning

---
 crates/edit/src/icu.rs | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/crates/edit/src/icu.rs b/crates/edit/src/icu.rs
index 8b8f783eeee9..a99acac7cc63 100644
--- a/crates/edit/src/icu.rs
+++ b/crates/edit/src/icu.rs
@@ -534,11 +534,7 @@ fn utext_access_impl<'a>(
             }
         }
 
-        loop {
-            let Some(c) = it.next() else {
-                break;
-            };
-
+        while let Some(c) = it.next() {
             // Thanks to our `if utf16_len >= UTF16_LEN_LIMIT` check,
             // we can safely assume that this will fit.
             unsafe {

From 948b4cd07c46e455ef15702ec6be44a4fd7706a1 Mon Sep 17 00:00:00 2001
From: Leonard Hecker <leonard@hecker.io>
Date: Mon, 26 Jan 2026 23:26:20 +0100
Subject: [PATCH 3/8] Fix more build failures

---
 crates/edit/src/bin/edit/draw_statusbar.rs | 2 +-
 crates/lsh-bin/src/main.rs                 | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/crates/edit/src/bin/edit/draw_statusbar.rs b/crates/edit/src/bin/edit/draw_statusbar.rs
index d812150922b7..c0e94b9e946e 100644
--- a/crates/edit/src/bin/edit/draw_statusbar.rs
+++ b/crates/edit/src/bin/edit/draw_statusbar.rs
@@ -358,7 +358,7 @@ fn encoding_picker_update_list(state: &mut State) {
         }
     }
 
-    matches.sort_unstable_by(|a, b| b.0.cmp(&a.0));
+    matches.sort_unstable_by_key(|b| std::cmp::Reverse(b.0));
     state.encoding_picker_results = Some(Vec::from_iter(matches.iter().map(|(_, enc)| *enc)));
 }
 
diff --git a/crates/lsh-bin/src/main.rs b/crates/lsh-bin/src/main.rs
index f7f9ac5bbd0a..c0540575c02d 100644
--- a/crates/lsh-bin/src/main.rs
+++ b/crates/lsh-bin/src/main.rs
@@ -179,7 +179,9 @@ fn run_render(generator: lsh::compiler::Generator, path: &Path) -> anyhow::Resul
         let scratch = scratch_arena(None);
         let highlights = engine.parse_next_line::<u32>(&scratch, line.as_bytes());
 
-        for [curr, next] in highlights.array_windows::<2>() {
+        for w in highlights.windows(2) {
+            let curr = &w[0];
+            let next = &w[1];
             let start = curr.start;
             let end = next.start;
             let kind = curr.kind;

From 1effea2358e4b9352e1b868d90980d41dcac14c7 Mon Sep 17 00:00:00 2001
From: Leonard Hecker <leonard@hecker.io>
Date: Tue, 27 Jan 2026 19:33:43 +0100
Subject: [PATCH 4/8] Restructured files, Added XML support

---
 assets/highlighting-tests/html.html  |  51 ++
 assets/highlighting-tests/xml.xml    |  39 ++
 crates/edit/src/lsh/cache.rs         |   6 +-
 crates/edit/src/lsh/highlighter.rs   |  16 +-
 crates/edit/src/lsh/mod.rs           |   2 +-
 crates/lsh-bin/src/main.rs           |  30 +-
 crates/lsh/definitions/xml.lsh       |  72 ++
 crates/lsh/src/compiler/backend.rs   |   9 +-
 crates/lsh/src/compiler/charset.rs   | 193 ++++++
 crates/lsh/src/compiler/frontend.rs  |  25 +-
 crates/lsh/src/compiler/generator.rs |  69 +-
 crates/lsh/src/compiler/helpers.rs   |  29 -
 crates/lsh/src/compiler/mod.rs       | 694 +------------------
 crates/lsh/src/compiler/optimizer.rs |  10 +-
 crates/lsh/src/compiler/regex.rs     |  87 ++-
 crates/lsh/src/compiler/tokenizer.rs |   6 +-
 crates/lsh/src/engine.rs             | 427 ------------
 crates/lsh/src/lib.rs                |  10 +-
 crates/lsh/src/runtime.rs            | 950 +++++++++++++++++++++++++++
 19 files changed, 1453 insertions(+), 1272 deletions(-)
 create mode 100644 assets/highlighting-tests/html.html
 create mode 100644 assets/highlighting-tests/xml.xml
 create mode 100644 crates/lsh/definitions/xml.lsh
 create mode 100644 crates/lsh/src/compiler/charset.rs
 delete mode 100644 crates/lsh/src/compiler/helpers.rs
 delete mode 100644 crates/lsh/src/engine.rs
 create mode 100644 crates/lsh/src/runtime.rs

diff --git a/assets/highlighting-tests/html.html b/assets/highlighting-tests/html.html
new file mode 100644
index 000000000000..35682eee1b34
--- /dev/null
+++ b/assets/highlighting-tests/html.html
@@ -0,0 +1,51 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>HTML Syntax Test &amp; Demo</title>
+    <!-- Comment with special chars: <>&"' -->
+    <style>
+        body {
+            margin: 0;
+            padding: 20px;
+        }
+
+        .test {
+            color: #ff0000;
+        }
+    </style>
+</head>
+
+<body>
+    <h1 id="main" class='header' data-value="123">Heading</h1>
+
+    <!-- Various elements -->
+    <p>Text with &lt; &gt; &amp; &quot; &apos; &#65; &#x41; entities</p>
+    <br />
+    <hr>
+    <img src="image.png" alt="Description" />
+
+    <!-- Attributes -->
+    <input type="text" disabled required value="" data-custom="attr">
+    <button onclick="test()">Click</button>
+
+    <!-- Nested elements -->
+    <div>
+        <span>Inline <strong>bold</strong> text</span>
+        <ul>
+            <li>Item 1</li>
+            <li>Item 2</li>
+        </ul>
+    </div>
+
+    <!-- Script -->
+    <script>
+        function test() {
+            return x < 10 && y > 5;
+        }
+    </script>
+</body>
+
+</html>
diff --git a/assets/highlighting-tests/xml.xml b/assets/highlighting-tests/xml.xml
new file mode 100644
index 000000000000..ad2c2aa09a0d
--- /dev/null
+++ b/assets/highlighting-tests/xml.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!DOCTYPE root [
+  <!ENTITY custom "Custom Entity">
+]>
+<!-- Root comment with special chars: <>&"' -->
+<?processing-instruction target="value"?>
+<root xmlns="http://default.namespace" xmlns:ns="http://custom.namespace">
+  <!-- Elements and attributes -->
+  <element attribute="value" ns:attr='single-quoted' empty-attr="">
+    <self-closing id="123" />
+    <nested>Text &amp; &lt; &gt; &quot; &apos; content</nested>
+    <entity>&custom;</entity>
+  </element>
+
+  <!-- CDATA section -->
+  <script><![CDATA[
+    function test() {
+      return x < 10 && y > 5;
+    }
+  ]]></script>
+
+  <!-- Mixed content -->
+  <mixed>Text before <tag>inline</tag> text after</mixed>
+
+  <!-- Numeric character references -->
+  <chars>&#65; &#x41; &#x1F600;</chars>
+
+  <!-- Edge cases -->
+  <edge-cases>
+    <empty></empty>
+    <whitespace>   </whitespace>
+    <special-chars>!@#$%^*()_+-={}[]|\:;"'&lt;&gt;,.</special-chars>
+  </edge-cases>
+
+  <!-- Nested namespaces -->
+  <ns:custom ns:id="namespaced">
+    <ns:child />
+  </ns:custom>
+</root>
diff --git a/crates/edit/src/lsh/cache.rs b/crates/edit/src/lsh/cache.rs
index 026221a15296..5c2ca96b8d0b 100644
--- a/crates/edit/src/lsh/cache.rs
+++ b/crates/edit/src/lsh/cache.rs
@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use lsh::engine::Higlight;
+use lsh::runtime::Highlight;
 use stdext::arena::{Arena, scratch_arena};
 
 use crate::helpers::CoordType;
@@ -33,7 +33,7 @@ impl HighlighterCache {
         arena: &'a Arena,
         highlighter: &mut Highlighter,
         line: CoordType,
-    ) -> Vec<Higlight<HighlightKind>, &'a Arena> {
+    ) -> Vec<Highlight<HighlightKind>, &'a Arena> {
         // Do we need to random seek?
         if line != highlighter.logical_pos_y() {
             // If so, restore the nearest, preceeding checkpoint...
@@ -65,7 +65,7 @@ impl HighlighterCache {
         &mut self,
         arena: &'a Arena,
         highlighter: &mut Highlighter,
-    ) -> Vec<Higlight<HighlightKind>, &'a Arena> {
+    ) -> Vec<Highlight<HighlightKind>, &'a Arena> {
         // If we need to store a checkpoint for the start of the next line, do so now.
         if Self::floor_line_to_offset(highlighter.logical_pos_y()) == self.checkpoints.len() {
             self.checkpoints.push(highlighter.snapshot());
diff --git a/crates/edit/src/lsh/highlighter.rs b/crates/edit/src/lsh/highlighter.rs
index b9fb9fc31332..bc0e3d2e9eb8 100644
--- a/crates/edit/src/lsh/highlighter.rs
+++ b/crates/edit/src/lsh/highlighter.rs
@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use lsh::engine::*;
+use lsh::runtime::*;
 use stdext::arena::{Arena, scratch_arena};
 
 use crate::document::ReadableDocument;
@@ -16,14 +16,14 @@ pub struct Highlighter<'a> {
     doc: &'a dyn ReadableDocument,
     offset: usize,
     logical_pos_y: CoordType,
-    engine: Engine<'static, 'static, 'static>,
+    runtime: Runtime<'static, 'static, 'static>,
 }
 
 #[derive(Clone)]
 pub struct HighlighterState {
     offset: usize,
     logical_pos_y: CoordType,
-    state: EngineState,
+    state: RuntimeState,
 }
 
 impl<'doc> Highlighter<'doc> {
@@ -32,7 +32,7 @@ impl<'doc> Highlighter<'doc> {
             doc,
             offset: 0,
             logical_pos_y: 0,
-            engine: Engine::new(&ASSEMBLY, &STRINGS, &CHARSETS, language.entrypoint),
+            runtime: Runtime::new(&ASSEMBLY, &STRINGS, &CHARSETS, language.entrypoint),
         }
     }
 
@@ -46,7 +46,7 @@ impl<'doc> Highlighter<'doc> {
         HighlighterState {
             offset: self.offset,
             logical_pos_y: self.logical_pos_y,
-            state: self.engine.snapshot(),
+            state: self.runtime.snapshot(),
         }
     }
 
@@ -54,13 +54,13 @@ impl<'doc> Highlighter<'doc> {
     pub fn restore(&mut self, snapshot: &HighlighterState) {
         self.offset = snapshot.offset;
         self.logical_pos_y = snapshot.logical_pos_y;
-        self.engine.restore(&snapshot.state);
+        self.runtime.restore(&snapshot.state);
     }
 
     pub fn parse_next_line<'a>(
         &mut self,
         arena: &'a Arena,
-    ) -> Vec<Higlight<HighlightKind>, &'a Arena> {
+    ) -> Vec<Highlight<HighlightKind>, &'a Arena> {
         let scratch = scratch_arena(Some(arena));
         let (line_beg, line) = self.read_next_line(&scratch);
 
@@ -73,7 +73,7 @@ impl<'doc> Highlighter<'doc> {
         }
 
         let line = unicode::strip_newline(line);
-        let mut res = self.engine.parse_next_line(arena, line);
+        let mut res = self.runtime.parse_next_line(arena, line);
 
         // Adjust the range to account for the line offset.
         for h in &mut res {
diff --git a/crates/edit/src/lsh/mod.rs b/crates/edit/src/lsh/mod.rs
index 1b47a24aa4b8..986083bfce85 100644
--- a/crates/edit/src/lsh/mod.rs
+++ b/crates/edit/src/lsh/mod.rs
@@ -11,7 +11,7 @@ use std::path::Path;
 
 pub use definitions::{FILE_ASSOCIATIONS, HighlightKind, LANGUAGES};
 pub use highlighter::*;
-pub use lsh::engine::Language;
+pub use lsh::runtime::Language;
 use stdext::glob::glob_match;
 
 pub fn process_file_associations<T>(
diff --git a/crates/lsh-bin/src/main.rs b/crates/lsh-bin/src/main.rs
index c0540575c02d..6e6a4ae7faf5 100644
--- a/crates/lsh-bin/src/main.rs
+++ b/crates/lsh-bin/src/main.rs
@@ -2,13 +2,14 @@
 // Licensed under the MIT License.
 
 use std::fs::File;
-use std::io::{BufRead, BufReader, IsTerminal, Write as _, stdout};
+use std::io::{BufRead, BufReader, BufWriter, IsTerminal, Write as _, stdout};
 use std::path::{Path, PathBuf};
 use std::process::exit;
 
 use anyhow::bail;
 use argh::FromArgs;
-use lsh::engine::Engine;
+use lsh::compiler::SerializedCharset;
+use lsh::runtime::Runtime;
 use stdext::arena::scratch_arena;
 use stdext::glob::glob_match;
 
@@ -148,36 +149,23 @@ fn run_render(generator: lsh::compiler::Generator, path: &Path) -> anyhow::Resul
 
     // Convert Assembly data to static references by leaking memory
     // This is fine for a CLI tool that runs once and exits
-    let charsets: Vec<[u16; 16]> = assembly
-        .charsets
-        .into_iter()
-        .map(|cs| {
-            let mut arr = [0u16; 16];
-            for lo in 0..16 {
-                let mut u = 0u16;
-                for hi in 0..16 {
-                    u |= (cs[hi * 16 + lo] as u16) << hi;
-                }
-                arr[lo] = u;
-            }
-            arr
-        })
-        .collect();
+    let charsets: Vec<SerializedCharset> =
+        assembly.charsets.into_iter().map(|cs| cs.serialize()).collect();
 
-    let mut engine = Engine::new(
+    let mut runtime = Runtime::new(
         &assembly.instructions,
         &assembly.strings,
         &charsets,
         entrypoint.address as u32,
     );
 
-    let reader = BufReader::new(File::open(path)?);
-    let mut stdout = stdout();
+    let reader = BufReader::with_capacity(128 * 1024, File::open(path)?);
+    let mut stdout = BufWriter::with_capacity(128 * 1024, stdout());
 
     for line in reader.lines() {
         let line = line?;
         let scratch = scratch_arena(None);
-        let highlights = engine.parse_next_line::<u32>(&scratch, line.as_bytes());
+        let highlights = runtime.parse_next_line::<u32>(&scratch, line.as_bytes());
 
         for w in highlights.windows(2) {
             let curr = &w[0];
diff --git a/crates/lsh/definitions/xml.lsh b/crates/lsh/definitions/xml.lsh
new file mode 100644
index 000000000000..71f3b459eb5e
--- /dev/null
+++ b/crates/lsh/definitions/xml.lsh
@@ -0,0 +1,72 @@
+#[display_name = "XML"]
+#[path = "**/*.csproj.user"]
+#[path = "**/*.csproj"]
+#[path = "**/*.html"]
+#[path = "**/*.nuspec"]
+#[path = "**/*.proj"]
+#[path = "**/*.props"]
+#[path = "**/*.rss"]
+#[path = "**/*.slnx"]
+#[path = "**/*.svg"]
+#[path = "**/*.targets"]
+#[path = "**/*.vcxproj.filters"]
+#[path = "**/*.vcxproj"]
+#[path = "**/*.xaml"]
+#[path = "**/*.xml"]
+#[path = "**/*.xml"]
+pub fn xml() {
+    until /$/ {
+        yield other;
+
+        if /<!--/ {
+            loop {
+                yield comment;
+                await input;
+                if /-->/ {
+                    yield comment;
+                    break;
+                }
+            }
+        } else if /<!\[CDATA\[/ {
+            loop {
+                yield string;
+                await input;
+                if /\]\]>/ {
+                    yield string;
+                    break;
+                }
+            }
+        } else if /<!(DOCTYPE)[^>]*>/ {
+            yield $1 as constant.language;
+        } else if /(?:<\?|<)([\w:.-]+)/ {
+            yield $1 as constant.language;
+
+            until />/ {
+                yield other;
+                await input;
+
+                if /([\w:.-]+)\s*=/ {
+                    yield $1 as variable;
+                } else if /"/ {
+                    until /"/ {
+                        yield string;
+                        await input;
+                    }
+                    yield string;
+                } else if /'/ {
+                    until /'/ {
+                        yield string;
+                        await input;
+                    }
+                    yield string;
+                }
+            }
+
+            yield other;
+        } else if /<\/([\w:.-]+)\s*>/ {
+            yield $1 as constant.language;
+        } else if /(?:&#|&)[\w:.-]+;/ {
+            yield constant.numeric;
+        }
+    }
+}
diff --git a/crates/lsh/src/compiler/backend.rs b/crates/lsh/src/compiler/backend.rs
index 93ea8f583aee..eeb2c77bc3f2 100644
--- a/crates/lsh/src/compiler/backend.rs
+++ b/crates/lsh/src/compiler/backend.rs
@@ -56,6 +56,7 @@ use std::collections::{HashMap, HashSet, VecDeque};
 use stdext::arena::scratch_arena;
 
 use super::*;
+use crate::runtime::Instruction;
 
 #[derive(Debug, Clone, Copy)]
 enum Relocation<'a> {
@@ -120,14 +121,6 @@ impl<'a> Backend<'a> {
             });
         }
 
-        // Normally the runtime/engine would need to do bounds checks at all times to be safe,
-        // since there may be malformed bytecode (e.g. a bug in this compiler).
-        // We can fix that by padding the instruction stream with invalid opcodes at the end.
-        // This works as long as the runtime checks for valid opcodes. Even if the last valid
-        // opcode is chopped off (due to a bug above), the runtime can do an unchecked read
-        // of `MAX_ENCODED_SIZE` bytes without risking OOB access.
-        self.assembly.instructions.extend(std::iter::repeat_n(0xff, Instruction::MAX_ENCODED_SIZE));
-
         self.assembly.entrypoints = compiler
             .functions
             .iter()
diff --git a/crates/lsh/src/compiler/charset.rs b/crates/lsh/src/compiler/charset.rs
new file mode 100644
index 000000000000..4b1cd3b0b0bd
--- /dev/null
+++ b/crates/lsh/src/compiler/charset.rs
@@ -0,0 +1,193 @@
+use std::fmt;
+use std::ops::RangeInclusive;
+
+use stdext::arena::Arena;
+
+use super::Intern;
+
+const WORD_BITS: usize = usize::BITS as usize;
+
+pub type SerializedCharset = [u16; 16];
+
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct Charset {
+    bits: [usize; 256 / WORD_BITS],
+}
+
+impl Charset {
+    pub const fn no() -> Self {
+        Charset { bits: [usize::MIN; _] }
+    }
+
+    pub fn invert(&mut self) {
+        for b in &mut self.bits {
+            *b = !*b;
+        }
+    }
+
+    pub fn get(&self, b: u8) -> bool {
+        let hi = b as usize / WORD_BITS;
+        let lo = b as usize % WORD_BITS;
+        (self.bits[hi] & (1 << lo)) != 0
+    }
+
+    pub fn covers_none(&self) -> bool {
+        self.bits.iter().all(|&b| b == usize::MIN)
+    }
+
+    pub fn covers_range(&self, range: RangeInclusive<u8>) -> bool {
+        range.into_iter().all(|b| self.get(b))
+    }
+
+    pub const fn set(&mut self, index: u8, value: bool) {
+        let hi = index as usize / WORD_BITS;
+        let lo = index as usize % WORD_BITS;
+        self.bits[hi] = if value { self.bits[hi] | (1 << lo) } else { self.bits[hi] & !(1 << lo) };
+    }
+
+    pub const fn set_range(&mut self, range: RangeInclusive<u8>, value: bool) {
+        let start = *range.start() as usize;
+        let end = *range.end() as usize;
+
+        let start_word = start / WORD_BITS;
+        let start_bit = start % WORD_BITS;
+        let end_word = end / WORD_BITS;
+        let end_bit = end % WORD_BITS;
+
+        // Write the starting bits of the first word
+        let mask = usize::MAX << start_bit;
+        self.bits[start_word] =
+            if value { self.bits[start_word] | mask } else { self.bits[start_word] & !mask };
+
+        // Fill the full words in between, if any
+        let mut word = start_word + 1;
+        while word <= end_word {
+            self.bits[word] = if value { usize::MAX } else { usize::MIN };
+            word += 1;
+        }
+
+        // Fix the trailing bits of the last word we wrote above
+        let shift = end_bit + 1;
+        let mask = if shift < WORD_BITS { usize::MAX << shift } else { 0 };
+        self.bits[end_word] =
+            if value { self.bits[end_word] & !mask } else { self.bits[end_word] | mask };
+    }
+
+    pub fn merge(&mut self, other: &Charset) {
+        for (a, b) in self.bits.iter_mut().zip(other.bits.iter()) {
+            *a |= *b;
+        }
+    }
+
+    pub fn is_superset_of(&self, other: &Charset) -> bool {
+        for (&s, &o) in self.bits.iter().zip(other.bits.iter()) {
+            // For self to be a superset, every bit in other must be in self
+            if (o & !s) != 0 {
+                return false;
+            }
+        }
+        true
+    }
+
+    pub fn serialize(&self) -> SerializedCharset {
+        let mut arr = [0u16; 16];
+        for lo in 0..16 {
+            let mut u = 0u16;
+            for hi in 0..16 {
+                u |= (self.get(hi * 16 + lo) as u16) << hi;
+            }
+            arr[lo as usize] = u;
+        }
+        arr
+    }
+}
+
+impl Default for Charset {
+    fn default() -> Self {
+        Self::no()
+    }
+}
+
+impl fmt::Debug for Charset {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let show_char = |f: &mut fmt::Formatter<'_>, b: usize| {
+            let b = b as u8;
+            if b == b'"' {
+                write!(f, "&quot;")
+            } else if b.is_ascii_graphic() {
+                let b = b as char;
+                write!(f, "{b}")
+            } else {
+                write!(f, "0x{b:02X}")
+            }
+        };
+
+        let get_bit = |index: usize| -> bool {
+            let hi = index / WORD_BITS;
+            let lo = index % WORD_BITS;
+            (self.bits[hi] & (1 << lo)) != 0
+        };
+
+        let mut beg = 0;
+        let mut first = true;
+
+        write!(f, "[")?;
+
+        while beg < 256 {
+            while beg < 256 && !get_bit(beg) {
+                beg += 1;
+            }
+            if beg >= 256 {
+                break;
+            }
+
+            let mut end = beg;
+            while end < 256 && get_bit(end) {
+                end += 1;
+            }
+
+            if !first {
+                write!(f, ", ")?;
+            }
+            show_char(f, beg)?;
+            if end - beg > 1 {
+                write!(f, "-")?;
+                show_char(f, end - 1)?;
+            }
+
+            beg = end;
+            first = false;
+        }
+
+        write!(f, "]")
+    }
+}
+
+impl<'a> Intern<'a, Charset> for Vec<&'a Charset> {
+    fn intern(&mut self, arena: &'a Arena, value: &Charset) -> &'a Charset {
+        if let Some(&s) = self.iter().find(|&&v| v == value) {
+            s
+        } else {
+            let s = arena.alloc_uninit().write(value.clone());
+            self.push(s);
+            s
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn set_range() {
+        for range in [5..=7, 10..=245, 0..=255] {
+            let mut cs = Charset::no();
+            cs.set_range(range.clone(), true);
+
+            for i in 0u8..=255 {
+                assert_eq!(cs.get(i), range.contains(&i), "range {range:?}, bit {i}");
+            }
+        }
+    }
+}
diff --git a/crates/lsh/src/compiler/frontend.rs b/crates/lsh/src/compiler/frontend.rs
index 6a9c64a662ae..1596407f18cd 100644
--- a/crates/lsh/src/compiler/frontend.rs
+++ b/crates/lsh/src/compiler/frontend.rs
@@ -37,18 +37,6 @@ struct RegexSpan<'a> {
     pub capture_groups: Vec<(IRRegCell<'a>, IRRegCell<'a>), &'a Arena>,
 }
 
-enum LoopFlowControlKind {
-    Break,
-    Continue,
-}
-
-struct LoopFlowControl<'a> {
-    line: usize,
-    column: usize,
-    kind: LoopFlowControlKind,
-    ir: IRCell<'a>,
-}
-
 struct Context<'a> {
     loop_start: Option<IRCell<'a>>,
     loop_exit: Option<IRCell<'a>>,
@@ -256,10 +244,16 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
             offset: usize::MAX,
         });
 
+        // NOTE: It's crucial that we connect the block with the loop before calling collect_interesting_charset,
+        // as the until statement's regex is not part of the loop but still counts as an "interesting charset",
+        // for the purpose of skipping uninteresting characters.
+        first.borrow_mut().set_next(loop_start);
+        loop_good.borrow_mut().set_next(block.first);
+
         // Skip any uninteresting characters before the next loop iteration.
         //   if /.*?/ {}
-        let interesting = self.compiler.collect_interesting_charset(block.first);
-        let fast_skip = if interesting.covers_all() {
+        let interesting = self.compiler.collect_interesting_charset(loop_start);
+        let fast_skip = if interesting.covers_none() {
             advance_check
         } else {
             let mut skip_charset = interesting.clone();
@@ -272,9 +266,6 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
             })
         };
 
-        first.borrow_mut().set_next(loop_start);
-        loop_good.borrow_mut().set_next(block.first);
-
         if let mut block_last = block.last.borrow_mut()
             && block_last.wants_next()
         {
diff --git a/crates/lsh/src/compiler/generator.rs b/crates/lsh/src/compiler/generator.rs
index 5ca94d948646..b97b7eb0ab45 100644
--- a/crates/lsh/src/compiler/generator.rs
+++ b/crates/lsh/src/compiler/generator.rs
@@ -10,7 +10,6 @@
 
 use std::collections::HashMap;
 use std::ffi::OsStr;
-use std::fmt::format;
 use std::fs::read_dir;
 use std::io;
 use std::path::PathBuf;
@@ -18,6 +17,7 @@ use std::path::PathBuf;
 use stdext::arena::scratch_arena;
 
 use super::*;
+use crate::runtime::{Instruction, MnemonicFormattingConfig};
 
 pub struct Generator<'a> {
     compiler: Compiler<'a>,
@@ -114,11 +114,13 @@ impl<'a> Generator<'a> {
                 _ = writeln!(output, "{label_prefix}{}:{label_suffix}", label);
             }
 
-            let (instr, len) = Instruction::decode(&assembly.instructions[off..]);
+            let (Some(instr), len) = Instruction::decode(&assembly.instructions[off..]) else {
+                break;
+            };
             let mnemonic_width = match instr {
-                Some(Instruction::JumpIfMatchCharset { .. })
-                | Some(Instruction::JumpIfMatchPrefix { .. })
-                | Some(Instruction::JumpIfMatchPrefixInsensitive { .. }) => {
+                Instruction::JumpIfMatchCharset { .. }
+                | Instruction::JumpIfMatchPrefix { .. }
+                | Instruction::JumpIfMatchPrefixInsensitive { .. } => {
                     if vt {
                         60
                     } else {
@@ -129,32 +131,22 @@ impl<'a> Generator<'a> {
             };
 
             let scratch = scratch_arena(None);
-            if let Some(instr) = instr {
-                _ = write!(
-                    output,
-                    "{line_prefix}{off:>line_num_width$}:  {mnemonic:mnemonic_width$}",
-                    mnemonic = instr.mnemonic(&scratch, &mnemonic_config),
-                );
-            } else {
-                _ = write!(
-                    output,
-                    "{line_prefix}{off:>line_num_width$}:  {_n}{opcode:#02x}{n_}",
-                    opcode = assembly.instructions[off],
-                    _n = mnemonic_config.numeric_prefix,
-                    n_ = mnemonic_config.numeric_suffix,
-                );
-            }
+            _ = write!(
+                output,
+                "{line_prefix}{off:>line_num_width$}:  {mnemonic:mnemonic_width$}",
+                mnemonic = instr.mnemonic(&scratch, &mnemonic_config),
+            );
 
             match instr {
-                Some(Instruction::JumpIfMatchCharset { idx, .. }) => {
+                Instruction::JumpIfMatchCharset { idx, .. } => {
                     _ = write!(
                         output,
                         " {comment_prefix}// {:?}{comment_suffix}",
                         assembly.charsets[idx as usize]
                     )
                 }
-                Some(Instruction::JumpIfMatchPrefix { idx, .. })
-                | Some(Instruction::JumpIfMatchPrefixInsensitive { idx, .. }) => {
+                Instruction::JumpIfMatchPrefix { idx, .. }
+                | Instruction::JumpIfMatchPrefixInsensitive { idx, .. } => {
                     _ = write!(
                         output,
                         " {comment_prefix}// {:?}{comment_suffix}",
@@ -176,7 +168,7 @@ impl<'a> Generator<'a> {
 
         let mut output = String::new();
         output.push_str("// This file is auto-generated. Do not edit it manually.\n\n");
-        output.push_str("use lsh::engine::Language;\n\n");
+        output.push_str("use lsh::runtime::Language;\n\n");
 
         output.push_str("#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub enum HighlightKind {\n");
         let members: Vec<_> = assembly
@@ -238,7 +230,7 @@ impl TryFrom<u32> for HighlightKind {{
         _ = writeln!(
             output,
             "\n#[rustfmt::skip] pub const ASSEMBLY: [u8; {len}] = [",
-            len = assembly.instructions.len(),
+            len = assembly.instructions.len() + Instruction::MAX_ENCODED_SIZE,
         );
         let line_num_width = assembly.instructions.len().checked_ilog10().unwrap_or(0) as usize + 1;
 
@@ -268,13 +260,25 @@ impl TryFrom<u32> for HighlightKind {{
                     output,
                     "{:<padding_width$}// {off:>line_num_width$}:  {mnemonic}",
                     "",
-                    padding_width = 9usize.saturating_sub(len) * 6,
+                    padding_width = Instruction::MAX_ENCODED_SIZE.saturating_sub(len) * 6,
                     mnemonic = instr.mnemonic(&scratch, &Default::default())
                 );
+            } else {
+                output.push('\n');
             }
 
             off += len;
         }
+        // Normally the runtime would need to do bounds checks at all times to be safe,
+        // since there may be malformed bytecode (e.g. a bug in this compiler).
+        // We can fix that by padding the instruction stream with invalid opcodes at the end.
+        // This works as long as the runtime checks for valid opcodes. Even if the last valid
+        // opcode is chopped off (due to a bug above), the runtime can do an unchecked read
+        // of `MAX_ENCODED_SIZE` bytes without risking OOB access.
+        output.push_str("\n    // padding\n");
+        for _ in 0..Instruction::MAX_ENCODED_SIZE {
+            _ = writeln!(output, "    0xff,");
+        }
         output.push_str("];\n");
 
         _ = writeln!(
@@ -283,16 +287,13 @@ impl TryFrom<u32> for HighlightKind {{
             len = assembly.charsets.len(),
         );
         for cs in assembly.charsets {
+            let cs = cs.serialize();
             output.push_str("    [");
-            for lo in 0..16 {
-                if lo > 0 {
-                    output.push_str(", ");
-                }
-                let mut u = 0u16;
-                for hi in 0..16 {
-                    u |= (cs[hi * 16 + lo] as u16) << hi;
+            for (i, &v) in cs.iter().enumerate() {
+                if i != 0 {
+                    _ = write!(output, ", ");
                 }
-                _ = write!(output, "0x{u:04x}");
+                _ = write!(output, "0x{v:04x}");
             }
             output.push_str("],\n");
         }
diff --git a/crates/lsh/src/compiler/helpers.rs b/crates/lsh/src/compiler/helpers.rs
deleted file mode 100644
index eb1cc319295e..000000000000
--- a/crates/lsh/src/compiler/helpers.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-use std::cell::RefCell;
-use std::collections::HashSet;
-use std::fmt;
-use std::fmt::Write as _;
-use std::ops::{Index, IndexMut};
-use std::path::{Path, PathBuf};
-use std::str::FromStr;
-
-use stdext::arena::*;
-
-pub fn arena_clone_str<'a>(arena: &'a Arena, s: &str) -> &'a str {
-    ArenaString::from_str(arena, s).leak()
-}
-
-pub trait Intern<'a, T: ?Sized> {
-    fn intern(&mut self, arena: &'a Arena, item: &T) -> &'a T;
-}
-
-impl<'a> Intern<'a, str> for Vec<&'a str> {
-    fn intern(&mut self, arena: &'a Arena, value: &str) -> &'a str {
-        if let Some(&s) = self.iter().find(|&&v| v == value) {
-            s
-        } else {
-            let s = arena_clone_str(arena, value);
-            self.push(s);
-            s
-        }
-    }
-}
diff --git a/crates/lsh/src/compiler/mod.rs b/crates/lsh/src/compiler/mod.rs
index 9a54624e56c7..17e603aff9f5 100644
--- a/crates/lsh/src/compiler/mod.rs
+++ b/crates/lsh/src/compiler/mod.rs
@@ -17,7 +17,7 @@
 //!
 //! ## Charset representation
 //!
-//! `Charset` is a 256-bit bitmap (`[bool; 256]`). The engine uses a transposed `[u16; 16]`
+//! `Charset` is a 256-bit bitmap (`[bool; 256]`). The runtime uses a transposed `[u16; 16]`
 //! layout for SIMD reasons - see `generator.rs` where the conversion happens.
 //!
 //! ## Instruction encoding
@@ -32,27 +32,26 @@
 //!   The backend has its own DFS traversal for liveness analysis.
 
 mod backend;
+mod charset;
 mod frontend;
 mod generator;
-mod helpers;
 mod optimizer;
 mod regex;
 mod tokenizer;
 
 use std::cell::{Cell, RefCell};
 use std::collections::{HashSet, VecDeque};
+use std::fmt;
 use std::fmt::Write as _;
-use std::marker::PhantomData;
-use std::mem::{MaybeUninit, transmute, zeroed};
-use std::ops::{Index, IndexMut};
+use std::mem::zeroed;
 use std::path::Path;
-use std::{fmt, ptr};
 
 use stdext::arena::{Arena, ArenaString};
 
+pub use self::charset::{Charset, SerializedCharset};
 use self::frontend::*;
 pub use self::generator::Generator;
-use crate::compiler::helpers::{Intern, arena_clone_str};
+use crate::runtime::Register;
 
 pub fn builtin_definitions_path() -> &'static Path {
     #[cfg(windows)]
@@ -302,7 +301,7 @@ impl<'a> Compiler<'a> {
                         _ = write!(output, "[{offset}: return]");
                     }
                     IRI::Flush { kind } => {
-                        _ = write!(output, "[{offset}: flush]");
+                        _ = write!(output, "[{offset}: flush {kind:?}]");
                     }
                     IRI::AwaitInput => {
                         _ = write!(output, "[{offset}: await input]");
@@ -369,6 +368,21 @@ impl<'a> Iterator for TreeVisitor<'a> {
     }
 }
 
+pub struct Assembly<'a> {
+    pub instructions: Vec<u8>,
+    pub entrypoints: Vec<Entrypoint>,
+    pub charsets: Vec<&'a Charset>,
+    pub strings: Vec<&'a str>,
+    pub highlight_kinds: Vec<HighlightKind<'a>>,
+}
+
+pub struct Entrypoint {
+    pub name: String,
+    pub display_name: String,
+    pub paths: Vec<String>,
+    pub address: usize,
+}
+
 #[derive(Clone)]
 pub struct HighlightKind<'a> {
     pub identifier: &'a str,
@@ -449,12 +463,12 @@ enum IRI<'a> {
 }
 
 #[derive(Default)]
-pub struct IRReg {
+struct IRReg {
     id: u32,
     physical: Option<Register>,
 }
 
-pub type IRRegCell<'a> = &'a RefCell<IRReg>;
+type IRRegCell<'a> = &'a RefCell<IRReg>;
 
 impl IRReg {
     fn new(id: u32) -> Self {
@@ -476,12 +490,12 @@ impl fmt::Debug for IRReg {
 
 #[derive(Clone, Copy)]
 struct IRSpan<'a> {
-    pub first: IRCell<'a>,
-    pub last: IRCell<'a>,
+    first: IRCell<'a>,
+    last: IRCell<'a>,
 }
 
 impl<'a> IRSpan<'a> {
-    pub fn single(node: IRCell<'a>) -> Self {
+    fn single(node: IRCell<'a>) -> Self {
         Self { first: node, last: node }
     }
 }
@@ -509,7 +523,7 @@ impl<'a, 's> IRChainBuilder<'a, 's> {
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum ComparisonOp {
+enum ComparisonOp {
     Eq,
     Ne,
     Lt,
@@ -538,660 +552,22 @@ impl<'a> IR<'a> {
     }
 }
 
-#[derive(Clone, PartialEq, Eq, Hash)]
-pub struct Charset {
-    bits: [bool; 256],
-}
-
-impl Charset {
-    const fn no() -> Self {
-        Charset { bits: [false; 256] }
-    }
-
-    const fn yes() -> Self {
-        Charset { bits: [true; 256] }
-    }
-
-    fn fill(&mut self, value: bool) {
-        self.bits.fill(value);
-    }
-
-    fn invert(&mut self) {
-        for b in &mut self.bits {
-            *b = !*b;
-        }
-    }
-
-    fn set(&mut self, index: u8, value: bool) {
-        self.bits[index as usize] = value;
-    }
-
-    fn merge(&mut self, other: &Charset) {
-        for (a, b) in self.bits.iter_mut().zip(other.bits.iter()) {
-            *a |= *b;
-        }
-    }
-
-    fn covers_all(&self) -> bool {
-        self.bits.iter().all(|&b| b)
-    }
-
-    fn covers_char(&self, b: u8) -> bool {
-        self.bits[b as usize]
-    }
-
-    fn covers_char_insensitive(&self, b: u8) -> bool {
-        self.bits[b.to_ascii_uppercase() as usize] && self.bits[b.to_ascii_lowercase() as usize]
-    }
-
-    fn covers_str(&self, s: &str) -> bool {
-        s.as_bytes().iter().all(|&b| self.bits[b as usize])
-    }
-
-    fn covers_str_insensitive(&self, s: &str) -> bool {
-        s.as_bytes().iter().all(|&b| {
-            self.bits[b.to_ascii_uppercase() as usize] && self.bits[b.to_ascii_lowercase() as usize]
-        })
-    }
-
-    fn is_superset(&self, other: &Charset) -> bool {
-        for (&s, &o) in self.bits.iter().zip(other.bits.iter()) {
-            if s && !o {
-                return false;
-            }
-        }
-        true
-    }
-
-    fn is_strict_superset(&self, other: &Charset) -> bool {
-        let mut has_extra = false;
-        for (&s, &o) in self.bits.iter().zip(other.bits.iter()) {
-            if !s && o {
-                return false;
-            }
-            has_extra |= s && !o;
-        }
-        has_extra
-    }
-}
-
-impl Default for Charset {
-    fn default() -> Self {
-        Self::no()
-    }
-}
-
-impl fmt::Debug for Charset {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let show_char = |f: &mut fmt::Formatter<'_>, b: usize| {
-            let b = b as u8;
-            if b == b'"' {
-                write!(f, "&quot;")
-            } else if b.is_ascii_graphic() {
-                let b = b as char;
-                write!(f, "{b}")
-            } else {
-                write!(f, "0x{b:02X}")
-            }
-        };
-
-        let mut beg = 0;
-        let mut first = true;
-
-        write!(f, "[")?;
-
-        while beg < 256 {
-            while beg < 256 && !self.bits[beg] {
-                beg += 1;
-            }
-            if beg >= 256 {
-                break;
-            }
-
-            let mut end = beg;
-            while end < 256 && self.bits[end] {
-                end += 1;
-            }
-
-            if !first {
-                write!(f, ", ")?;
-            }
-            show_char(f, beg)?;
-            if end - beg > 1 {
-                write!(f, "-")?;
-                show_char(f, end - 1)?;
-            }
-
-            beg = end;
-            first = false;
-        }
-
-        write!(f, "]")
-    }
-}
-
-impl<I> Index<I> for Charset
-where
-    [bool]: Index<I>,
-{
-    type Output = <[bool] as Index<I>>::Output;
-
-    #[inline]
-    fn index(&self, index: I) -> &Self::Output {
-        self.bits.index(index)
-    }
+fn arena_clone_str<'a>(arena: &'a Arena, s: &str) -> &'a str {
+    ArenaString::from_str(arena, s).leak()
 }
 
-impl<I> IndexMut<I> for Charset
-where
-    [bool]: IndexMut<I>,
-{
-    #[inline]
-    fn index_mut(&mut self, index: I) -> &mut Self::Output {
-        self.bits.index_mut(index)
-    }
+trait Intern<'a, T: ?Sized> {
+    fn intern(&mut self, arena: &'a Arena, item: &T) -> &'a T;
 }
 
-impl<'a> Intern<'a, Charset> for Vec<&'a Charset> {
-    fn intern(&mut self, arena: &'a Arena, value: &Charset) -> &'a Charset {
+impl<'a> Intern<'a, str> for Vec<&'a str> {
+    fn intern(&mut self, arena: &'a Arena, value: &str) -> &'a str {
         if let Some(&s) = self.iter().find(|&&v| v == value) {
             s
         } else {
-            let s = arena.alloc_uninit().write(value.clone());
+            let s = arena_clone_str(arena, value);
             self.push(s);
             s
         }
     }
 }
-
-pub struct Assembly<'a> {
-    pub instructions: Vec<u8>,
-    pub entrypoints: Vec<Entrypoint>,
-    pub charsets: Vec<&'a Charset>,
-    pub strings: Vec<&'a str>,
-    pub highlight_kinds: Vec<HighlightKind<'a>>,
-}
-
-pub struct Entrypoint {
-    pub name: String,
-    pub display_name: String,
-    pub paths: Vec<String>,
-    pub address: usize,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub enum Register {
-    ProgramCounter,
-    InputOffset,
-    HighlightStart,
-    X3,
-    X4,
-    X5,
-    X6,
-    X7,
-    X8,
-    X9,
-    X10,
-    X11,
-    X12,
-    X13,
-    X14,
-    X15,
-}
-
-impl Register {
-    const FIRST_USER_REG: usize = 3; // aka x3
-    const COUNT: usize = 16;
-
-    fn from_usize(value: usize) -> Self {
-        debug_assert!(value < Self::COUNT);
-        unsafe { transmute::<u8, Register>(value as u8) }
-    }
-
-    fn mnemonic(&self) -> &'static str {
-        match self {
-            Register::ProgramCounter => "pc",
-            Register::InputOffset => "off",
-            Register::HighlightStart => "hs",
-            Register::X3 => "x3",
-            Register::X4 => "x4",
-            Register::X5 => "x5",
-            Register::X6 => "x6",
-            Register::X7 => "x7",
-            Register::X8 => "x8",
-            Register::X9 => "x9",
-            Register::X10 => "x10",
-            Register::X11 => "x11",
-            Register::X12 => "x12",
-            Register::X13 => "x13",
-            Register::X14 => "x14",
-            Register::X15 => "x15",
-        }
-    }
-}
-
-impl fmt::Display for Register {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.mnemonic())
-    }
-}
-
-#[repr(C)]
-#[derive(Default, Clone, Copy)]
-pub struct Registers {
-    pub pc: u32,  // x0 = ProgramCounter
-    pub off: u32, // x1 = InputOffset
-    pub hs: u32,  // x2 = HighlightStart
-    pub x3: u32,
-    pub x4: u32,
-    pub x5: u32,
-    pub x6: u32,
-    pub x7: u32,
-    pub x8: u32,
-    pub x9: u32,
-    pub x10: u32,
-    pub x11: u32,
-    pub x12: u32,
-    pub x13: u32,
-    pub x14: u32,
-    pub x15: u32,
-}
-
-impl Registers {
-    const COUNT: usize = 16;
-
-    #[inline(always)]
-    pub fn get(&self, reg: usize) -> u32 {
-        debug_assert!(reg < Self::COUNT);
-        unsafe { (self as *const _ as *const u32).add(reg).read() }
-    }
-
-    #[inline(always)]
-    pub fn set(&mut self, reg: usize, val: u32) {
-        debug_assert!(reg < Self::COUNT);
-        unsafe { (self as *mut _ as *mut u32).add(reg).write(val) }
-    }
-}
-
-struct RegisterAllocator(u16);
-
-impl RegisterAllocator {
-    fn new() -> Self {
-        RegisterAllocator((1 << Register::FIRST_USER_REG) - 1)
-    }
-
-    fn alloc(&mut self) -> Option<Register> {
-        let available = !self.0;
-        if available == 0 {
-            return None;
-        }
-
-        let idx = available.trailing_zeros() as usize;
-        self.0 |= 1 << idx;
-        Some(Register::from_usize(idx))
-    }
-
-    fn dealloc(&mut self, reg: Register) {
-        let idx = reg as usize;
-        if idx >= Register::FIRST_USER_REG {
-            self.0 &= !(1 << idx);
-        }
-    }
-}
-
-impl Default for RegisterAllocator {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-pub struct AnnotatedInstruction<'a> {
-    pub instr: Instruction,
-    pub label: &'a str,
-}
-
-#[allow(dead_code)]
-#[repr(u8)]
-#[derive(Debug, Clone, Copy)]
-pub enum Instruction {
-    // NOTE: This allows for jumps by manipulating Register::ProgramCounter.
-    Mov { dst: Register, src: Register },
-    Add { dst: Register, src: Register },
-    Sub { dst: Register, src: Register },
-    MovImm { dst: Register, imm: u32 },
-    AddImm { dst: Register, imm: u32 },
-    SubImm { dst: Register, imm: u32 },
-
-    Call { tgt: u32 },
-    Return,
-
-    JumpEQ { lhs: Register, rhs: Register, tgt: u32 }, // ==
-    JumpNE { lhs: Register, rhs: Register, tgt: u32 }, // !=
-    JumpLT { lhs: Register, rhs: Register, tgt: u32 }, // <
-    JumpLE { lhs: Register, rhs: Register, tgt: u32 }, // <=
-    JumpGT { lhs: Register, rhs: Register, tgt: u32 }, // >
-    JumpGE { lhs: Register, rhs: Register, tgt: u32 }, // >=
-
-    // Jumps to `tgt` if we're at the end of the line.
-    JumpIfEndOfLine { tgt: u32 },
-
-    // Jumps to `tgt` if the test succeeds.
-    // `idx` specifies the charset/string to use.
-    JumpIfMatchCharset { idx: u32, tgt: u32 },
-    JumpIfMatchPrefix { idx: u32, tgt: u32 },
-    JumpIfMatchPrefixInsensitive { idx: u32, tgt: u32 },
-
-    // Flushes the current HighlightKind to the output.
-    FlushHighlight { kind: Register },
-
-    // Awaits more input to be available.
-    AwaitInput,
-}
-
-impl Instruction {
-    // JumpIfMatchCharset, etc., are 1 byte opcode + two u32 parameters.
-    pub const MAX_ENCODED_SIZE: usize = 9;
-
-    pub fn address_offset(&self) -> Option<usize> {
-        match *self {
-            Instruction::MovImm { .. }
-            | Instruction::AddImm { .. }
-            | Instruction::SubImm { .. } => Some(2), // opcode + dst
-
-            Instruction::Call { .. } => Some(1), // opcode
-
-            Instruction::JumpEQ { .. }
-            | Instruction::JumpNE { .. }
-            | Instruction::JumpLT { .. }
-            | Instruction::JumpLE { .. }
-            | Instruction::JumpGT { .. }
-            | Instruction::JumpGE { .. } => Some(2), // opcode + lhs/rhs pair
-
-            Instruction::JumpIfEndOfLine { .. } => Some(1), // opcode
-
-            Instruction::JumpIfMatchCharset { .. }
-            | Instruction::JumpIfMatchPrefix { .. }
-            | Instruction::JumpIfMatchPrefixInsensitive { .. } => Some(5), // opcode + idx
-
-            _ => None,
-        }
-    }
-
-    #[allow(clippy::identity_op)]
-    pub fn encode<'a>(&self, arena: &'a Arena) -> Vec<u8, &'a Arena> {
-        fn enc_reg_pair(lo: Register, hi: Register) -> u8 {
-            ((hi as u8) << 4) | (lo as u8)
-        }
-
-        fn enc_reg_single(lo: Register) -> u8 {
-            lo as u8
-        }
-
-        fn enc_u16(val: u16) -> [u8; 2] {
-            val.to_le_bytes()
-        }
-
-        fn enc_u32(val: u32) -> [u8; 4] {
-            val.to_le_bytes()
-        }
-
-        let mut bytes = Vec::with_capacity_in(16, arena);
-        #[allow(clippy::missing_transmute_annotations)]
-        bytes.push(unsafe { std::mem::transmute(std::mem::discriminant(self)) });
-
-        match *self {
-            Instruction::Mov { dst, src }
-            | Instruction::Add { dst, src }
-            | Instruction::Sub { dst, src } => {
-                bytes.push(enc_reg_pair(dst, src));
-            }
-            Instruction::MovImm { dst, imm }
-            | Instruction::AddImm { dst, imm }
-            | Instruction::SubImm { dst, imm } => {
-                bytes.push(enc_reg_single(dst));
-                bytes.extend_from_slice(&enc_u32(imm));
-            }
-
-            Instruction::Call { tgt } => {
-                bytes.extend_from_slice(&enc_u32(tgt));
-            }
-            Instruction::Return => {}
-
-            Instruction::JumpEQ { lhs, rhs, tgt }
-            | Instruction::JumpNE { lhs, rhs, tgt }
-            | Instruction::JumpLT { lhs, rhs, tgt }
-            | Instruction::JumpLE { lhs, rhs, tgt }
-            | Instruction::JumpGT { lhs, rhs, tgt }
-            | Instruction::JumpGE { lhs, rhs, tgt } => {
-                bytes.push(enc_reg_pair(lhs, rhs));
-                bytes.extend_from_slice(&enc_u32(tgt));
-            }
-
-            Instruction::JumpIfEndOfLine { tgt } => {
-                bytes.extend_from_slice(&enc_u32(tgt));
-            }
-            Instruction::JumpIfMatchCharset { idx, tgt }
-            | Instruction::JumpIfMatchPrefix { idx, tgt }
-            | Instruction::JumpIfMatchPrefixInsensitive { idx, tgt } => {
-                bytes.extend_from_slice(&enc_u32(idx));
-                bytes.extend_from_slice(&enc_u32(tgt));
-            }
-
-            Instruction::FlushHighlight { kind } => {
-                bytes.push(enc_reg_single(kind));
-            }
-            Instruction::AwaitInput => {}
-        }
-
-        bytes
-    }
-
-    pub fn decode(bytes: &[u8]) -> (Option<Self>, usize) {
-        fn dec_reg_pair(b: u8) -> (Register, Register) {
-            let hi = Register::from_usize((b >> 4) as usize);
-            let lo = Register::from_usize((b & 0xF) as usize);
-            (lo, hi)
-        }
-
-        fn dec_reg_single(b: u8) -> Register {
-            Register::from_usize(b as usize)
-        }
-
-        fn dec_u32(bytes: &[u8]) -> u32 {
-            u32::from_le_bytes(bytes[..4].try_into().unwrap())
-        }
-
-        let opcode = bytes[0];
-        match opcode {
-            0 => {
-                let (dst, src) = dec_reg_pair(bytes[1]);
-                (Some(Instruction::Mov { dst, src }), 2)
-            }
-            1 => {
-                let (dst, src) = dec_reg_pair(bytes[1]);
-                (Some(Instruction::Add { dst, src }), 2)
-            }
-            2 => {
-                let (dst, src) = dec_reg_pair(bytes[1]);
-                (Some(Instruction::Sub { dst, src }), 2)
-            }
-            3 => {
-                let dst = dec_reg_single(bytes[1]);
-                let imm = dec_u32(&bytes[2..]);
-                (Some(Instruction::MovImm { dst, imm }), 6)
-            }
-            4 => {
-                let dst = dec_reg_single(bytes[1]);
-                let imm = dec_u32(&bytes[2..]);
-                (Some(Instruction::AddImm { dst, imm }), 6)
-            }
-            5 => {
-                let dst = dec_reg_single(bytes[1]);
-                let imm = dec_u32(&bytes[2..]);
-                (Some(Instruction::SubImm { dst, imm }), 6)
-            }
-            6 => (Some(Instruction::Call { tgt: dec_u32(&bytes[1..]) }), 5),
-            7 => (Some(Instruction::Return), 1),
-            8 => {
-                let (lhs, rhs) = dec_reg_pair(bytes[1]);
-                let tgt = dec_u32(&bytes[2..]);
-                (Some(Instruction::JumpEQ { lhs, rhs, tgt }), 6)
-            }
-            9 => {
-                let (lhs, rhs) = dec_reg_pair(bytes[1]);
-                let tgt = dec_u32(&bytes[2..]);
-                (Some(Instruction::JumpNE { lhs, rhs, tgt }), 6)
-            }
-            10 => {
-                let (lhs, rhs) = dec_reg_pair(bytes[1]);
-                let tgt = dec_u32(&bytes[2..]);
-                (Some(Instruction::JumpLT { lhs, rhs, tgt }), 6)
-            }
-            11 => {
-                let (lhs, rhs) = dec_reg_pair(bytes[1]);
-                let tgt = dec_u32(&bytes[2..]);
-                (Some(Instruction::JumpLE { lhs, rhs, tgt }), 6)
-            }
-            12 => {
-                let (lhs, rhs) = dec_reg_pair(bytes[1]);
-                let tgt = dec_u32(&bytes[2..]);
-                (Some(Instruction::JumpGT { lhs, rhs, tgt }), 6)
-            }
-            13 => {
-                let (lhs, rhs) = dec_reg_pair(bytes[1]);
-                let tgt = dec_u32(&bytes[2..]);
-                (Some(Instruction::JumpGE { lhs, rhs, tgt }), 6)
-            }
-            14 => (Some(Instruction::JumpIfEndOfLine { tgt: dec_u32(&bytes[1..]) }), 5),
-            15 => {
-                let idx = dec_u32(&bytes[1..]);
-                let tgt = dec_u32(&bytes[5..]);
-                (Some(Instruction::JumpIfMatchCharset { idx, tgt }), 9)
-            }
-            16 => {
-                let idx = dec_u32(&bytes[1..]);
-                let tgt = dec_u32(&bytes[5..]);
-                (Some(Instruction::JumpIfMatchPrefix { idx, tgt }), 9)
-            }
-            17 => {
-                let idx = dec_u32(&bytes[1..]);
-                let tgt = dec_u32(&bytes[5..]);
-                (Some(Instruction::JumpIfMatchPrefixInsensitive { idx, tgt }), 9)
-            }
-            18 => {
-                let kind = dec_reg_single(bytes[1]);
-                (Some(Instruction::FlushHighlight { kind }), 2)
-            }
-            19 => (Some(Instruction::AwaitInput), 1),
-            _ => (None, 1),
-        }
-    }
-
-    pub fn mnemonic<'a>(
-        &self,
-        arena: &'a Arena,
-        config: &MnemonicFormattingConfig,
-    ) -> ArenaString<'a> {
-        let mut str = ArenaString::new_in(arena);
-        let _i = config.instruction_prefix;
-        let i_ = config.instruction_suffix;
-        let _r = config.register_prefix;
-        let r_ = config.register_suffix;
-        let _a = config.address_prefix;
-        let a_ = config.address_suffix;
-        let _n = config.numeric_prefix;
-        let n_ = config.numeric_suffix;
-
-        match *self {
-            Instruction::Mov { dst, src } => {
-                _ = write!(str, "{_i}mov{i_}    {_r}{dst}{r_}, {_r}{src}{r_}");
-            }
-            Instruction::Add { dst, src } => {
-                _ = write!(str, "{_i}add{i_}    {_r}{dst}{r_}, {_r}{src}{r_}");
-            }
-            Instruction::Sub { dst, src } => {
-                _ = write!(str, "{_i}sub{i_}    {_r}{dst}{r_}, {_r}{src}{r_}");
-            }
-            Instruction::MovImm { dst, imm } => {
-                if dst == Register::ProgramCounter {
-                    _ = write!(str, "{_i}movi{i_}   {_r}{dst}{r_}, {_a}{imm}{a_}");
-                } else {
-                    _ = write!(str, "{_i}movi{i_}   {_r}{dst}{r_}, {_n}{imm}{n_}");
-                }
-            }
-            Instruction::AddImm { dst, imm } => {
-                _ = write!(str, "{_i}addi{i_}   {_r}{dst}{r_}, {_n}{imm}{n_}");
-            }
-            Instruction::SubImm { dst, imm } => {
-                _ = write!(str, "{_i}subi{i_}   {_r}{dst}{r_}, {_n}{imm}{n_}");
-            }
-
-            Instruction::Call { tgt } => {
-                _ = write!(str, "{_i}call{i_}   {_a}{tgt}{a_}");
-            }
-            Instruction::Return => {
-                _ = write!(str, "{_i}ret{i_}");
-            }
-
-            Instruction::JumpEQ { lhs, rhs, tgt } => {
-                _ = write!(str, "{_i}jeq{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
-            }
-            Instruction::JumpNE { lhs, rhs, tgt } => {
-                _ = write!(str, "{_i}jne{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
-            }
-            Instruction::JumpLT { lhs, rhs, tgt } => {
-                _ = write!(str, "{_i}jlt{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
-            }
-            Instruction::JumpLE { lhs, rhs, tgt } => {
-                _ = write!(str, "{_i}jle{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
-            }
-            Instruction::JumpGT { lhs, rhs, tgt } => {
-                _ = write!(str, "{_i}jgt{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
-            }
-            Instruction::JumpGE { lhs, rhs, tgt } => {
-                _ = write!(str, "{_i}jge{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
-            }
-
-            Instruction::JumpIfEndOfLine { tgt } => {
-                _ = write!(str, "{_i}jeol{i_}   {_a}{tgt}{a_}");
-            }
-            Instruction::JumpIfMatchCharset { idx, tgt } => {
-                _ = write!(str, "{_i}jc{i_}     {_n}{idx}{n_}, {_a}{tgt}{a_}");
-            }
-            Instruction::JumpIfMatchPrefix { idx, tgt } => {
-                _ = write!(str, "{_i}jp{i_}     {_n}{idx}{n_}, {_a}{tgt}{a_}");
-            }
-            Instruction::JumpIfMatchPrefixInsensitive { idx, tgt } => {
-                _ = write!(str, "{_i}jpi{i_}    {_n}{idx}{n_}, {_a}{tgt}{a_}");
-            }
-
-            Instruction::FlushHighlight { kind } => {
-                _ = write!(str, "{_i}flush{i_}  {_r}{kind}{r_}");
-            }
-            Instruction::AwaitInput => {
-                _ = write!(str, "{_i}await{i_}");
-            }
-        }
-
-        str
-    }
-}
-
-#[derive(Default)]
-pub struct MnemonicFormattingConfig<'a> {
-    // Color used for highlighting the instruction.
-    pub instruction_prefix: &'a str,
-    pub instruction_suffix: &'a str,
-
-    // Color used for highlighting a register name.
-    pub register_prefix: &'a str,
-    pub register_suffix: &'a str,
-
-    // Color used for highlighting an immediate value.
-    pub address_prefix: &'a str,
-    pub address_suffix: &'a str,
-
-    // Color used for highlighting an immediate value.
-    pub numeric_prefix: &'a str,
-    pub numeric_suffix: &'a str,
-}
diff --git a/crates/lsh/src/compiler/optimizer.rs b/crates/lsh/src/compiler/optimizer.rs
index 01f0fa8c222d..39d15ab252c0 100644
--- a/crates/lsh/src/compiler/optimizer.rs
+++ b/crates/lsh/src/compiler/optimizer.rs
@@ -13,8 +13,6 @@
 //! - Could merge consecutive `Add { off, off, 1 }` instructions.
 //! - Could eliminate unreachable code after `Return`.
 
-use std::cell::RefMut;
-use std::iter::repeat_n;
 use std::ptr;
 
 use stdext::arena::scratch_arena;
@@ -48,7 +46,7 @@ fn optimize_noop<'a>(compiler: &mut Compiler<'a>) {
             {
                 // `IRI::If` nodes have an additional "next" pointer.
                 let current = &mut *current;
-                let mut nexts = [
+                let nexts = [
                     current.next.as_mut(),
                     match &mut current.instr {
                         IRI::If { then, .. } => Some(then),
@@ -94,7 +92,7 @@ fn optimize_redundant_offset_backup_restore<'a>(compiler: &mut Compiler<'a>) {
     for function in &compiler.functions {
         for current_cell in compiler.visit_nodes_from(function.body) {
             // First, filter down to nodes that assign the `off` to a virtual register.
-            if let mut save = current_cell.borrow_mut()
+            if let save = current_cell.borrow()
                 && let IRI::Mov { dst: backup_reg, src } = save.instr
                 && ptr::eq(src, off_reg)
                 && backup_reg.borrow().physical.is_none()
@@ -106,7 +104,7 @@ fn optimize_redundant_offset_backup_restore<'a>(compiler: &mut Compiler<'a>) {
                     && let mut cond = cond.borrow_mut()
                     && matches!(cond.instr, IRI::If { .. })
                     && let Some(restore) = cond.next
-                    && let mut restore = restore.borrow_mut()
+                    && let restore = restore.borrow()
                     && matches!(restore.instr, IRI::Mov { dst, src } if ptr::eq(dst, off_reg) && ptr::eq(src, backup_reg))
                 {
                     cond.next = restore.next;
@@ -202,7 +200,7 @@ fn optimize_highlight_kind_values<'a>(compiler: &mut Compiler<'a>) {
         for current in compiler.visit_nodes_from(function.body) {
             let mut current = current.borrow_mut();
 
-            if let IRI::MovKind { dst, kind } = &mut current.instr {
+            if let IRI::MovKind { kind, .. } = &mut current.instr {
                 *kind = mapping[*kind as usize];
             }
         }
diff --git a/crates/lsh/src/compiler/regex.rs b/crates/lsh/src/compiler/regex.rs
index b6b91a30f012..43af40813985 100644
--- a/crates/lsh/src/compiler/regex.rs
+++ b/crates/lsh/src/compiler/regex.rs
@@ -16,6 +16,11 @@
 //!
 //! ## Gotchas
 //!
+//! - IMPORTANT (also the most important TODO):
+//!   Alternatives are not properly implemented, because backtracking doesn't exist.
+//!   E.g. matching `(a|ab)[0-9]+` will NOT match `ab123` because after matching `a` from the
+//!   first alternative, it immediately tries to match `[0-9]+` and fails.
+//!
 //! - Single-character classes like `[eE]` get expanded into a chain of `Prefix` conditions
 //!   rather than using `Charset`. This is because prefix matching advances `off` by exactly
 //!   the matched length, while charset matching is greedy (matches as many as possible).
@@ -30,17 +35,25 @@
 //!
 //! ## TODO
 //!
-//! - Returning `Err(String)` but the caller wraps it in `CompileError`.
-//!   Should just return `CompileError` directly.
-//! - The rest of the fucking owl.
+//! - Delete the entire file.
+//! - Rewrite it based on a proper TDFA architecture.
 
-use std::ops::RangeInclusive;
 use std::ptr;
 
 use regex_syntax::hir::{Class, ClassBytes, ClassBytesRange, Hir, HirKind, Look};
 
 use super::*;
 
+const ASCII_WORD_CHARSET: Charset = {
+    let mut charset = Charset::no();
+    charset.set_range(b'0'..=b'9', true);
+    charset.set_range(b'A'..=b'Z', true);
+    charset.set_range(b'_'..=b'_', true);
+    charset.set_range(b'a'..=b'z', true);
+    charset.set_range(0xC2..=0xF4, true);
+    charset
+};
+
 pub fn parse<'a>(
     compiler: &mut Compiler<'a>,
     pattern: &str,
@@ -190,7 +203,7 @@ fn transform_literal_plus<'a>(
 ) -> Result<IRCell<'a>, String> {
     assert!(lit.0.len() == 1);
     let mut c = Charset::default();
-    c[lit.0[0] as usize] = true;
+    c.set(lit.0[0], true);
     let c = compiler.intern_charset(&c);
     Ok(compiler.alloc_iri(IRI::If { condition: Condition::Charset(c), then: dst }))
 }
@@ -216,26 +229,30 @@ fn transform_class<'a>(
     let mut first: Option<IRCell<'a>> = None;
     let mut last: Option<IRCell<'a>> = None;
 
-    for i in 0..256 {
-        if !charset[i] {
+    for ch in 0..=255 {
+        if !charset.get(ch) {
             continue;
         }
 
-        if i >= 128 {
-            panic!("Invalid non-ASCII class character {i}");
+        if ch >= 128 {
+            panic!("Invalid non-ASCII class character {ch}");
         }
 
-        let ch = i as u8;
         let mut str = String::new();
         str.push(ch as char);
 
-        let upper = ch.to_ascii_lowercase() as usize;
-        // NOTE: Uppercase chars have a lower numeric value than lowercase chars.
-        // As such, we need to test for `is_ascii_uppercase`.
-        let insensitive = ch.is_ascii_uppercase() && charset[upper];
+        // Check if our given `class` covers lower- and uppercase of an ASCII letter,
+        // and if so, we have to turn that into a `PrefixInsensitive` condition.
+        //
+        // NOTE: Uppercase letters have a lower numerical value than lowercase letters.
+        // As such, we check for is_ascii_uppercase(), turn that into `insensitive == true`,
+        // and clear out the (numerically higher) lowercase bit from `class`, so we ignore it.
+        // We already handled it with our `PrefixInsensitive` condition after all.
+        let lower = ch.to_ascii_lowercase();
+        let insensitive = ch.is_ascii_uppercase() && charset.get(lower);
 
         if insensitive {
-            charset[upper] = false;
+            charset.set(lower, false);
             str.make_ascii_lowercase();
         }
 
@@ -409,7 +426,7 @@ fn append_fallback_recursive<'a>(
     if let IRI::If { condition: Condition::Charset(charset), then } = &node_ref.instr {
         // If the fallback charset is a superset of this node's charset,
         // we can append the fallback to this node's failure path
-        if is_superset(fallback_charset, charset) {
+        if fallback_charset.is_superset_of(charset) {
             // Recursively try to append to the success path first
             let appended_to_then = append_fallback_recursive(then, fallback, fallback_charset);
 
@@ -450,52 +467,22 @@ fn get_initial_charset<'a>(node: IRCell<'a>) -> Option<&'a Charset> {
     }
 }
 
-// Check if `superset` is a superset of `subset`
-fn is_superset(superset: &Charset, subset: &Charset) -> bool {
-    for i in 0..256 {
-        if subset[i] && !superset[i] {
-            return false;
-        }
-    }
-    true
-}
-
 // [a-z] -> 256-ary LUT
 fn class_to_charset(class: &ClassBytes) -> Charset {
     let mut charset = Charset::default();
 
     for r in class.iter() {
-        charset[r.start() as usize..=r.end() as usize].fill(true);
+        charset.set_range(r.start()..=r.end(), true);
     }
 
     // If the class includes \w, we also set any non-ASCII UTF8 starters.
     // That's not how Unicode works, but it simplifies the implementation.
-    if [(b'0', b'9'), (b'A', b'Z'), (b'_', b'_'), (b'a', b'z')]
+    if [b'0'..=b'9', b'A'..=b'Z', b'_'..=b'_', b'a'..=b'z']
         .iter()
-        .all(|&(beg, end)| charset[beg as usize..=end as usize].iter().all(|&b| b))
+        .all(|r| charset.covers_range(r.clone()))
     {
-        charset[0xC2..=0xF4].fill(true);
+        charset.set_range(0xC2..=0xF4, true);
     }
 
     charset
 }
-
-const ASCII_WORD_CHARSET: Charset = {
-    const fn fill(bits: &mut [bool; 256], range: RangeInclusive<u8>) {
-        let beg = *range.start() as usize;
-        let end = *range.end() as usize;
-        let mut i = beg;
-        while i <= end {
-            bits[i] = true;
-            i += 1;
-        }
-    }
-
-    let mut charset = Charset::no();
-    fill(&mut charset.bits, b'0'..=b'9');
-    fill(&mut charset.bits, b'A'..=b'Z');
-    fill(&mut charset.bits, b'_'..=b'_');
-    fill(&mut charset.bits, b'a'..=b'z');
-    fill(&mut charset.bits, 0xC2..=0xF4);
-    charset
-};
diff --git a/crates/lsh/src/compiler/tokenizer.rs b/crates/lsh/src/compiler/tokenizer.rs
index 97996c9e19af..22d1628386eb 100644
--- a/crates/lsh/src/compiler/tokenizer.rs
+++ b/crates/lsh/src/compiler/tokenizer.rs
@@ -112,7 +112,7 @@ impl<'a> Tokenizer<'a> {
                 ',' => Token::Comma,
                 '$' => {
                     self.start_pos += 1; // Skip the '$'
-                    match self.read_integer(ch) {
+                    match self.read_integer() {
                         Token::Integer(idx) => Token::Submatch(idx),
                         other => other,
                     }
@@ -160,7 +160,7 @@ impl<'a> Tokenizer<'a> {
                         Token::Plus
                     }
                 }
-                '0'..='9' => self.read_integer(ch),
+                '0'..='9' => self.read_integer(),
                 c => self.read_identifier_or_keyword(c),
             },
         }
@@ -230,7 +230,7 @@ impl<'a> Tokenizer<'a> {
         Token::Regex(&self.input[beg..end])
     }
 
-    fn read_integer(&mut self, first: char) -> Token<'a> {
+    fn read_integer(&mut self) -> Token<'a> {
         while let Some(ch) = self.peek()
             && ch.is_ascii_digit()
         {
diff --git a/crates/lsh/src/engine.rs b/crates/lsh/src/engine.rs
deleted file mode 100644
index dfceca46782f..000000000000
--- a/crates/lsh/src/engine.rs
+++ /dev/null
@@ -1,427 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-//! LSH bytecode interpreter.
-//!
-//! ## Performance notes
-//!
-//! - The main loop is "unsafe". Profile before "cleaning" it up.
-//! - `charset_gobble`, `inlined_mem(i)cmp` are hot paths.
-//!
-//! ## Gotchas
-//!
-//! - `Return` with empty stack resets the VM to entrypoint and clears registers.
-//!   This is how the DSL returns to the "idle" state between tokens.
-//! - `AwaitInput` only breaks the loop if `off >= line.len()`. If not at EOL, it's a no-op.
-//!   This allows the DSL to say "wait for more input OR continue if there is some".
-//! - The result always has a sentinel span at `line.len()`. Consumers can rely on this.
-
-use std::fmt::Debug;
-use std::mem;
-use std::path::Path;
-
-use stdext::arena::{Arena, scratch_arena};
-
-use crate::compiler::Registers;
-
-/// A compiled language definition with its bytecode entrypoint.
-pub struct Language {
-    /// Unique identifier (e.g., "rust", "markdown").
-    pub id: &'static str,
-    /// Human-readable display name.
-    pub name: &'static str,
-    /// Bytecode address where execution begins for this language.
-    pub entrypoint: u32,
-}
-
-impl PartialEq for &'static Language {
-    fn eq(&self, other: &Self) -> bool {
-        std::ptr::eq(*self, *other)
-    }
-}
-
-/// A highlight span indicating that text from `start` to the next span has the given `kind`.
-///
-/// Spans are half-open: `[start, next.start)`. The final span in a line extends to EOL.
-#[derive(Clone, PartialEq, Eq)]
-pub struct Higlight<T> {
-    /// Byte offset where this highlight begins.
-    pub start: usize,
-    /// The token/highlight type (e.g., keyword, string, comment).
-    pub kind: T,
-}
-
-impl<T: Debug> Debug for Higlight<T> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "({}, {:?})", self.start, self.kind)
-    }
-}
-
-/// The bytecode interpreter for syntax highlighting.
-#[derive(Clone)]
-pub struct Engine<'pa, 'ps, 'pc> {
-    assembly: &'pa [u8],
-    strings: &'ps [&'ps str],
-    charsets: &'pc [[u16; 16]],
-    entrypoint: u32,
-    stack: Vec<u32>,
-    registers: Registers,
-}
-
-/// Snapshot of engine state for incremental re-highlighting.
-///
-/// Save with [`Engine::snapshot`], restore with [`Engine::restore`].
-/// This allows re-highlighting from a known state when only part of a file changes.
-#[derive(Clone)]
-pub struct EngineState {
-    stack: Vec<u32>,
-    registers: Registers,
-}
-
-impl<'pa, 'ps, 'pc> Engine<'pa, 'ps, 'pc> {
-    pub fn new(
-        assembly: &'pa [u8],
-        strings: &'ps [&'ps str],
-        charsets: &'pc [[u16; 16]],
-        entrypoint: u32,
-    ) -> Self {
-        Engine {
-            assembly,
-            strings,
-            charsets,
-            entrypoint,
-            stack: Default::default(),
-            registers: Registers { pc: entrypoint, ..Default::default() },
-        }
-    }
-
-    pub fn snapshot(&self) -> EngineState {
-        EngineState { stack: self.stack.clone(), registers: self.registers }
-    }
-
-    pub fn restore(&mut self, state: &EngineState) {
-        self.stack = state.stack.clone();
-        self.registers = state.registers;
-    }
-
-    /// Parse a single line and return highlight spans.
-    ///
-    /// Executes bytecode until the line is fully consumed or a `Return` resets the VM.
-    /// The returned spans partition the line into highlighted regions.
-    ///
-    /// # Arguments
-    /// * `arena` - Allocator for the result vector
-    /// * `line` - The line bytes to highlight (without trailing newline)
-    ///
-    /// # Returns
-    /// A vector of [`Higlight`] spans. Always contains at least two spans:
-    /// one at offset 0 and one at `line.len()` as a sentinel.
-    pub fn parse_next_line<'a, T: PartialEq + TryFrom<u32>>(
-        &mut self,
-        arena: &'a Arena,
-        line: &[u8],
-    ) -> Vec<Higlight<T>, &'a Arena> {
-        let mut res: Vec<Higlight<T>, &Arena> = Vec::new_in(arena);
-        let mut reset = false;
-
-        self.registers.off = 0;
-        self.registers.hs = 0;
-
-        // By default, any line starts with HighlightKind::Other.
-        // If the DSL yields anything, this will be overwritten.
-        res.push(Higlight { start: 0, kind: unsafe { mem::zeroed() } });
-
-        loop {
-            let pc = self.registers.pc as usize;
-            let op = self.assembly[pc];
-            self.registers.pc += 1;
-
-            match op {
-                0 => {
-                    // Mov { dst: Register, src: Register }
-                    let (dst, src) = self.read_reg_pair();
-                    let s = self.registers.get(src);
-                    self.registers.set(dst, s);
-                }
-                1 => {
-                    // Add { dst: Register, src: Register }
-                    let (dst, src) = self.read_reg_pair();
-                    let d = self.registers.get(dst);
-                    let s = self.registers.get(src);
-                    self.registers.set(dst, d.saturating_add(s));
-                }
-                2 => {
-                    // Sub { dst: Register, src: Register }
-                    let (dst, src) = self.read_reg_pair();
-                    let d = self.registers.get(dst);
-                    let s = self.registers.get(src);
-                    self.registers.set(dst, d.saturating_sub(s));
-                }
-                3 => {
-                    // MovImm { dst: Register, imm: u32 }
-                    let (dst, _) = self.read_reg_pair();
-                    let imm = self.read::<u32>();
-                    self.registers.set(dst, imm);
-                }
-                4 => {
-                    // AddImm { dst: Register, imm: u32 }
-                    let (dst, _) = self.read_reg_pair();
-                    let imm = self.read::<u32>();
-                    let d = self.registers.get(dst);
-                    self.registers.set(dst, d.saturating_add(imm));
-                }
-                5 => {
-                    // SubImm { dst: Register, imm: u32 }
-                    let (dst, _) = self.read_reg_pair();
-                    let imm = self.read::<u32>();
-                    let d = self.registers.get(dst);
-                    self.registers.set(dst, d.saturating_sub(imm));
-                }
-
-                6 => {
-                    // Call { tgt: u32 }
-                    let tgt = self.read::<u32>();
-                    self.stack.push(self.registers.pc);
-                    self.registers.pc = tgt;
-                }
-                7 => {
-                    // Return
-                    if let Some(pc) = self.stack.pop() {
-                        self.registers.pc = pc;
-                    } else {
-                        reset = true;
-                        break;
-                    }
-                }
-
-                8 => {
-                    // JumpEQ { lhs: Register, rhs: Register, tgt: u32 }
-                    let (dst, src) = self.read_reg_pair();
-                    let tgt = self.read::<u32>();
-                    if self.registers.get(dst) == self.registers.get(src) {
-                        self.registers.pc = tgt;
-                    }
-                }
-                9 => {
-                    // JumpNE { lhs: Register, rhs: Register, tgt: u32 }
-                    let (dst, src) = self.read_reg_pair();
-                    let tgt = self.read::<u32>();
-                    if self.registers.get(dst) != self.registers.get(src) {
-                        self.registers.pc = tgt;
-                    }
-                }
-                10 => {
-                    // JumpLT { lhs: Register, rhs: Register, tgt: u32 }
-                    let (dst, src) = self.read_reg_pair();
-                    let tgt = self.read::<u32>();
-                    if self.registers.get(dst) < self.registers.get(src) {
-                        self.registers.pc = tgt;
-                    }
-                }
-                11 => {
-                    // JumpLE { lhs: Register, rhs: Register, tgt: u32 }
-                    let (dst, src) = self.read_reg_pair();
-                    let tgt = self.read::<u32>();
-                    if self.registers.get(dst) <= self.registers.get(src) {
-                        self.registers.pc = tgt;
-                    }
-                }
-                12 => {
-                    // JumpGT { lhs: Register, rhs: Register, tgt: u32 }
-                    let (dst, src) = self.read_reg_pair();
-                    let tgt = self.read::<u32>();
-                    if self.registers.get(dst) > self.registers.get(src) {
-                        self.registers.pc = tgt;
-                    }
-                }
-                13 => {
-                    // JumpGE { lhs: Register, rhs: Register, tgt: u32 }
-                    let (dst, src) = self.read_reg_pair();
-                    let tgt = self.read::<u32>();
-                    if self.registers.get(dst) >= self.registers.get(src) {
-                        self.registers.pc = tgt;
-                    }
-                }
-
-                14 => {
-                    // JumpIfEndOfLine { tgt: u32 }
-                    let tgt = self.read::<u32>();
-                    if (self.registers.off as usize) >= line.len() {
-                        self.registers.pc = tgt;
-                    }
-                }
-
-                15 => {
-                    // JumpIfMatchCharset { idx: u32, tgt: u32 }
-                    let idx = self.read::<u32>();
-                    let tgt = self.read::<u32>();
-                    let off = self.registers.off as usize;
-                    let cs = &self.charsets[idx as usize];
-
-                    if let Some(off) = Self::charset_gobble(line, off, cs) {
-                        self.registers.off = off as u32;
-                        self.registers.pc = tgt;
-                    }
-                }
-                16 => {
-                    // JumpIfMatchPrefix { idx: u32, tgt: u32 }
-                    let idx = self.read::<u32>();
-                    let tgt = self.read::<u32>();
-                    let off = self.registers.off as usize;
-                    let str = self.strings[idx as usize].as_bytes();
-
-                    if Self::inlined_memcmp(line, off, str) {
-                        self.registers.off = (off + str.len()) as u32;
-                        self.registers.pc = tgt;
-                    }
-                }
-                17 => {
-                    // JumpIfMatchPrefixInsensitive { idx: u32, tgt: u32 }
-                    let idx = self.read::<u32>();
-                    let tgt = self.read::<u32>();
-                    let off = self.registers.off as usize;
-                    let str = self.strings[idx as usize].as_bytes();
-
-                    if Self::inlined_memicmp(line, off, str) {
-                        self.registers.off = (off + str.len()) as u32;
-                        self.registers.pc = tgt;
-                    }
-                }
-
-                18 => {
-                    // FlushHighlight
-                    let (kind, _) = self.read_reg_pair();
-                    let kind = self.registers.get(kind);
-                    let kind = unsafe { kind.try_into().unwrap_unchecked() };
-                    let start = (self.registers.hs as usize).min(line.len());
-
-                    if let Some(last) = res.last_mut()
-                        && (last.start == start || last.kind == kind)
-                    {
-                        last.kind = kind;
-                    } else {
-                        res.push(Higlight { start, kind });
-                    }
-
-                    self.registers.hs = self.registers.off;
-                }
-                19 => {
-                    // AwaitInput
-                    let off = self.registers.off as usize;
-                    if off >= line.len() {
-                        break;
-                    }
-                }
-
-                _ => unreachable!(),
-            }
-        }
-
-        // Ensure that there's a past-the-end highlight.
-        if res.last().is_none_or(|last| last.start < line.len()) {
-            res.push(Higlight { start: line.len(), kind: unsafe { mem::zeroed() } });
-        }
-
-        if reset {
-            self.registers = unsafe { mem::zeroed() };
-            self.registers.pc = self.entrypoint;
-        }
-
-        res
-    }
-
-    // TODO: http://0x80.pl/notesen/2018-10-18-simd-byte-lookup.html#alternative-implementation
-    #[inline]
-    fn charset_gobble(haystack: &[u8], mut off: usize, cs: &[u16; 16]) -> Option<usize> {
-        if off >= haystack.len() || !Self::in_set(cs, haystack[off]) {
-            return None;
-        }
-
-        while {
-            off += 1;
-            off < haystack.len() && Self::in_set(cs, haystack[off])
-        } {}
-
-        Some(off)
-    }
-
-    /// A mini-memcmp implementation for short needles.
-    /// Compares the `haystack` at `off` with the `needle`.
-    #[inline]
-    fn inlined_memcmp(haystack: &[u8], off: usize, needle: &[u8]) -> bool {
-        unsafe {
-            if off >= haystack.len() || haystack.len() - off < needle.len() {
-                return false;
-            }
-
-            let a = haystack.as_ptr().add(off);
-            let b = needle.as_ptr();
-            let mut i = 0;
-
-            while i < needle.len() {
-                let a = *a.add(i);
-                let b = *b.add(i);
-                i += 1;
-                if a != b {
-                    return false;
-                }
-            }
-
-            true
-        }
-    }
-
-    /// Like `inlined_memcmp`, but case-insensitive.
-    #[inline]
-    fn inlined_memicmp(haystack: &[u8], off: usize, needle: &[u8]) -> bool {
-        unsafe {
-            if off >= haystack.len() || haystack.len() - off < needle.len() {
-                return false;
-            }
-
-            let a = haystack.as_ptr().add(off);
-            let b = needle.as_ptr();
-            let mut i = 0;
-
-            while i < needle.len() {
-                // str in PrefixInsensitive(str) is expected to be lowercase, printable ASCII.
-                let a = a.add(i).read().to_ascii_lowercase();
-                let b = b.add(i).read();
-                i += 1;
-                if a != b {
-                    return false;
-                }
-            }
-
-            true
-        }
-    }
-
-    #[inline]
-    fn in_set(bitmap: &[u16; 16], byte: u8) -> bool {
-        let lo_nibble = byte & 0xf;
-        let hi_nibble = byte >> 4;
-
-        let bitset = bitmap[lo_nibble as usize];
-        let bitmask = 1u16 << hi_nibble;
-
-        (bitset & bitmask) != 0
-    }
-
-    #[inline]
-    fn read<T: Copy>(&mut self) -> T {
-        let pc = self.registers.pc as usize;
-        self.registers.pc += mem::size_of::<T>() as u32;
-        // The LSH backend emits extra 0xff instructions at the end, to allow
-        // us to blindly read any instruction and only later check if it was valid.
-        unsafe { (self.assembly.as_ptr().add(pc) as *const T).read_unaligned() }
-    }
-
-    #[inline]
-    fn read_reg_pair(&mut self) -> (usize, usize) {
-        let reg_pair = self.read::<u8>();
-        let dst = (reg_pair & 0xf) as usize;
-        let src = (reg_pair >> 4) as usize;
-        (dst, src)
-    }
-}
diff --git a/crates/lsh/src/lib.rs b/crates/lsh/src/lib.rs
index e31254fb56f6..1a3a995d672a 100644
--- a/crates/lsh/src/lib.rs
+++ b/crates/lsh/src/lib.rs
@@ -10,7 +10,7 @@
 //! → `frontend` (parse) → IR
 //! → `optimizer` → IR
 //! → `backend` (regalloc + codegen) → bytecode
-//! → `engine` (execute)
+//! → `runtime` (execute)
 //!
 //! The IR is a graph of `RefCell<IR>` nodes. Each node has a `.next` pointer and `If` nodes
 //! in particular have a `.then` pointer. This makes CFG manipulation trivial but means you can't
@@ -21,7 +21,7 @@
 //! - `off` advances only on successful regex matches. Failed matches leave it alone.
 //!   This is why the frontend emits backup/restore pairs around regex chains.
 //! - `hs` (highlight start) is used to track the input offset of the last yield statement,
-//!   which permits the engine (runtime) to highlight everything inbetween with the next yield's highlight kind.
+//!   which permits the runtime to highlight everything inbetween with the next yield's highlight kind.
 //!
 //! ## Charset encoding
 //!
@@ -49,9 +49,7 @@
 //! - No include statements, and all functions across all files share a single namespace.
 
 #![feature(allocator_api)]
-#![allow(irrefutable_let_patterns, unused, clippy::upper_case_acronyms)]
+#![allow(irrefutable_let_patterns, clippy::upper_case_acronyms)]
 
 pub mod compiler;
-pub mod engine;
-
-pub use crate::compiler::CompileResult;
+pub mod runtime;
diff --git a/crates/lsh/src/runtime.rs b/crates/lsh/src/runtime.rs
new file mode 100644
index 000000000000..2cb9ea6dd38c
--- /dev/null
+++ b/crates/lsh/src/runtime.rs
@@ -0,0 +1,950 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! LSH bytecode interpreter.
+//!
+//! ## Performance notes
+//!
+//! - The main loop is "unsafe". Profile before "cleaning" it up.
+//! - `charset_gobble`, `inlined_mem(i)cmp` are hot paths.
+//!
+//! ## Gotchas
+//!
+//! - `Return` with empty stack resets the VM to entrypoint and clears registers.
+//!   This is how the DSL returns to the "idle" state between tokens.
+//! - `AwaitInput` only breaks the loop if `off >= line.len()`. If not at EOL, it's a no-op.
+//!   This allows the DSL to say "wait for more input OR continue if there is some".
+//! - The result always has a sentinel span at `line.len()`. Consumers can rely on this.
+
+use std::fmt::{self, Debug, Write as _};
+use std::mem;
+
+use stdext::arena::{Arena, ArenaString};
+
+/// A compiled language definition with its bytecode entrypoint.
+pub struct Language {
+    /// Unique identifier (e.g., "rust", "markdown").
+    pub id: &'static str,
+    /// Human-readable display name.
+    pub name: &'static str,
+    /// Bytecode address where execution begins for this language.
+    pub entrypoint: u32,
+}
+
+impl PartialEq for &'static Language {
+    fn eq(&self, other: &Self) -> bool {
+        std::ptr::eq(*self, *other)
+    }
+}
+
+/// A highlight span indicating that text from `start` to the next span has the given `kind`.
+///
+/// Spans are half-open: `[start, next.start)`. The final span in a line extends to EOL.
+#[derive(Clone, PartialEq, Eq)]
+pub struct Highlight<T> {
+    /// Byte offset where this highlight begins.
+    pub start: usize,
+    /// The token/highlight type (e.g., keyword, string, comment).
+    pub kind: T,
+}
+
+impl<T: Debug> Debug for Highlight<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "({}, {:?})", self.start, self.kind)
+    }
+}
+
+/// The bytecode interpreter for syntax highlighting.
+#[derive(Clone)]
+pub struct Runtime<'pa, 'ps, 'pc> {
+    assembly: &'pa [u8],
+    strings: &'ps [&'ps str],
+    charsets: &'pc [[u16; 16]],
+    entrypoint: u32,
+    stack: Vec<Registers>,
+    registers: Registers,
+}
+
+/// Snapshot of the runtime state for incremental re-highlighting.
+#[derive(Clone)]
+pub struct RuntimeState {
+    stack: Vec<Registers>,
+    registers: Registers,
+}
+
+impl<'pa, 'ps, 'pc> Runtime<'pa, 'ps, 'pc> {
+    pub fn new(
+        assembly: &'pa [u8],
+        strings: &'ps [&'ps str],
+        charsets: &'pc [[u16; 16]],
+        entrypoint: u32,
+    ) -> Self {
+        Runtime {
+            assembly,
+            strings,
+            charsets,
+            entrypoint,
+            stack: Default::default(),
+            registers: Registers { pc: entrypoint, ..Default::default() },
+        }
+    }
+
+    pub fn snapshot(&self) -> RuntimeState {
+        RuntimeState { stack: self.stack.clone(), registers: self.registers }
+    }
+
+    pub fn restore(&mut self, state: &RuntimeState) {
+        self.stack = state.stack.clone();
+        self.registers = state.registers;
+    }
+
+    /// Parse a single line and return highlight spans.
+    ///
+    /// Executes bytecode until the line is fully consumed or a `Return` resets the VM.
+    /// The returned spans partition the line into highlighted regions.
+    ///
+    /// # Returns
+    /// A vector of [`Highlight`] spans. Always contains at least two spans:
+    /// one at offset 0 and one at `line.len()` as a sentinel.
+    pub fn parse_next_line<'a, T: PartialEq + TryFrom<u32>>(
+        &mut self,
+        arena: &'a Arena,
+        line: &[u8],
+    ) -> Vec<Highlight<T>, &'a Arena> {
+        let mut res: Vec<Highlight<T>, &Arena> = Vec::new_in(arena);
+
+        self.registers.off = 0;
+        self.registers.hs = 0;
+
+        // By default, any line starts with HighlightKind::Other.
+        // If the DSL yields anything, this will be overwritten.
+        res.push(Highlight { start: 0, kind: unsafe { mem::zeroed() } });
+
+        loop {
+            instruction_decode!(self.assembly, self.registers.pc, {
+                Mov { dst, src } => {
+                    let s = self.registers.get(src);
+                    self.registers.set(dst, s);
+                }
+                Add { dst, src } => {
+                    let d = self.registers.get(dst);
+                    let s = self.registers.get(src);
+                    self.registers.set(dst, d.saturating_add(s));
+                }
+                Sub { dst, src } => {
+                    let d = self.registers.get(dst);
+                    let s = self.registers.get(src);
+                    self.registers.set(dst, d.saturating_sub(s));
+                }
+                MovImm { dst, imm } => {
+                    self.registers.set(dst, imm);
+                }
+                AddImm { dst, imm } => {
+                    let d = self.registers.get(dst);
+                    self.registers.set(dst, d.saturating_add(imm));
+                }
+                SubImm { dst, imm } => {
+                    let d = self.registers.get(dst);
+                    self.registers.set(dst, d.saturating_sub(imm));
+                }
+
+                Call { tgt } => {
+                    // PC already points to the next instruction (return address)
+                    self.stack.push(self.registers);
+                    self.registers.pc = tgt;
+                }
+                Return => {
+                    if let Some(registers) = self.stack.pop() {
+                        self.registers = registers;
+                    } else {
+                        self.registers = unsafe { mem::zeroed() };
+                        self.registers.pc = self.entrypoint;
+                        break;
+                    }
+                }
+
+                JumpEQ { lhs, rhs, tgt } => {
+                    if self.registers.get(lhs) == self.registers.get(rhs) {
+                        self.registers.pc = tgt;
+                    }
+                }
+                JumpNE { lhs, rhs, tgt } => {
+                    if self.registers.get(lhs) != self.registers.get(rhs) {
+                        self.registers.pc = tgt;
+                    }
+                }
+                JumpLT { lhs, rhs, tgt } => {
+                    if self.registers.get(lhs) < self.registers.get(rhs) {
+                        self.registers.pc = tgt;
+                    }
+                }
+                JumpLE { lhs, rhs, tgt } => {
+                    if self.registers.get(lhs) <= self.registers.get(rhs) {
+                        self.registers.pc = tgt;
+                    }
+                }
+                JumpGT { lhs, rhs, tgt } => {
+                    if self.registers.get(lhs) > self.registers.get(rhs) {
+                        self.registers.pc = tgt;
+                    }
+                }
+                JumpGE { lhs, rhs, tgt } => {
+                    if self.registers.get(lhs) >= self.registers.get(rhs) {
+                        self.registers.pc = tgt;
+                    }
+                }
+
+                JumpIfEndOfLine { tgt } => {
+                    if (self.registers.off as usize) >= line.len() {
+                        self.registers.pc = tgt;
+                    }
+                }
+
+                JumpIfMatchCharset { idx, tgt } => {
+                    let off = self.registers.off as usize;
+                    let cs = &self.charsets[idx as usize];
+
+                    if let Some(off) = Self::charset_gobble(line, off, cs) {
+                        self.registers.off = off as u32;
+                        self.registers.pc = tgt;
+                    }
+                }
+                JumpIfMatchPrefix { idx, tgt } => {
+                    let off = self.registers.off as usize;
+                    let str = self.strings[idx as usize].as_bytes();
+
+                    if Self::inlined_memcmp(line, off, str) {
+                        self.registers.off = (off + str.len()) as u32;
+                        self.registers.pc = tgt;
+                    }
+                }
+                JumpIfMatchPrefixInsensitive { idx, tgt } => {
+                    let off = self.registers.off as usize;
+                    let str = self.strings[idx as usize].as_bytes();
+
+                    if Self::inlined_memicmp(line, off, str) {
+                        self.registers.off = (off + str.len()) as u32;
+                        self.registers.pc = tgt;
+                    }
+                }
+
+                FlushHighlight { kind } => {
+                    let kind = self.registers.get(kind);
+                    let kind = unsafe { kind.try_into().unwrap_unchecked() };
+                    let start = (self.registers.hs as usize).min(line.len());
+
+                    if let Some(last) = res.last_mut()
+                        && (last.start == start || last.kind == kind)
+                    {
+                        last.kind = kind;
+                    } else {
+                        res.push(Highlight { start, kind });
+                    }
+
+                    self.registers.hs = self.registers.off;
+                }
+                AwaitInput => {
+                    let off = self.registers.off as usize;
+                    if off >= line.len() {
+                        break;
+                    }
+                }
+
+                _ => unreachable!(),
+            });
+        }
+
+        // Ensure that there's a past-the-end highlight.
+        if res.last().is_none_or(|last| last.start < line.len()) {
+            res.push(Highlight { start: line.len(), kind: unsafe { mem::zeroed() } });
+        }
+
+        res
+    }
+
+    // TODO: http://0x80.pl/notesen/2018-10-18-simd-byte-lookup.html#alternative-implementation
+    #[inline]
+    fn charset_gobble(haystack: &[u8], mut off: usize, cs: &[u16; 16]) -> Option<usize> {
+        if off >= haystack.len() || !Self::in_set(cs, haystack[off]) {
+            return None;
+        }
+
+        while {
+            off += 1;
+            off < haystack.len() && Self::in_set(cs, haystack[off])
+        } {}
+
+        Some(off)
+    }
+
+    /// A mini-memcmp implementation for short needles.
+    /// Compares the `haystack` at `off` with the `needle`.
+    #[inline]
+    fn inlined_memcmp(haystack: &[u8], off: usize, needle: &[u8]) -> bool {
+        unsafe {
+            if off >= haystack.len() || haystack.len() - off < needle.len() {
+                return false;
+            }
+
+            let a = haystack.as_ptr().add(off);
+            let b = needle.as_ptr();
+            let mut i = 0;
+
+            while i < needle.len() {
+                let a = *a.add(i);
+                let b = *b.add(i);
+                i += 1;
+                if a != b {
+                    return false;
+                }
+            }
+
+            true
+        }
+    }
+
+    /// Like `inlined_memcmp`, but case-insensitive.
+    #[inline]
+    fn inlined_memicmp(haystack: &[u8], off: usize, needle: &[u8]) -> bool {
+        unsafe {
+            if off >= haystack.len() || haystack.len() - off < needle.len() {
+                return false;
+            }
+
+            let a = haystack.as_ptr().add(off);
+            let b = needle.as_ptr();
+            let mut i = 0;
+
+            while i < needle.len() {
+                // str in PrefixInsensitive(str) is expected to be lowercase, printable ASCII.
+                let a = a.add(i).read().to_ascii_lowercase();
+                let b = b.add(i).read();
+                i += 1;
+                if a != b {
+                    return false;
+                }
+            }
+
+            true
+        }
+    }
+
+    #[inline]
+    fn in_set(bitmap: &[u16; 16], byte: u8) -> bool {
+        let lo_nibble = byte & 0xf;
+        let hi_nibble = byte >> 4;
+
+        let bitset = bitmap[lo_nibble as usize];
+        let bitmask = 1u16 << hi_nibble;
+
+        (bitset & bitmask) != 0
+    }
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Register {
+    ProgramCounter,
+    InputOffset,
+    HighlightStart,
+    X3,
+    X4,
+    X5,
+    X6,
+    X7,
+    X8,
+    X9,
+    X10,
+    X11,
+    X12,
+    X13,
+    X14,
+    X15,
+}
+
+impl Register {
+    pub const FIRST_USER_REG: usize = 3; // aka x3
+    pub const COUNT: usize = 16;
+
+    #[inline(always)]
+    pub fn from_usize(value: usize) -> Self {
+        debug_assert!(value < Self::COUNT);
+        unsafe { std::mem::transmute::<u8, Register>(value as u8) }
+    }
+
+    pub fn mnemonic(&self) -> &'static str {
+        match self {
+            Register::ProgramCounter => "pc",
+            Register::InputOffset => "off",
+            Register::HighlightStart => "hs",
+            Register::X3 => "x3",
+            Register::X4 => "x4",
+            Register::X5 => "x5",
+            Register::X6 => "x6",
+            Register::X7 => "x7",
+            Register::X8 => "x8",
+            Register::X9 => "x9",
+            Register::X10 => "x10",
+            Register::X11 => "x11",
+            Register::X12 => "x12",
+            Register::X13 => "x13",
+            Register::X14 => "x14",
+            Register::X15 => "x15",
+        }
+    }
+}
+
+impl fmt::Display for Register {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.mnemonic())
+    }
+}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy)]
+pub struct Registers {
+    pub pc: u32,  // x0 = ProgramCounter
+    pub off: u32, // x1 = InputOffset
+    pub hs: u32,  // x2 = HighlightStart
+    pub x3: u32,
+    pub x4: u32,
+    pub x5: u32,
+    pub x6: u32,
+    pub x7: u32,
+    pub x8: u32,
+    pub x9: u32,
+    pub x10: u32,
+    pub x11: u32,
+    pub x12: u32,
+    pub x13: u32,
+    pub x14: u32,
+    pub x15: u32,
+}
+
+impl Registers {
+    #[inline(always)]
+    pub fn get(&self, reg: Register) -> u32 {
+        unsafe { (self as *const _ as *const u32).add(reg as usize).read() }
+    }
+
+    #[inline(always)]
+    pub fn set(&mut self, reg: Register, val: u32) {
+        unsafe { (self as *mut _ as *mut u32).add(reg as usize).write(val) }
+    }
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy)]
+pub enum Instruction {
+    // NOTE: This allows for jumps by manipulating Register::ProgramCounter.
+    Mov { dst: Register, src: Register },
+    Add { dst: Register, src: Register },
+    Sub { dst: Register, src: Register },
+    MovImm { dst: Register, imm: u32 },
+    AddImm { dst: Register, imm: u32 },
+    SubImm { dst: Register, imm: u32 },
+
+    Call { tgt: u32 },
+    Return,
+
+    JumpEQ { lhs: Register, rhs: Register, tgt: u32 }, // ==
+    JumpNE { lhs: Register, rhs: Register, tgt: u32 }, // !=
+    JumpLT { lhs: Register, rhs: Register, tgt: u32 }, // <
+    JumpLE { lhs: Register, rhs: Register, tgt: u32 }, // <=
+    JumpGT { lhs: Register, rhs: Register, tgt: u32 }, // >
+    JumpGE { lhs: Register, rhs: Register, tgt: u32 }, // >=
+
+    // Jumps to `tgt` if we're at the end of the line.
+    JumpIfEndOfLine { tgt: u32 },
+
+    // Jumps to `tgt` if the test succeeds.
+    // `idx` specifies the charset/string to use.
+    JumpIfMatchCharset { idx: u32, tgt: u32 },
+    JumpIfMatchPrefix { idx: u32, tgt: u32 },
+    JumpIfMatchPrefixInsensitive { idx: u32, tgt: u32 },
+
+    // Flushes the current HighlightKind to the output.
+    FlushHighlight { kind: Register },
+
+    // Awaits more input to be available.
+    AwaitInput,
+}
+
+macro_rules! instruction_decode {
+    ($assembly:expr, $pc:expr, {
+        Mov { $mov_dst:ident, $mov_src:ident } => $mov_handler:block
+        Add { $add_dst:ident, $add_src:ident } => $add_handler:block
+        Sub { $sub_dst:ident, $sub_src:ident } => $sub_handler:block
+        MovImm { $movi_dst:ident, $movi_imm:ident } => $movi_handler:block
+        AddImm { $addi_dst:ident, $addi_imm:ident } => $addi_handler:block
+        SubImm { $subi_dst:ident, $subi_imm:ident } => $subi_handler:block
+
+        Call { $call_tgt:ident } => $call_handler:block
+        Return => $ret_handler:block
+
+        JumpEQ { $jeq_lhs:ident, $jeq_rhs:ident, $jeq_tgt:ident } => $jeq_handler:block
+        JumpNE { $jne_lhs:ident, $jne_rhs:ident, $jne_tgt:ident } => $jne_handler:block
+        JumpLT { $jlt_lhs:ident, $jlt_rhs:ident, $jlt_tgt:ident } => $jlt_handler:block
+        JumpLE { $jle_lhs:ident, $jle_rhs:ident, $jle_tgt:ident } => $jle_handler:block
+        JumpGT { $jgt_lhs:ident, $jgt_rhs:ident, $jgt_tgt:ident } => $jgt_handler:block
+        JumpGE { $jge_lhs:ident, $jge_rhs:ident, $jge_tgt:ident } => $jge_handler:block
+
+        JumpIfEndOfLine { $jeol_tgt:ident } => $jeol_handler:block
+
+        JumpIfMatchCharset { $jc_idx:ident, $jc_tgt:ident } => $jc_handler:block
+        JumpIfMatchPrefix { $jp_idx:ident, $jp_tgt:ident } => $jp_handler:block
+        JumpIfMatchPrefixInsensitive { $jpi_idx:ident, $jpi_tgt:ident } => $jpi_handler:block
+
+        FlushHighlight { $flush_kind:ident } => $flush_handler:block
+        AwaitInput => $await_handler:block
+
+        _ => $bad_opcode:expr $(,)?
+    }) => {{
+        #[inline(always)]
+        fn dec_reg_single(bytes: &[u8], off: usize) -> Register {
+            let b = unsafe { *bytes.as_ptr().add(off) as usize };
+            Register::from_usize(b & 0xf)
+        }
+
+        #[inline(always)]
+        fn dec_reg_pair(bytes: &[u8], off: usize) -> (Register, Register) {
+            let b = unsafe { *bytes.as_ptr().add(off) as usize };
+            let dst = Register::from_usize(b & 0xf);
+            let src = Register::from_usize(b >> 4);
+            (dst, src)
+        }
+
+        #[inline(always)]
+        fn dec_u32(bytes: &[u8], off: usize) -> u32 {
+            unsafe { (bytes.as_ptr().add(off) as *const u32).read_unaligned() }
+        }
+
+        let __asm: &[u8] = $assembly;
+        let __off = $pc as usize;
+
+        // The use of unsafe code above boosts performance by about 10%. We rely on the code
+        // generator to emit invalid 0xff opcodes at the end of the instruction stream as padding.
+        // This way we can read past-the-end, even if the last non-0xff instruction is truncated.
+        let __opcode = __asm[__off];
+
+        match __opcode {
+            0 => {
+                $pc += 2;
+                let ($mov_dst, $mov_src) = dec_reg_pair(__asm, __off + 1);
+                $mov_handler
+            }
+            1 => {
+                $pc += 2;
+                let ($add_dst, $add_src) = dec_reg_pair(__asm, __off + 1);
+                $add_handler
+            }
+            2 => {
+                $pc += 2;
+                let ($sub_dst, $sub_src) = dec_reg_pair(__asm, __off + 1);
+                $sub_handler
+            }
+            3 => {
+                $pc += 6;
+                let $movi_dst = dec_reg_single(__asm, __off + 1);
+                let $movi_imm = dec_u32(__asm, __off + 2);
+                $movi_handler
+            }
+            4 => {
+                $pc += 6;
+                let $addi_dst = dec_reg_single(__asm, __off + 1);
+                let $addi_imm = dec_u32(__asm, __off + 2);
+                $addi_handler
+            }
+            5 => {
+                $pc += 6;
+                let $subi_dst = dec_reg_single(__asm, __off + 1);
+                let $subi_imm = dec_u32(__asm, __off + 2);
+                $subi_handler
+            }
+            6 => {
+                $pc += 5;
+                let $call_tgt = dec_u32(__asm, __off + 1);
+                $call_handler
+            }
+            7 => {
+                $pc += 1;
+                $ret_handler
+            }
+            8 => {
+                $pc += 6;
+                let ($jeq_lhs, $jeq_rhs) = dec_reg_pair(__asm, __off + 1);
+                let $jeq_tgt = dec_u32(__asm, __off + 2);
+                $jeq_handler
+            }
+            9 => {
+                $pc += 6;
+                let ($jne_lhs, $jne_rhs) = dec_reg_pair(__asm, __off + 1);
+                let $jne_tgt = dec_u32(__asm, __off + 2);
+                $jne_handler
+            }
+            10 => {
+                $pc += 6;
+                let ($jlt_lhs, $jlt_rhs) = dec_reg_pair(__asm, __off + 1);
+                let $jlt_tgt = dec_u32(__asm, __off + 2);
+                $jlt_handler
+            }
+            11 => {
+                $pc += 6;
+                let ($jle_lhs, $jle_rhs) = dec_reg_pair(__asm, __off + 1);
+                let $jle_tgt = dec_u32(__asm, __off + 2);
+                $jle_handler
+            }
+            12 => {
+                $pc += 6;
+                let ($jgt_lhs, $jgt_rhs) = dec_reg_pair(__asm, __off + 1);
+                let $jgt_tgt = dec_u32(__asm, __off + 2);
+                $jgt_handler
+            }
+            13 => {
+                $pc += 6;
+                let ($jge_lhs, $jge_rhs) = dec_reg_pair(__asm, __off + 1);
+                let $jge_tgt = dec_u32(__asm, __off + 2);
+                $jge_handler
+            }
+            14 => {
+                $pc += 5;
+                let $jeol_tgt = dec_u32(__asm, __off + 1);
+                $jeol_handler
+            }
+            15 => {
+                $pc += 9;
+                let $jc_idx = dec_u32(__asm, __off + 1);
+                let $jc_tgt = dec_u32(__asm, __off + 5);
+                $jc_handler
+            }
+            16 => {
+                $pc += 9;
+                let $jp_idx = dec_u32(__asm, __off + 1);
+                let $jp_tgt = dec_u32(__asm, __off + 5);
+                $jp_handler
+            }
+            17 => {
+                $pc += 9;
+                let $jpi_idx = dec_u32(__asm, __off + 1);
+                let $jpi_tgt = dec_u32(__asm, __off + 5);
+                $jpi_handler
+            }
+            18 => {
+                $pc += 2;
+                let $flush_kind = dec_reg_single(__asm, __off + 1);
+                $flush_handler
+            }
+            19 => {
+                $pc += 1;
+                $await_handler
+            }
+            _ => $bad_opcode,
+        }
+    }};
+}
+
+use instruction_decode;
+
+impl Instruction {
+    // JumpIfMatchCharset, etc., are 1 byte opcode + two u32 parameters.
+    pub const MAX_ENCODED_SIZE: usize = 9;
+
+    pub fn address_offset(&self) -> Option<usize> {
+        match *self {
+            Instruction::MovImm { .. }
+            | Instruction::AddImm { .. }
+            | Instruction::SubImm { .. } => Some(2), // opcode + dst
+
+            Instruction::Call { .. } => Some(1), // opcode
+
+            Instruction::JumpEQ { .. }
+            | Instruction::JumpNE { .. }
+            | Instruction::JumpLT { .. }
+            | Instruction::JumpLE { .. }
+            | Instruction::JumpGT { .. }
+            | Instruction::JumpGE { .. } => Some(2), // opcode + lhs/rhs pair
+
+            Instruction::JumpIfEndOfLine { .. } => Some(1), // opcode
+
+            Instruction::JumpIfMatchCharset { .. }
+            | Instruction::JumpIfMatchPrefix { .. }
+            | Instruction::JumpIfMatchPrefixInsensitive { .. } => Some(5), // opcode + idx
+
+            _ => None,
+        }
+    }
+
+    #[allow(clippy::identity_op)]
+    pub fn encode<'a>(&self, arena: &'a Arena) -> Vec<u8, &'a Arena> {
+        fn enc_reg_pair(lo: Register, hi: Register) -> u8 {
+            ((hi as u8) << 4) | (lo as u8)
+        }
+
+        fn enc_reg_single(lo: Register) -> u8 {
+            lo as u8
+        }
+
+        fn enc_u32(val: u32) -> [u8; 4] {
+            val.to_le_bytes()
+        }
+
+        let mut bytes = Vec::with_capacity_in(16, arena);
+        #[allow(clippy::missing_transmute_annotations)]
+        bytes.push(unsafe { std::mem::transmute(std::mem::discriminant(self)) });
+
+        match *self {
+            Instruction::Mov { dst, src }
+            | Instruction::Add { dst, src }
+            | Instruction::Sub { dst, src } => {
+                bytes.push(enc_reg_pair(dst, src));
+            }
+            Instruction::MovImm { dst, imm }
+            | Instruction::AddImm { dst, imm }
+            | Instruction::SubImm { dst, imm } => {
+                bytes.push(enc_reg_single(dst));
+                bytes.extend_from_slice(&enc_u32(imm));
+            }
+
+            Instruction::Call { tgt } => {
+                bytes.extend_from_slice(&enc_u32(tgt));
+            }
+            Instruction::Return => {}
+
+            Instruction::JumpEQ { lhs, rhs, tgt }
+            | Instruction::JumpNE { lhs, rhs, tgt }
+            | Instruction::JumpLT { lhs, rhs, tgt }
+            | Instruction::JumpLE { lhs, rhs, tgt }
+            | Instruction::JumpGT { lhs, rhs, tgt }
+            | Instruction::JumpGE { lhs, rhs, tgt } => {
+                bytes.push(enc_reg_pair(lhs, rhs));
+                bytes.extend_from_slice(&enc_u32(tgt));
+            }
+
+            Instruction::JumpIfEndOfLine { tgt } => {
+                bytes.extend_from_slice(&enc_u32(tgt));
+            }
+            Instruction::JumpIfMatchCharset { idx, tgt }
+            | Instruction::JumpIfMatchPrefix { idx, tgt }
+            | Instruction::JumpIfMatchPrefixInsensitive { idx, tgt } => {
+                bytes.extend_from_slice(&enc_u32(idx));
+                bytes.extend_from_slice(&enc_u32(tgt));
+            }
+
+            Instruction::FlushHighlight { kind } => {
+                bytes.push(enc_reg_single(kind));
+            }
+            Instruction::AwaitInput => {}
+        }
+
+        bytes
+    }
+
+    pub fn decode(bytes: &[u8]) -> (Option<Self>, usize) {
+        fn dec_reg_pair(b: u8) -> (Register, Register) {
+            let hi = Register::from_usize((b >> 4) as usize);
+            let lo = Register::from_usize((b & 0xF) as usize);
+            (lo, hi)
+        }
+
+        fn dec_reg_single(b: u8) -> Register {
+            Register::from_usize(b as usize)
+        }
+
+        fn dec_u32(bytes: &[u8]) -> u32 {
+            u32::from_le_bytes(bytes[..4].try_into().unwrap())
+        }
+
+        let opcode = bytes[0];
+        match opcode {
+            0 => {
+                let (dst, src) = dec_reg_pair(bytes[1]);
+                (Some(Instruction::Mov { dst, src }), 2)
+            }
+            1 => {
+                let (dst, src) = dec_reg_pair(bytes[1]);
+                (Some(Instruction::Add { dst, src }), 2)
+            }
+            2 => {
+                let (dst, src) = dec_reg_pair(bytes[1]);
+                (Some(Instruction::Sub { dst, src }), 2)
+            }
+            3 => {
+                let dst = dec_reg_single(bytes[1]);
+                let imm = dec_u32(&bytes[2..]);
+                (Some(Instruction::MovImm { dst, imm }), 6)
+            }
+            4 => {
+                let dst = dec_reg_single(bytes[1]);
+                let imm = dec_u32(&bytes[2..]);
+                (Some(Instruction::AddImm { dst, imm }), 6)
+            }
+            5 => {
+                let dst = dec_reg_single(bytes[1]);
+                let imm = dec_u32(&bytes[2..]);
+                (Some(Instruction::SubImm { dst, imm }), 6)
+            }
+            6 => (Some(Instruction::Call { tgt: dec_u32(&bytes[1..]) }), 5),
+            7 => (Some(Instruction::Return), 1),
+            8 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpEQ { lhs, rhs, tgt }), 6)
+            }
+            9 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpNE { lhs, rhs, tgt }), 6)
+            }
+            10 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpLT { lhs, rhs, tgt }), 6)
+            }
+            11 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpLE { lhs, rhs, tgt }), 6)
+            }
+            12 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpGT { lhs, rhs, tgt }), 6)
+            }
+            13 => {
+                let (lhs, rhs) = dec_reg_pair(bytes[1]);
+                let tgt = dec_u32(&bytes[2..]);
+                (Some(Instruction::JumpGE { lhs, rhs, tgt }), 6)
+            }
+            14 => (Some(Instruction::JumpIfEndOfLine { tgt: dec_u32(&bytes[1..]) }), 5),
+            15 => {
+                let idx = dec_u32(&bytes[1..]);
+                let tgt = dec_u32(&bytes[5..]);
+                (Some(Instruction::JumpIfMatchCharset { idx, tgt }), 9)
+            }
+            16 => {
+                let idx = dec_u32(&bytes[1..]);
+                let tgt = dec_u32(&bytes[5..]);
+                (Some(Instruction::JumpIfMatchPrefix { idx, tgt }), 9)
+            }
+            17 => {
+                let idx = dec_u32(&bytes[1..]);
+                let tgt = dec_u32(&bytes[5..]);
+                (Some(Instruction::JumpIfMatchPrefixInsensitive { idx, tgt }), 9)
+            }
+            18 => {
+                let kind = dec_reg_single(bytes[1]);
+                (Some(Instruction::FlushHighlight { kind }), 2)
+            }
+            19 => (Some(Instruction::AwaitInput), 1),
+            _ => (None, 1),
+        }
+    }
+
+    pub fn mnemonic<'a>(
+        &self,
+        arena: &'a Arena,
+        config: &MnemonicFormattingConfig,
+    ) -> ArenaString<'a> {
+        let mut str = ArenaString::new_in(arena);
+        let _i = config.instruction_prefix;
+        let i_ = config.instruction_suffix;
+        let _r = config.register_prefix;
+        let r_ = config.register_suffix;
+        let _a = config.address_prefix;
+        let a_ = config.address_suffix;
+        let _n = config.numeric_prefix;
+        let n_ = config.numeric_suffix;
+
+        match *self {
+            Instruction::Mov { dst, src } => {
+                _ = write!(str, "{_i}mov{i_}    {_r}{dst}{r_}, {_r}{src}{r_}");
+            }
+            Instruction::Add { dst, src } => {
+                _ = write!(str, "{_i}add{i_}    {_r}{dst}{r_}, {_r}{src}{r_}");
+            }
+            Instruction::Sub { dst, src } => {
+                _ = write!(str, "{_i}sub{i_}    {_r}{dst}{r_}, {_r}{src}{r_}");
+            }
+            Instruction::MovImm { dst, imm } => {
+                if dst == Register::ProgramCounter {
+                    _ = write!(str, "{_i}movi{i_}   {_r}{dst}{r_}, {_a}{imm}{a_}");
+                } else {
+                    _ = write!(str, "{_i}movi{i_}   {_r}{dst}{r_}, {_n}{imm}{n_}");
+                }
+            }
+            Instruction::AddImm { dst, imm } => {
+                _ = write!(str, "{_i}addi{i_}   {_r}{dst}{r_}, {_n}{imm}{n_}");
+            }
+            Instruction::SubImm { dst, imm } => {
+                _ = write!(str, "{_i}subi{i_}   {_r}{dst}{r_}, {_n}{imm}{n_}");
+            }
+
+            Instruction::Call { tgt } => {
+                _ = write!(str, "{_i}call{i_}   {_a}{tgt}{a_}");
+            }
+            Instruction::Return => {
+                _ = write!(str, "{_i}ret{i_}");
+            }
+
+            Instruction::JumpEQ { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jeq{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpNE { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jne{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpLT { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jlt{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpLE { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jle{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpGT { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jgt{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpGE { lhs, rhs, tgt } => {
+                _ = write!(str, "{_i}jge{i_}    {_r}{lhs}{r_}, {_r}{rhs}{r_}, {_a}{tgt}{a_}");
+            }
+
+            Instruction::JumpIfEndOfLine { tgt } => {
+                _ = write!(str, "{_i}jeol{i_}   {_a}{tgt}{a_}");
+            }
+            Instruction::JumpIfMatchCharset { idx, tgt } => {
+                _ = write!(str, "{_i}jc{i_}     {_n}{idx}{n_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpIfMatchPrefix { idx, tgt } => {
+                _ = write!(str, "{_i}jp{i_}     {_n}{idx}{n_}, {_a}{tgt}{a_}");
+            }
+            Instruction::JumpIfMatchPrefixInsensitive { idx, tgt } => {
+                _ = write!(str, "{_i}jpi{i_}    {_n}{idx}{n_}, {_a}{tgt}{a_}");
+            }
+
+            Instruction::FlushHighlight { kind } => {
+                _ = write!(str, "{_i}flush{i_}  {_r}{kind}{r_}");
+            }
+            Instruction::AwaitInput => {
+                _ = write!(str, "{_i}await{i_}");
+            }
+        }
+
+        str
+    }
+}
+
+#[derive(Default)]
+pub struct MnemonicFormattingConfig<'a> {
+    // Color used for highlighting the instruction.
+    pub instruction_prefix: &'a str,
+    pub instruction_suffix: &'a str,
+
+    // Color used for highlighting a register name.
+    pub register_prefix: &'a str,
+    pub register_suffix: &'a str,
+
+    // Color used for highlighting an immediate value.
+    pub address_prefix: &'a str,
+    pub address_suffix: &'a str,
+
+    // Color used for highlighting an immediate value.
+    pub numeric_prefix: &'a str,
+    pub numeric_suffix: &'a str,
+}

From 532178f477dc39d30e58f920a48f8296420b2700 Mon Sep 17 00:00:00 2001
From: Leonard Hecker <leonard@hecker.io>
Date: Tue, 27 Jan 2026 20:17:08 +0100
Subject: [PATCH 5/8] Sorta fix word wrap

---
 crates/edit/src/buffer/mod.rs | 134 +++++++++++++++++++---------------
 crates/lsh/src/runtime.rs     |  61 ++++++++++++----
 2 files changed, 122 insertions(+), 73 deletions(-)

diff --git a/crates/edit/src/buffer/mod.rs b/crates/edit/src/buffer/mod.rs
index 9e9ef6600634..84414a4780fc 100644
--- a/crates/edit/src/buffer/mod.rs
+++ b/crates/edit/src/buffer/mod.rs
@@ -1991,70 +1991,90 @@ impl TextBuffer {
                     global_off += chunk.len();
                 }
 
-                if let Some(highlighter) = &mut highlighter {
-                    let highlights = self.highlighter_cache.parse_line(
+                // If we have a highlighter...
+                if let Some(highlighter) = &mut highlighter
+                    && let highlights = self.highlighter_cache.parse_line(
                         &scratch,
                         highlighter,
                         cursor_beg.logical_pos.y,
-                    );
-                    let mut highlights = highlights.iter();
-
-                    if let Some(first) = highlights.next() {
-                        let mut highlight_kind = first.kind;
-                        let mut highlight_beg =
-                            self.cursor_move_to_offset_internal(cursor_beg, first.start);
-
-                        for next in highlights {
-                            let kind = highlight_kind;
-                            highlight_kind = next.kind;
-
-                            let beg = highlight_beg.visual_pos;
-                            highlight_beg =
-                                self.cursor_move_to_offset_internal(highlight_beg, next.start);
-                            let end = highlight_beg.visual_pos;
-                            let target = Rect {
-                                left: destination.left + self.margin_width + beg.x - origin.x,
-                                top: destination.top + y,
-                                right: destination.left + self.margin_width + end.x - origin.x,
-                                bottom: destination.top + y + 1,
+                    )
+                    // ...and it yields something for this line...
+                    && let Some(first) = highlights.first()
+                {
+                    let mut highlight_beg =
+                        self.cursor_move_to_offset_internal(cursor_beg, first.start);
+
+                    for pair in highlights.windows(2) {
+                        let curr = &pair[0];
+                        let next = &pair[1];
+                        let highlight_end =
+                            self.cursor_move_to_offset_internal(highlight_beg, next.start);
+
+                        if curr.kind != HighlightKind::Other {
+                            let color = match curr.kind {
+                                HighlightKind::Other => None,
+
+                                HighlightKind::Comment => Some(IndexedColor::Green),
+                                HighlightKind::Method => Some(IndexedColor::BrightYellow),
+                                HighlightKind::String => Some(IndexedColor::BrightRed),
+                                HighlightKind::Variable => Some(IndexedColor::BrightCyan),
+
+                                HighlightKind::ConstantLanguage => Some(IndexedColor::BrightBlue),
+                                HighlightKind::ConstantNumeric => Some(IndexedColor::BrightGreen),
+                                HighlightKind::KeywordControl => Some(IndexedColor::BrightMagenta),
+                                HighlightKind::KeywordOther => Some(IndexedColor::BrightBlue),
+                                HighlightKind::MarkupBold => None,
+                                HighlightKind::MarkupChanged => Some(IndexedColor::BrightBlue),
+                                HighlightKind::MarkupDeleted => Some(IndexedColor::BrightRed),
+                                HighlightKind::MarkupHeading => Some(IndexedColor::BrightBlue),
+                                HighlightKind::MarkupInserted => Some(IndexedColor::BrightGreen),
+                                HighlightKind::MarkupItalic => None,
+                                HighlightKind::MarkupLink => None,
+                                HighlightKind::MarkupList => Some(IndexedColor::BrightBlue),
+                                HighlightKind::MarkupStrikethrough => None,
+                                HighlightKind::MetaHeader => Some(IndexedColor::BrightBlue),
+                            };
+                            let attr = match curr.kind {
+                                HighlightKind::MarkupBold => Some(Attributes::Bold),
+                                HighlightKind::MarkupItalic => Some(Attributes::Italic),
+                                HighlightKind::MarkupLink => Some(Attributes::Underlined),
+                                HighlightKind::MarkupStrikethrough => {
+                                    Some(Attributes::Strikethrough)
+                                }
+                                _ => None,
                             };
 
-                            'block: {
-                                let color = match kind {
-                                    HighlightKind::Other => break 'block,
-
-                                    HighlightKind::Comment => IndexedColor::Green,
-                                    HighlightKind::Method => IndexedColor::BrightYellow,
-                                    HighlightKind::String => IndexedColor::BrightRed,
-                                    HighlightKind::Variable => IndexedColor::BrightCyan,
-
-                                    HighlightKind::ConstantLanguage => IndexedColor::BrightBlue,
-                                    HighlightKind::ConstantNumeric => IndexedColor::BrightGreen,
-                                    HighlightKind::KeywordControl => IndexedColor::BrightMagenta,
-                                    HighlightKind::KeywordOther => IndexedColor::BrightBlue,
-                                    HighlightKind::MarkupBold => break 'block,
-                                    HighlightKind::MarkupChanged => IndexedColor::BrightBlue,
-                                    HighlightKind::MarkupDeleted => IndexedColor::BrightRed,
-                                    HighlightKind::MarkupHeading => IndexedColor::BrightBlue,
-                                    HighlightKind::MarkupInserted => IndexedColor::BrightGreen,
-                                    HighlightKind::MarkupItalic => break 'block,
-                                    HighlightKind::MarkupLink => break 'block,
-                                    HighlightKind::MarkupList => IndexedColor::BrightBlue,
-                                    HighlightKind::MarkupStrikethrough => break 'block,
-                                    HighlightKind::MetaHeader => IndexedColor::BrightBlue,
+                            for y in highlight_beg.visual_pos.y..=highlight_end.visual_pos.y {
+                                let left = if y == highlight_beg.visual_pos.y {
+                                    destination.left
+                                        + self.margin_width
+                                        + highlight_beg.visual_pos.x
+                                        - origin.x
+                                } else {
+                                    destination.left + self.margin_width - origin.x
                                 };
-                                fb.blend_fg(target, fb.indexed(color));
-                            }
-
-                            'block: {
-                                let attr = match kind {
-                                    HighlightKind::MarkupBold => Attributes::Bold,
-                                    HighlightKind::MarkupItalic => Attributes::Italic,
-                                    HighlightKind::MarkupLink => Attributes::Underlined,
-                                    HighlightKind::MarkupStrikethrough => Attributes::Strikethrough,
-                                    _ => break 'block,
+                                let right = if y == highlight_end.visual_pos.y {
+                                    destination.left
+                                        + self.margin_width
+                                        + highlight_end.visual_pos.x
+                                        - origin.x
+                                } else {
+                                    CoordType::MAX
+                                };
+                                let right = right.min(destination.right);
+                                let line_target = Rect {
+                                    left,
+                                    top: destination.top + y - origin.y,
+                                    right,
+                                    bottom: destination.top + y + 1 - origin.y,
                                 };
-                                fb.replace_attr(target, Attributes::All, attr);
+
+                                if let Some(color) = color {
+                                    fb.blend_fg(line_target, fb.indexed(color));
+                                }
+                                if let Some(attr) = attr {
+                                    fb.replace_attr(line_target, Attributes::All, attr);
+                                }
                             }
                         }
                     }
diff --git a/crates/lsh/src/runtime.rs b/crates/lsh/src/runtime.rs
index 2cb9ea6dd38c..37256926b1c8 100644
--- a/crates/lsh/src/runtime.rs
+++ b/crates/lsh/src/runtime.rs
@@ -61,14 +61,14 @@ pub struct Runtime<'pa, 'ps, 'pc> {
     strings: &'ps [&'ps str],
     charsets: &'pc [[u16; 16]],
     entrypoint: u32,
-    stack: Vec<Registers>,
+    stack: Vec<u32>,
     registers: Registers,
 }
 
 /// Snapshot of the runtime state for incremental re-highlighting.
 #[derive(Clone)]
 pub struct RuntimeState {
-    stack: Vec<Registers>,
+    stack: Vec<u32>,
     registers: Registers,
 }
 
@@ -149,16 +149,13 @@ impl<'pa, 'ps, 'pc> Runtime<'pa, 'ps, 'pc> {
                 }
 
                 Call { tgt } => {
-                    // PC already points to the next instruction (return address)
-                    self.stack.push(self.registers);
+                    // PC already points to the next instruction (= return address)
+                    self.registers.save_registers(&mut self.stack);
                     self.registers.pc = tgt;
                 }
                 Return => {
-                    if let Some(registers) = self.stack.pop() {
-                        self.registers = registers;
-                    } else {
-                        self.registers = unsafe { mem::zeroed() };
-                        self.registers.pc = self.entrypoint;
+                    if !self.registers.load_registers(&mut self.stack) {
+                        self.registers = Registers { pc: self.entrypoint, ..Default::default() };
                         break;
                     }
                 }
@@ -344,9 +341,11 @@ impl<'pa, 'ps, 'pc> Runtime<'pa, 'ps, 'pc> {
 #[repr(u8)]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub enum Register {
-    ProgramCounter,
+    // These two registers are shared across function calls...
     InputOffset,
     HighlightStart,
+    // ...and the rest is caller-saved.
+    ProgramCounter,
     X3,
     X4,
     X5,
@@ -374,9 +373,9 @@ impl Register {
 
     pub fn mnemonic(&self) -> &'static str {
         match self {
-            Register::ProgramCounter => "pc",
             Register::InputOffset => "off",
             Register::HighlightStart => "hs",
+            Register::ProgramCounter => "pc",
             Register::X3 => "x3",
             Register::X4 => "x4",
             Register::X5 => "x5",
@@ -403,9 +402,9 @@ impl fmt::Display for Register {
 #[repr(C)]
 #[derive(Default, Clone, Copy)]
 pub struct Registers {
-    pub pc: u32,  // x0 = ProgramCounter
-    pub off: u32, // x1 = InputOffset
-    pub hs: u32,  // x2 = HighlightStart
+    pub off: u32, // x0 = InputOffset
+    pub hs: u32,  // x1 = HighlightStart
+    pub pc: u32,  // x2 = ProgramCounter
     pub x3: u32,
     pub x4: u32,
     pub x5: u32,
@@ -424,12 +423,42 @@ pub struct Registers {
 impl Registers {
     #[inline(always)]
     pub fn get(&self, reg: Register) -> u32 {
-        unsafe { (self as *const _ as *const u32).add(reg as usize).read() }
+        unsafe { self.as_ptr().add(reg as usize).read() }
     }
 
     #[inline(always)]
     pub fn set(&mut self, reg: Register, val: u32) {
-        unsafe { (self as *mut _ as *mut u32).add(reg as usize).write(val) }
+        unsafe { self.as_mut_ptr().add(reg as usize).write(val) }
+    }
+
+    #[inline(always)]
+    fn save_registers(&self, vec: &mut Vec<u32>) {
+        unsafe { vec.extend_from_slice(std::slice::from_raw_parts(self.as_ptr().add(2), 14)) };
+    }
+
+    #[inline(always)]
+    fn load_registers(&mut self, vec: &mut Vec<u32>) -> bool {
+        unsafe {
+            if vec.len() < 14 {
+                return false;
+            }
+
+            let src = vec.as_ptr().add(vec.len() - 14);
+            let dst = self.as_mut_ptr().add(2);
+            std::ptr::copy_nonoverlapping(src, dst, 14);
+            vec.truncate(vec.len() - 14);
+            true
+        }
+    }
+
+    #[inline(always)]
+    unsafe fn as_ptr(&self) -> *const u32 {
+        self as *const _ as *const u32
+    }
+
+    #[inline(always)]
+    unsafe fn as_mut_ptr(&mut self) -> *mut u32 {
+        self as *mut _ as *mut u32
     }
 }
 

From 72f40e9a28e749e7246d11a4679289ac552879a5 Mon Sep 17 00:00:00 2001
From: Leonard Hecker <leonard@hecker.io>
Date: Tue, 27 Jan 2026 22:05:25 +0100
Subject: [PATCH 6/8] wip

---
 crates/edit/src/buffer/mod.rs | 187 ++++++++++++++++++----------------
 1 file changed, 97 insertions(+), 90 deletions(-)

diff --git a/crates/edit/src/buffer/mod.rs b/crates/edit/src/buffer/mod.rs
index 84414a4780fc..1ccbe34d9964 100644
--- a/crates/edit/src/buffer/mod.rs
+++ b/crates/edit/src/buffer/mod.rs
@@ -1752,7 +1752,6 @@ impl TextBuffer {
         let text_width = width - self.margin_width;
         let mut visualizer_buf = [0xE2, 0x90, 0x80]; // U+2400 in UTF8
         let mut visual_pos_x_max = 0;
-        let mut highlighter = self.language.map(|l| Highlighter::new(&self.buffer, l));
 
         // Pick the cursor closer to the `origin.y`.
         let mut cursor = {
@@ -1991,95 +1990,6 @@ impl TextBuffer {
                     global_off += chunk.len();
                 }
 
-                // If we have a highlighter...
-                if let Some(highlighter) = &mut highlighter
-                    && let highlights = self.highlighter_cache.parse_line(
-                        &scratch,
-                        highlighter,
-                        cursor_beg.logical_pos.y,
-                    )
-                    // ...and it yields something for this line...
-                    && let Some(first) = highlights.first()
-                {
-                    let mut highlight_beg =
-                        self.cursor_move_to_offset_internal(cursor_beg, first.start);
-
-                    for pair in highlights.windows(2) {
-                        let curr = &pair[0];
-                        let next = &pair[1];
-                        let highlight_end =
-                            self.cursor_move_to_offset_internal(highlight_beg, next.start);
-
-                        if curr.kind != HighlightKind::Other {
-                            let color = match curr.kind {
-                                HighlightKind::Other => None,
-
-                                HighlightKind::Comment => Some(IndexedColor::Green),
-                                HighlightKind::Method => Some(IndexedColor::BrightYellow),
-                                HighlightKind::String => Some(IndexedColor::BrightRed),
-                                HighlightKind::Variable => Some(IndexedColor::BrightCyan),
-
-                                HighlightKind::ConstantLanguage => Some(IndexedColor::BrightBlue),
-                                HighlightKind::ConstantNumeric => Some(IndexedColor::BrightGreen),
-                                HighlightKind::KeywordControl => Some(IndexedColor::BrightMagenta),
-                                HighlightKind::KeywordOther => Some(IndexedColor::BrightBlue),
-                                HighlightKind::MarkupBold => None,
-                                HighlightKind::MarkupChanged => Some(IndexedColor::BrightBlue),
-                                HighlightKind::MarkupDeleted => Some(IndexedColor::BrightRed),
-                                HighlightKind::MarkupHeading => Some(IndexedColor::BrightBlue),
-                                HighlightKind::MarkupInserted => Some(IndexedColor::BrightGreen),
-                                HighlightKind::MarkupItalic => None,
-                                HighlightKind::MarkupLink => None,
-                                HighlightKind::MarkupList => Some(IndexedColor::BrightBlue),
-                                HighlightKind::MarkupStrikethrough => None,
-                                HighlightKind::MetaHeader => Some(IndexedColor::BrightBlue),
-                            };
-                            let attr = match curr.kind {
-                                HighlightKind::MarkupBold => Some(Attributes::Bold),
-                                HighlightKind::MarkupItalic => Some(Attributes::Italic),
-                                HighlightKind::MarkupLink => Some(Attributes::Underlined),
-                                HighlightKind::MarkupStrikethrough => {
-                                    Some(Attributes::Strikethrough)
-                                }
-                                _ => None,
-                            };
-
-                            for y in highlight_beg.visual_pos.y..=highlight_end.visual_pos.y {
-                                let left = if y == highlight_beg.visual_pos.y {
-                                    destination.left
-                                        + self.margin_width
-                                        + highlight_beg.visual_pos.x
-                                        - origin.x
-                                } else {
-                                    destination.left + self.margin_width - origin.x
-                                };
-                                let right = if y == highlight_end.visual_pos.y {
-                                    destination.left
-                                        + self.margin_width
-                                        + highlight_end.visual_pos.x
-                                        - origin.x
-                                } else {
-                                    CoordType::MAX
-                                };
-                                let right = right.min(destination.right);
-                                let line_target = Rect {
-                                    left,
-                                    top: destination.top + y - origin.y,
-                                    right,
-                                    bottom: destination.top + y + 1 - origin.y,
-                                };
-
-                                if let Some(color) = color {
-                                    fb.blend_fg(line_target, fb.indexed(color));
-                                }
-                                if let Some(attr) = attr {
-                                    fb.replace_attr(line_target, Attributes::All, attr);
-                                }
-                            }
-                        }
-                    }
-                }
-
                 visual_pos_x_max = visual_pos_x_max.max(cursor_end.visual_pos.x);
             }
 
@@ -2088,6 +1998,10 @@ impl TextBuffer {
             cursor = cursor_end;
         }
 
+        let logical_y_beg = self.cursor_for_rendering.unwrap().logical_pos.y;
+        let logical_y_end = cursor.logical_pos.y + 1;
+        self.render_apply_highlights(origin, destination, logical_y_beg..logical_y_end, fb);
+
         // Colorize the margin that we wrote above.
         if self.margin_width > 0 {
             let margin = Rect {
@@ -2153,6 +2067,99 @@ impl TextBuffer {
         Some(RenderResult { visual_pos_x_max })
     }
 
+    fn render_apply_highlights(
+        &mut self,
+        origin: Point,
+        destination: Rect,
+        logical_y_range: Range<CoordType>,
+        fb: &mut Framebuffer,
+    ) {
+        let Some(language) = self.language else {
+            return;
+        };
+
+        let mut highlighter = Highlighter::new(&self.buffer, language);
+        let cursor = self.cursor_for_rendering.unwrap();
+
+        for logical_y in logical_y_range {
+            let scratch = scratch_arena(None);
+            let highlights =
+                self.highlighter_cache.parse_line(&scratch, &mut highlighter, logical_y);
+
+            for pair in highlights.windows(2) {
+                let curr = &pair[0];
+                let next = &pair[1];
+
+                if curr.start < cursor.offset {
+                    continue;
+                }
+
+                let beg = self.cursor_move_to_offset_internal(cursor, curr.start);
+                let end = self.cursor_move_to_offset_internal(cursor, next.start);
+
+                if curr.kind != HighlightKind::Other {
+                    let color = match curr.kind {
+                        HighlightKind::Other => None,
+
+                        HighlightKind::Comment => Some(IndexedColor::Green),
+                        HighlightKind::Method => Some(IndexedColor::BrightYellow),
+                        HighlightKind::String => Some(IndexedColor::BrightRed),
+                        HighlightKind::Variable => Some(IndexedColor::BrightCyan),
+
+                        HighlightKind::ConstantLanguage => Some(IndexedColor::BrightBlue),
+                        HighlightKind::ConstantNumeric => Some(IndexedColor::BrightGreen),
+                        HighlightKind::KeywordControl => Some(IndexedColor::BrightMagenta),
+                        HighlightKind::KeywordOther => Some(IndexedColor::BrightBlue),
+                        HighlightKind::MarkupBold => None,
+                        HighlightKind::MarkupChanged => Some(IndexedColor::BrightBlue),
+                        HighlightKind::MarkupDeleted => Some(IndexedColor::BrightRed),
+                        HighlightKind::MarkupHeading => Some(IndexedColor::BrightBlue),
+                        HighlightKind::MarkupInserted => Some(IndexedColor::BrightGreen),
+                        HighlightKind::MarkupItalic => None,
+                        HighlightKind::MarkupLink => None,
+                        HighlightKind::MarkupList => Some(IndexedColor::BrightBlue),
+                        HighlightKind::MarkupStrikethrough => None,
+                        HighlightKind::MetaHeader => Some(IndexedColor::BrightBlue),
+                    };
+                    let attr = match curr.kind {
+                        HighlightKind::MarkupBold => Some(Attributes::Bold),
+                        HighlightKind::MarkupItalic => Some(Attributes::Italic),
+                        HighlightKind::MarkupLink => Some(Attributes::Underlined),
+                        HighlightKind::MarkupStrikethrough => Some(Attributes::Strikethrough),
+                        _ => None,
+                    };
+
+                    for y in beg.visual_pos.y..=end.visual_pos.y {
+                        let left = if y == beg.visual_pos.y {
+                            destination.left + self.margin_width + beg.visual_pos.x - origin.x
+                        } else {
+                            destination.left + self.margin_width - origin.x
+                        };
+                        let right = if y == end.visual_pos.y {
+                            destination.left + self.margin_width + end.visual_pos.x - origin.x
+                        } else {
+                            CoordType::MAX
+                        };
+                        let right = right.min(destination.right);
+                        let line_target = Rect {
+                            left,
+                            top: destination.top + y - origin.y,
+                            right,
+                            bottom: destination.top + y + 1 - origin.y,
+                        };
+
+                        if let Some(color) = color {
+                            fb.blend_fg(line_target, fb.indexed(color));
+                        }
+                        if let Some(attr) = attr {
+                            fb.replace_attr(line_target, Attributes::All, attr);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     pub fn cut(&mut self, clipboard: &mut Clipboard) {
         self.cut_copy(clipboard, true);
     }

From d9ece615d3ac442424e15f62838d99bc88c8178f Mon Sep 17 00:00:00 2001
From: Leonard Hecker <leonard@hecker.io>
Date: Tue, 27 Jan 2026 22:24:32 +0100
Subject: [PATCH 7/8] Robust word-wrap handling

---
 crates/edit/src/buffer/mod.rs | 168 ++++++++++++++++++++++------------
 1 file changed, 111 insertions(+), 57 deletions(-)

diff --git a/crates/edit/src/buffer/mod.rs b/crates/edit/src/buffer/mod.rs
index 1ccbe34d9964..7b30381b8a36 100644
--- a/crates/edit/src/buffer/mod.rs
+++ b/crates/edit/src/buffer/mod.rs
@@ -2079,9 +2079,25 @@ impl TextBuffer {
         };
 
         let mut highlighter = Highlighter::new(&self.buffer, language);
-        let cursor = self.cursor_for_rendering.unwrap();
+
+        // Track cursor position for efficient offset-to-position conversions.
+        // Start from the rendering cursor which is at the beginning of the visible area.
+        let mut cursor = self.cursor_for_rendering.unwrap();
+
+        // Visible vertical range in visual coordinates.
+        let visible_top = origin.y;
+        let visible_bottom = origin.y + destination.height();
+
+        // Text area boundaries in screen coordinates (excluding margin).
+        let text_left = destination.left + self.margin_width;
+        let text_right = destination.right;
 
         for logical_y in logical_y_range {
+            // Seek cursor to the start of this logical line for efficient lookups.
+            // This is important because highlights are sorted by offset within
+            // each logical line.
+            cursor = self.goto_line_start(cursor, logical_y);
+
             let scratch = scratch_arena(None);
             let highlights =
                 self.highlighter_cache.parse_line(&scratch, &mut highlighter, logical_y);
@@ -2090,70 +2106,108 @@ impl TextBuffer {
                 let curr = &pair[0];
                 let next = &pair[1];
 
-                if curr.start < cursor.offset {
+                // Skip highlights with no visual effect.
+                if curr.kind == HighlightKind::Other {
                     continue;
                 }
 
+                // Convert byte offsets to cursor positions. Since highlights are
+                // sorted by offset, we chain from cursor -> beg -> end for efficiency.
                 let beg = self.cursor_move_to_offset_internal(cursor, curr.start);
-                let end = self.cursor_move_to_offset_internal(cursor, next.start);
-
-                if curr.kind != HighlightKind::Other {
-                    let color = match curr.kind {
-                        HighlightKind::Other => None,
-
-                        HighlightKind::Comment => Some(IndexedColor::Green),
-                        HighlightKind::Method => Some(IndexedColor::BrightYellow),
-                        HighlightKind::String => Some(IndexedColor::BrightRed),
-                        HighlightKind::Variable => Some(IndexedColor::BrightCyan),
-
-                        HighlightKind::ConstantLanguage => Some(IndexedColor::BrightBlue),
-                        HighlightKind::ConstantNumeric => Some(IndexedColor::BrightGreen),
-                        HighlightKind::KeywordControl => Some(IndexedColor::BrightMagenta),
-                        HighlightKind::KeywordOther => Some(IndexedColor::BrightBlue),
-                        HighlightKind::MarkupBold => None,
-                        HighlightKind::MarkupChanged => Some(IndexedColor::BrightBlue),
-                        HighlightKind::MarkupDeleted => Some(IndexedColor::BrightRed),
-                        HighlightKind::MarkupHeading => Some(IndexedColor::BrightBlue),
-                        HighlightKind::MarkupInserted => Some(IndexedColor::BrightGreen),
-                        HighlightKind::MarkupItalic => None,
-                        HighlightKind::MarkupLink => None,
-                        HighlightKind::MarkupList => Some(IndexedColor::BrightBlue),
-                        HighlightKind::MarkupStrikethrough => None,
-                        HighlightKind::MetaHeader => Some(IndexedColor::BrightBlue),
+                let end = self.cursor_move_to_offset_internal(beg, next.start);
+                cursor = end;
+
+                let color = match curr.kind {
+                    HighlightKind::Other => None,
+                    HighlightKind::Comment => Some(IndexedColor::Green),
+                    HighlightKind::Method => Some(IndexedColor::BrightYellow),
+                    HighlightKind::String => Some(IndexedColor::BrightRed),
+                    HighlightKind::Variable => Some(IndexedColor::BrightCyan),
+                    HighlightKind::ConstantLanguage => Some(IndexedColor::BrightBlue),
+                    HighlightKind::ConstantNumeric => Some(IndexedColor::BrightGreen),
+                    HighlightKind::KeywordControl => Some(IndexedColor::BrightMagenta),
+                    HighlightKind::KeywordOther => Some(IndexedColor::BrightBlue),
+                    HighlightKind::MarkupBold => None,
+                    HighlightKind::MarkupChanged => Some(IndexedColor::BrightBlue),
+                    HighlightKind::MarkupDeleted => Some(IndexedColor::BrightRed),
+                    HighlightKind::MarkupHeading => Some(IndexedColor::BrightBlue),
+                    HighlightKind::MarkupInserted => Some(IndexedColor::BrightGreen),
+                    HighlightKind::MarkupItalic => None,
+                    HighlightKind::MarkupLink => None,
+                    HighlightKind::MarkupList => Some(IndexedColor::BrightBlue),
+                    HighlightKind::MarkupStrikethrough => None,
+                    HighlightKind::MetaHeader => Some(IndexedColor::BrightBlue),
+                };
+                let attr = match curr.kind {
+                    HighlightKind::MarkupBold => Some(Attributes::Bold),
+                    HighlightKind::MarkupItalic => Some(Attributes::Italic),
+                    HighlightKind::MarkupLink => Some(Attributes::Underlined),
+                    HighlightKind::MarkupStrikethrough => Some(Attributes::Strikethrough),
+                    _ => None,
+                };
+
+                // Handle the case where the highlight spans multiple visual lines
+                // due to word wrapping. The range is [beg, end) in terms of offsets,
+                // which maps to visual lines [beg.visual_pos.y, end.visual_pos.y].
+                //
+                // When beg and end are on the same visual line, we highlight
+                // [beg.visual_pos.x, end.visual_pos.x).
+                //
+                // When they span multiple lines:
+                // - First line: [beg.visual_pos.x, end_of_line)
+                // - Middle lines: [0, end_of_line)
+                // - Last line: [0, end.visual_pos.x)
+                //
+                // However, if end.visual_pos.x == 0, the last line has no content
+                // to highlight (the span ends exactly at the line boundary).
+                let visual_y_end = if end.visual_pos.x == 0 && end.visual_pos.y > beg.visual_pos.y {
+                    // The span ends at position 0 of a new visual line, meaning
+                    // it actually ends at the end of the previous visual line.
+                    end.visual_pos.y - 1
+                } else {
+                    end.visual_pos.y
+                };
+
+                // Use min/max to skip visual lines outside the visible vertical range.
+                for visual_y in
+                    beg.visual_pos.y.max(visible_top)..(visual_y_end + 1).min(visible_bottom)
+                {
+                    let vis_left = if visual_y == beg.visual_pos.y {
+                        beg.visual_pos.x
+                    } else {
+                        // Wrapped continuation lines start at visual x=0.
+                        0
                     };
-                    let attr = match curr.kind {
-                        HighlightKind::MarkupBold => Some(Attributes::Bold),
-                        HighlightKind::MarkupItalic => Some(Attributes::Italic),
-                        HighlightKind::MarkupLink => Some(Attributes::Underlined),
-                        HighlightKind::MarkupStrikethrough => Some(Attributes::Strikethrough),
-                        _ => None,
+                    let vis_right = if visual_y == end.visual_pos.y {
+                        end.visual_pos.x
+                    } else {
+                        // Line extends to the word wrap column or beyond.
+                        COORD_TYPE_SAFE_MAX
                     };
 
-                    for y in beg.visual_pos.y..=end.visual_pos.y {
-                        let left = if y == beg.visual_pos.y {
-                            destination.left + self.margin_width + beg.visual_pos.x - origin.x
-                        } else {
-                            destination.left + self.margin_width - origin.x
-                        };
-                        let right = if y == end.visual_pos.y {
-                            destination.left + self.margin_width + end.visual_pos.x - origin.x
-                        } else {
-                            CoordType::MAX
-                        };
-                        let right = right.min(destination.right);
-                        let line_target = Rect {
-                            left,
-                            top: destination.top + y - origin.y,
-                            right,
-                            bottom: destination.top + y + 1 - origin.y,
-                        };
+                    // Convert to screen coordinates.
+                    let screen_left = text_left + vis_left - origin.x;
+                    let screen_right = (text_left + vis_right - origin.x).min(text_right);
+                    let screen_y = destination.top + visual_y - origin.y;
+
+                    // Create the target rectangle, clamped to the text area.
+                    let rect = Rect {
+                        left: screen_left.max(text_left),
+                        top: screen_y,
+                        right: screen_right,
+                        bottom: screen_y + 1,
+                    };
 
-                        if let Some(color) = color {
-                            fb.blend_fg(line_target, fb.indexed(color));
-                        }
-                        if let Some(attr) = attr {
-                            fb.replace_attr(line_target, Attributes::All, attr);
-                        }
+                    // Skip empty or invalid rectangles.
+                    if rect.left >= rect.right {
+                        continue;
+                    }
+
+                    if let Some(color) = color {
+                        fb.blend_fg(rect, fb.indexed(color));
+                    }
+                    if let Some(attr) = attr {
+                        fb.replace_attr(rect, Attributes::All, attr);
                     }
                 }
             }

From 1c34e36b635c76e2394cd5e1f0ad1171ac66b467 Mon Sep 17 00:00:00 2001
From: Leonard Hecker <leonard@hecker.io>
Date: Tue, 27 Jan 2026 23:48:10 +0100
Subject: [PATCH 8/8] Inline the tokenizer code

---
 crates/lsh/src/compiler/frontend.rs  | 476 +++++++++++++++++----------
 crates/lsh/src/compiler/mod.rs       |   1 -
 crates/lsh/src/compiler/tokenizer.rs | 281 ----------------
 3 files changed, 295 insertions(+), 463 deletions(-)
 delete mode 100644 crates/lsh/src/compiler/tokenizer.rs

diff --git a/crates/lsh/src/compiler/frontend.rs b/crates/lsh/src/compiler/frontend.rs
index 1596407f18cd..6cef01773b65 100644
--- a/crates/lsh/src/compiler/frontend.rs
+++ b/crates/lsh/src/compiler/frontend.rs
@@ -9,23 +9,22 @@
 //!   second `x`. But the variable *name* still maps to the new vreg, so later reads see it.
 //! - Physical registers (off, etc.) don't follow SSA. Reading them doesn't create a vreg;
 //!   the `parse_expression` function handles this by copying to a fresh vreg.
-//! - The `raise!` macro captures tokenizer position at call time. Don't advance() before raising.
+//! - The `raise!` macro captures token_start position at call time. Don't advance before raising.
 
 use std::collections::HashMap;
 use std::collections::hash_map::Entry;
 
-use super::tokenizer::*;
 use super::*;
 
 macro_rules! raise {
     ($self:ident, $msg:literal) => {{
         let path = $self.path.to_string();
-        let (line, column) = $self.tokenizer.position();
+        let (line, column) = $self.position();
         return Err(CompileError { path, line, column, message: $msg.to_string() })
     }};
     ($self:ident, $($arg:tt)*) => {{
         let path = $self.path.to_string();
-        let (line, column) = $self.tokenizer.position();
+        let (line, column) = $self.position();
         return Err(CompileError { path, line, column, message: format!($($arg)*) })
     }};
 }
@@ -46,8 +45,9 @@ struct Context<'a> {
 pub struct Parser<'a, 'c, 'src> {
     compiler: &'c mut Compiler<'a>,
     path: &'src str,
-    tokenizer: Tokenizer<'src>,
-    current_token: Token<'src>,
+    src: &'src str,
+    pos: usize,
+    token_start: usize,
     context: Vec<Context<'a>, &'a Arena>,
     variables: HashMap<&'src str, IRRegCell<'a>>,
 }
@@ -55,19 +55,11 @@ pub struct Parser<'a, 'c, 'src> {
 impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
     pub fn new(compiler: &'c mut Compiler<'a>, path: &'src str, src: &'src str) -> Self {
         let context = Vec::new_in(compiler.arena);
-        Self {
-            compiler,
-            path,
-            tokenizer: Tokenizer::new(src),
-            current_token: Token::Eof,
-            context,
-            variables: Default::default(),
-        }
+        Self { compiler, path, src, pos: 0, token_start: 0, context, variables: Default::default() }
     }
 
     pub fn run(&mut self) -> CompileResult<()> {
-        self.advance();
-        while !matches!(self.current_token, Token::Eof) {
+        while !self.is_at_eof() {
             let f = self.parse_function()?;
             self.compiler.functions.push(f);
         }
@@ -77,25 +69,14 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
     fn parse_attributes(&mut self) -> CompileResult<FunctionAttributes<'a>> {
         let mut attributes = FunctionAttributes::default();
 
-        while self.current_token == Token::Hash {
-            self.advance();
-            self.expect_token(Token::LeftBracket)?;
-
-            let key = match self.current_token {
-                Token::Identifier(k) => k,
-                _ => raise!(self, "expected attribute name"),
-            };
-            self.advance();
-
-            self.expect_token(Token::Equals)?;
-
-            let value = match self.current_token {
-                Token::String(s) => arena_clone_str(self.compiler.arena, s),
-                _ => raise!(self, "expected attribute value"),
-            };
-            self.advance();
+        while self.peek() == Some('#') {
+            self.pos += 1;
+            self.expect('[')?;
 
-            self.expect_token(Token::RightBracket)?;
+            let key = self.read_identifier()?;
+            self.expect('=')?;
+            let value = arena_clone_str(self.compiler.arena, self.read_string()?);
+            self.expect(']')?;
 
             match key {
                 "display_name" => attributes.display_name = Some(value),
@@ -114,23 +95,19 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
 
         let attributes = self.parse_attributes()?;
 
-        let public = if matches!(self.current_token, Token::Pub) {
-            self.advance();
+        let public = if self.is_keyword("pub") {
+            self.pos += 3;
             true
         } else {
             false
         };
 
-        self.expect_token(Token::Fn)?;
+        self.expect_keyword("fn")?;
 
-        let name = match self.current_token {
-            Token::Identifier(n) => arena_clone_str(self.compiler.arena, n),
-            _ => raise!(self, "expected function name"),
-        };
-        self.advance();
+        let name = arena_clone_str(self.compiler.arena, self.read_identifier()?);
 
-        self.expect_token(Token::LeftParen)?;
-        self.expect_token(Token::RightParen)?;
+        self.expect('(')?;
+        self.expect(')')?;
 
         let span = self.parse_block()?;
 
@@ -144,12 +121,12 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
     }
 
     fn parse_block(&mut self) -> CompileResult<IRSpan<'a>> {
-        self.expect_token(Token::LeftBrace)?;
+        self.expect('{')?;
 
         // TODO: a bit inoptimal to always allocate a noop node
         let mut result: Option<IRSpan> = None;
 
-        while !matches!(self.current_token, Token::RightBrace | Token::Eof) {
+        while !matches!(self.peek(), Some('}') | None) {
             let s = self.parse_statement()?;
             if let Some(span) = &mut result {
                 span.last.borrow_mut().set_next(s.first);
@@ -159,7 +136,7 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
             }
         }
 
-        self.expect_token(Token::RightBrace)?;
+        self.expect('}')?;
         Ok(match result {
             Some(span) => span,
             None => IRSpan::single(self.compiler.alloc_noop()),
@@ -167,23 +144,42 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
     }
 
     fn parse_statement(&mut self) -> CompileResult<IRSpan<'a>> {
-        match self.current_token {
-            Token::Var => self.parse_var_declaration(),
-            Token::Loop => self.parse_loop(),
-            Token::Until => self.parse_until(),
-            Token::Break => self.parse_break(),
-            Token::Continue => self.parse_continue(),
-            Token::Return => self.parse_return(),
-            Token::If => self.parse_if(),
-            Token::Await => self.parse_await(),
-            Token::Yield => self.parse_yield(),
-            Token::Identifier(_) => self.parse_identifier(),
-            _ => raise!(self, "unexpected token: {:?}", self.current_token),
+        if self.is_keyword("var") {
+            return self.parse_var_declaration();
+        }
+        if self.is_keyword("loop") {
+            return self.parse_loop();
+        }
+        if self.is_keyword("until") {
+            return self.parse_until();
+        }
+        if self.is_keyword("break") {
+            return self.parse_break();
+        }
+        if self.is_keyword("continue") {
+            return self.parse_continue();
         }
+        if self.is_keyword("return") {
+            return self.parse_return();
+        }
+        if self.is_keyword("if") {
+            return self.parse_if();
+        }
+        if self.is_keyword("await") {
+            return self.parse_await();
+        }
+        if self.is_keyword("yield") {
+            return self.parse_yield();
+        }
+        if self.peek().is_some_and(Self::is_ident_start) {
+            return self.parse_identifier_stmt();
+        }
+        self.mark();
+        raise!(self, "unexpected token")
     }
 
     fn parse_loop(&mut self) -> CompileResult<IRSpan<'a>> {
-        self.expect_token(Token::Loop)?;
+        self.expect_keyword("loop")?;
 
         let loop_start = self.compiler.alloc_noop();
         let loop_exit = self.compiler.alloc_noop();
@@ -191,7 +187,7 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
     }
 
     fn parse_until(&mut self) -> CompileResult<IRSpan<'a>> {
-        self.expect_token(Token::Until)?;
+        self.expect_keyword("until")?;
 
         let re = self.parse_if_regex()?;
 
@@ -276,8 +272,8 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
     }
 
     fn parse_break(&mut self) -> CompileResult<IRSpan<'a>> {
-        self.expect_token(Token::Break)?;
-        self.expect_token(Token::Semicolon)?;
+        self.expect_keyword("break")?;
+        self.expect(';')?;
 
         if let Some(exit) = self.context.last_mut().and_then(|ctx| ctx.loop_exit) {
             let ir = self.compiler.alloc_noop();
@@ -289,8 +285,8 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
     }
 
     fn parse_continue(&mut self) -> CompileResult<IRSpan<'a>> {
-        self.expect_token(Token::Continue)?;
-        self.expect_token(Token::Semicolon)?;
+        self.expect_keyword("continue")?;
+        self.expect(';')?;
 
         if let Some(start) = self.context.last_mut().and_then(|ctx| ctx.loop_start) {
             let ir = self.compiler.alloc_noop();
@@ -302,15 +298,15 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
     }
 
     fn parse_return(&mut self) -> CompileResult<IRSpan<'a>> {
-        self.expect_token(Token::Return)?;
-        self.expect_token(Token::Semicolon)?;
+        self.expect_keyword("return")?;
+        self.expect(';')?;
         Ok(IRSpan::single(self.compiler.alloc_iri(IRI::Return)))
     }
 
     fn parse_if(&mut self) -> CompileResult<IRSpan<'a>> {
-        self.expect_token(Token::If)?;
+        self.expect_keyword("if")?;
 
-        if matches!(self.current_token, Token::Regex(_)) {
+        if self.peek() == Some('/') {
             self.parse_if_regex_chain()
         } else {
             self.parse_if_comparison()
@@ -365,16 +361,16 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
             re.dst_bad.borrow_mut().set_next(dst_bad);
 
             // No else branch? dst_bad (= no hit) means we're done, so make that connection.
-            if !matches!(self.current_token, Token::Else) {
+            if !self.is_keyword("else") {
                 dst_bad.borrow_mut().set_next(last);
                 break;
             }
 
-            // Gobble the "else" token.
-            self.advance();
+            // Gobble the "else" keyword.
+            self.pos += 4;
 
             // The else branch has a block? That's our dst_bad.
-            if !matches!(self.current_token, Token::If) {
+            if !self.is_keyword("if") {
                 let bl = self.parse_block()?;
                 dst_bad.borrow_mut().set_next(bl.first);
                 // Connect the end of the {} to the end of the if/else chain.
@@ -387,7 +383,7 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
             }
 
             // Otherwise, we expect an "if" in the next iteration to form an "else if".
-            self.expect_token(Token::If)?;
+            self.expect_keyword("if")?;
             prev = Some(dst_bad);
         }
 
@@ -396,30 +392,34 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
 
     fn parse_if_comparison(&mut self) -> CompileResult<IRSpan<'a>> {
         // Parse: if var1 OP var2 { block } where OP is ==, !=, <, >, <=, >=
-        let lhs_name = match self.current_token {
-            Token::Identifier(n) => n,
-            _ => raise!(self, "expected variable name"),
-        };
+        let lhs_name = self.read_identifier()?;
         let lhs_vreg = self.get_variable(lhs_name)?;
-        self.advance();
-
-        let op = match self.current_token {
-            Token::EqualsEquals => ComparisonOp::Eq,
-            Token::NotEquals => ComparisonOp::Ne,
-            Token::LessThan => ComparisonOp::Lt,
-            Token::GreaterThan => ComparisonOp::Gt,
-            Token::LessThanEquals => ComparisonOp::Le,
-            Token::GreaterThanEquals => ComparisonOp::Ge,
-            _ => raise!(self, "expected comparison operator (==, !=, <, >, <=, >=)"),
-        };
-        self.advance();
 
-        let rhs_name = match self.current_token {
-            Token::Identifier(n) => n,
-            _ => raise!(self, "expected variable name"),
+        self.mark();
+        let op = if self.is_str("==") {
+            self.pos += 2;
+            ComparisonOp::Eq
+        } else if self.is_str("!=") {
+            self.pos += 2;
+            ComparisonOp::Ne
+        } else if self.is_str("<=") {
+            self.pos += 2;
+            ComparisonOp::Le
+        } else if self.is_str(">=") {
+            self.pos += 2;
+            ComparisonOp::Ge
+        } else if self.is_str("<") {
+            self.pos += 1;
+            ComparisonOp::Lt
+        } else if self.is_str(">") {
+            self.pos += 1;
+            ComparisonOp::Gt
+        } else {
+            raise!(self, "expected comparison operator (==, !=, <, >, <=, >=)")
         };
+
+        let rhs_name = self.read_identifier()?;
         let rhs_vreg = self.get_variable(rhs_name)?;
-        self.advance();
 
         let dst_good = self.compiler.alloc_noop();
         let dst_bad = self.compiler.alloc_noop();
@@ -438,11 +438,11 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
         let end = self.compiler.alloc_noop();
 
         // Handle optional else branch
-        if matches!(self.current_token, Token::Else) {
-            self.advance();
+        if self.is_keyword("else") {
+            self.pos += 4;
 
             // Check for else if
-            if matches!(self.current_token, Token::If) {
+            if self.is_keyword("if") {
                 let else_if_span = self.parse_if()?;
                 dst_bad.borrow_mut().set_next(else_if_span.first);
                 if let mut block_last = bl.last.borrow_mut()
@@ -483,10 +483,7 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
     }
 
     fn parse_if_regex(&mut self) -> CompileResult<RegexSpan<'a>> {
-        let pattern = match self.current_token {
-            Token::Regex(s) => s,
-            _ => raise!(self, "expected regex"),
-        };
+        let pattern = self.read_regex()?;
         let dst_good = self.compiler.alloc_noop();
         let dst_bad = self.compiler.alloc_noop();
         let mut capture_groups = Vec::new_in(self.compiler.arena);
@@ -496,46 +493,40 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
             Err(e) => raise!(self, "{}", e),
         };
 
-        self.advance();
-
         Ok(RegexSpan { src, dst_good, dst_bad, capture_groups })
     }
 
     fn parse_await(&mut self) -> CompileResult<IRSpan<'a>> {
-        self.expect_token(Token::Await)?;
+        self.expect_keyword("await")?;
 
-        match self.current_token {
-            Token::Identifier("input") => self.advance(),
-            _ => raise!(self, "expected 'input' after await"),
+        let ident = self.read_identifier()?;
+        if ident != "input" {
+            raise!(self, "expected 'input' after await");
         }
 
-        self.expect_token(Token::Semicolon)?;
+        self.expect(';')?;
 
         let ir = self.compiler.alloc_iri(IRI::AwaitInput);
         Ok(IRSpan::single(ir))
     }
 
     fn parse_yield(&mut self) -> CompileResult<IRSpan<'a>> {
-        self.expect_token(Token::Yield)?;
+        self.expect_keyword("yield")?;
 
         // Check if this is a capture group reference: yield $n as color
-        if let Token::Submatch(capture_index) = self.current_token {
-            let capture_index = capture_index as usize - 1;
-            self.advance();
+        if self.peek() == Some('$') {
+            self.pos += 1;
+            let capture_index = self.read_integer()? as usize - 1;
 
             // Expect "as"
-            match self.current_token {
-                Token::Identifier("as") => self.advance(),
-                _ => raise!(self, "expected 'as' after capture group reference"),
+            let kw = self.read_identifier()?;
+            if kw != "as" {
+                raise!(self, "expected 'as' after capture group reference");
             }
 
-            let color = match self.current_token {
-                Token::Identifier(c) => c,
-                _ => raise!(self, "expected color name after 'as'"),
-            };
+            let color = self.read_identifier()?;
             let kind = self.compiler.intern_highlight_kind(color).value;
-            self.advance();
-            self.expect_token(Token::Semicolon)?;
+            self.expect(';')?;
 
             let (start_vreg, end_vreg) = match self.context.last() {
                 Some(ctx) if capture_index < ctx.capture_groups.len() => {
@@ -571,14 +562,10 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
             Ok(span)
         } else {
             // Normal yield: yield color;
-            let color = match self.current_token {
-                Token::Identifier(c) => c,
-                _ => raise!(self, "expected color name after yield"),
-            };
+            let color = self.read_identifier()?;
             let kind = self.compiler.intern_highlight_kind(color).value;
 
-            self.advance();
-            self.expect_token(Token::Semicolon)?;
+            self.expect(';')?;
 
             let vreg = self.compiler.alloc_vreg();
             let span = self
@@ -592,17 +579,12 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
     }
 
     fn parse_var_declaration(&mut self) -> CompileResult<IRSpan<'a>> {
-        self.expect_token(Token::Var)?;
-
-        let name = match self.current_token {
-            Token::Identifier(n) => n,
-            _ => raise!(self, "expected variable name"),
-        };
-        self.advance();
+        self.expect_keyword("var")?;
 
-        self.expect_token(Token::Equals)?;
+        let name = self.read_identifier()?;
+        self.expect('=')?;
         let (expr, vreg) = self.parse_expression()?;
-        self.expect_token(Token::Semicolon)?;
+        self.expect(';')?;
 
         match self.variables.entry(name) {
             Entry::Vacant(e) => _ = e.insert(vreg),
@@ -612,80 +594,71 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
         Ok(expr)
     }
 
-    fn parse_identifier(&mut self) -> CompileResult<IRSpan<'a>> {
-        let name = match self.current_token {
-            Token::Identifier(n) => n,
-            _ => raise!(self, "expected identifier"),
-        };
-        self.advance();
+    fn parse_identifier_stmt(&mut self) -> CompileResult<IRSpan<'a>> {
+        let name = self.read_identifier()?;
 
-        match self.current_token {
+        match self.peek() {
             // foo();
-            Token::LeftParen => {
-                self.advance();
-                self.expect_token(Token::RightParen)?;
-                self.expect_token(Token::Semicolon)?;
+            Some('(') => {
+                self.pos += 1;
+                self.expect(')')?;
+                self.expect(';')?;
                 let name = self.compiler.strings.intern(self.compiler.arena, name);
                 Ok(IRSpan::single(self.compiler.alloc_iri(IRI::Call { name })))
             }
             // foo = expr;
-            Token::Equals => {
-                self.advance();
+            Some('=') if !self.is_str("==") => {
+                self.pos += 1;
                 let (expr, vreg) = self.parse_expression()?;
-                self.expect_token(Token::Semicolon)?;
+                self.expect(';')?;
                 self.variables.insert(name, vreg);
                 Ok(expr)
             }
             // foo += expr;
-            Token::PlusEquals => {
+            Some('+') if self.is_str("+=") => {
                 let lhs_vreg = self.get_variable(name)?;
-                self.advance();
-
-                let Token::Integer(val) = self.current_token else {
-                    raise!(self, "expected integer literal after '+='");
-                };
-                self.advance();
+                self.pos += 2;
 
-                self.expect_token(Token::Semicolon)?;
+                let val = self.read_integer()?;
+                self.expect(';')?;
 
                 let ir = self.compiler.alloc_iri(IRI::AddImm { dst: lhs_vreg, imm: val });
                 self.variables.insert(name, lhs_vreg);
                 Ok(IRSpan::single(ir))
             }
-            _ => raise!(self, "expected '(', or '=', '+=' after identifier"),
+            _ => {
+                self.mark();
+                raise!(self, "expected '(', '=' or '+=' after identifier")
+            }
         }
     }
 
     fn parse_expression(&mut self) -> CompileResult<(IRSpan<'a>, IRRegCell<'a>)> {
-        let (lhs_span, lhs_vreg) = match self.current_token {
-            Token::Integer(val) => {
+        self.mark();
+        let (lhs_span, lhs_vreg) = match self.peek() {
+            Some('0'..='9') => {
+                let val = self.read_integer()?;
                 let vreg = self.compiler.alloc_vreg();
                 let ir = self.compiler.alloc_iri(IRI::MovImm { dst: vreg, imm: val });
-                self.advance();
                 (IRSpan::single(ir), vreg)
             }
-            Token::Identifier(name) => {
+            Some(c) if Self::is_ident_start(c) => {
+                let name = self.read_identifier()?;
                 let vreg = self.get_variable(name)?;
-                self.advance();
                 (IRSpan::single(self.compiler.alloc_noop()), vreg)
             }
             _ => raise!(self, "expected integer or identifier in expression"),
         };
 
         // Check for binary operator
-        if matches!(self.current_token, Token::Plus) {
-            self.advance();
+        if self.peek() == Some('+') && !self.is_str("+=") {
+            self.pos += 1;
 
             // Parse right-hand side - only integer literals supported
-            match self.current_token {
-                Token::Integer(val) => {
-                    self.advance();
-                    let add_ir = self.compiler.alloc_iri(IRI::AddImm { dst: lhs_vreg, imm: val });
-                    lhs_span.last.borrow_mut().set_next(add_ir);
-                    Ok((IRSpan { first: lhs_span.first, last: add_ir }, lhs_vreg))
-                }
-                _ => raise!(self, "expected integer literal after '+'"),
-            }
+            let val = self.read_integer()?;
+            let add_ir = self.compiler.alloc_iri(IRI::AddImm { dst: lhs_vreg, imm: val });
+            lhs_span.last.borrow_mut().set_next(add_ir);
+            Ok((IRSpan { first: lhs_span.first, last: add_ir }, lhs_vreg))
         } else if lhs_vreg.borrow().physical.is_some() {
             // For expressions of type `var virtual = phyiscal;`, we need to ensure
             // that we actually copy the physical register into a new virtual one.
@@ -705,16 +678,157 @@ impl<'a, 'c, 'src> Parser<'a, 'c, 'src> {
         }
     }
 
-    fn expect_token(&mut self, expected: Token) -> CompileResult<()> {
-        if std::mem::discriminant(&self.current_token) == std::mem::discriminant(&expected) {
-            self.advance();
+    //
+    // vvv Tokenization helpers start here vvv
+    //
+
+    fn is_ident_start(ch: char) -> bool {
+        ch.is_ascii_alphabetic() || ch == '_'
+    }
+
+    fn is_ident_char(ch: char) -> bool {
+        ch.is_ascii_alphanumeric() || ch == '_' || ch == '.'
+    }
+
+    fn rest(&self) -> &'src str {
+        &self.src[self.pos..]
+    }
+
+    fn skip_whitespace_comments(&mut self) {
+        loop {
+            let rest = self.rest();
+            let trimmed = rest.trim_ascii_start();
+            self.pos = self.src.len() - trimmed.len();
+
+            if let Some(after_comment) = trimmed.strip_prefix("//") {
+                self.pos += 2 + after_comment.find('\n').unwrap_or(after_comment.len());
+            } else {
+                break;
+            }
+        }
+    }
+
+    fn peek(&mut self) -> Option<char> {
+        self.skip_whitespace_comments();
+        self.rest().chars().next()
+    }
+
+    fn is_str(&self, s: &str) -> bool {
+        self.rest().starts_with(s)
+    }
+
+    fn is_at_eof(&mut self) -> bool {
+        self.skip_whitespace_comments();
+        self.pos >= self.src.len()
+    }
+
+    /// Mark current position for error reporting.
+    fn mark(&mut self) {
+        self.skip_whitespace_comments();
+        self.token_start = self.pos;
+    }
+
+    fn position(&self) -> (usize, usize) {
+        let before = &self.src[..self.token_start];
+        let line = before.bytes().filter(|&b| b == b'\n').count() + 1;
+        let column = before.rfind('\n').map_or(before.len(), |i| before.len() - i - 1) + 1;
+        (line, column)
+    }
+
+    fn expect(&mut self, ch: char) -> CompileResult<()> {
+        self.mark();
+        if self.rest().starts_with(ch) {
+            self.pos += ch.len_utf8();
+            Ok(())
+        } else {
+            raise!(self, "expected '{}'", ch)
+        }
+    }
+
+    fn expect_keyword(&mut self, kw: &str) -> CompileResult<()> {
+        self.mark();
+        if let Some(rest) = self.rest().strip_prefix(kw)
+            && !rest.chars().next().is_some_and(Self::is_ident_char)
+        {
+            self.pos += kw.len();
             Ok(())
         } else {
-            raise!(self, "expected {:?}, found {:?}", expected, self.current_token)
+            raise!(self, "expected '{}'", kw)
         }
     }
 
-    fn advance(&mut self) {
-        self.current_token = self.tokenizer.next();
+    /// Check if next token is keyword (without consuming).
+    fn is_keyword(&mut self, kw: &str) -> bool {
+        self.skip_whitespace_comments();
+        self.rest()
+            .strip_prefix(kw)
+            .is_some_and(|rest| !rest.chars().next().is_some_and(Self::is_ident_char))
+    }
+
+    fn read_identifier(&mut self) -> CompileResult<&'src str> {
+        self.mark();
+        let start = self.pos;
+
+        if !self.rest().chars().next().is_some_and(Self::is_ident_start) {
+            raise!(self, "expected identifier");
+        }
+
+        let rest = self.rest();
+        let len = rest.find(|c| !Self::is_ident_char(c)).unwrap_or(rest.len());
+        self.pos += len;
+        Ok(&self.src[start..self.pos])
+    }
+
+    fn read_integer(&mut self) -> CompileResult<u32> {
+        self.mark();
+        let start = self.pos;
+
+        let rest = self.rest();
+        let len = rest.find(|c: char| !c.is_ascii_digit()).unwrap_or(rest.len());
+        self.pos += len;
+
+        if start == self.pos {
+            raise!(self, "expected integer");
+        }
+
+        self.src[start..self.pos].parse().map_err(|_| {
+            let path = self.path.to_string();
+            let (line, column) = self.position();
+            CompileError { path, line, column, message: "invalid integer".to_string() }
+        })
+    }
+
+    fn read_string(&mut self) -> CompileResult<&'src str> {
+        self.mark();
+        self.expect('"')?;
+        let start = self.pos;
+        let end = self.find_closing_delimiter(b'"');
+        self.pos = end;
+        self.expect('"')?;
+        Ok(&self.src[start..end])
+    }
+
+    fn read_regex(&mut self) -> CompileResult<&'src str> {
+        self.mark();
+        self.expect('/')?;
+        let start = self.pos;
+        let end = self.find_closing_delimiter(b'/');
+        self.pos = end;
+        self.expect('/')?;
+        Ok(&self.src[start..end])
+    }
+
+    /// Find unescaped closing delimiter (handles `\x` escapes).
+    fn find_closing_delimiter(&self, delim: u8) -> usize {
+        let bytes = self.rest().as_bytes();
+        let mut i = 0;
+        while i < bytes.len() {
+            match bytes[i] {
+                b'\\' => i += 2,
+                b if b == delim => return self.pos + i,
+                _ => i += 1,
+            }
+        }
+        self.pos + bytes.len()
     }
 }
diff --git a/crates/lsh/src/compiler/mod.rs b/crates/lsh/src/compiler/mod.rs
index 17e603aff9f5..321547a73f54 100644
--- a/crates/lsh/src/compiler/mod.rs
+++ b/crates/lsh/src/compiler/mod.rs
@@ -37,7 +37,6 @@ mod frontend;
 mod generator;
 mod optimizer;
 mod regex;
-mod tokenizer;
 
 use std::cell::{Cell, RefCell};
 use std::collections::{HashSet, VecDeque};
diff --git a/crates/lsh/src/compiler/tokenizer.rs b/crates/lsh/src/compiler/tokenizer.rs
deleted file mode 100644
index 22d1628386eb..000000000000
--- a/crates/lsh/src/compiler/tokenizer.rs
+++ /dev/null
@@ -1,281 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-//! Lexer for the LSH DSL.
-//!
-//! ## Quirks
-//!
-//! - Regex and string tokens include escape sequences verbatim. The frontend/regex compiler
-//!   handles unescaping. This means `/\/foo/` tokenizes as `Regex("\\/foo")`.
-//! - `position()` computes line/column by scanning from the start.
-//!   Fine for error reporting, but don't call it in hot paths.
-//! - Identifiers can contain dots (e.g., `string.quoted`), which is unusual.
-//!   This is for hierarchical highlight kind names. Affects `is_ident_continuation`.
-
-use std::iter::Peekable;
-use std::str::Chars;
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum Token<'a> {
-    // Literals
-    Identifier(&'a str),
-    Integer(u32),
-    String(&'a str),
-    Regex(&'a str),
-    Submatch(u32),
-
-    // Keywords
-    Pub,
-    Fn,
-    Var,
-    If,
-    Else,
-    Loop,
-    Until,
-    Break,
-    Continue,
-    Return,
-    Await,
-    Yield,
-
-    // Operators
-    Equals,
-    EqualsEquals,
-    NotEquals,
-    LessThan,
-    LessThanEquals,
-    GreaterThan,
-    GreaterThanEquals,
-    Plus,
-    PlusEquals,
-
-    // Punctuation
-    LeftBrace,
-    RightBrace,
-    LeftParen,
-    RightParen,
-    Semicolon,
-    LeftBracket,
-    RightBracket,
-    Hash,
-    Comma,
-
-    // End of file
-    Eof,
-
-    // Error
-    Error(String),
-}
-
-pub struct Tokenizer<'a> {
-    input: &'a str,
-    chars: Peekable<Chars<'a>>,
-    current_pos: usize,
-    start_pos: usize,
-}
-
-impl<'a> Tokenizer<'a> {
-    pub fn new(input: &'a str) -> Self {
-        Self { input, chars: input.chars().peekable(), current_pos: 0, start_pos: 0 }
-    }
-
-    pub fn position(&self) -> (usize, usize) {
-        let mut line = 1;
-        let mut column = 1;
-
-        for ch in self.input[..self.start_pos].chars() {
-            column += 1;
-            if ch == '\n' {
-                line += 1;
-                column = 1;
-            }
-        }
-
-        (line, column)
-    }
-
-    pub fn next(&mut self) -> Token<'a> {
-        self.skip_whitespace();
-
-        self.start_pos = self.current_pos;
-
-        match self.advance() {
-            None => Token::Eof,
-            Some(ch) => match ch {
-                '{' => Token::LeftBrace,
-                '}' => Token::RightBrace,
-                '(' => Token::LeftParen,
-                ')' => Token::RightParen,
-                '[' => Token::LeftBracket,
-                ']' => Token::RightBracket,
-                '#' => Token::Hash,
-                ',' => Token::Comma,
-                '$' => {
-                    self.start_pos += 1; // Skip the '$'
-                    match self.read_integer() {
-                        Token::Integer(idx) => Token::Submatch(idx),
-                        other => other,
-                    }
-                }
-                ';' => Token::Semicolon,
-                '"' => self.read_string(),
-                '/' => self.read_regex(),
-                '=' => {
-                    if self.peek() == Some('=') {
-                        self.advance();
-                        Token::EqualsEquals
-                    } else {
-                        Token::Equals
-                    }
-                }
-                '!' => {
-                    if self.peek() == Some('=') {
-                        self.advance();
-                        Token::NotEquals
-                    } else {
-                        Token::Error("Unexpected character: '!'".to_string())
-                    }
-                }
-                '<' => {
-                    if self.peek() == Some('=') {
-                        self.advance();
-                        Token::LessThanEquals
-                    } else {
-                        Token::LessThan
-                    }
-                }
-                '>' => {
-                    if self.peek() == Some('=') {
-                        self.advance();
-                        Token::GreaterThanEquals
-                    } else {
-                        Token::GreaterThan
-                    }
-                }
-                '+' => {
-                    if self.peek() == Some('=') {
-                        self.advance();
-                        Token::PlusEquals
-                    } else {
-                        Token::Plus
-                    }
-                }
-                '0'..='9' => self.read_integer(),
-                c => self.read_identifier_or_keyword(c),
-            },
-        }
-    }
-
-    fn advance(&mut self) -> Option<char> {
-        if let Some(ch) = self.chars.next() {
-            self.current_pos += ch.len_utf8();
-            Some(ch)
-        } else {
-            None
-        }
-    }
-
-    fn peek(&mut self) -> Option<char> {
-        self.chars.peek().copied()
-    }
-
-    fn skip_whitespace(&mut self) {
-        loop {
-            // Skip whitespace. advance() has been inlined with ASCII assumptions.
-            while let Some(ch) = self.peek()
-                && ch.is_ascii_whitespace()
-            {
-                self.chars.next();
-                self.current_pos += 1;
-            }
-
-            // Check for "//" comments. Skip the line if found.
-            if !self.input.get(self.current_pos..).is_some_and(|s| s.starts_with("//")) {
-                break;
-            }
-            for ch in &mut self.chars {
-                self.current_pos += 1;
-                if ch == '\n' {
-                    break;
-                }
-            }
-        }
-    }
-
-    fn read_string(&mut self) -> Token<'a> {
-        let beg = self.current_pos;
-        let mut last_char = '\0';
-
-        while let Some(ch) = self.advance()
-            && (ch != '"' || last_char == '\\')
-        {
-            last_char = ch;
-        }
-
-        let end = self.current_pos - 1;
-        Token::String(&self.input[beg..end])
-    }
-
-    fn read_regex(&mut self) -> Token<'a> {
-        let beg = self.current_pos;
-        let mut last_char = '\0';
-
-        while let Some(ch) = self.advance()
-            && (ch != '/' || last_char == '\\')
-        {
-            last_char = ch;
-        }
-
-        let end = self.current_pos - 1;
-        Token::Regex(&self.input[beg..end])
-    }
-
-    fn read_integer(&mut self) -> Token<'a> {
-        while let Some(ch) = self.peek()
-            && ch.is_ascii_digit()
-        {
-            self.advance();
-        }
-
-        let s = &self.input[self.start_pos..self.current_pos];
-        match s.parse::<u32>() {
-            Ok(val) => Token::Integer(val),
-            Err(_) => Token::Error(format!("Invalid integer: {s}")),
-        }
-    }
-
-    fn read_identifier_or_keyword(&mut self, ch: char) -> Token<'a> {
-        fn is_ident_start(ch: char) -> bool {
-            ch.is_ascii_alphabetic() || ch == '_'
-        }
-        fn is_ident_continuation(ch: char) -> bool {
-            ch.is_ascii_alphanumeric() || ch == '_' || ch == '.'
-        }
-
-        if !is_ident_start(ch) {
-            return Token::Error(format!("Unexpected character: '{ch}'"));
-        }
-
-        while let Some(ch) = self.peek()
-            && is_ident_continuation(ch)
-        {
-            self.advance();
-        }
-
-        match self.input.get(self.start_pos..self.current_pos).unwrap_or("") {
-            "pub" => Token::Pub,
-            "fn" => Token::Fn,
-            "var" => Token::Var,
-            "if" => Token::If,
-            "else" => Token::Else,
-            "loop" => Token::Loop,
-            "until" => Token::Until,
-            "break" => Token::Break,
-            "continue" => Token::Continue,
-            "return" => Token::Return,
-            "await" => Token::Await,
-            "yield" => Token::Yield,
-            ident => Token::Identifier(ident),
-        }
-    }
-}