feat: perform highlighting

Cargo.toml

@@ -1,20 +1,30 @@
 [package]
 name = "karnaugh"
-version = "0.1.3"
+version = "0.1.5"
 edition = "2024"
 
 [dependencies]
-axum = "0.8.7"
-flate2 = "1.1.5"
-itertools = "0.14.0"
-tar = "0.4.44"
-time = "0.3.47"
-tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread"] }
-tower-http = { version = "0.6.8", features = ["compression-br", "fs", "trace"] }
-tracing = "0.1.43"
-tracing-subscriber = "0.3.22"
-typst-html = { git = "https://github.com/mkorje/typst.git", branch = "mathml" }
-typst-kit = { git = "https://github.com/mkorje/typst.git", branch = "mathml" }
-typst = { git = "https://github.com/mkorje/typst.git", branch = "mathml" }
-ureq = "3.1.4"
-clap = { version = "4.6.0", features = ["derive"] }
+axum = "0.8.9"
+flate2 = "1.1.9"
+itertools = "0.15.0"
+tar = "0.4.46"
+time = "0.3.53"
+tokio = { version = "1.53.0", features = ["macros", "rt-multi-thread"] }
+tower-http = { version = "0.7.0", features = [
+    "compression-br",
+    "fs",
+    "trace",
+    "set-header",
+] }
+tracing = "0.1.44"
+tracing-subscriber = "0.3.23"
+typst-html = "0.15.1"
+typst-kit = "0.15.1"
+typst = "0.15.1"
+ureq = "3.3.0"
+clap = { version = "4.6.2", features = ["derive"] }
+typst-library = "0.15.1"
+typst-utils = "0.15.1"
+tree-house = "0.4.0"
+tree-house-bindings = { version = "0.3.2", features = ["ropey"] }
+ropey = "1.6.1"

flake.lock

@@ -18,13 +18,48 @@
         "type": "github"
       }
     },
+    "helix": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1784219418,
+        "narHash": "sha256-yfNBfV6HWAQU/xZXwrFoedsSYsDmCDIlTHRxAZI6teY=",
+        "owner": "helix-editor",
+        "repo": "helix",
+        "rev": "f6f3eb1fe4a73933466f30175843059a4381a985",
+        "type": "github"
+      },
+      "original": {
+        "owner": "helix-editor",
+        "repo": "helix",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1773821835,
-        "narHash": "sha256-TJ3lSQtW0E2JrznGVm8hOQGVpXjJyXY2guAxku2O9A4=",
+        "lastModified": 1775036866,
+        "narHash": "sha256-ZojAnPuCdy657PbTq5V0Y+AHKhZAIwSIT2cb8UgAz/U=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "b40629efe5d6ec48dd1efba650c797ddbd39ace0",
+        "rev": "6201e203d09599479a3b3450ed24fa81537ebc4e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1784120854,
+        "narHash": "sha256-KesHgItiZPgGX740axSiQLcIQ8D24MDqNpkKYWIek8k=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "753cc8a3a87467296ddd1fa93f0cc3e81120ee46",
         "type": "github"
       },
       "original": {
@@ -37,7 +72,29 @@
     "root": {
       "inputs": {
         "flake-utils": "flake-utils",
-        "nixpkgs": "nixpkgs"
+        "helix": "helix",
+        "nixpkgs": "nixpkgs_2"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "helix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1775358767,
+        "narHash": "sha256-f2eC+WIfhjevCPQILuV08i/kmKZzYZpUvkom/33VxCA=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "20fd44bc663daa53a2575e01293e24e681d62244",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
       }
     },
     "systems": {

flake.nix

@@ -2,6 +2,7 @@
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
     flake-utils.url = "github:numtide/flake-utils";
+    helix.url = "github:helix-editor/helix";
   };
 
   outputs =
@@ -9,6 +10,7 @@
       self,
       nixpkgs,
       flake-utils,
+      helix,
       ...
     }:
     flake-utils.lib.eachDefaultSystem (
@@ -19,11 +21,36 @@
         };
       in
       rec {
+        packages.syntax =
+          let
+            grammars = pkgs.callPackage "${helix}/grammars.nix" { };
+          in
+          pkgs.stdenv.mkDerivation {
+            name = "syntax";
+            phases = [
+              "buildPhase"
+              "installPhase"
+            ];
+            buildPhase = ''
+              cp -r ${helix}/runtime/queries/* ./
+              chmod u+w ./* -R
+              for so in ${grammars}/*.so; do
+                name=$(basename "$so" .so)
+                if [ -d "./$name" ]; then
+                  cp "$so" "./$name/$name.so"
+                fi
+              done
+            '';
+            installPhase = ''
+              mkdir -p $out
+              cp -r ./* $out/
+            '';
+          };
         packages.karnaugh = pkgs.rustPlatform.buildRustPackage (final: {
           pname = "karnaugh";
-          version = "0.1.3";
+          version = "0.1.5";
           src = self;
-          cargoHash = "sha256-HwwvbYf9hKBANbIoxBQFOgya3K7SwWZCPZKvLGB057U=";
+          cargoLock.lockFile = ./Cargo.lock;
         });
         packages.container = pkgs.dockerTools.buildImage {
           name = "karnaugh";
@@ -47,6 +74,8 @@
             rustc
             rustfmt
             clippy
+            cargo-edit
+            cargo-audit
           ];
         };
       }

src/main.rs

@@ -8,7 +8,7 @@ use ::typst::syntax::VirtualPath;
 use axum::{
     Router,
     extract::{Path, State},
-    http::StatusCode,
+    http::{HeaderValue, StatusCode, header::CACHE_CONTROL},
     response::{Html, IntoResponse, Response},
     routing::get,
 };
@@ -17,6 +17,7 @@ use itertools::Itertools;
 use tower_http::{
     compression::CompressionLayer,
     services::{ServeDir, ServeFile},
+    set_header::SetResponseHeaderLayer,
     trace::TraceLayer,
 };
 use tracing::{Level, info};
@@ -24,6 +25,7 @@ use tracing_subscriber::FmtSubscriber;
 
 use crate::typst::TypstContext;
 
+mod syntax;
 mod typst;
 
 #[derive(Clone)]
@@ -58,6 +60,15 @@ struct AppConfig {
     #[arg(short = 'd', long, value_name = "DIR", default_value = "/tmp/karnaugh")]
     packages_root: PathBuf,
 
+    /// The directory where the syntax files are stored.
+    #[arg(short = 'g', long, value_name = "DIR", default_value = None)]
+    syntax_root: Option<PathBuf>,
+
+    /// Grammars to load, separated by commas.
+    /// Useful to avoid the memory required to load all languages.
+    #[arg(short = 'L', long = "lang", value_name = "NAME", value_delimiter = ',')]
+    languages: Vec<String>,
+
     /// The log level to use
     #[arg(short = 'l', long, value_name = "LEVEL", default_value_t = Level::INFO)]
     log_level: Level,
@@ -93,24 +104,32 @@ async fn main() {
         .with_max_level(state.config.log_level)
         .init();
 
+    syntax::init(&state.config);
+
     // TODO: Replace with config option
     let mut favicon = state.config.get_full_assets_path();
     favicon.push(PathBuf::from("favicon.svg"));
 
+    let assets_service = ServeDir::new(state.config.get_full_assets_path());
+
     let app = Router::new()
         .route("/", get(root_handler))
         .route("/{*path}", get(typst_handler))
         .route_service("/favicon.ico", ServeFile::new(favicon))
-        .nest_service("/assets", ServeDir::new(&state.config.assets_root))
+        .nest_service("/assets", assets_service)
         .with_state(state.clone())
         .layer(TraceLayer::new_for_http())
-        .layer(CompressionLayer::new());
+        .layer(CompressionLayer::new())
+        .layer(SetResponseHeaderLayer::overriding(
+            CACHE_CONTROL,
+            HeaderValue::from_static("public, max-age=86400, immutable"),
+        ));
 
     let socket = SocketAddrV4::new(Ipv4Addr::new(0, 0, 0, 0), state.config.port);
     info!("Serving Karnaugh on socket {}", socket);
     let listener = tokio::net::TcpListener::bind(socket)
         .await
-        .expect(&format!("Failed to bind to socket {}", socket));
+        .unwrap_or_else(|_| panic!("Failed to bind to socket {}", socket));
     axum::serve(listener, app)
         .await
         .expect("Failed to serve site");

src/syntax.rs

@@ -0,0 +1,259 @@
+use std::{
+    borrow::Cow,
+    collections::{HashMap, HashSet},
+    error::Error,
+    fs,
+    path::Path,
+    sync::OnceLock,
+    time::Duration,
+};
+
+use ropey::Rope;
+use tracing::{info, warn};
+use tree_house::{
+    InjectionLanguageMarker, Language, LanguageConfig, LanguageLoader, Syntax,
+    highlighter::{Highlight, HighlightEvent, Highlighter},
+    read_query,
+};
+use tree_house_bindings::Grammar;
+use typst::{
+    foundations::{Content, Func, NativeElement, NativeFunc, func},
+    text::TextElem,
+};
+use typst_html::{HtmlElem, attr, tag};
+
+use crate::AppConfig;
+
+static REGISTRY: OnceLock<LanguageRegistry> = OnceLock::new();
+
+/// Class names used in highlighted code.
+const HIGHLIGHT_NAMES: &[&str] = &[
+    "attribute",
+    "comment",
+    "comment.block",
+    "comment.line",
+    "constant",
+    "constant.builtin",
+    "constructor",
+    "function",
+    "function.builtin",
+    "function.macro",
+    "function.method",
+    "keyword",
+    "keyword.control",
+    "keyword.operator",
+    "label",
+    "namespace",
+    "number",
+    "operator",
+    "property",
+    "punctuation",
+    "punctuation.bracket",
+    "punctuation.delimiter",
+    "special",
+    "string",
+    "string.special",
+    "tag",
+    "type",
+    "type.builtin",
+    "variable",
+    "variable.builtin",
+    "variable.parameter",
+];
+
+/// Get the longest matching highlight name for a given scope.
+fn highlight_for_scope(scope: &str) -> Option<Highlight> {
+    let mut candidate = scope;
+    loop {
+        if let Some(i) = HIGHLIGHT_NAMES.iter().position(|&n| n == candidate) {
+            return Some(Highlight::new(i as u32));
+        }
+        match candidate.rfind('.') {
+            Some(dot) => candidate = &candidate[..dot],
+            None => return None,
+        }
+    }
+}
+
+struct LanguageRegistry {
+    by_name: HashMap<String, Language>,
+    configs: Vec<LanguageConfig>,
+}
+
+impl LanguageRegistry {
+    /// Loads grammars from `syntax_root`. When `languages` is non-empty, only
+    /// grammars with those names are loaded; otherwise every grammar found is.
+    fn new(syntax_root: &Path, languages: &[String]) -> Self {
+        let filter: Option<HashSet<&str>> =
+            (!languages.is_empty()).then(|| languages.iter().map(String::as_str).collect());
+
+        let mut by_name = HashMap::new();
+        let mut configs = Vec::new();
+
+        for entry in fs::read_dir(syntax_root).into_iter().flatten().flatten() {
+            let name = entry.file_name().to_string_lossy().into_owned();
+            if filter.as_ref().is_some_and(|f| !f.contains(name.as_str())) {
+                continue;
+            }
+            let so_path = entry.path().join(format!("{name}.so"));
+            if !so_path.exists() {
+                continue;
+            }
+
+            match load_language(&name, &entry.path(), syntax_root) {
+                Ok(config) => {
+                    by_name.insert(name, Language::new(configs.len() as u32));
+                    configs.push(config);
+                }
+                Err(e) => warn!("Failed to load grammar {name}: {e}"),
+            }
+        }
+
+        // Warn about explicitly requested grammars that were not found.
+        for lang in languages {
+            if !by_name.contains_key(lang) {
+                warn!("Requested grammar {lang} was not found in the syntax root");
+            }
+        }
+
+        info!("Loaded {} syntax highlighting grammars", configs.len());
+        Self { by_name, configs }
+    }
+}
+
+fn load_language(
+    name: &str,
+    lang_dir: &Path,
+    syntax_root: &Path,
+) -> Result<LanguageConfig, Box<dyn Error>> {
+    let grammar = unsafe { Grammar::new(name, &lang_dir.join(format!("{name}.so")))? };
+
+    let read = |kind: &str| {
+        read_query(name, |lang| {
+            fs::read_to_string(syntax_root.join(lang).join(format!("{kind}.scm")))
+                .unwrap_or_default()
+        })
+    };
+
+    let config = LanguageConfig::new(
+        grammar,
+        &read("highlights"),
+        &read("injections"),
+        &read("locals"),
+    )?;
+    config.configure(highlight_for_scope);
+
+    Ok(config)
+}
+
+impl LanguageLoader for LanguageRegistry {
+    fn language_for_marker(&self, marker: InjectionLanguageMarker<'_>) -> Option<Language> {
+        match marker {
+            InjectionLanguageMarker::Name(name) => self.by_name.get(name).copied(),
+            InjectionLanguageMarker::Match(text) => {
+                let name: Cow<str> = text.into();
+                self.by_name.get(name.as_ref()).copied()
+            }
+            _ => None,
+        }
+    }
+
+    fn get_config(&self, lang: Language) -> Option<&LanguageConfig> {
+        self.configs.get(lang.idx())
+    }
+}
+
+pub fn init(config: &AppConfig) {
+    match &config.syntax_root {
+        Some(syntax_root) => {
+            info!("Loading syntax files from {}", syntax_root.display());
+            REGISTRY.get_or_init(|| LanguageRegistry::new(syntax_root, &config.languages));
+        }
+        None => {
+            warn!(
+                "No syntax root has been configured, improved tree-sitter highlighting will not be possible."
+            )
+        }
+    }
+}
+
+pub fn highlight_func() -> Func {
+    highlight::func()
+}
+
+#[func]
+fn highlight(
+    text: String,
+    #[named]
+    #[default(false)]
+    block: bool,
+    #[named]
+    #[default(None)]
+    lang: Option<String>,
+) -> Content {
+    let highlighted = highlight_code(&text, lang.as_deref()).unwrap_or(TextElem::packed(text));
+
+    if block {
+        HtmlElem::new(tag::pre).with_body(Some(highlighted)).pack()
+    } else {
+        highlighted
+    }
+}
+
+/// Highlights `source` as `lang`.
+fn highlight_code(source: &str, lang: Option<&str>) -> Option<Content> {
+    let registry = REGISTRY.get()?;
+    let language = *registry.by_name.get(lang?)?;
+
+    let rope = Rope::from_str(source);
+    let slice = rope.slice(..);
+    let syntax = Syntax::new(slice, language, Duration::from_millis(500), registry).ok()?;
+    let mut hl = Highlighter::new(&syntax, slice, registry, 0..source.len() as u32);
+
+    let src = source.as_bytes();
+    let src_len = source.len() as u32;
+    let mut pos: u32 = 0;
+    let mut classes = String::new();
+    let mut nodes = Vec::<Content>::new();
+
+    loop {
+        let next = hl.next_event_offset().min(src_len);
+
+        if next > pos {
+            let text = std::str::from_utf8(&src[pos as usize..next as usize]).ok()?;
+            let node = TextElem::packed(text);
+            nodes.push(if classes.is_empty() {
+                node
+            } else {
+                HtmlElem::new(tag::span)
+                    .with_attr(attr::class, classes.clone())
+                    .with_body(Some(node))
+                    .pack()
+            });
+            pos = next;
+        }
+
+        if pos >= src_len {
+            break;
+        }
+
+        let (event, highlights) = hl.advance();
+        let base = match event {
+            // Replace the active highlights
+            HighlightEvent::Refresh => String::new(),
+            // Extend the active highlights
+            HighlightEvent::Push => classes.clone(),
+        };
+        classes = highlights.fold(base, |mut acc, h| {
+            let name = HIGHLIGHT_NAMES[h.idx()].replace('.', "-");
+            if !acc.is_empty() {
+                acc.push(' ');
+            }
+            acc.push_str("hl-");
+            acc.push_str(&name);
+            acc
+        });
+    }
+
+    Some(Content::sequence(nodes))
+}

src/typst.rs

@@ -5,11 +5,12 @@ use tracing::info;
 use typst::{
     Library, LibraryExt, World,
     diag::{FileError, FileResult, PackageError, PackageResult, SourceDiagnostic},
-    foundations::{Bytes, Datetime, Duration},
+    foundations::{Bytes, Datetime, Dict, Duration, Value},
     syntax::{FileId, RootedPath, Source, VirtualPath, VirtualRoot, package::PackageSpec},
     text::{Font, FontBook},
     utils::LazyHash,
 };
+use typst_html::HtmlOptions;
 use typst_kit::fonts::FontStore;
 
 use crate::AppConfig;
@@ -22,7 +23,15 @@ pub struct TypstContext {
 
 impl TypstContext {
     pub fn new(config: AppConfig) -> Self {
+        let mut inputs = Dict::new();
+        if config.syntax_root.is_some() {
+            inputs.insert(
+                "highlight".into(),
+                Value::Func(crate::syntax::highlight_func()),
+            );
+        }
         let library = Library::builder()
+            .with_inputs(inputs)
             .with_features([typst::Feature::Html].into_iter().collect())
             .build();
         let library = LazyHash::new(library);
@@ -41,12 +50,14 @@ impl TypstContext {
         let world = DocumentWorld::new(path, self);
         let compiled = typst::compile(&world);
 
+        let html_options = HtmlOptions::default();
+
         // TODO: Log warnings
 
         // TODO: Log errors
         compiled
             .output
-            .and_then(|doc| typst_html::html(&doc))
+            .and_then(|doc| typst_html::html(&doc, &html_options))
             .map_err(|err| err.into_iter().collect())
     }
 }
@@ -116,7 +127,8 @@ impl<'a> DocumentWorld<'a> {
     }
 
     fn resolve_project_file(&self, path: &VirtualPath) -> FileResult<PathBuf> {
-        Ok(path.realize(&self.context.config.content_root))
+        path.realize(&self.context.config.content_root)
+            .map_err(FileError::Realize)
     }
 
     fn resolve_package_file(&self, path: &VirtualPath, spec: &PackageSpec) -> FileResult<PathBuf> {
@@ -129,12 +141,13 @@ impl<'a> DocumentWorld<'a> {
 
     fn resolve_common_file(&self, path: &VirtualPath, spec: &PackageSpec) -> FileResult<PathBuf> {
         let path = get_package_path(path, spec);
-        Ok(path.realize(&self.context.config.get_full_common_path()))
+        path.realize(&self.context.config.get_full_common_path())
+            .map_err(FileError::Realize)
     }
 
     fn resolve_preview_file(&self, path: &VirtualPath, spec: &PackageSpec) -> FileResult<PathBuf> {
         let package_path = self.get_package(spec)?;
-        Ok(path.realize(&package_path))
+        path.realize(&package_path).map_err(FileError::Realize)
     }
 
     fn get_package(&self, spec: &PackageSpec) -> PackageResult<PathBuf> {
@@ -142,7 +155,9 @@ impl<'a> DocumentWorld<'a> {
             &VirtualPath::new("").expect("Failed to create empty virtual path"),
             spec,
         );
-        let path = path.realize(&self.context.config.packages_root);
+        let path = path
+            .realize(&self.context.config.packages_root)
+            .map_err(|_| PackageError::NotFound(spec.clone()))?;
 
         if path.exists() {
             return Ok(path);