feat: generate scopes based on applied theme

src/main.rs

@@ -69,11 +69,6 @@ struct AppConfig {
     #[arg(short = 't', long, value_name = "DIR", default_value = None)]
     themes_root: Option<PathBuf>,
 
-    /// Grammars to load, separated by commas.
-    /// Useful to avoid the memory required to load all languages.
-    #[arg(short = 'L', long = "lang", value_name = "NAME", value_delimiter = ',')]
-    languages: Vec<String>,
-
     /// The log level to use
     #[arg(short = 'l', long, value_name = "LEVEL", default_value_t = Level::INFO)]
     log_level: Level,

src/syntax.rs

@@ -1,10 +1,10 @@
 use std::{
     borrow::Cow,
-    collections::{HashMap, HashSet},
+    collections::{BTreeSet, HashMap},
     error::Error,
     fs,
-    path::Path,
+    path::{Path, PathBuf},
-    sync::OnceLock,
+    sync::{Arc, OnceLock, RwLock},
     time::Duration,
 };
 
@@ -24,48 +24,18 @@ use typst_html::{HtmlAttr, HtmlElem, attr, tag};
 
 use crate::AppConfig;
 
-static REGISTRY: OnceLock<LanguageRegistry> = OnceLock::new();
+/// Where grammars are loaded from, captured at `init`.
+static SYNTAX_ROOT: OnceLock<PathBuf> = OnceLock::new();
 
-/// Class names used in highlighted code.
+/// LanguageRegistries keyed by ScopeSet
-const HIGHLIGHT_NAMES: &[&str] = &[
+static REGISTRIES: OnceLock<RwLock<HashMap<BTreeSet<String>, Arc<LanguageRegistry>>>> =
-    "attribute",
+    OnceLock::new();
-    "comment",
-    "comment.block",
-    "comment.line",
-    "constant",
-    "constant.builtin",
-    "constructor",
-    "function",
-    "function.builtin",
-    "function.macro",
-    "function.method",
-    "keyword",
-    "keyword.control",
-    "keyword.operator",
-    "label",
-    "namespace",
-    "number",
-    "operator",
-    "property",
-    "punctuation",
-    "punctuation.bracket",
-    "punctuation.delimiter",
-    "special",
-    "string",
-    "string.special",
-    "tag",
-    "type",
-    "type.builtin",
-    "variable",
-    "variable.builtin",
-    "variable.parameter",
-];
 
-/// Get the longest matching highlight name for a given scope.
+/// Get the index of the longest prefix of `scope` present in `scopes`.
-fn highlight_for_scope(scope: &str) -> Option<Highlight> {
+fn highlight_for_scope(scope: &str, scopes: &[String]) -> Option<Highlight> {
     let mut candidate = scope;
     loop {
-        if let Some(i) = HIGHLIGHT_NAMES.iter().position(|&n| n == candidate) {
+        if let Some(i) = scopes.iter().position(|n| n == candidate) {
             return Some(Highlight::new(i as u32));
         }
         match candidate.rfind('.') {
@@ -76,49 +46,88 @@ fn highlight_for_scope(scope: &str) -> Option<Highlight> {
 }
 
 struct LanguageRegistry {
-    by_name: HashMap<String, Language>,
+    syntax_root: PathBuf,
-    configs: Vec<LanguageConfig>,
+    /// The scope vocabulary.
+    scopes: Vec<String>,
+    /// Name to Language mapping, Option to allow storing None for failed
+    /// configuration and preventing reloading.
+    by_name: RwLock<HashMap<String, Option<Language>>>,
+    configs: RwLock<Vec<&'static LanguageConfig>>,
 }
 
 impl LanguageRegistry {
-    /// Loads grammars from `syntax_root`. When `languages` is non-empty, only
+    fn new(syntax_root: PathBuf, scopes: Vec<String>) -> Self {
-    /// grammars with those names are loaded; otherwise every grammar found is.
+        Self {
-    fn new(syntax_root: &Path, languages: &[String]) -> Self {
+            syntax_root,
-        let filter: Option<HashSet<&str>> =
+            scopes,
-            (!languages.is_empty()).then(|| languages.iter().map(String::as_str).collect());
+            by_name: RwLock::new(HashMap::new()),
-
+            configs: RwLock::new(Vec::new()),
-        let mut by_name = HashMap::new();
-        let mut configs = Vec::new();
-
-        for entry in fs::read_dir(syntax_root).into_iter().flatten().flatten() {
-            let name = entry.file_name().to_string_lossy().into_owned();
-            if filter.as_ref().is_some_and(|f| !f.contains(name.as_str())) {
-                continue;
         }
-            let so_path = entry.path().join(format!("{name}.so"));
-            if !so_path.exists() {
-                continue;
     }
 
-            match load_language(&name, &entry.path(), syntax_root) {
+    /// The `Language` for `name`, loading and configuring its grammar on first
-                Ok(config) => {
+    /// request. Returns `None` if the grammar is missing or fails to load.
-                    by_name.insert(name, Language::new(configs.len() as u32));
+    fn language(&self, name: &str) -> Option<Language> {
+        if let Some(cached) = self.by_name.read().unwrap().get(name) {
+            return *cached;
+        }
+
+        let loaded = self.load(name);
+
+        let mut by_name = self.by_name.write().unwrap();
+        if let Some(cached) = by_name.get(name) {
+            return *cached;
+        }
+        let language = loaded.map(|config| {
+            let mut configs = self.configs.write().unwrap();
             configs.push(config);
+            Language::new((configs.len() - 1) as u32)
+        });
+        by_name.insert(name.to_owned(), language);
+        language
+    }
+
+    /// Loads and configures the grammar named `name`, leaking the config to a
+    /// `'static` borrow. `None` if the `.so` is absent or parsing fails.
+    fn load(&self, name: &str) -> Option<&'static LanguageConfig> {
+        let lang_dir = self.syntax_root.join(name);
+        if !lang_dir.join(format!("{name}.so")).exists() {
+            return None;
+        }
+        match load_language(name, &lang_dir, &self.syntax_root) {
+            Ok(config) => {
+                config.configure(|scope| highlight_for_scope(scope, &self.scopes));
+                Some(Box::leak(Box::new(config)))
+            }
+            Err(e) => {
+                warn!("Failed to load grammar {name}: {e}");
+                None
+            }
         }
-                Err(e) => warn!("Failed to load grammar {name}: {e}"),
     }
 }
 
-        // Warn about explicitly requested grammars that were not found.
+/// Fetch or build the registry configured for `scopes`.
-        for lang in languages {
+fn registry_for(scopes: BTreeSet<String>) -> Option<Arc<LanguageRegistry>> {
-            if !by_name.contains_key(lang) {
+    let registries = REGISTRIES.get()?;
-                warn!("Requested grammar {lang} was not found in the syntax root");
+
-            }
+    // Fast path: already built.
+    if let Some(registry) = registries.read().unwrap().get(&scopes) {
+        return Some(registry.clone());
     }
 
-        info!("Loaded {} syntax highlighting grammars", configs.len());
+    let syntax_root = SYNTAX_ROOT.get()?;
-        Self { by_name, configs }
+    let ordered = scopes.iter().cloned().collect();
-    }
+    let registry = Arc::new(LanguageRegistry::new(syntax_root.clone(), ordered));
+
+    Some(
+        registries
+            .write()
+            .unwrap()
+            .entry(scopes)
+            .or_insert(registry)
+            .clone(),
+    )
 }
 
 fn load_language(
@@ -141,7 +150,6 @@ fn load_language(
         &read("injections"),
         &read("locals"),
     )?;
-    config.configure(highlight_for_scope);
 
     Ok(config)
 }
@@ -149,17 +157,17 @@ fn load_language(
 impl LanguageLoader for LanguageRegistry {
     fn language_for_marker(&self, marker: InjectionLanguageMarker<'_>) -> Option<Language> {
         match marker {
-            InjectionLanguageMarker::Name(name) => self.by_name.get(name).copied(),
+            InjectionLanguageMarker::Name(name) => self.language(name),
             InjectionLanguageMarker::Match(text) => {
                 let name: Cow<str> = text.into();
-                self.by_name.get(name.as_ref()).copied()
+                self.language(name.as_ref())
             }
             _ => None,
         }
     }
 
     fn get_config(&self, lang: Language) -> Option<&LanguageConfig> {
-        self.configs.get(lang.idx())
+        self.configs.read().unwrap().get(lang.idx()).copied()
     }
 }
 
@@ -167,7 +175,8 @@ pub fn init(config: &AppConfig) {
     match &config.syntax_root {
         Some(syntax_root) => {
             info!("Loading syntax files from {}", syntax_root.display());
-            REGISTRY.get_or_init(|| LanguageRegistry::new(syntax_root, &config.languages));
+            SYNTAX_ROOT.get_or_init(|| syntax_root.clone());
+            REGISTRIES.get_or_init(|| RwLock::new(HashMap::new()));
         }
         None => {
             warn!(
@@ -190,8 +199,15 @@ fn highlight(
     #[named]
     #[default(None)]
     lang: Option<String>,
+    #[named]
+    #[default(None)]
+    light: Option<String>,
+    #[named]
+    #[default(None)]
+    dark: Option<String>,
 ) -> Content {
-    let highlighted = highlight_code(&text, lang.as_deref()).unwrap_or(TextElem::packed(text));
+    let highlighted = highlight_code(&text, lang.as_deref(), light.as_deref(), dark.as_deref())
+        .unwrap_or_else(|| TextElem::packed(text));
 
     if block {
         HtmlElem::new(tag::pre)
@@ -207,14 +223,21 @@ fn highlight(
 }
 
 /// Highlights `source` as `lang`.
-fn highlight_code(source: &str, lang: Option<&str>) -> Option<Content> {
+fn highlight_code(
-    let registry = REGISTRY.get()?;
+    source: &str,
-    let language = *registry.by_name.get(lang?)?;
+    lang: Option<&str>,
+    light: Option<&str>,
+    dark: Option<&str>,
+) -> Option<Content> {
+    let scopes = crate::themes::shared_scopes(light?, dark?)?;
+    let registry = registry_for(scopes)?;
+    let language = registry.language(lang?)?;
 
     let rope = Rope::from_str(source);
     let slice = rope.slice(..);
-    let syntax = Syntax::new(slice, language, Duration::from_millis(500), registry).ok()?;
+    let loader = registry.as_ref();
-    let mut hl = Highlighter::new(&syntax, slice, registry, 0..source.len() as u32);
+    let syntax = Syntax::new(slice, language, Duration::from_millis(500), loader).ok()?;
+    let mut hl = Highlighter::new(&syntax, slice, loader, 0..source.len() as u32);
 
     let src = source.as_bytes();
     let src_len = source.len() as u32;
@@ -251,7 +274,7 @@ fn highlight_code(source: &str, lang: Option<&str>) -> Option<Content> {
             HighlightEvent::Push => classes.clone(),
         };
         classes = highlights.fold(base, |mut acc, h| {
-            let name = HIGHLIGHT_NAMES[h.idx()].replace('.', "-");
+            let name = registry.scopes[h.idx()].replace('.', "-");
             if !acc.is_empty() {
                 acc.push(' ');
             }