Jan/karnaugh
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: generate scopes based on applied theme

Browse Source
This commit is contained in:
Jan-Bulthuis 2026-07-19 20:28:57 +02:00
parent 3711e99528
commit 9b50509910
2 changed files with 114 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main.rs
View File

@ -69,11 +69,6 @@ struct AppConfig {
#[arg(short = 't', long, value_name = "DIR", default_value = None)]
themes_root: Option<PathBuf>,
/// Grammars to load, separated by commas.
/// Useful to avoid the memory required to load all languages.
#[arg(short = 'L', long = "lang", value_name = "NAME", value_delimiter = ',')]
languages: Vec<String>,
/// The log level to use
#[arg(short = 'l', long, value_name = "LEVEL", default_value_t = Level::INFO)]
log_level: Level,

189
src/syntax.rs
View File

@ -1,10 +1,10 @@
use std::{
borrow::Cow,
collections::{HashMap, HashSet},
collections::{BTreeSet, HashMap},
error::Error,
fs,
path::Path,
sync::OnceLock,
path::{Path, PathBuf},
sync::{Arc, OnceLock, RwLock},
time::Duration,
};
@ -24,48 +24,18 @@ use typst_html::{HtmlAttr, HtmlElem, attr, tag};
use crate::AppConfig;
static REGISTRY: OnceLock<LanguageRegistry> = OnceLock::new();
/// Where grammars are loaded from, captured at `init`.
static SYNTAX_ROOT: OnceLock<PathBuf> = OnceLock::new();
/// Class names used in highlighted code.
const HIGHLIGHT_NAMES: &[&str] = &[
"attribute",
"comment",
"comment.block",
"comment.line",
"constant",
"constant.builtin",
"constructor",
"function",
"function.builtin",
"function.macro",
"function.method",
"keyword",
"keyword.control",
"keyword.operator",
"label",
"namespace",
"number",
"operator",
"property",
"punctuation",
"punctuation.bracket",
"punctuation.delimiter",
"special",
"string",
"string.special",
"tag",
"type",
"type.builtin",
"variable",
"variable.builtin",
"variable.parameter",
];
/// LanguageRegistries keyed by ScopeSet
static REGISTRIES: OnceLock<RwLock<HashMap<BTreeSet<String>, Arc<LanguageRegistry>>>> =
OnceLock::new();
/// Get the longest matching highlight name for a given scope.
fn highlight_for_scope(scope: &str) -> Option<Highlight> {
/// Get the index of the longest prefix of `scope` present in `scopes`.
fn highlight_for_scope(scope: &str, scopes: &[String]) -> Option<Highlight> {
let mut candidate = scope;
loop {
if let Some(i) = HIGHLIGHT_NAMES.iter().position(|&n| n == candidate) {
if let Some(i) = scopes.iter().position(|n| n == candidate) {
return Some(Highlight::new(i as u32));
}
match candidate.rfind('.') {
@ -76,49 +46,88 @@ fn highlight_for_scope(scope: &str) -> Option<Highlight> {
}
struct LanguageRegistry {
by_name: HashMap<String, Language>,
configs: Vec<LanguageConfig>,
syntax_root: PathBuf,
/// The scope vocabulary.
scopes: Vec<String>,
/// Name to Language mapping, Option to allow storing None for failed
/// configuration and preventing reloading.
by_name: RwLock<HashMap<String, Option<Language>>>,
configs: RwLock<Vec<&'static LanguageConfig>>,
}
impl LanguageRegistry {
/// Loads grammars from `syntax_root`. When `languages` is non-empty, only
/// grammars with those names are loaded; otherwise every grammar found is.
fn new(syntax_root: &Path, languages: &[String]) -> Self {
let filter: Option<HashSet<&str>> =
(!languages.is_empty()).then(|| languages.iter().map(String::as_str).collect());
let mut by_name = HashMap::new();
let mut configs = Vec::new();
for entry in fs::read_dir(syntax_root).into_iter().flatten().flatten() {
let name = entry.file_name().to_string_lossy().into_owned();
if filter.as_ref().is_some_and(|f| !f.contains(name.as_str())) {
continue;
fn new(syntax_root: PathBuf, scopes: Vec<String>) -> Self {
Self {
syntax_root,
scopes,
by_name: RwLock::new(HashMap::new()),
configs: RwLock::new(Vec::new()),
}
let so_path = entry.path().join(format!("{name}.so"));
if !so_path.exists() {
continue;
}
match load_language(&name, &entry.path(), syntax_root) {
Ok(config) => {
by_name.insert(name, Language::new(configs.len() as u32));
/// The `Language` for `name`, loading and configuring its grammar on first
/// request. Returns `None` if the grammar is missing or fails to load.
fn language(&self, name: &str) -> Option<Language> {
if let Some(cached) = self.by_name.read().unwrap().get(name) {
return *cached;
}
let loaded = self.load(name);
let mut by_name = self.by_name.write().unwrap();
if let Some(cached) = by_name.get(name) {
return *cached;
}
let language = loaded.map(|config| {
let mut configs = self.configs.write().unwrap();
configs.push(config);
Language::new((configs.len() - 1) as u32)
});
by_name.insert(name.to_owned(), language);
language
}
/// Loads and configures the grammar named `name`, leaking the config to a
/// `'static` borrow. `None` if the `.so` is absent or parsing fails.
fn load(&self, name: &str) -> Option<&'static LanguageConfig> {
let lang_dir = self.syntax_root.join(name);
if !lang_dir.join(format!("{name}.so")).exists() {
return None;
}
match load_language(name, &lang_dir, &self.syntax_root) {
Ok(config) => {
config.configure(|scope| highlight_for_scope(scope, &self.scopes));
Some(Box::leak(Box::new(config)))
}
Err(e) => {
warn!("Failed to load grammar {name}: {e}");
None
}
}
Err(e) => warn!("Failed to load grammar {name}: {e}"),
}
}
// Warn about explicitly requested grammars that were not found.
for lang in languages {
if !by_name.contains_key(lang) {
warn!("Requested grammar {lang} was not found in the syntax root");
}
/// Fetch or build the registry configured for `scopes`.
fn registry_for(scopes: BTreeSet<String>) -> Option<Arc<LanguageRegistry>> {
let registries = REGISTRIES.get()?;
// Fast path: already built.
if let Some(registry) = registries.read().unwrap().get(&scopes) {
return Some(registry.clone());
}
info!("Loaded {} syntax highlighting grammars", configs.len());
Self { by_name, configs }
}
let syntax_root = SYNTAX_ROOT.get()?;
let ordered = scopes.iter().cloned().collect();
let registry = Arc::new(LanguageRegistry::new(syntax_root.clone(), ordered));
Some(
registries
.write()
.unwrap()
.entry(scopes)
.or_insert(registry)
.clone(),
)
}
fn load_language(
@ -141,7 +150,6 @@ fn load_language(
&read("injections"),
&read("locals"),
)?;
config.configure(highlight_for_scope);
Ok(config)
}
@ -149,17 +157,17 @@ fn load_language(
impl LanguageLoader for LanguageRegistry {
fn language_for_marker(&self, marker: InjectionLanguageMarker<'_>) -> Option<Language> {
match marker {
InjectionLanguageMarker::Name(name) => self.by_name.get(name).copied(),
InjectionLanguageMarker::Name(name) => self.language(name),
InjectionLanguageMarker::Match(text) => {
let name: Cow<str> = text.into();
self.by_name.get(name.as_ref()).copied()
self.language(name.as_ref())
}
_ => None,
}
}
fn get_config(&self, lang: Language) -> Option<&LanguageConfig> {
self.configs.get(lang.idx())
self.configs.read().unwrap().get(lang.idx()).copied()
}
}
@ -167,7 +175,8 @@ pub fn init(config: &AppConfig) {
match &config.syntax_root {
Some(syntax_root) => {
info!("Loading syntax files from {}", syntax_root.display());
REGISTRY.get_or_init(|| LanguageRegistry::new(syntax_root, &config.languages));
SYNTAX_ROOT.get_or_init(|| syntax_root.clone());
REGISTRIES.get_or_init(|| RwLock::new(HashMap::new()));
}
None => {
warn!(
@ -190,8 +199,15 @@ fn highlight(
#[named]
#[default(None)]
lang: Option<String>,
#[named]
#[default(None)]
light: Option<String>,
#[named]
#[default(None)]
dark: Option<String>,
) -> Content {
let highlighted = highlight_code(&text, lang.as_deref()).unwrap_or(TextElem::packed(text));
let highlighted = highlight_code(&text, lang.as_deref(), light.as_deref(), dark.as_deref())
.unwrap_or_else(|| TextElem::packed(text));
if block {
HtmlElem::new(tag::pre)
@ -207,14 +223,21 @@ fn highlight(
}
/// Highlights `source` as `lang`.
fn highlight_code(source: &str, lang: Option<&str>) -> Option<Content> {
let registry = REGISTRY.get()?;
let language = *registry.by_name.get(lang?)?;
fn highlight_code(
source: &str,
lang: Option<&str>,
light: Option<&str>,
dark: Option<&str>,
) -> Option<Content> {
let scopes = crate::themes::shared_scopes(light?, dark?)?;
let registry = registry_for(scopes)?;
let language = registry.language(lang?)?;
let rope = Rope::from_str(source);
let slice = rope.slice(..);
let syntax = Syntax::new(slice, language, Duration::from_millis(500), registry).ok()?;
let mut hl = Highlighter::new(&syntax, slice, registry, 0..source.len() as u32);
let loader = registry.as_ref();
let syntax = Syntax::new(slice, language, Duration::from_millis(500), loader).ok()?;
let mut hl = Highlighter::new(&syntax, slice, loader, 0..source.len() as u32);
let src = source.as_bytes();
let src_len = source.len() as u32;
@ -251,7 +274,7 @@ fn highlight_code(source: &str, lang: Option<&str>) -> Option<Content> {
HighlightEvent::Push => classes.clone(),
};
classes = highlights.fold(base, |mut acc, h| {
let name = HIGHLIGHT_NAMES[h.idx()].replace('.', "-");
let name = registry.scopes[h.idx()].replace('.', "-");
if !acc.is_empty() {
acc.push(' ');
}