From 99d781b886f55b2c3de356000aa24df7f6fac6c3 Mon Sep 17 00:00:00 2001 From: Keith Cirkel Date: Thu, 19 Dec 2024 16:10:38 +0000 Subject: [PATCH] (lsp) use string ropes for incremental edits, scoped per-file threads for parsing --- Cargo.lock | 17 ++ Cargo.toml | 5 +- crates/hdx_lsp/Cargo.toml | 5 +- crates/hdx_lsp/src/server.rs | 6 +- crates/hdx_lsp/src/server/handler.rs | 1 - crates/hdx_lsp/src/service.rs | 240 +++++++++++++++++++++------ 6 files changed, 213 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc967f30..17dc56a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -707,6 +707,7 @@ dependencies = [ "lsp-types", "miette", "pprof", + "ropey", "serde", "serde_json", "similar", @@ -1342,6 +1343,16 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "ropey" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93411e420bcd1a75ddd1dc3caf18c23155eda2c090631a85af21ba19e97093b5" +dependencies = [ + "smallvec", + "str_indices", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -1490,6 +1501,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "str_indices" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6" + [[package]] name = "str_stack" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4e70229b..260c36c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,10 +29,11 @@ hdx_lsp = { version = "0.0.0", path = "crates/hdx_lsp" } bumpalo = { version = "3.16.0" } # Data structure libraries/helpers -smallvec = { version = "1.13.2" } +bitmask-enum = { version = "2.2.5" } itertools = { version = "0.13.0" } +ropey = { version = "1.6.1" } +smallvec = { version = "1.13.2" } strum = { version = "0.26.3" } -bitmask-enum = { version = "2.2.5" } # CLI clap = { version = "4.5.23" } diff --git a/crates/hdx_lsp/Cargo.toml b/crates/hdx_lsp/Cargo.toml index ca9d21e3..84b722d8 100644 --- a/crates/hdx_lsp/Cargo.toml +++ b/crates/hdx_lsp/Cargo.toml @@ -20,10 +20,11 @@ hdx_highlight = { workspace = true } bumpalo = { workspace = true, features = ["collections", "boxed"] } miette = { workspace = true, features = ["derive"] } -smallvec = { workspace = true } bitmask-enum = { workspace = true } -strum = { workspace = true, features = ["derive"] } itertools = { workspace = true } +ropey = { workspace = true } +smallvec = { workspace = true } +strum = { workspace = true, features = ["derive"] } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/hdx_lsp/src/server.rs b/crates/hdx_lsp/src/server.rs index ac0e1749..c433dc71 100644 --- a/crates/hdx_lsp/src/server.rs +++ b/crates/hdx_lsp/src/server.rs @@ -110,8 +110,8 @@ mod tests { use super::*; use lsp_types::{ - request::{GotoDeclaration, GotoDeclarationParams, Initialize, Request as RequestTrait}, - GotoDefinitionResponse, InitializeParams, InitializeResult, + request::{GotoDeclaration, Initialize, Request as RequestTrait}, + InitializeParams, InitializeResult, }; use serde_json::{json, to_value, Value}; use tracing::level_filters::LevelFilter; @@ -122,7 +122,7 @@ mod tests { let stderr_log = fmt::layer().with_writer(io::stderr).with_filter(LevelFilter::TRACE); struct TestHandler {} impl Handler for TestHandler { - fn initialize(&self, req: InitializeParams) -> Result { + fn initialize(&self, _req: InitializeParams) -> Result { Ok(InitializeResult { ..Default::default() }) } } diff --git a/crates/hdx_lsp/src/server/handler.rs b/crates/hdx_lsp/src/server/handler.rs index 9e964c50..00163fac 100644 --- a/crates/hdx_lsp/src/server/handler.rs +++ b/crates/hdx_lsp/src/server/handler.rs @@ -24,7 +24,6 @@ pub trait Handler: Sized + Send + Sync + 'static { let span = trace_span!("Handling request", "{:#?}", message); let _ = span.enter(); let id = message.id().unwrap_or_default(); - debug!("LspMessageHandler -> {:#?}", &message); if message.is_exit_notification() { return None; } diff --git a/crates/hdx_lsp/src/service.rs b/crates/hdx_lsp/src/service.rs index 817ea898..379824d8 100644 --- a/crates/hdx_lsp/src/service.rs +++ b/crates/hdx_lsp/src/service.rs @@ -1,22 +1,135 @@ use bumpalo::Bump; +use crossbeam_channel::{bounded, Receiver, Sender}; use dashmap::DashMap; use hdx_ast::css::{StyleSheet, Visitable}; -use hdx_highlight::{SemanticKind, SemanticModifier, TokenHighlighter}; -use hdx_parser::{Features, Parser}; +use hdx_highlight::{Highlight, SemanticKind, SemanticModifier, TokenHighlighter}; +use hdx_parser::{Features, Parser, ParserReturn}; use itertools::Itertools; use lsp_types::Uri; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, +use ropey::Rope; +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread::{Builder, JoinHandle}, }; use strum::VariantNames; -use tracing::trace; +use tracing::{instrument, trace}; use crate::{ErrorCode, Handler}; +type Line = u32; +type Col = u32; + +#[derive(Debug)] +enum FileCall { + // Re-parse the document based on changes + RopeChange(Rope), + // Highlight a document, returning the semantic highlights + Highlight, +} + +#[derive(Debug)] +enum FileReturn { + Highlights(Vec<(Highlight, Line, Col)>), +} + +#[derive(Debug)] +pub struct File { + pub content: Rope, + thread: JoinHandle<()>, + sender: Sender, + receiver: Receiver, +} + +impl File { + fn new() -> Self { + let (sender, read_receiver) = bounded::(0); + let (write_sender, receiver) = bounded::(0); + Self { + content: Rope::new(), + sender, + receiver, + thread: Builder::new() + .name("LspDocumentHandler".into()) + .spawn(move || { + let mut bump = Bump::default(); + let mut string: String = "".into(); + let mut result: ParserReturn<'_, StyleSheet<'_>> = + Parser::new(&bump, "", Features::default()).parse_entirely::(); + while let Ok(call) = read_receiver.recv() { + trace!("String is currently {:?}", string); + match call { + FileCall::RopeChange(rope) => { + trace!("Parsing document"); + // TODO! we should be able to optimize this by parsing a subset of the tree and mutating in + // place. For now though a partial parse request re-parses it all. + drop(result); + bump.reset(); + string = rope.clone().into(); + result = + Parser::new(&bump, &string, Features::default()).parse_entirely::(); + if let Some(stylesheet) = &result.output { + trace!("Sucessfully parsed stylesheet: {:#?}", &stylesheet); + } + } + FileCall::Highlight => { + trace!("Highlighting document"); + let mut highlighter = TokenHighlighter::new(); + if let Some(stylesheet) = &result.output { + stylesheet.accept(&mut highlighter); + let mut current_line = 0; + let mut current_start = 0; + let data = highlighter + .highlights() + .sorted_by(|a, b| Ord::cmp(&a.span(), &b.span())) + .map(|h| { + // TODO: figure out a more efficient way to get line/col + let span_contents = h.span().span_contents(&string); + let (line, start) = span_contents.line_and_column(); + let delta_line: Line = line - current_line; + current_line = line; + let delta_start: Col = + if delta_line == 0 { start - current_start } else { start }; + current_start = start; + (*h, delta_line, delta_start) + }); + write_sender.send(FileReturn::Highlights(data.collect())).ok(); + } + } + } + } + }) + .expect("Failed to document thread Reader"), + } + } + + fn to_string(&self) -> String { + self.content.clone().into() + } + + fn commit(&mut self, rope: Rope) { + self.content = rope; + self.sender.send(FileCall::RopeChange(self.content.clone())).unwrap(); + } + + #[instrument] + fn get_highlights(&self) -> Vec<(Highlight, Line, Col)> { + self.sender.send(FileCall::Highlight).unwrap(); + while let Ok(ret) = self.receiver.recv() { + if let FileReturn::Highlights(highlights) = ret { + return highlights; + } + } + return vec![]; + } +} + +#[derive(Debug)] pub struct LSPService { version: String, - files: Arc>, + files: Arc>, initialized: AtomicBool, } @@ -27,10 +140,12 @@ impl LSPService { } impl Handler for LSPService { + #[instrument] fn initialized(&self) -> bool { self.initialized.load(Ordering::SeqCst) } + #[instrument] fn initialize(&self, req: lsp_types::InitializeParams) -> Result { self.initialized.swap(true, Ordering::SeqCst); Ok(lsp_types::InitializeResult { @@ -112,71 +227,90 @@ impl Handler for LSPService { }) } + #[instrument] fn semantic_tokens_full_request( &self, req: lsp_types::SemanticTokensParams, ) -> Result, ErrorCode> { let uri = req.text_document.uri; - let allocator = Bump::default(); - if let Some(source_text) = self.files.get(&uri) { - trace!("Asked for SemanticTokens"); - let result = - Parser::new(&allocator, source_text.as_str(), Features::default()).parse_entirely::(); - if let Some(stylesheet) = result.output { - trace!("Sucessfully parsed stylesheet: {:#?}", &stylesheet); - let mut highlighter = TokenHighlighter::new(); - stylesheet.accept(&mut highlighter); - let mut current_line = 0; - let mut current_start = 0; - let data = highlighter - .highlights() - .sorted_by(|a, b| Ord::cmp(&a.span(), &b.span())) - .map(|highlight| { - let span_contents = highlight.span().span_contents(source_text.as_str()); - let (line, start) = span_contents.line_and_column(); - let delta_line = line - current_line; - current_line = line; - let delta_start = if delta_line == 0 { start - current_start } else { start }; - current_start = start; - lsp_types::SemanticToken { - token_type: highlight.kind().bits() as u32, - token_modifiers_bitset: highlight.modifier().bits() as u32, - delta_line, - delta_start, - length: span_contents.size(), - } - }) - .collect(); - return Ok(Some(lsp_types::SemanticTokensResult::Tokens(lsp_types::SemanticTokens { - result_id: None, - data, - }))); - } else if !result.errors.is_empty() { - trace!("\n\nParse on {:?} failed. Saw error {:?}", &uri, result.errors); - } + trace!("Asked for SemanticTokens for {:?}", &uri); + if let Some(document) = self.files.get(&uri) { + let mut current_line = 0; + let mut current_start = 0; + // TODO: remove this, figure out a more efficient way to get line/col + let str = document.to_string(); + let data = document + .get_highlights() + .into_iter() + .map(|(highlight, delta_line, delta_start)| lsp_types::SemanticToken { + token_type: highlight.kind().bits() as u32, + token_modifiers_bitset: highlight.modifier().bits() as u32, + delta_line, + delta_start, + length: highlight.span().size(), + }) + .collect(); + Ok(Some(lsp_types::SemanticTokensResult::Tokens(lsp_types::SemanticTokens { result_id: None, data }))) + } else { + Err(ErrorCode::InternalError) } - Err(ErrorCode::InternalError) } + #[instrument] fn completion(&self, req: lsp_types::CompletionParams) -> Result, ErrorCode> { - // let uri = req.text_document.uri; - // let position = req.text_document_position; - // let context = req.context; - Err(ErrorCode::UnknownErrorCode) + let uri = req.text_document_position.text_document.uri; + let position = req.text_document_position.position; + let context = req.context; + Ok(None) } + #[instrument] fn on_did_open_text_document(&self, req: lsp_types::DidOpenTextDocumentParams) { let uri = req.text_document.uri; let source_text = req.text_document.text; - self.files.clone().insert(uri, source_text); + let mut doc = File::new(); + let mut rope = doc.content.clone(); + rope.remove(0..); + rope.insert(0, &source_text); + trace!("comitting new document {:?} {:?}", &uri, rope); + doc.commit(rope); + self.files.clone().insert(uri, doc); } + #[instrument] fn on_did_change_text_document(&self, req: lsp_types::DidChangeTextDocumentParams) { let uri = req.text_document.uri; let changes = req.content_changes; - if changes.len() == 1 && changes[0].range.is_none() { - let source_text = &changes[0].text; - self.files.clone().insert(uri, source_text.into()); + if let Some(mut file) = self.files.clone().get_mut(&uri) { + let mut rope = file.content.clone(); + for change in changes { + let range = if let Some(range) = change.range { + rope.try_line_to_char(range.start.line as usize).map_or_else( + |_| (0, None), + |start| { + rope.try_line_to_char(range.end.line as usize).map_or_else( + |_| (start + range.start.character as usize, None), + |end| { + (start + range.start.character as usize, Some(end + range.end.character as usize)) + }, + ) + }, + ) + } else { + (0, None) + }; + match range { + (start, None) => { + rope.try_remove(start..).ok(); + rope.try_insert(start, &change.text).ok(); + } + (start, Some(end)) => { + rope.try_remove(start..end).ok(); + rope.try_insert(start, &change.text).ok(); + } + } + } + file.commit(rope) } } }