From c4817f572db8de6fe56a9a7762c0f1bacd24f1dd Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 17 Oct 2023 11:10:35 +0100 Subject: [PATCH 01/35] WIP --- docker-images/syntax-highlighter/Cargo.lock | 16 ++++++++++++++++ docker-images/syntax-highlighter/Cargo.toml | 2 ++ .../crates/scip-treesitter-cli/Cargo.toml | 18 ++++++++++++++++++ .../crates/scip-treesitter-cli/src/main.rs | 4 ++++ 4 files changed, 40 insertions(+) create mode 100644 docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml create mode 100644 docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs diff --git a/docker-images/syntax-highlighter/Cargo.lock b/docker-images/syntax-highlighter/Cargo.lock index 6b1fa6722675..22c4374f5a98 100644 --- a/docker-images/syntax-highlighter/Cargo.lock +++ b/docker-images/syntax-highlighter/Cargo.lock @@ -1755,6 +1755,21 @@ dependencies = [ "tree-sitter", ] +[[package]] +name = "scip-treesitter-cli" +version = "0.1.0" +dependencies = [ + "clap 4.3.23", + "lazy_static", + "paste", + "protobuf", + "scip", + "scip-syntax", + "scip-treesitter", + "scip-treesitter-languages", + "sg-syntax", +] + [[package]] name = "scip-treesitter-languages" version = "0.1.0" @@ -2019,6 +2034,7 @@ dependencies = [ "scip", "scip-syntax", "scip-treesitter", + "scip-treesitter-cli", "scip-treesitter-languages", "serde", "serde_json", diff --git a/docker-images/syntax-highlighter/Cargo.toml b/docker-images/syntax-highlighter/Cargo.toml index 076691e58763..4a2cc7e7c396 100644 --- a/docker-images/syntax-highlighter/Cargo.toml +++ b/docker-images/syntax-highlighter/Cargo.toml @@ -23,6 +23,7 @@ base64 = "0.13.0" sg-syntax = { path = "./crates/sg-syntax" } scip-treesitter = { path = "./crates/scip-treesitter" } scip-treesitter-languages = { path = "./crates/scip-treesitter-languages" } +scip-treesitter-cli = { path = "./crates/scip-treesitter-cli" } scip-syntax = { path = "./crates/scip-syntax" } # March 20, 2023 - Pinned explicitly with features that match the features # required by rocket. Once bazel rules correctly roll up all the features, @@ -45,6 +46,7 @@ members = [ "crates/scip-syntax", "crates/scip-treesitter", "crates/scip-treesitter-languages", + "crates/scip-treesitter-cli", ] [workspace.dependencies] diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml new file mode 100644 index 000000000000..5a9f62572b3d --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "scip-treesitter-cli" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +lazy_static = "1.0" +paste = "1.0.6" +protobuf.workspace = true +scip.workspace = true + +sg-syntax = { path = "../sg-syntax" } +scip-syntax = { path = "../scip-syntax" } +scip-treesitter-languages = { path = "../scip-treesitter-languages" } +scip-treesitter= { path = "../scip-treesitter" } +clap = { workspace = true, features = ["derive"] } diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs new file mode 100644 index 000000000000..6c63536b6abe --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -0,0 +1,4 @@ + +pub fn main() { + println!("hello!"); +} From 0ee4716cfe72bce9c3a1e4d0293e6489f0822eab Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 17 Oct 2023 11:28:01 +0100 Subject: [PATCH 02/35] Don't move BundlerParser in get_locals --- .../crates/scip-syntax/src/bin/scip-perf.rs | 2 +- .../crates/scip-syntax/src/languages.rs | 2 +- .../syntax-highlighter/crates/scip-syntax/src/lib.rs | 2 +- .../crates/scip-syntax/src/locals.rs | 10 +++++----- .../crates/sg-syntax/src/sg_treesitter.rs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-perf.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-perf.rs index d10c69d2441f..07b34052d68b 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-perf.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-perf.rs @@ -18,7 +18,7 @@ struct ParseTiming { } fn parse_files(dir: &Path) -> Vec { - let config = scip_syntax::languages::get_local_configuration(BundledParser::Go).unwrap(); + let config = scip_syntax::languages::get_local_configuration(&BundledParser::Go).unwrap(); let extension = "go"; let mut timings = vec![]; diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs index 4a2e7942b2b8..6604c3e90d53 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs @@ -287,7 +287,7 @@ mod locals { create_locals_configuration!(matlab, BundledParser::Matlab, "matlab"); create_locals_configuration!(java, BundledParser::Java, "java"); - pub fn get_local_configuration(parser: BundledParser) -> Option<&'static LocalConfiguration> { + pub fn get_local_configuration(parser: &BundledParser) -> Option<&'static LocalConfiguration> { match parser { BundledParser::Go => Some(go()), BundledParser::Perl => Some(perl()), diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs index ea2eced085e1..592631b2dbdb 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs @@ -31,7 +31,7 @@ pub fn get_globals( Some(globals::parse_tree(config, &tree, source_bytes)) } -pub fn get_locals(parser: BundledParser, source_bytes: &[u8]) -> Option>> { +pub fn get_locals(parser: &BundledParser, source_bytes: &[u8]) -> Option>> { let config = languages::get_local_configuration(parser)?; let mut parser = config.get_parser(); let tree = parser.parse(source_bytes, None).unwrap(); diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs index 621f0eb1f30a..697eb0526e75 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs @@ -540,7 +540,7 @@ mod test { #[test] fn test_can_do_go() -> Result<()> { - let mut config = crate::languages::get_local_configuration(BundledParser::Go).unwrap(); + let mut config = crate::languages::get_local_configuration(&BundledParser::Go).unwrap(); let source_code = include_str!("../testdata/locals.go"); let doc = parse_file_for_lang(&mut config, source_code)?; @@ -552,7 +552,7 @@ mod test { #[test] fn test_can_do_nested_locals() -> Result<()> { - let mut config = crate::languages::get_local_configuration(BundledParser::Go).unwrap(); + let mut config = crate::languages::get_local_configuration(&BundledParser::Go).unwrap(); let source_code = include_str!("../testdata/locals-nested.go"); let doc = parse_file_for_lang(&mut config, source_code)?; @@ -564,7 +564,7 @@ mod test { #[test] fn test_can_do_functions() -> Result<()> { - let mut config = crate::languages::get_local_configuration(BundledParser::Go).unwrap(); + let mut config = crate::languages::get_local_configuration(&BundledParser::Go).unwrap(); let source_code = include_str!("../testdata/funcs.go"); let doc = parse_file_for_lang(&mut config, source_code)?; @@ -576,7 +576,7 @@ mod test { #[test] fn test_can_do_perl() -> Result<()> { - let mut config = crate::languages::get_local_configuration(BundledParser::Perl).unwrap(); + let mut config = crate::languages::get_local_configuration(&BundledParser::Perl).unwrap(); let source_code = include_str!("../testdata/perl.pm"); let doc = parse_file_for_lang(&mut config, source_code)?; @@ -588,7 +588,7 @@ mod test { #[test] fn test_can_do_matlab() -> Result<()> { - let mut config = crate::languages::get_local_configuration(BundledParser::Matlab).unwrap(); + let mut config = crate::languages::get_local_configuration(&BundledParser::Matlab).unwrap(); let source_code = include_str!("../testdata/locals.m"); let doc = parse_file_for_lang(&mut config, source_code)?; diff --git a/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs b/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs index ed22daa81df7..7f1fe7d7ea38 100644 --- a/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs +++ b/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs @@ -83,7 +83,7 @@ pub fn index_language_with_config( let parser = scip_treesitter_languages::parsers::BundledParser::get_parser(filetype); if let Some(parser) = parser { // TODO: Could probably write this in a much better way. - let mut local_occs = scip_syntax::get_locals(parser, code.as_bytes()) + let mut local_occs = scip_syntax::get_locals(&parser, code.as_bytes()) .unwrap_or(Ok(vec![])) .unwrap_or(vec![]); From a60bdd8e4aea0b8d3943f027eec11f9f5516f0f2 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 17 Oct 2023 11:28:48 +0100 Subject: [PATCH 03/35] Add cargo feature to clap dependency --- docker-images/syntax-highlighter/Cargo.lock | 1 + docker-images/syntax-highlighter/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-images/syntax-highlighter/Cargo.lock b/docker-images/syntax-highlighter/Cargo.lock index 22c4374f5a98..cf0cfab72405 100644 --- a/docker-images/syntax-highlighter/Cargo.lock +++ b/docker-images/syntax-highlighter/Cargo.lock @@ -301,6 +301,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex 0.5.0", + "once_cell", "strsim", ] diff --git a/docker-images/syntax-highlighter/Cargo.toml b/docker-images/syntax-highlighter/Cargo.toml index 4a2cc7e7c396..23ffde0df5ae 100644 --- a/docker-images/syntax-highlighter/Cargo.toml +++ b/docker-images/syntax-highlighter/Cargo.toml @@ -51,7 +51,7 @@ members = [ [workspace.dependencies] anyhow = "1" -clap = { version = "4.1.11", features = [ "derive" ] } +clap = { version = "4.1.11", features = [ "derive", "cargo"] } itertools = "0.10.5" rocket = { version = "0.5.0-rc.1", features = ["json"] } # See https://stackoverflow.com/questions/76905691/bazel-unable-to-build-external-rust-dependency-with-feature-serde1 From 0018ce21cc297f3615c5a315512fceb3f9469f85 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 17 Oct 2023 11:29:04 +0100 Subject: [PATCH 04/35] Basic indexing command in scip-treesitter-cli This command takes language name and a list of files and produces a SCIP file containing both global symbols (from scip-ctags) and locals (if they're available) --- .../crates/scip-treesitter-cli/src/main.rs | 107 +++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index 6c63536b6abe..93228820455a 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -1,4 +1,109 @@ +use clap::{Parser, Subcommand}; +use scip_syntax::{get_locals, get_symbols}; +use scip_treesitter_languages::parsers::BundledParser; + +use std::path::Path; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +#[command(propagate_version = true)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Adds files to myapp + Index { + #[arg(short, long)] + language: String, + + #[arg(short, long)] + out: Option, + filenames: Vec, + }, +} pub fn main() { - println!("hello!"); + let cli = Cli::parse(); + + match cli.command { + Commands::Index { + language, + out, + filenames, + } => index_impl(&language, &filenames, &out), + } +} + +fn index_impl(language: &String, filenames: &Vec, out: &Option) { + let p = BundledParser::get_parser(language).unwrap(); + + let working_directory = Path::new("./"); + let mut index = scip::types::Index { + metadata: Some(scip::types::Metadata { + tool_info: Some(scip::types::ToolInfo { + name: "scip-treesitter".to_string(), + version: clap::crate_version!().to_string(), + arguments: vec![], + ..Default::default() + }) + .into(), + project_root: format!("file://{}", working_directory.to_str().unwrap()), + ..Default::default() + }) + .into(), + ..Default::default() + }; + + for (_, filename) in filenames.iter().enumerate() { + let contents = std::fs::read(filename).unwrap(); + let mut document = get_symbols(&p, &contents).unwrap(); + + document.relative_path = filename.clone(); + let locals = get_locals(&p, &contents); + + match locals { + Some(Ok(occurrences)) => { + for occ in occurrences { + document.occurrences.push(occ); + } + } + Some(Err(msg)) => { + println!("Error extracting locals: {}", msg); + } + None => {} + } + + index.documents.push(document); + } + + let out_name = out.clone().unwrap_or("index.scip".to_string()); + let path = working_directory.join(out_name); + + println!( + "Writing index for {} documents into {}", + index.documents.len(), + path.display() + ); + + write_message_to_file(path, index).expect("to write the file"); +} + +pub fn write_message_to_file

( + path: P, + msg: impl protobuf::Message, +) -> Result<(), Box> +where + P: AsRef, +{ + use std::io::Write; + + let res = msg.write_to_bytes()?; + let output = std::fs::File::create(path)?; + let mut writer = std::io::BufWriter::new(output); + writer.write_all(&res)?; + + Ok(()) } From 33a04218c9e117122f383d7a6a7b786248948270 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 17 Oct 2023 11:51:42 +0100 Subject: [PATCH 05/35] Add Bazel configuration --- WORKSPACE | 3 ++- docker-images/syntax-highlighter/BUILD.bazel | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 199c486e87bb..5c3f1fbb7f48 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -347,6 +347,7 @@ crates_repository( "//docker-images/syntax-highlighter:crates/scip-syntax/Cargo.toml", "//docker-images/syntax-highlighter:crates/scip-treesitter/Cargo.toml", "//docker-images/syntax-highlighter:crates/scip-treesitter-languages/Cargo.toml", + "//docker-images/syntax-highlighter:crates/scip-treesitter-cli/Cargo.toml", "//docker-images/syntax-highlighter:crates/sg-syntax/Cargo.toml", ], ) @@ -364,7 +365,7 @@ load("@rules_oci//oci:dependencies.bzl", "rules_oci_dependencies") rules_oci_dependencies() -load("@rules_oci//oci:repositories.bzl", "LATEST_CRANE_VERSION", "LATEST_ZOT_VERSION", "oci_register_toolchains") +load("@rules_oci//oci:repositories.bzl", "LATEST_CRANE_VERSION", "oci_register_toolchains") oci_register_toolchains( name = "oci", diff --git a/docker-images/syntax-highlighter/BUILD.bazel b/docker-images/syntax-highlighter/BUILD.bazel index 65fca2251e7b..e653b56e58d0 100644 --- a/docker-images/syntax-highlighter/BUILD.bazel +++ b/docker-images/syntax-highlighter/BUILD.bazel @@ -53,6 +53,19 @@ rust_binary( ], ) +rust_binary( + name = "scip-treesitter-cli", + srcs = ["src/main.rs"], + aliases = aliases(), + proc_macro_deps = all_crate_deps( + proc_macro = True, + ), + visibility = ["//visibility:public"], + deps = all_crate_deps( + normal = True, + ), +) + pkg_tar( name = "tar_syntect_server", srcs = [":syntect_server"], From 6d1d23d23c67c7be5b38d2ba27a996a28e10aba9 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 17 Oct 2023 12:46:45 +0100 Subject: [PATCH 06/35] Update bazel build --- docker-images/syntax-highlighter/BUILD.bazel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-images/syntax-highlighter/BUILD.bazel b/docker-images/syntax-highlighter/BUILD.bazel index e653b56e58d0..784098157067 100644 --- a/docker-images/syntax-highlighter/BUILD.bazel +++ b/docker-images/syntax-highlighter/BUILD.bazel @@ -63,7 +63,10 @@ rust_binary( visibility = ["//visibility:public"], deps = all_crate_deps( normal = True, - ), + ) + [ + "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", + "//docker-images/syntax-highlighter/crates/scip-syntax", + ], ) pkg_tar( From a6eafbcd817c6e853b9b733fbd9b05c07a0bb995 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 17 Oct 2023 12:53:41 +0100 Subject: [PATCH 07/35] Satisfy bazel pre-check.. --- docker-images/syntax-highlighter/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-images/syntax-highlighter/BUILD.bazel b/docker-images/syntax-highlighter/BUILD.bazel index 784098157067..8ad6bb5a9bf4 100644 --- a/docker-images/syntax-highlighter/BUILD.bazel +++ b/docker-images/syntax-highlighter/BUILD.bazel @@ -64,8 +64,8 @@ rust_binary( deps = all_crate_deps( normal = True, ) + [ - "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", "//docker-images/syntax-highlighter/crates/scip-syntax", + "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", ], ) From bde3d155dc654a5ce2a950b18ec4c4b80ab26c98 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 17 Oct 2023 13:01:00 +0100 Subject: [PATCH 08/35] Daddy Bazel let me be --- docker-images/syntax-highlighter/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-images/syntax-highlighter/BUILD.bazel b/docker-images/syntax-highlighter/BUILD.bazel index 8ad6bb5a9bf4..aba92c8858eb 100644 --- a/docker-images/syntax-highlighter/BUILD.bazel +++ b/docker-images/syntax-highlighter/BUILD.bazel @@ -65,6 +65,7 @@ rust_binary( normal = True, ) + [ "//docker-images/syntax-highlighter/crates/scip-syntax", + "//docker-images/syntax-highlighter/crates/sg-syntax", "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", ], ) From 5a4f73b06468cc110aecd0fe1956d2e259a660dc Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 17 Oct 2023 13:22:39 +0100 Subject: [PATCH 09/35] Alphabet.. --- docker-images/syntax-highlighter/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-images/syntax-highlighter/BUILD.bazel b/docker-images/syntax-highlighter/BUILD.bazel index aba92c8858eb..81c3642f1e18 100644 --- a/docker-images/syntax-highlighter/BUILD.bazel +++ b/docker-images/syntax-highlighter/BUILD.bazel @@ -65,8 +65,8 @@ rust_binary( normal = True, ) + [ "//docker-images/syntax-highlighter/crates/scip-syntax", - "//docker-images/syntax-highlighter/crates/sg-syntax", "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", + "//docker-images/syntax-highlighter/crates/sg-syntax", ], ) From 84a50afb4253d55408a48fa41c371e55628ef2f6 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 18 Oct 2023 12:31:34 +0100 Subject: [PATCH 10/35] WIP --- .../crates/scip-treesitter-cli/src/main.rs | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index 93228820455a..ea4e83cbb6bf 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -33,11 +33,11 @@ pub fn main() { language, out, filenames, - } => index_impl(&language, &filenames, &out), + } => index_command(&language, &filenames, &out), } } -fn index_impl(language: &String, filenames: &Vec, out: &Option) { +fn index_command(language: &String, filenames: &Vec, out: &Option) { let p = BundledParser::get_parser(language).unwrap(); let working_directory = Path::new("./"); @@ -91,7 +91,7 @@ fn index_impl(language: &String, filenames: &Vec, out: &Option) write_message_to_file(path, index).expect("to write the file"); } -pub fn write_message_to_file

( +fn write_message_to_file

( path: P, msg: impl protobuf::Message, ) -> Result<(), Box> @@ -107,3 +107,18 @@ where Ok(()) } + +#[cfg(test)] +mod tests { + use std::env::temp_dir; + + #[test] + fn basic_test() { + let out_dir = temp_dir(); + let java = include_str!("../../scip-syntax/testdata/globals.java"); + + + let result = 2 + 2; + assert_eq!(result, 4); + } +} From 86e022d77a0c95c0430f0decd7e2f5e0c4a8a117 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Mon, 23 Oct 2023 11:50:11 +0100 Subject: [PATCH 11/35] Allow ignoring locals/globals through CLI options --- .../crates/scip-treesitter-cli/src/main.rs | 51 +++++++++++++------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index ea4e83cbb6bf..e6c8a158699a 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -1,4 +1,5 @@ use clap::{Parser, Subcommand}; +use scip::types::Document; use scip_syntax::{get_locals, get_symbols}; use scip_treesitter_languages::parsers::BundledParser; @@ -22,6 +23,10 @@ enum Commands { #[arg(short, long)] out: Option, filenames: Vec, + #[arg(long)] + no_locals: bool, + #[arg(long)] + no_globals: bool, }, } @@ -33,11 +38,19 @@ pub fn main() { language, out, filenames, - } => index_command(&language, &filenames, &out), + no_locals, + no_globals, + } => index_command(&language, &filenames, &out, !no_locals, !no_globals), } } -fn index_command(language: &String, filenames: &Vec, out: &Option) { +fn index_command( + language: &String, + filenames: &Vec, + out: &Option, + locals: bool, + globals: bool, +) { let p = BundledParser::get_parser(language).unwrap(); let working_directory = Path::new("./"); @@ -59,24 +72,33 @@ fn index_command(language: &String, filenames: &Vec, out: &Option { - for occ in occurrences { - document.occurrences.push(occ); + if locals { + let locals = get_locals(&p, &contents); + + match locals { + Some(Ok(occurrences)) => { + for occ in occurrences { + document.occurrences.push(occ); + } } + Some(Err(msg)) => { + println!("Error extracting locals: {}", msg); + } + None => {} } - Some(Err(msg)) => { - println!("Error extracting locals: {}", msg); - } - None => {} - } - index.documents.push(document); + index.documents.push(document); + } } let out_name = out.clone().unwrap_or("index.scip".to_string()); @@ -117,7 +139,6 @@ mod tests { let out_dir = temp_dir(); let java = include_str!("../../scip-syntax/testdata/globals.java"); - let result = 2 + 2; assert_eq!(result, 4); } From ace2205f06f4feb1853768f7e2e90d9e2791833b Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Mon, 23 Oct 2023 13:03:10 +0100 Subject: [PATCH 12/35] Upgrade Rust to 1.69 and add command-testing dependencies predicates crate requires at least 1.69 --- docker-images/syntax-highlighter/Cargo.lock | 117 +++++++++++++++++- .../crates/scip-treesitter-cli/Cargo.toml | 4 + .../syntax-highlighter/rust-toolchain.toml | 2 +- 3 files changed, 119 insertions(+), 4 deletions(-) diff --git a/docker-images/syntax-highlighter/Cargo.lock b/docker-images/syntax-highlighter/Cargo.lock index cf0cfab72405..006d557e4396 100644 --- a/docker-images/syntax-highlighter/Cargo.lock +++ b/docker-images/syntax-highlighter/Cargo.lock @@ -87,6 +87,21 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "assert_cmd" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" +dependencies = [ + "anstyle", + "bstr", + "doc-comment", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -209,6 +224,17 @@ dependencies = [ "wyz", ] +[[package]] +name = "bstr" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" +dependencies = [ + "memchr", + "regex-automata 0.3.6", + "serde", +] + [[package]] name = "bumpalo" version = "3.13.0" @@ -393,7 +419,7 @@ dependencies = [ "ciborium", "clap 3.2.25", "criterion-plot", - "itertools", + "itertools 0.10.5", "lazy_static", "num-traits", "oorandom", @@ -414,7 +440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -505,6 +531,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "dirs-next" version = "2.0.0" @@ -526,6 +558,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "either" version = "1.9.0" @@ -631,6 +669,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -920,6 +967,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -1110,6 +1166,12 @@ dependencies = [ "memoffset 0.6.5", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1318,6 +1380,37 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "predicates" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dfc28575c2e3f19cb3c73b93af36460ae898d426eba6fc15b9bd2a5220758a0" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "itertools 0.11.0", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "pretty_assertions" version = "1.4.0" @@ -1731,7 +1824,7 @@ dependencies = [ "bitvec", "clap 4.3.23", "insta", - "itertools", + "itertools 0.10.5", "once_cell", "protobuf", "regex", @@ -1760,9 +1853,12 @@ dependencies = [ name = "scip-treesitter-cli" version = "0.1.0" dependencies = [ + "anyhow", + "assert_cmd", "clap 4.3.23", "lazy_static", "paste", + "predicates", "protobuf", "scip", "scip-syntax", @@ -2062,6 +2158,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + [[package]] name = "textwrap" version = "0.16.0" @@ -2571,6 +2673,15 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.3.3" diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml index 5a9f62572b3d..026f06e3c0c1 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml @@ -8,8 +8,12 @@ edition = "2021" [dependencies] lazy_static = "1.0" paste = "1.0.6" +assert_cmd = "2.0.12" +predicates = "3.0.4" + protobuf.workspace = true scip.workspace = true +anyhow.workspace = true sg-syntax = { path = "../sg-syntax" } scip-syntax = { path = "../scip-syntax" } diff --git a/docker-images/syntax-highlighter/rust-toolchain.toml b/docker-images/syntax-highlighter/rust-toolchain.toml index 75fed73379d6..4fc41ee80d3a 100644 --- a/docker-images/syntax-highlighter/rust-toolchain.toml +++ b/docker-images/syntax-highlighter/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.68.0" +channel = "1.69.0" # Keep in sync with /WORKSPACE and docker-images/syntax-highlighter/Dockerfile From 530fcaf7fc8ccd8d65863e1c68a851949eb4b117 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Mon, 23 Oct 2023 13:03:42 +0100 Subject: [PATCH 13/35] Add basic end to end tests, refactor slightly --- .../crates/scip-treesitter-cli/src/main.rs | 163 ++++++++++++++---- 1 file changed, 126 insertions(+), 37 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index e6c8a158699a..128fedb9e68b 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -1,9 +1,11 @@ use clap::{Parser, Subcommand}; +use protobuf::{CodedInputStream, Message}; use scip::types::Document; use scip_syntax::{get_locals, get_symbols}; use scip_treesitter_languages::parsers::BundledParser; -use std::path::Path; +use anyhow::Result; +use std::{fs::File, io::BufReader, path::Path}; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -27,9 +29,19 @@ enum Commands { no_locals: bool, #[arg(long)] no_globals: bool, + #[arg(long)] + strict: bool, + #[arg(long)] + cwd: Option, }, } +struct Options { + locals: bool, + globals: bool, + strict: bool, +} + pub fn main() { let cli = Cli::parse(); @@ -40,7 +52,19 @@ pub fn main() { filenames, no_locals, no_globals, - } => index_command(&language, &filenames, &out, !no_locals, !no_globals), + strict, + cwd, + } => index_command( + &language, + &filenames, + &out, + &cwd, + &Options { + locals: !no_locals, + globals: !no_globals, + strict, + }, + ), } } @@ -48,12 +72,14 @@ fn index_command( language: &String, filenames: &Vec, out: &Option, - locals: bool, - globals: bool, + cwd: &Option, + options: &Options, ) { let p = BundledParser::get_parser(language).unwrap(); - let working_directory = Path::new("./"); + let working_directory: String = cwd.clone().unwrap_or("./".to_string()); //= cwd.map(|p| Path::new(p.as_str())).unwrap_or(Path::new("./")); + let working_path = Path::new(&working_directory); + let mut index = scip::types::Index { metadata: Some(scip::types::Metadata { tool_info: Some(scip::types::ToolInfo { @@ -63,7 +89,7 @@ fn index_command( ..Default::default() }) .into(), - project_root: format!("file://{}", working_directory.to_str().unwrap()), + project_root: format!("file://{}", working_directory), ..Default::default() }) .into(), @@ -72,39 +98,26 @@ fn index_command( for (_, filename) in filenames.iter().enumerate() { let contents = std::fs::read(filename).unwrap(); - let mut document: Document; //= get_symbols(&p, &contents).unwrap(); - - if globals { - document = get_symbols(&p, &contents).unwrap(); - } else { - document = Document::new(); - } - - document.relative_path = filename.clone(); - - if locals { - let locals = get_locals(&p, &contents); - - match locals { - Some(Ok(occurrences)) => { - for occ in occurrences { - document.occurrences.push(occ); - } - } - Some(Err(msg)) => { - println!("Error extracting locals: {}", msg); + eprintln!("Processing {filename}"); + match index_content(contents, &p, options) { + Ok(mut document) => { + document.relative_path = filename.to_string(); + index.documents.push(document); + } + other => { + if options.strict { + other.unwrap(); + } else { + eprintln!("Failed to extract locals: {:?}", other) } - None => {} } - - index.documents.push(document); } } let out_name = out.clone().unwrap_or("index.scip".to_string()); - let path = working_directory.join(out_name); + let path = working_path.join(out_name); - println!( + eprintln!( "Writing index for {} documents into {}", index.documents.len(), path.display() @@ -113,6 +126,38 @@ fn index_command( write_message_to_file(path, index).expect("to write the file"); } +fn index_content(contents: Vec, parser: &BundledParser, options: &Options) -> Result { + let mut document: Document; + + if options.globals { + document = get_symbols(parser, &contents).unwrap(); + } else { + document = Document::new(); + } + + if options.locals { + let locals = get_locals(parser, &contents); + + match locals { + Some(Ok(occurrences)) => { + for occ in occurrences { + document.occurrences.push(occ); + } + } + Some(other) => { + if options.strict { + other.unwrap(); + } else { + eprintln!("Failed to extract locals: {:?}", other) + } + } + None => {} + } + } + + return Ok(document); +} + fn write_message_to_file

( path: P, msg: impl protobuf::Message, @@ -130,16 +175,60 @@ where Ok(()) } +fn read_index_from_file(file: &str) -> scip::types::Index { + let mut candidate_idx = scip::types::Index::new(); + let candidate_f = File::open(file).unwrap(); + + let mut reader = BufReader::new(candidate_f); + let mut cis = CodedInputStream::from_buf_read(&mut reader); + + candidate_idx.merge_from(&mut cis).unwrap(); + return candidate_idx; +} + #[cfg(test)] mod tests { - use std::env::temp_dir; + use crate::read_index_from_file; + use assert_cmd::prelude::*; // Add methods on commands + use std::process::Command; + use std::{env::temp_dir, path::Path}; // Run programs #[test] - fn basic_test() { + fn e2e() { + let mut cmd = Command::cargo_bin("scip-treesitter-cli").unwrap(); let out_dir = temp_dir(); - let java = include_str!("../../scip-syntax/testdata/globals.java"); + let path = out_dir.join("globals.java"); + let out_path = out_dir + .join("index-java.scip") + .to_str() + .unwrap() + .to_string(); + + write_file( + &path, + include_str!("../../scip-syntax/testdata/globals.java").to_string(), + ); + + cmd.current_dir(out_dir) + .arg("index") + .args(["-l", "java", "-o", &out_path]) + .arg("globals.java"); + + cmd.assert().success(); + + let index = read_index_from_file(&out_path); + + assert!(index.documents.len() == 1); + assert_eq!(index.documents[0].relative_path, "globals.java"); + assert!(index.documents[0].symbols.len() > 0); + } + + fn write_file(path: &Path, contents: String) { + use std::io::Write; - let result = 2 + 2; - assert_eq!(result, 4); + let res = contents.into_bytes(); + let output = std::fs::File::create(path).unwrap(); + let mut writer = std::io::BufWriter::new(output); + writer.write_all(&res).unwrap(); } } From 6241ab6aadf34d8d62e449ed0bfc186fd0e9676e Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 24 Oct 2023 09:18:03 +0100 Subject: [PATCH 14/35] Cleanup --- .../crates/scip-treesitter-cli/src/main.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index 128fedb9e68b..581d36d18e47 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -77,7 +77,7 @@ fn index_command( ) { let p = BundledParser::get_parser(language).unwrap(); - let working_directory: String = cwd.clone().unwrap_or("./".to_string()); //= cwd.map(|p| Path::new(p.as_str())).unwrap_or(Path::new("./")); + let working_directory: String = cwd.clone().unwrap_or("./".to_string()); let working_path = Path::new(&working_directory); let mut index = scip::types::Index { @@ -210,9 +210,7 @@ mod tests { ); cmd.current_dir(out_dir) - .arg("index") - .args(["-l", "java", "-o", &out_path]) - .arg("globals.java"); + .args(["index", "-l", "java", "-o", &out_path, "globals.java"]); cmd.assert().success(); From 89cedbc90dac297388ff7c32839497d6148b4a4f Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 24 Oct 2023 09:43:58 +0100 Subject: [PATCH 15/35] Sync bazel lock --- .../syntax-highlighter/Cargo.Bazel.lock | 607 +++++++++++++++++- 1 file changed, 606 insertions(+), 1 deletion(-) diff --git a/docker-images/syntax-highlighter/Cargo.Bazel.lock b/docker-images/syntax-highlighter/Cargo.Bazel.lock index 540b51dc06df..6c075c41b4c9 100644 --- a/docker-images/syntax-highlighter/Cargo.Bazel.lock +++ b/docker-images/syntax-highlighter/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "71ecf92a04073e6a00ea002279ba6a43bf1d28cbbeb6399099fadd979b289c67", + "checksum": "a9ad7637adc5f733c0847427d9fc062a905b727781d8ee3a777c754a11478880", "crates": { "addr2line 0.20.0": { "name": "addr2line", @@ -457,6 +457,69 @@ }, "license": "MIT OR Apache-2.0" }, + "assert_cmd 2.0.12": { + "name": "assert_cmd", + "version": "2.0.12", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/assert_cmd/2.0.12/download", + "sha256": "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" + } + }, + "targets": [ + { + "Library": { + "crate_name": "assert_cmd", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "assert_cmd", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "anstyle 1.0.1", + "target": "anstyle" + }, + { + "id": "bstr 1.6.0", + "target": "bstr" + }, + { + "id": "doc-comment 0.3.3", + "target": "doc_comment" + }, + { + "id": "predicates 3.0.4", + "target": "predicates" + }, + { + "id": "predicates-core 1.0.6", + "target": "predicates_core" + }, + { + "id": "predicates-tree 1.0.9", + "target": "predicates_tree" + }, + { + "id": "wait-timeout 0.2.0", + "target": "wait_timeout" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "2.0.12" + }, + "license": "MIT OR Apache-2.0" + }, "async-stream 0.3.5": { "name": "async-stream", "version": "0.3.5", @@ -1110,6 +1173,58 @@ }, "license": "MIT" }, + "bstr 1.6.0": { + "name": "bstr", + "version": "1.6.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/bstr/1.6.0/download", + "sha256": "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" + } + }, + "targets": [ + { + "Library": { + "crate_name": "bstr", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "bstr", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "default", + "std", + "unicode" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "memchr 2.5.0", + "target": "memchr" + }, + { + "id": "regex-automata 0.3.6", + "target": "regex_automata" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.6.0" + }, + "license": "MIT OR Apache-2.0" + }, "bumpalo 3.13.0": { "name": "bumpalo", "version": "3.13.0", @@ -1502,6 +1617,7 @@ ], "crate_features": { "common": [ + "cargo", "color", "default", "derive", @@ -1567,6 +1683,7 @@ ], "crate_features": { "common": [ + "cargo", "color", "error-context", "help", @@ -1590,6 +1707,10 @@ "id": "clap_lex 0.5.0", "target": "clap_lex" }, + { + "id": "once_cell 1.18.0", + "target": "once_cell" + }, { "id": "strsim 0.10.0", "target": "strsim" @@ -2636,6 +2757,36 @@ }, "license": "MIT OR Apache-2.0" }, + "difflib 0.4.0": { + "name": "difflib", + "version": "0.4.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/difflib/0.4.0/download", + "sha256": "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + } + }, + "targets": [ + { + "Library": { + "crate_name": "difflib", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "difflib", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "0.4.0" + }, + "license": "MIT" + }, "dirs-next 2.0.0": { "name": "dirs-next", "version": "2.0.0", @@ -2732,6 +2883,59 @@ }, "license": "MIT OR Apache-2.0" }, + "doc-comment 0.3.3": { + "name": "doc-comment", + "version": "0.3.3", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/doc-comment/0.3.3/download", + "sha256": "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + } + }, + "targets": [ + { + "Library": { + "crate_name": "doc_comment", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "doc_comment", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "doc-comment 0.3.3", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.3.3" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT" + }, "either 1.9.0": { "name": "either", "version": "1.9.0", @@ -3322,6 +3526,53 @@ }, "license": "MIT OR Apache-2.0" }, + "float-cmp 0.9.0": { + "name": "float-cmp", + "version": "0.9.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/float-cmp/0.9.0/download", + "sha256": "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" + } + }, + "targets": [ + { + "Library": { + "crate_name": "float_cmp", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "float_cmp", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "num-traits", + "ratio" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "num-traits 0.2.16", + "target": "num_traits" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.9.0" + }, + "license": "MIT" + }, "fnv 1.0.7": { "name": "fnv", "version": "1.0.7", @@ -4934,6 +5185,53 @@ }, "license": "MIT/Apache-2.0" }, + "itertools 0.11.0": { + "name": "itertools", + "version": "0.11.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/itertools/0.11.0/download", + "sha256": "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" + } + }, + "targets": [ + { + "Library": { + "crate_name": "itertools", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "itertools", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "use_alloc", + "use_std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "either 1.9.0", + "target": "either" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.11.0" + }, + "license": "MIT OR Apache-2.0" + }, "itoa 1.0.9": { "name": "itoa", "version": "1.0.9", @@ -6103,6 +6401,36 @@ }, "license": "MIT" }, + "normalize-line-endings 0.3.0": { + "name": "normalize-line-endings", + "version": "0.3.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/normalize-line-endings/0.3.0/download", + "sha256": "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + } + }, + "targets": [ + { + "Library": { + "crate_name": "normalize_line_endings", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "normalize_line_endings", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "0.3.0" + }, + "license": "Apache-2.0" + }, "nu-ansi-term 0.46.0": { "name": "nu-ansi-term", "version": "0.46.0", @@ -7220,6 +7548,153 @@ }, "license": "MIT/Apache-2.0" }, + "predicates 3.0.4": { + "name": "predicates", + "version": "3.0.4", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/predicates/3.0.4/download", + "sha256": "6dfc28575c2e3f19cb3c73b93af36460ae898d426eba6fc15b9bd2a5220758a0" + } + }, + "targets": [ + { + "Library": { + "crate_name": "predicates", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "predicates", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "color", + "default", + "diff", + "float-cmp", + "normalize-line-endings", + "regex" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "anstyle 1.0.1", + "target": "anstyle" + }, + { + "id": "difflib 0.4.0", + "target": "difflib" + }, + { + "id": "float-cmp 0.9.0", + "target": "float_cmp" + }, + { + "id": "itertools 0.11.0", + "target": "itertools" + }, + { + "id": "normalize-line-endings 0.3.0", + "target": "normalize_line_endings" + }, + { + "id": "predicates-core 1.0.6", + "target": "predicates_core" + }, + { + "id": "regex 1.9.3", + "target": "regex" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "3.0.4" + }, + "license": "MIT OR Apache-2.0" + }, + "predicates-core 1.0.6": { + "name": "predicates-core", + "version": "1.0.6", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/predicates-core/1.0.6/download", + "sha256": "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + } + }, + "targets": [ + { + "Library": { + "crate_name": "predicates_core", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "predicates_core", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "1.0.6" + }, + "license": "MIT OR Apache-2.0" + }, + "predicates-tree 1.0.9": { + "name": "predicates-tree", + "version": "1.0.9", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/predicates-tree/1.0.9/download", + "sha256": "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" + } + }, + "targets": [ + { + "Library": { + "crate_name": "predicates_tree", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "predicates_tree", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "predicates-core 1.0.6", + "target": "predicates_core" + }, + { + "id": "termtree 0.4.1", + "target": "termtree" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.0.9" + }, + "license": "MIT OR Apache-2.0" + }, "pretty_assertions 1.4.0": { "name": "pretty_assertions", "version": "1.4.0", @@ -8363,6 +8838,7 @@ "common": [ "alloc", "dfa-onepass", + "dfa-search", "hybrid", "meta", "nfa-backtrack", @@ -9649,6 +10125,63 @@ }, "license": null }, + "scip-treesitter-cli 0.1.0": { + "name": "scip-treesitter-cli", + "version": "0.1.0", + "repository": null, + "targets": [], + "library_target_name": null, + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "anyhow 1.0.75", + "target": "anyhow" + }, + { + "id": "assert_cmd 2.0.12", + "target": "assert_cmd" + }, + { + "id": "clap 4.3.23", + "target": "clap" + }, + { + "id": "lazy_static 1.4.0", + "target": "lazy_static" + }, + { + "id": "predicates 3.0.4", + "target": "predicates" + }, + { + "id": "protobuf 3.2.0", + "target": "protobuf" + }, + { + "id": "scip 0.1.1", + "target": "scip" + } + ], + "selects": {} + }, + "edition": "2021", + "proc_macro_deps": { + "common": [ + { + "id": "paste 1.0.14", + "target": "paste" + } + ], + "selects": {} + }, + "version": "0.1.0" + }, + "license": null + }, "scip-treesitter-languages 0.1.0": { "name": "scip-treesitter-languages", "version": "0.1.0", @@ -11145,6 +11678,36 @@ }, "license": "MIT OR Apache-2.0" }, + "termtree 0.4.1": { + "name": "termtree", + "version": "0.4.1", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/termtree/0.4.1/download", + "sha256": "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + } + }, + "targets": [ + { + "Library": { + "crate_name": "termtree", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "termtree", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "0.4.1" + }, + "license": "MIT" + }, "textwrap 0.16.0": { "name": "textwrap", "version": "0.16.0", @@ -14138,6 +14701,47 @@ }, "license": "MIT/Apache-2.0" }, + "wait-timeout 0.2.0": { + "name": "wait-timeout", + "version": "0.2.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/wait-timeout/0.2.0/download", + "sha256": "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" + } + }, + "targets": [ + { + "Library": { + "crate_name": "wait_timeout", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "wait_timeout", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "cfg(unix)": [ + { + "id": "libc 0.2.147", + "target": "libc" + } + ] + } + }, + "edition": "2015", + "version": "0.2.0" + }, + "license": "MIT/Apache-2.0" + }, "walkdir 2.3.3": { "name": "walkdir", "version": "2.3.3", @@ -16140,6 +16744,7 @@ "scip-macros 0.1.0": "docker-images/syntax-highlighter/crates/scip-macros", "scip-syntax 0.1.0": "docker-images/syntax-highlighter/crates/scip-syntax", "scip-treesitter 0.1.0": "docker-images/syntax-highlighter/crates/scip-treesitter", + "scip-treesitter-cli 0.1.0": "docker-images/syntax-highlighter/crates/scip-treesitter-cli", "scip-treesitter-languages 0.1.0": "docker-images/syntax-highlighter/crates/scip-treesitter-languages", "sg-syntax 0.1.0": "docker-images/syntax-highlighter/crates/sg-syntax", "syntect_server 1.0.1": "docker-images/syntax-highlighter" From 3b357d2006bca8a9f5e8e126c4da42292802f07b Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 24 Oct 2023 10:17:26 +0100 Subject: [PATCH 16/35] Use references everywhere --- .../crates/scip-syntax/src/bin/scip-local-nav.rs | 2 +- .../syntax-highlighter/crates/scip-syntax/src/locals.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-local-nav.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-local-nav.rs index 85e319335580..804d219d03db 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-local-nav.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-local-nav.rs @@ -91,7 +91,7 @@ fn main() { ..Default::default() }; - let config = scip_syntax::languages::get_local_configuration(BundledParser::Go).unwrap(); + let config = scip_syntax::languages::get_local_configuration(&BundledParser::Go).unwrap(); index .documents .extend(parse_files(config, directory, directory)); diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs index 697eb0526e75..4c418d781bdc 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs @@ -600,7 +600,7 @@ mod test { #[test] fn test_can_do_java() -> Result<()> { - let mut config = crate::languages::get_local_configuration(BundledParser::Java).unwrap(); + let mut config = crate::languages::get_local_configuration(&BundledParser::Java).unwrap(); let source_code = include_str!("../testdata/locals.java"); let doc = parse_file_for_lang(&mut config, source_code)?; From e8b3a03b747fa6f81dac325bec944f3eec9abacf Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 24 Oct 2023 10:23:48 +0100 Subject: [PATCH 17/35] Cleanup references --- .../crates/scip-treesitter-cli/src/main.rs | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index 581d36d18e47..abffe570ac8f 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -17,7 +17,6 @@ struct Cli { #[derive(Subcommand)] enum Commands { - /// Adds files to myapp Index { #[arg(short, long)] language: String, @@ -25,12 +24,16 @@ enum Commands { #[arg(short, long)] out: Option, filenames: Vec, + #[arg(long)] no_locals: bool, + #[arg(long)] no_globals: bool, + #[arg(long)] strict: bool, + #[arg(long)] cwd: Option, }, @@ -55,11 +58,11 @@ pub fn main() { strict, cwd, } => index_command( - &language, - &filenames, - &out, - &cwd, - &Options { + language, + filenames, + out, + cwd, + Options { locals: !no_locals, globals: !no_globals, strict, @@ -69,13 +72,13 @@ pub fn main() { } fn index_command( - language: &String, - filenames: &Vec, - out: &Option, - cwd: &Option, - options: &Options, + language: String, + filenames: Vec, + out: Option, + cwd: Option, + options: Options, ) { - let p = BundledParser::get_parser(language).unwrap(); + let p = BundledParser::get_parser(&language).unwrap(); let working_directory: String = cwd.clone().unwrap_or("./".to_string()); let working_path = Path::new(&working_directory); @@ -99,7 +102,7 @@ fn index_command( for (_, filename) in filenames.iter().enumerate() { let contents = std::fs::read(filename).unwrap(); eprintln!("Processing {filename}"); - match index_content(contents, &p, options) { + match index_content(contents, &p, &options) { Ok(mut document) => { document.relative_path = filename.to_string(); index.documents.push(document); @@ -189,9 +192,9 @@ fn read_index_from_file(file: &str) -> scip::types::Index { #[cfg(test)] mod tests { use crate::read_index_from_file; - use assert_cmd::prelude::*; // Add methods on commands + use assert_cmd::prelude::*; use std::process::Command; - use std::{env::temp_dir, path::Path}; // Run programs + use std::{env::temp_dir, path::Path}; #[test] fn e2e() { From ebadf4866c4263fec8251a665d2948e8b74683b2 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 24 Oct 2023 10:57:52 +0100 Subject: [PATCH 18/35] Move bazel definitions around --- docker-images/syntax-highlighter/BUILD.bazel | 17 ------------ .../crates/scip-treesitter-cli/BUILD.bazel | 27 +++++++++++++++++++ 2 files changed, 27 insertions(+), 17 deletions(-) create mode 100644 docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel diff --git a/docker-images/syntax-highlighter/BUILD.bazel b/docker-images/syntax-highlighter/BUILD.bazel index 81c3642f1e18..65fca2251e7b 100644 --- a/docker-images/syntax-highlighter/BUILD.bazel +++ b/docker-images/syntax-highlighter/BUILD.bazel @@ -53,23 +53,6 @@ rust_binary( ], ) -rust_binary( - name = "scip-treesitter-cli", - srcs = ["src/main.rs"], - aliases = aliases(), - proc_macro_deps = all_crate_deps( - proc_macro = True, - ), - visibility = ["//visibility:public"], - deps = all_crate_deps( - normal = True, - ) + [ - "//docker-images/syntax-highlighter/crates/scip-syntax", - "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", - "//docker-images/syntax-highlighter/crates/sg-syntax", - ], -) - pkg_tar( name = "tar_syntect_server", srcs = [":syntect_server"], diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel new file mode 100644 index 000000000000..c3b50e1f0cfe --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel @@ -0,0 +1,27 @@ +load("@crate_index//:defs.bzl", "aliases", "all_crate_deps") +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test") + +rust_binary( + name = "scip-treesitter-cli", + srcs = ["src/main.rs"], + aliases = aliases(), + proc_macro_deps = all_crate_deps( + proc_macro = True, + ), + visibility = ["//visibility:public"], + deps = all_crate_deps( + normal = True, + ) + [ + "//docker-images/syntax-highlighter/crates/scip-syntax", + "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", + "//docker-images/syntax-highlighter/crates/sg-syntax", + ], +) + +rust_test( + name = "unit_test", + srcs = ["src/main.rs"], + env = { + "RUST_BACKTRACE": "1", + }, +) From 8dd5f271a45f824d50e43e10a4ce2e7bab1d5fad Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 24 Oct 2023 11:30:53 +0100 Subject: [PATCH 19/35] Use own test data for the CLI crate --- .../crates/scip-treesitter-cli/BUILD.bazel | 10 +++++ .../crates/scip-treesitter-cli/src/main.rs | 2 +- .../scip-treesitter-cli/testdata/globals.java | 41 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 docker-images/syntax-highlighter/crates/scip-treesitter-cli/testdata/globals.java diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel index c3b50e1f0cfe..7711d06c5e8f 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel @@ -21,7 +21,17 @@ rust_binary( rust_test( name = "unit_test", srcs = ["src/main.rs"], + data = glob( + ["testdata/**"], + ), env = { "RUST_BACKTRACE": "1", }, + deps = all_crate_deps( + normal = True, + ) + [ + "//docker-images/syntax-highlighter/crates/scip-syntax", + "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", + "//docker-images/syntax-highlighter/crates/sg-syntax", + ], ) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index abffe570ac8f..4113dec0dcee 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -209,7 +209,7 @@ mod tests { write_file( &path, - include_str!("../../scip-syntax/testdata/globals.java").to_string(), + include_str!("../testdata/globals.java").to_string(), ); cmd.current_dir(out_dir) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/testdata/globals.java b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/testdata/globals.java new file mode 100644 index 000000000000..782bba6b5e6c --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/testdata/globals.java @@ -0,0 +1,41 @@ +package MyPackage; + +public class globals { + private static int field1; + protected static int field2; + public static int field3; + private int field4; + protected int field5; + public int field6; + + private static void method1() {} + protected static void method2() {} + public static void method3() {} + private void method4() {} + protected void method5() {} + public void method6() {} + + public static final String COOLEST_STRING = "probably this one"; + + public class ClassInAClass { + boolean classy = true; + + public static enum Enum { + these, + should, + be, + recognized, + as, + terms + } + + public interface Goated { + boolean withTheSauce(); + } + + public void myCoolMethod() { + class WhatIsGoingOn {} + boolean iThinkThisIsAllowedButWeDontReallyCare = true; + } + } +} From 07a85524350b793f1971e72890538ff158bf5ab4 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 24 Oct 2023 13:30:15 +0100 Subject: [PATCH 20/35] WIP: try to make tests runnable with Bazel --- .../crates/scip-treesitter-cli/BUILD.bazel | 3 ++- .../crates/scip-treesitter-cli/Cargo.toml | 2 +- .../crates/scip-treesitter-cli/src/main.rs | 16 ++++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel index 7711d06c5e8f..3b0af86af094 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel @@ -23,9 +23,10 @@ rust_test( srcs = ["src/main.rs"], data = glob( ["testdata/**"], - ), + ) + [":scip-treesitter-cli"], env = { "RUST_BACKTRACE": "1", + "SCIP_CLI_LOCATION": "$(execpath :scip-treesitter-cli)", }, deps = all_crate_deps( normal = True, diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml index 026f06e3c0c1..01f19371b8aa 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml @@ -14,9 +14,9 @@ predicates = "3.0.4" protobuf.workspace = true scip.workspace = true anyhow.workspace = true +clap = { workspace = true, features = ["derive"] } sg-syntax = { path = "../sg-syntax" } scip-syntax = { path = "../scip-syntax" } scip-treesitter-languages = { path = "../scip-treesitter-languages" } scip-treesitter= { path = "../scip-treesitter" } -clap = { workspace = true, features = ["derive"] } diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index 4113dec0dcee..766d45f9f066 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -198,7 +198,14 @@ mod tests { #[test] fn e2e() { - let mut cmd = Command::cargo_bin("scip-treesitter-cli").unwrap(); + let mut cmd: Command; + match std::env::var("SCIP_CLI_LOCATION") { + Ok(va) => cmd = Command::new(va), + _ => cmd = Command::cargo_bin("scip-treesitter-cli").unwrap(), + } + + println!("{:?}", std::env::var("SCIP_CLI_LOCATION")); + let out_dir = temp_dir(); let path = out_dir.join("globals.java"); let out_path = out_dir @@ -207,12 +214,9 @@ mod tests { .unwrap() .to_string(); - write_file( - &path, - include_str!("../testdata/globals.java").to_string(), - ); + write_file(&path, include_str!("../testdata/globals.java").to_string()); - cmd.current_dir(out_dir) + cmd .args(["index", "-l", "java", "-o", &out_path, "globals.java"]); cmd.assert().success(); From 57487f68c3df1c8bf9d8c93537308a1e9672fd39 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Tue, 24 Oct 2023 14:36:23 +0200 Subject: [PATCH 21/35] bzl: use rootpath instead of execpath --- .../syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel index 3b0af86af094..ccd91ef269b4 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel @@ -26,7 +26,7 @@ rust_test( ) + [":scip-treesitter-cli"], env = { "RUST_BACKTRACE": "1", - "SCIP_CLI_LOCATION": "$(execpath :scip-treesitter-cli)", + "SCIP_CLI_LOCATION": "$(rootpath :scip-treesitter-cli)", }, deps = all_crate_deps( normal = True, From ac2dfa89a98b0c06ded436fa307f603f8d8d105e Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 24 Oct 2023 13:41:50 +0100 Subject: [PATCH 22/35] Resolve absolute path when running through Bazel --- .../crates/scip-treesitter-cli/src/main.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index 766d45f9f066..ad222b148969 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -200,7 +200,10 @@ mod tests { fn e2e() { let mut cmd: Command; match std::env::var("SCIP_CLI_LOCATION") { - Ok(va) => cmd = Command::new(va), + Ok(va) => cmd = { + let cwd = std::env::current_dir().unwrap().join(va); + Command::new(cwd) + }, _ => cmd = Command::cargo_bin("scip-treesitter-cli").unwrap(), } @@ -216,10 +219,11 @@ mod tests { write_file(&path, include_str!("../testdata/globals.java").to_string()); - cmd - .args(["index", "-l", "java", "-o", &out_path, "globals.java"]); - cmd.assert().success(); + cmd.args(["index", "-l", "java", "-o", &out_path, "globals.java"]) + .current_dir(out_dir) + .assert() + .success(); let index = read_index_from_file(&out_path); From 97074e5a43d56ac2bfee394f641c49a11ce16718 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Tue, 24 Oct 2023 16:21:13 +0100 Subject: [PATCH 23/35] Don't pass BundledParser enum by reference --- .../crates/scip-syntax/src/bin/scip-local-nav.rs | 2 +- .../crates/scip-syntax/src/bin/scip-perf.rs | 2 +- .../crates/scip-syntax/src/ctags.rs | 2 +- .../crates/scip-syntax/src/globals.rs | 2 +- .../crates/scip-syntax/src/languages.rs | 6 +++--- .../syntax-highlighter/crates/scip-syntax/src/lib.rs | 8 ++++---- .../crates/scip-syntax/src/locals.rs | 12 ++++++------ .../crates/scip-syntax/src/symbols.rs | 2 +- .../crates/scip-treesitter-cli/src/main.rs | 4 ++-- .../crates/scip-treesitter-languages/src/parsers.rs | 4 ++-- .../crates/sg-syntax/src/sg_treesitter.rs | 2 +- docker-images/syntax-highlighter/src/main.rs | 2 +- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-local-nav.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-local-nav.rs index 804d219d03db..85e319335580 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-local-nav.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-local-nav.rs @@ -91,7 +91,7 @@ fn main() { ..Default::default() }; - let config = scip_syntax::languages::get_local_configuration(&BundledParser::Go).unwrap(); + let config = scip_syntax::languages::get_local_configuration(BundledParser::Go).unwrap(); index .documents .extend(parse_files(config, directory, directory)); diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-perf.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-perf.rs index 07b34052d68b..d10c69d2441f 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-perf.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/bin/scip-perf.rs @@ -18,7 +18,7 @@ struct ParseTiming { } fn parse_files(dir: &Path) -> Vec { - let config = scip_syntax::languages::get_local_configuration(&BundledParser::Go).unwrap(); + let config = scip_syntax::languages::get_local_configuration(BundledParser::Go).unwrap(); let extension = "go"; let mut timings = vec![]; diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/ctags.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/ctags.rs index 31fb404abb67..d676e4de4873 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/ctags.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/ctags.rs @@ -194,7 +194,7 @@ pub fn generate_tags( let filepath = path.file_name()?.to_str()?; let parser = BundledParser::get_parser_from_extension(extension)?; - let (root_scope, _) = match get_globals(&parser, file_data)? { + let (root_scope, _) = match get_globals(parser, file_data)? { Ok(vals) => vals, Err(err) => { // TODO: Not sure I want to keep this or not diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/globals.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/globals.rs index 93fc6d56cceb..419044933fbb 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/globals.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/globals.rs @@ -330,7 +330,7 @@ pub mod test { #[test] fn test_enclosing_range() -> Result<()> { let config = - crate::languages::get_tag_configuration(&BundledParser::Go).expect("to have parser"); + crate::languages::get_tag_configuration(BundledParser::Go).expect("to have parser"); let source_code = include_str!("../testdata/scopes_of_go.go"); let doc = parse_file_for_lang(config, source_code)?; diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs index 6604c3e90d53..8fcf368e6cc0 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs @@ -42,7 +42,7 @@ impl TagConfiguration { let included_lang = included_lang.trim(); let parser = BundledParser::get_parser(included_lang).expect("valid language"); - let configuration = get_tag_configuration(&parser).expect("valid config"); + let configuration = get_tag_configuration(parser).expect("valid config"); format!("{}\n{}", configuration.query_text, tag_query) } @@ -241,7 +241,7 @@ mod tags { create_tags_configuration!(go, BundledParser::Go, "go", "go"); create_tags_configuration!(zig, BundledParser::Zig, "zig"); - pub fn get_tag_configuration(parser: &BundledParser) -> Option<&'static TagConfiguration> { + pub fn get_tag_configuration(parser: BundledParser) -> Option<&'static TagConfiguration> { match parser { BundledParser::C => Some(c()), BundledParser::Javascript => Some(javascript()), @@ -287,7 +287,7 @@ mod locals { create_locals_configuration!(matlab, BundledParser::Matlab, "matlab"); create_locals_configuration!(java, BundledParser::Java, "java"); - pub fn get_local_configuration(parser: &BundledParser) -> Option<&'static LocalConfiguration> { + pub fn get_local_configuration(parser: BundledParser) -> Option<&'static LocalConfiguration> { match parser { BundledParser::Go => Some(go()), BundledParser::Perl => Some(perl()), diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs index 592631b2dbdb..95be029ccc2b 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs @@ -9,7 +9,7 @@ pub mod locals; pub mod symbols; pub mod ts_scip; -pub fn get_symbols(parser: &BundledParser, source_bytes: &[u8]) -> Result { +pub fn get_symbols(parser: BundledParser, source_bytes: &[u8]) -> Result { let config = match crate::languages::get_tag_configuration(parser) { Some(config) => config, None => return Err(anyhow::anyhow!("Missing config for language")), @@ -22,7 +22,7 @@ pub fn get_symbols(parser: &BundledParser, source_bytes: &[u8]) -> Result Option> { let config = languages::get_tag_configuration(parser)?; @@ -31,7 +31,7 @@ pub fn get_globals( Some(globals::parse_tree(config, &tree, source_bytes)) } -pub fn get_locals(parser: &BundledParser, source_bytes: &[u8]) -> Option>> { +pub fn get_locals(parser: BundledParser, source_bytes: &[u8]) -> Option>> { let config = languages::get_local_configuration(parser)?; let mut parser = config.get_parser(); let tree = parser.parse(source_bytes, None).unwrap(); @@ -64,7 +64,7 @@ mod test { let parser = BundledParser::get_parser_from_extension(extension).expect("to have parser"); let config = - crate::languages::get_tag_configuration(&parser).expect("to have rust parser"); + crate::languages::get_tag_configuration(parser).expect("to have rust parser"); let doc = crate::globals::test::parse_file_for_lang(config, &source_code) .expect("to parse document"); let dumped = dump_document(&doc, &source_code).expect("to dumb document"); diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs index 4c418d781bdc..621f0eb1f30a 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs @@ -540,7 +540,7 @@ mod test { #[test] fn test_can_do_go() -> Result<()> { - let mut config = crate::languages::get_local_configuration(&BundledParser::Go).unwrap(); + let mut config = crate::languages::get_local_configuration(BundledParser::Go).unwrap(); let source_code = include_str!("../testdata/locals.go"); let doc = parse_file_for_lang(&mut config, source_code)?; @@ -552,7 +552,7 @@ mod test { #[test] fn test_can_do_nested_locals() -> Result<()> { - let mut config = crate::languages::get_local_configuration(&BundledParser::Go).unwrap(); + let mut config = crate::languages::get_local_configuration(BundledParser::Go).unwrap(); let source_code = include_str!("../testdata/locals-nested.go"); let doc = parse_file_for_lang(&mut config, source_code)?; @@ -564,7 +564,7 @@ mod test { #[test] fn test_can_do_functions() -> Result<()> { - let mut config = crate::languages::get_local_configuration(&BundledParser::Go).unwrap(); + let mut config = crate::languages::get_local_configuration(BundledParser::Go).unwrap(); let source_code = include_str!("../testdata/funcs.go"); let doc = parse_file_for_lang(&mut config, source_code)?; @@ -576,7 +576,7 @@ mod test { #[test] fn test_can_do_perl() -> Result<()> { - let mut config = crate::languages::get_local_configuration(&BundledParser::Perl).unwrap(); + let mut config = crate::languages::get_local_configuration(BundledParser::Perl).unwrap(); let source_code = include_str!("../testdata/perl.pm"); let doc = parse_file_for_lang(&mut config, source_code)?; @@ -588,7 +588,7 @@ mod test { #[test] fn test_can_do_matlab() -> Result<()> { - let mut config = crate::languages::get_local_configuration(&BundledParser::Matlab).unwrap(); + let mut config = crate::languages::get_local_configuration(BundledParser::Matlab).unwrap(); let source_code = include_str!("../testdata/locals.m"); let doc = parse_file_for_lang(&mut config, source_code)?; @@ -600,7 +600,7 @@ mod test { #[test] fn test_can_do_java() -> Result<()> { - let mut config = crate::languages::get_local_configuration(&BundledParser::Java).unwrap(); + let mut config = crate::languages::get_local_configuration(BundledParser::Java).unwrap(); let source_code = include_str!("../testdata/locals.java"); let doc = parse_file_for_lang(&mut config, source_code)?; diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/symbols.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/symbols.rs index c81a0074b4d2..8260f8fb5633 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/symbols.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/symbols.rs @@ -349,7 +349,7 @@ mod test { #[test] fn generates_some_go_symbols() -> Result<()> { - let config = crate::languages::get_tag_configuration(&BundledParser::Go).unwrap(); + let config = crate::languages::get_tag_configuration(BundledParser::Go).unwrap(); let source_code = include_str!("../testdata/symbols.go"); let document = parse_file_for_lang(config, source_code)?; let dumped = dump_document(&document, source_code)?; diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index ad222b148969..47256ea2f70e 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -102,7 +102,7 @@ fn index_command( for (_, filename) in filenames.iter().enumerate() { let contents = std::fs::read(filename).unwrap(); eprintln!("Processing {filename}"); - match index_content(contents, &p, &options) { + match index_content(contents, p, &options) { Ok(mut document) => { document.relative_path = filename.to_string(); index.documents.push(document); @@ -129,7 +129,7 @@ fn index_command( write_message_to_file(path, index).expect("to write the file"); } -fn index_content(contents: Vec, parser: &BundledParser, options: &Options) -> Result { +fn index_content(contents: Vec, parser: BundledParser, options: &Options) -> Result { let mut document: Document; if options.globals { diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-languages/src/parsers.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-languages/src/parsers.rs index ae6c52a47580..1cd8c4c85563 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-languages/src/parsers.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-languages/src/parsers.rs @@ -1,6 +1,6 @@ use tree_sitter::Language; -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BundledParser { C, Cpp, @@ -29,7 +29,7 @@ pub enum BundledParser { } impl BundledParser { - pub fn get_language(&self) -> Language { + pub fn get_language(self) -> Language { match self { BundledParser::C => tree_sitter_c::language(), BundledParser::Cpp => tree_sitter_cpp::language(), diff --git a/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs b/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs index 7f1fe7d7ea38..ed22daa81df7 100644 --- a/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs +++ b/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs @@ -83,7 +83,7 @@ pub fn index_language_with_config( let parser = scip_treesitter_languages::parsers::BundledParser::get_parser(filetype); if let Some(parser) = parser { // TODO: Could probably write this in a much better way. - let mut local_occs = scip_syntax::get_locals(&parser, code.as_bytes()) + let mut local_occs = scip_syntax::get_locals(parser, code.as_bytes()) .unwrap_or(Ok(vec![])) .unwrap_or(vec![]); diff --git a/docker-images/syntax-highlighter/src/main.rs b/docker-images/syntax-highlighter/src/main.rs index 77ea258060bd..35aac5082887 100644 --- a/docker-images/syntax-highlighter/src/main.rs +++ b/docker-images/syntax-highlighter/src/main.rs @@ -75,7 +75,7 @@ fn symbols(q: Json) -> JsonValue { None => return json!({"error": "Could not infer parser from extension"}), }; - let document = match scip_syntax::get_symbols(&parser, q.content.as_bytes()) { + let document = match scip_syntax::get_symbols(parser, q.content.as_bytes()) { Ok(vals) => vals, Err(err) => { return jsonify_err(err); From 770b3e6e696c0c7508329b93be1f973a36cba4ec Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 10:26:27 +0100 Subject: [PATCH 24/35] Use consistent syntax in Cargo.toml --- .../crates/scip-treesitter-cli/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml index 01f19371b8aa..3b5a228ecc40 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml @@ -11,10 +11,10 @@ paste = "1.0.6" assert_cmd = "2.0.12" predicates = "3.0.4" -protobuf.workspace = true -scip.workspace = true -anyhow.workspace = true -clap = { workspace = true, features = ["derive"] } +protobuf = { workspace = true } +scip = { workspace = true } +anyhow = { workspace = true } +clap = { workspace = true } sg-syntax = { path = "../sg-syntax" } scip-syntax = { path = "../scip-syntax" } From 5f4f2b6ceb415334162acc63dd73f431db81e209 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 10:27:03 +0100 Subject: [PATCH 25/35] Improve handling of analysis mode and document CLI options --- .../crates/scip-treesitter-cli/src/main.rs | 90 ++++++++++++------- 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index 47256ea2f70e..2450cc36f4d0 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -1,11 +1,11 @@ -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; use protobuf::{CodedInputStream, Message}; use scip::types::Document; use scip_syntax::{get_locals, get_symbols}; use scip_treesitter_languages::parsers::BundledParser; use anyhow::Result; -use std::{fs::File, io::BufReader, path::Path}; +use std::{fs::File, io::BufReader, path::PathBuf}; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -15,33 +15,57 @@ struct Cli { command: Commands, } +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +enum AnalysisMode { + /// Only extract occurrences of local definitions + Locals, + /// Only extract globally-accessible symbols + Globals, + /// Locals + Globals, extract everything + Full, +} + +impl AnalysisMode { + fn locals(self) -> bool { + return self == AnalysisMode::Locals || self == AnalysisMode::Full; + } + fn globals(self) -> bool { + return self == AnalysisMode::Globals || self == AnalysisMode::Full; + } +} + #[derive(Subcommand)] enum Commands { + /// Index source files using Tree Sitter parser for a given language + /// and produce a SCIP file Index { + /// #[arg(short, long)] language: String, - #[arg(short, long)] - out: Option, - filenames: Vec, + /// Path where the SCIP index will be written + #[arg(short, long, default_value = "./index.scip")] + out: String, - #[arg(long)] - no_locals: bool, + /// List of files to analyse + filenames: Vec, - #[arg(long)] - no_globals: bool, + /// Analysis mode + #[arg(short, long, default_value = "full")] + mode: AnalysisMode, - #[arg(long)] + /// Fail on first error + #[arg(long, default_value_t = false)] strict: bool, - #[arg(long)] - cwd: Option, + /// Project root to write to SCIP index + #[arg(short, long, default_value = "./")] + project_root: String, }, } struct Options { - locals: bool, - globals: bool, + analysis_mode: AnalysisMode, strict: bool, } @@ -53,18 +77,16 @@ pub fn main() { language, out, filenames, - no_locals, - no_globals, + mode, strict, - cwd, + project_root, } => index_command( language, filenames, - out, - cwd, + PathBuf::from(out), + PathBuf::from(project_root), Options { - locals: !no_locals, - globals: !no_globals, + analysis_mode: mode, strict, }, ), @@ -74,15 +96,12 @@ pub fn main() { fn index_command( language: String, filenames: Vec, - out: Option, - cwd: Option, + out: PathBuf, + project_root: PathBuf, options: Options, ) { let p = BundledParser::get_parser(&language).unwrap(); - let working_directory: String = cwd.clone().unwrap_or("./".to_string()); - let working_path = Path::new(&working_directory); - let mut index = scip::types::Index { metadata: Some(scip::types::Metadata { tool_info: Some(scip::types::ToolInfo { @@ -92,7 +111,13 @@ fn index_command( ..Default::default() }) .into(), - project_root: format!("file://{}", working_directory), + project_root: format!( + "file://{}", + project_root + .canonicalize() + .expect("Failed to canonicalize project root") + .display() + ), ..Default::default() }) .into(), @@ -117,28 +142,25 @@ fn index_command( } } - let out_name = out.clone().unwrap_or("index.scip".to_string()); - let path = working_path.join(out_name); - eprintln!( "Writing index for {} documents into {}", index.documents.len(), - path.display() + out.display() ); - write_message_to_file(path, index).expect("to write the file"); + write_message_to_file(out, index).expect("to write the file"); } fn index_content(contents: Vec, parser: BundledParser, options: &Options) -> Result { let mut document: Document; - if options.globals { + if options.analysis_mode.globals() { document = get_symbols(parser, &contents).unwrap(); } else { document = Document::new(); } - if options.locals { + if options.analysis_mode.locals() { let locals = get_locals(parser, &contents); match locals { From ee6704324000a983d9486ffd5df4dba6855c5df5 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 11:01:41 +0100 Subject: [PATCH 26/35] Use snapshot testing, and refactor CLI tests --- docker-images/syntax-highlighter/Cargo.lock | 5 +- .../crates/scip-treesitter-cli/Cargo.toml | 1 + .../crates/scip-treesitter-cli/src/main.rs | 103 +++++++++++++----- ...p_treesitter_cli__tests__globals.java.snap | 88 +++++++++++++++ 4 files changed, 165 insertions(+), 32 deletions(-) create mode 100644 docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/snapshots/scip_treesitter_cli__tests__globals.java.snap diff --git a/docker-images/syntax-highlighter/Cargo.lock b/docker-images/syntax-highlighter/Cargo.lock index 006d557e4396..d9a272a013ca 100644 --- a/docker-images/syntax-highlighter/Cargo.lock +++ b/docker-images/syntax-highlighter/Cargo.lock @@ -936,9 +936,9 @@ checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" [[package]] name = "insta" -version = "1.31.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0770b0a3d4c70567f0d58331f3088b0e4c4f56c9b8d764efe654b4a5d46de3a" +checksum = "5d64600be34b2fcfc267740a243fa7744441bb4947a619ac4e5bb6507f35fbfc" dependencies = [ "console", "lazy_static", @@ -1856,6 +1856,7 @@ dependencies = [ "anyhow", "assert_cmd", "clap 4.3.23", + "insta", "lazy_static", "paste", "predicates", diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml index 3b5a228ecc40..096b4e7fb240 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml @@ -10,6 +10,7 @@ lazy_static = "1.0" paste = "1.0.6" assert_cmd = "2.0.12" predicates = "3.0.4" +insta = "1.34.0" protobuf = { workspace = true } scip = { workspace = true } diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index 2450cc36f4d0..3ee9a084d44d 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -200,7 +200,7 @@ where Ok(()) } -fn read_index_from_file(file: &str) -> scip::types::Index { +fn read_index_from_file(file: PathBuf) -> scip::types::Index { let mut candidate_idx = scip::types::Index::new(); let candidate_f = File::open(file).unwrap(); @@ -214,52 +214,95 @@ fn read_index_from_file(file: &str) -> scip::types::Index { #[cfg(test)] mod tests { use crate::read_index_from_file; + use assert_cmd::cargo::cargo_bin; use assert_cmd::prelude::*; + use std::collections::HashMap; use std::process::Command; - use std::{env::temp_dir, path::Path}; + use std::{env::temp_dir, path::PathBuf}; + + lazy_static::lazy_static! { + static ref BINARY_LOCATION: PathBuf = { + let mut c: PathBuf; + match std::env::var("SCIP_CLI_LOCATION") { + Ok(va) => { + c = { + std::env::current_dir().unwrap().join(va) + } + } + _ => c = cargo_bin("scip-treesitter-cli") + } - #[test] - fn e2e() { - let mut cmd: Command; - match std::env::var("SCIP_CLI_LOCATION") { - Ok(va) => cmd = { - let cwd = std::env::current_dir().unwrap().join(va); - Command::new(cwd) + c + }; + } + + use scip_treesitter::snapshot::{dump_document_with_config, EmitSymbol, SnapshotOptions}; + + fn snapshot_syntax_document(doc: &scip::types::Document, source: &str) -> String { + dump_document_with_config( + doc, + source, + SnapshotOptions { + emit_symbol: EmitSymbol::All, + ..Default::default() }, - _ => cmd = Command::cargo_bin("scip-treesitter-cli").unwrap(), + ) + .expect("dump document") + } + + #[test] + fn java_e2e() { + let out_dir = temp_dir(); + let setup = HashMap::from([( + PathBuf::from("globals.java"), + include_str!("../testdata/globals.java").to_string(), + )]); + + run_index(&out_dir, &setup, vec!["--language", "java"]); + + let index = read_index_from_file(out_dir.join("index.scip")); + + for doc in &index.documents { + let path = &doc.relative_path; + let dumped = + snapshot_syntax_document(doc, setup.get(&PathBuf::from(&path)).expect("??")); + + insta::assert_snapshot!(path.clone(), dumped); } - println!("{:?}", std::env::var("SCIP_CLI_LOCATION")); + } - let out_dir = temp_dir(); - let path = out_dir.join("globals.java"); - let out_path = out_dir - .join("index-java.scip") - .to_str() - .unwrap() - .to_string(); + fn prepare(temp: &PathBuf, files: &HashMap) { + for (path, contents) in files.iter() { + let file_path = temp.join(path); + write_file(&file_path, contents); + } + } + + fn run_index(location: &PathBuf, files: &HashMap, extra_arguments: Vec<&str>) { + prepare(location, files); - write_file(&path, include_str!("../testdata/globals.java").to_string()); + let mut base_args = vec!["index"]; + base_args.extend(extra_arguments); + let mut cmd = Command::new(BINARY_LOCATION.to_str().unwrap()); - cmd.args(["index", "-l", "java", "-o", &out_path, "globals.java"]) - .current_dir(out_dir) - .assert() - .success(); + cmd.args(base_args); + + for (path, _) in files.iter() { + cmd.arg(path.to_str().unwrap()); + } - let index = read_index_from_file(&out_path); + cmd.current_dir(location); - assert!(index.documents.len() == 1); - assert_eq!(index.documents[0].relative_path, "globals.java"); - assert!(index.documents[0].symbols.len() > 0); + cmd.assert().success(); } - fn write_file(path: &Path, contents: String) { + fn write_file(path: &PathBuf, contents: &String) { use std::io::Write; - let res = contents.into_bytes(); let output = std::fs::File::create(path).unwrap(); let mut writer = std::io::BufWriter::new(output); - writer.write_all(&res).unwrap(); + writer.write_all(contents.as_bytes()).unwrap(); } } diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/snapshots/scip_treesitter_cli__tests__globals.java.snap b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/snapshots/scip_treesitter_cli__tests__globals.java.snap new file mode 100644 index 000000000000..fb8a7c2a6b71 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/snapshots/scip_treesitter_cli__tests__globals.java.snap @@ -0,0 +1,88 @@ +--- +source: crates/scip-treesitter-cli/src/main.rs +expression: dumped +--- + package MyPackage; + + public class globals { +// ^^^^^^^ definition scip-ctags globals# + private static int field1; +// ^^^^^^ definition scip-ctags globals#field1. +// ^^^^^^ definition local 1 + protected static int field2; +// ^^^^^^ definition scip-ctags globals#field2. +// ^^^^^^ definition local 2 + public static int field3; +// ^^^^^^ definition scip-ctags globals#field3. +// ^^^^^^ definition local 3 + private int field4; +// ^^^^^^ definition scip-ctags globals#field4. +// ^^^^^^ definition local 4 + protected int field5; +// ^^^^^^ definition scip-ctags globals#field5. +// ^^^^^^ definition local 5 + public int field6; +// ^^^^^^ definition scip-ctags globals#field6. +// ^^^^^^ definition local 6 + + private static void method1() {} +// ^^^^^^^ definition scip-ctags globals#method1(). + protected static void method2() {} +// ^^^^^^^ definition scip-ctags globals#method2(). + public static void method3() {} +// ^^^^^^^ definition scip-ctags globals#method3(). + private void method4() {} +// ^^^^^^^ definition scip-ctags globals#method4(). + protected void method5() {} +// ^^^^^^^ definition scip-ctags globals#method5(). + public void method6() {} +// ^^^^^^^ definition scip-ctags globals#method6(). + + public static final String COOLEST_STRING = "probably this one"; +// ^^^^^^^^^^^^^^ definition scip-ctags globals#COOLEST_STRING. +// ^^^^^^^^^^^^^^ definition local 7 + + public class ClassInAClass { +// ^^^^^^^^^^^^^ definition scip-ctags globals#ClassInAClass# + boolean classy = true; +// ^^^^^^ definition scip-ctags globals#ClassInAClass#classy. +// ^^^^^^ definition local 8 + + public static enum Enum { +// ^^^^ definition scip-ctags globals#ClassInAClass#Enum# + these, +// ^^^^^ definition scip-ctags globals#ClassInAClass#Enum#these. +// ^^^^^ definition local 9 + should, +// ^^^^^^ definition scip-ctags globals#ClassInAClass#Enum#should. +// ^^^^^^ definition local 10 + be, +// ^^ definition scip-ctags globals#ClassInAClass#Enum#be. +// ^^ definition local 11 + recognized, +// ^^^^^^^^^^ definition scip-ctags globals#ClassInAClass#Enum#recognized. +// ^^^^^^^^^^ definition local 12 + as, +// ^^ definition scip-ctags globals#ClassInAClass#Enum#as. +// ^^ definition local 13 + terms +// ^^^^^ definition scip-ctags globals#ClassInAClass#Enum#terms. +// ^^^^^ definition local 14 + } + + public interface Goated { +// ^^^^^^ definition scip-ctags globals#ClassInAClass#Goated# + boolean withTheSauce(); +// ^^^^^^^^^^^^ definition scip-ctags globals#ClassInAClass#Goated#withTheSauce(). + } + + public void myCoolMethod() { +// ^^^^^^^^^^^^ definition scip-ctags globals#ClassInAClass#myCoolMethod(). + class WhatIsGoingOn {} +// ^^^^^^^^^^^^^ definition scip-ctags globals#ClassInAClass#WhatIsGoingOn# + boolean iThinkThisIsAllowedButWeDontReallyCare = true; +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition local 15 + } + } + } + From 197874e611e9421945c880f1db4a7cf6d0142de5 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 11:21:52 +0100 Subject: [PATCH 27/35] Use indicatif for progress reporting --- .../syntax-highlighter/Cargo.Bazel.lock | 240 +++++++++++++++++- docker-images/syntax-highlighter/Cargo.lock | 36 +++ docker-images/syntax-highlighter/Cargo.toml | 1 + .../crates/scip-treesitter-cli/Cargo.toml | 1 + .../crates/scip-treesitter-cli/src/main.rs | 23 +- 5 files changed, 289 insertions(+), 12 deletions(-) diff --git a/docker-images/syntax-highlighter/Cargo.Bazel.lock b/docker-images/syntax-highlighter/Cargo.Bazel.lock index 6c075c41b4c9..006fa4b0c8cb 100644 --- a/docker-images/syntax-highlighter/Cargo.Bazel.lock +++ b/docker-images/syntax-highlighter/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "a9ad7637adc5f733c0847427d9fc062a905b727781d8ee3a777c754a11478880", + "checksum": "7c3b88f110f55336ad2c100aed4f36d0c862c706f6b296f33cb26a78f9feac71", "crates": { "addr2line 0.20.0": { "name": "addr2line", @@ -1953,6 +1953,13 @@ "compile_data_glob": [ "**" ], + "crate_features": { + "common": [ + "ansi-parsing", + "unicode-width" + ], + "selects": {} + }, "deps": { "common": [ { @@ -1962,6 +1969,10 @@ { "id": "libc 0.2.147", "target": "libc" + }, + { + "id": "unicode-width 0.1.10", + "target": "unicode_width" } ], "selects": { @@ -4992,6 +5003,71 @@ }, "license": "Apache-2.0 OR MIT" }, + "indicatif 0.17.7": { + "name": "indicatif", + "version": "0.17.7", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/indicatif/0.17.7/download", + "sha256": "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" + } + }, + "targets": [ + { + "Library": { + "crate_name": "indicatif", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "indicatif", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "unicode-width" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "console 0.15.7", + "target": "console" + }, + { + "id": "number_prefix 0.4.0", + "target": "number_prefix" + }, + { + "id": "portable-atomic 1.5.0", + "target": "portable_atomic" + }, + { + "id": "unicode-width 0.1.10", + "target": "unicode_width" + } + ], + "selects": { + "cfg(target_arch = \"wasm32\")": [ + { + "id": "instant 0.1.12", + "target": "instant" + } + ] + } + }, + "edition": "2021", + "version": "0.17.7" + }, + "license": "MIT" + }, "inlinable_string 0.1.15": { "name": "inlinable_string", "version": "0.1.15", @@ -5022,13 +5098,13 @@ }, "license": "Apache-2.0/MIT" }, - "insta 1.31.0": { + "insta 1.34.0": { "name": "insta", - "version": "1.31.0", + "version": "1.34.0", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/insta/1.31.0/download", - "sha256": "a0770b0a3d4c70567f0d58331f3088b0e4c4f56c9b8d764efe654b4a5d46de3a" + "url": "https://crates.io/api/v1/crates/insta/1.34.0/download", + "sha256": "5d64600be34b2fcfc267740a243fa7744441bb4947a619ac4e5bb6507f35fbfc" } }, "targets": [ @@ -5081,10 +5157,49 @@ "selects": {} }, "edition": "2018", - "version": "1.31.0" + "version": "1.34.0" }, "license": "Apache-2.0" }, + "instant 0.1.12": { + "name": "instant", + "version": "0.1.12", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/instant/0.1.12/download", + "sha256": "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "instant", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "instant", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.1.12" + }, + "license": "BSD-3-Clause" + }, "is-terminal 0.4.9": { "name": "is-terminal", "version": "0.4.9", @@ -6593,6 +6708,43 @@ }, "license": "MIT OR Apache-2.0" }, + "number_prefix 0.4.0": { + "name": "number_prefix", + "version": "0.4.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/number_prefix/0.4.0/download", + "sha256": "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + } + }, + "targets": [ + { + "Library": { + "crate_name": "number_prefix", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "number_prefix", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "edition": "2015", + "version": "0.4.0" + }, + "license": "MIT" + }, "object 0.31.1": { "name": "object", "version": "0.31.1", @@ -7511,6 +7663,66 @@ }, "license": "MIT" }, + "portable-atomic 1.5.0": { + "name": "portable-atomic", + "version": "1.5.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/portable-atomic/1.5.0/download", + "sha256": "b559898e0b4931ed2d3b959ab0c2da4d99cc644c4b0b1a35b4d344027f474023" + } + }, + "targets": [ + { + "Library": { + "crate_name": "portable_atomic", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "portable_atomic", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "fallback" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "portable-atomic 1.5.0", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "1.5.0" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "Apache-2.0 OR MIT" + }, "ppv-lite86 0.2.17": { "name": "ppv-lite86", "version": "0.2.17", @@ -10028,7 +10240,7 @@ "target": "clap" }, { - "id": "insta 1.31.0", + "id": "insta 1.34.0", "target": "insta" }, { @@ -10149,6 +10361,14 @@ "id": "clap 4.3.23", "target": "clap" }, + { + "id": "indicatif 0.17.7", + "target": "indicatif" + }, + { + "id": "insta 1.34.0", + "target": "insta" + }, { "id": "lazy_static 1.4.0", "target": "lazy_static" @@ -10716,7 +10936,7 @@ "deps_dev": { "common": [ { - "id": "insta 1.31.0", + "id": "insta 1.34.0", "target": "insta" }, { @@ -17037,6 +17257,10 @@ "x86_64-unknown-linux-gnu", "x86_64-unknown-none" ], + "cfg(target_arch = \"wasm32\")": [ + "wasm32-unknown-unknown", + "wasm32-wasi" + ], "cfg(target_os = \"dragonfly\")": [], "cfg(target_os = \"hermit\")": [], "cfg(target_os = \"redox\")": [], diff --git a/docker-images/syntax-highlighter/Cargo.lock b/docker-images/syntax-highlighter/Cargo.lock index d9a272a013ca..51eb83ee0276 100644 --- a/docker-images/syntax-highlighter/Cargo.lock +++ b/docker-images/syntax-highlighter/Cargo.lock @@ -384,6 +384,7 @@ dependencies = [ "encode_unicode", "lazy_static", "libc", + "unicode-width", "windows-sys 0.45.0", ] @@ -928,6 +929,19 @@ dependencies = [ "hashbrown 0.14.0", ] +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "inlinable_string" version = "0.1.15" @@ -947,6 +961,15 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + [[package]] name = "is-terminal" version = "0.4.9" @@ -1201,6 +1224,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.31.1" @@ -1374,6 +1403,12 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "portable-atomic" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b559898e0b4931ed2d3b959ab0c2da4d99cc644c4b0b1a35b4d344027f474023" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1856,6 +1891,7 @@ dependencies = [ "anyhow", "assert_cmd", "clap 4.3.23", + "indicatif", "insta", "lazy_static", "paste", diff --git a/docker-images/syntax-highlighter/Cargo.toml b/docker-images/syntax-highlighter/Cargo.toml index 23ffde0df5ae..1d08f898da74 100644 --- a/docker-images/syntax-highlighter/Cargo.toml +++ b/docker-images/syntax-highlighter/Cargo.toml @@ -51,6 +51,7 @@ members = [ [workspace.dependencies] anyhow = "1" +# "cargo" feature is enabled for using the `crate_version` macro clap = { version = "4.1.11", features = [ "derive", "cargo"] } itertools = "0.10.5" rocket = { version = "0.5.0-rc.1", features = ["json"] } diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml index 096b4e7fb240..c77e44123a76 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml @@ -11,6 +11,7 @@ paste = "1.0.6" assert_cmd = "2.0.12" predicates = "3.0.4" insta = "1.34.0" +indicatif = "0.17.7" protobuf = { workspace = true } scip = { workspace = true } diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index 3ee9a084d44d..0018210a37d7 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -1,4 +1,5 @@ use clap::{Parser, Subcommand, ValueEnum}; +use indicatif::{ProgressBar, ProgressStyle}; use protobuf::{CodedInputStream, Message}; use scip::types::Document; use scip_syntax::{get_locals, get_symbols}; @@ -105,7 +106,7 @@ fn index_command( let mut index = scip::types::Index { metadata: Some(scip::types::Metadata { tool_info: Some(scip::types::ToolInfo { - name: "scip-treesitter".to_string(), + name: "scip-treesitter-cli".to_string(), version: clap::crate_version!().to_string(), arguments: vec![], ..Default::default() @@ -124,9 +125,20 @@ fn index_command( ..Default::default() }; + let bar = ProgressBar::new(filenames.len() as u64); + + bar.set_style( + ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7}\n {msg}", + ) + .unwrap() + .progress_chars("##-"), + ); + for (_, filename) in filenames.iter().enumerate() { let contents = std::fs::read(filename).unwrap(); - eprintln!("Processing {filename}"); + bar.set_message(filename.clone()); + bar.inc(1); match index_content(contents, p, &options) { Ok(mut document) => { document.relative_path = filename.to_string(); @@ -136,12 +148,16 @@ fn index_command( if options.strict { other.unwrap(); } else { - eprintln!("Failed to extract locals: {:?}", other) + eprintln!("Failed to extract locals from {filename}: {:?}", other) } } } } + bar.finish(); + + eprintln!(""); + eprintln!( "Writing index for {} documents into {}", index.documents.len(), @@ -269,7 +285,6 @@ mod tests { insta::assert_snapshot!(path.clone(), dumped); } - } fn prepare(temp: &PathBuf, files: &HashMap) { From 89c244bf79c7f82efdeca671577882a1b69a69d0 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 11:25:03 +0100 Subject: [PATCH 28/35] Add document on strict field in options --- .../syntax-highlighter/crates/scip-treesitter-cli/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index 0018210a37d7..f7c88cdc865c 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -67,6 +67,9 @@ enum Commands { struct Options { analysis_mode: AnalysisMode, + /// When true, fail on first encountered error + /// Otherwise errors are logged but they don't + /// interrupt the process strict: bool, } From 7c4f735de1d992694e152d3dd6778064f15ff255 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 11:33:02 +0100 Subject: [PATCH 29/35] Remove dependency on sg-syntax --- docker-images/syntax-highlighter/Cargo.Bazel.lock | 2 +- docker-images/syntax-highlighter/Cargo.lock | 1 - .../syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-images/syntax-highlighter/Cargo.Bazel.lock b/docker-images/syntax-highlighter/Cargo.Bazel.lock index 006fa4b0c8cb..c81c83f54d15 100644 --- a/docker-images/syntax-highlighter/Cargo.Bazel.lock +++ b/docker-images/syntax-highlighter/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "7c3b88f110f55336ad2c100aed4f36d0c862c706f6b296f33cb26a78f9feac71", + "checksum": "32e9598236d6478220e2b34daa81ea3d8c85f1740c7d9a0955fedef68bc54d3b", "crates": { "addr2line 0.20.0": { "name": "addr2line", diff --git a/docker-images/syntax-highlighter/Cargo.lock b/docker-images/syntax-highlighter/Cargo.lock index 51eb83ee0276..24c5a84f8868 100644 --- a/docker-images/syntax-highlighter/Cargo.lock +++ b/docker-images/syntax-highlighter/Cargo.lock @@ -1901,7 +1901,6 @@ dependencies = [ "scip-syntax", "scip-treesitter", "scip-treesitter-languages", - "sg-syntax", ] [[package]] diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml index c77e44123a76..1f596c0f829f 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/Cargo.toml @@ -18,7 +18,6 @@ scip = { workspace = true } anyhow = { workspace = true } clap = { workspace = true } -sg-syntax = { path = "../sg-syntax" } scip-syntax = { path = "../scip-syntax" } scip-treesitter-languages = { path = "../scip-treesitter-languages" } scip-treesitter= { path = "../scip-treesitter" } From a4406e9bbf184a01aa4727d7077f273337b18a1f Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 12:34:53 +0100 Subject: [PATCH 30/35] Fix Bazel build now that we use Insta.. --- .../crates/scip-treesitter-cli/BUILD.bazel | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel index ccd91ef269b4..ee7cd7e3f9a8 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel @@ -14,17 +14,29 @@ rust_binary( ) + [ "//docker-images/syntax-highlighter/crates/scip-syntax", "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", - "//docker-images/syntax-highlighter/crates/sg-syntax", + "//docker-images/syntax-highlighter/crates/scip-treesitter", ], ) rust_test( name = "unit_test", srcs = ["src/main.rs"], + compile_data = glob( + [ + "testdata/**", + ], + allow_empty = False, + ), + crate = ":scip-treesitter-cli", data = glob( - ["testdata/**"], - ) + [":scip-treesitter-cli"], + ["testdata/**"], + ) + [":scip-treesitter-cli"] + + glob( + ["src/snapshots/**"], + allow_empty = False, + ), env = { + "INSTA_WORKSPACE_ROOT": ".", "RUST_BACKTRACE": "1", "SCIP_CLI_LOCATION": "$(rootpath :scip-treesitter-cli)", }, @@ -33,6 +45,6 @@ rust_test( ) + [ "//docker-images/syntax-highlighter/crates/scip-syntax", "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", - "//docker-images/syntax-highlighter/crates/sg-syntax", + "//docker-images/syntax-highlighter/crates/scip-treesitter", ], ) From a11ba4e3966137f7ad3b2855747ad8ac11f9f0f8 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 12:38:41 +0100 Subject: [PATCH 31/35] Add README --- docker-images/syntax-highlighter/README.md | 1 + .../crates/scip-treesitter-cli/README.md | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 docker-images/syntax-highlighter/crates/scip-treesitter-cli/README.md diff --git a/docker-images/syntax-highlighter/README.md b/docker-images/syntax-highlighter/README.md index 3361b73651db..7dc470ea2bbc 100644 --- a/docker-images/syntax-highlighter/README.md +++ b/docker-images/syntax-highlighter/README.md @@ -5,6 +5,7 @@ Crates: - The main `syntect_server` executable - `crates/scip-treesitter-languages/`: All the grammars and parsers live here to make shipping parsers with the same tree-sitter version (and associated build tooling) very easy. All new grammars and parsers should be added here. - `crates/scip-treesitter/`: Associated utilities for tree-sitter and scip. Not required to be used for other projects +- `crates/scip-treesitter-cli/`: Command line tool that produces a SCIP index file from the analysis performed by other `scip-*` crates in this project - `crates/scip-syntax/`: local navigation calculation (and some other utilities) live here. - `crates/sg-syntax/`: Sourcegraph code to glue together whatever from the above crates to be used for our purposes. diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/README.md b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/README.md new file mode 100644 index 000000000000..7f0d1286fc77 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/README.md @@ -0,0 +1,28 @@ +# scip-treesitter-cli + +A command line tool that uses other scip-* crates to produce a SCIP file containing +global and local symbols discovered during analysis. + +## Usage + +**Indexing** (`index --help` for more details) + +``` +scip-treesitter-cli index --language java --out ./index.scip file1.java file2.java ... +``` + +## Contributing + +This is a standard Rust CLI project, with a single runnable entry point - the CLI itself. + +1. Run tests: `cargo test` + + We use Insta for snapshot testing, if you're changing the output produced by the CLI, + run `cargo test` and then `cargo insta review` to accept/reject changes in symbols + +2. Run CLI: `cargo run -- index --language java --out ./index.scip file1.java file2.java ...` + +On CI, Bazel is used instead of Cargo. + +1. Run tests: `bazel test //docker-images/syntax-highlighter/crates/scip-treesitter-cli:unit_test` +2. Build CLI: `bazel build //docker-images/syntax-highlighter/crates/scip-treesitter-cli:scip-treesitter-cli` From 6226a9480e924c5b9028865652ba1b887bf5a9f2 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 12:47:00 +0100 Subject: [PATCH 32/35] Update Cargo bazel lock --- docker-images/syntax-highlighter/Cargo.Bazel.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-images/syntax-highlighter/Cargo.Bazel.lock b/docker-images/syntax-highlighter/Cargo.Bazel.lock index 9e8618f4f9fa..7ad584782e2b 100644 --- a/docker-images/syntax-highlighter/Cargo.Bazel.lock +++ b/docker-images/syntax-highlighter/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "32e9598236d6478220e2b34daa81ea3d8c85f1740c7d9a0955fedef68bc54d3b", + "checksum": "a30200fddef89273f32721d5e5e6b1e44a4d56104443ffb9e2bc722f6ca165ab", "crates": { "addr2line 0.20.0": { "name": "addr2line", From 478dc1d87108d53252e8060d6ad2d48fe98e0716 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 12:48:03 +0100 Subject: [PATCH 33/35] Document the hair-pulling moment around Insta and names under Bazel --- .../syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel index ee7cd7e3f9a8..e64203988c60 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel @@ -27,6 +27,9 @@ rust_test( ], allow_empty = False, ), + # Make sure this name matches the crate name in cargo configuration + # This is to ensure that snapshots produced by Bazel tests and + # ones produced by `cargo test` command - otherwise insta testing will fail! crate = ":scip-treesitter-cli", data = glob( ["testdata/**"], From c414977e3a14ca50035621a8b8750175e85b3f2f Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 12:56:30 +0100 Subject: [PATCH 34/35] Run bazel configure --- .../syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel index e64203988c60..0d615483f507 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/BUILD.bazel @@ -13,8 +13,8 @@ rust_binary( normal = True, ) + [ "//docker-images/syntax-highlighter/crates/scip-syntax", - "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", "//docker-images/syntax-highlighter/crates/scip-treesitter", + "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", ], ) @@ -47,7 +47,7 @@ rust_test( normal = True, ) + [ "//docker-images/syntax-highlighter/crates/scip-syntax", - "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", "//docker-images/syntax-highlighter/crates/scip-treesitter", + "//docker-images/syntax-highlighter/crates/scip-treesitter-languages", ], ) From e6e184fda6730bdf3067f43ea8fc2618a5bc4695 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Wed, 25 Oct 2023 13:34:34 +0100 Subject: [PATCH 35/35] Error handling improvements, rename strict -> fail_fast --- .../crates/scip-treesitter-cli/src/main.rs | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs index f7c88cdc865c..bcaf7330a321 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-cli/src/main.rs @@ -57,7 +57,7 @@ enum Commands { /// Fail on first error #[arg(long, default_value_t = false)] - strict: bool, + fail_fast: bool, /// Project root to write to SCIP index #[arg(short, long, default_value = "./")] @@ -70,7 +70,7 @@ struct Options { /// When true, fail on first encountered error /// Otherwise errors are logged but they don't /// interrupt the process - strict: bool, + fail_fast: bool, } pub fn main() { @@ -82,7 +82,7 @@ pub fn main() { out, filenames, mode, - strict, + fail_fast, project_root, } => index_command( language, @@ -91,7 +91,7 @@ pub fn main() { PathBuf::from(project_root), Options { analysis_mode: mode, - strict, + fail_fast, }, ), } @@ -147,11 +147,11 @@ fn index_command( document.relative_path = filename.to_string(); index.documents.push(document); } - other => { - if options.strict { - other.unwrap(); + Err(error) => { + if options.fail_fast { + panic!("Failed to index {filename}: {:?}", error); } else { - eprintln!("Failed to extract locals from {filename}: {:?}", other) + eprintln!("Failed to index {filename}: {:?}", error) } } } @@ -188,13 +188,7 @@ fn index_content(contents: Vec, parser: BundledParser, options: &Options) -> document.occurrences.push(occ); } } - Some(other) => { - if options.strict { - other.unwrap(); - } else { - eprintln!("Failed to extract locals: {:?}", other) - } - } + Some(Err(e)) => return Err(e), None => {} } }