From 1e920b033107584235f21ccbe3f5c930d3662c37 Mon Sep 17 00:00:00 2001 From: maciektr Date: Tue, 11 Jun 2024 17:32:05 +0200 Subject: [PATCH] Scarb expand initial impl (#1344) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit-id:8273b556 --- **Stack**: - #1348 - #1347 - #1344 ⬅ ⚠️ *Part of a stack created by [spr](https://github.com/ejoffe/spr). Do not merge manually using the UI - doing so may have unexpected results.* --- Cargo.lock | 1 + Cargo.toml | 1 + scarb/Cargo.toml | 1 + scarb/src/bin/scarb/args.rs | 17 ++ scarb/src/bin/scarb/commands/expand.rs | 17 ++ scarb/src/bin/scarb/commands/mod.rs | 2 + scarb/src/ops/compile.rs | 2 +- scarb/src/ops/expand.rs | 214 +++++++++++++++++++++++++ scarb/src/ops/mod.rs | 2 + 9 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 scarb/src/bin/scarb/commands/expand.rs create mode 100644 scarb/src/ops/expand.rs diff --git a/Cargo.lock b/Cargo.lock index 051d06d70..28b80ee58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4555,6 +4555,7 @@ dependencies = [ "cairo-lang-formatter", "cairo-lang-macro", "cairo-lang-macro-stable", + "cairo-lang-parser", "cairo-lang-semantic", "cairo-lang-sierra", "cairo-lang-sierra-to-casm", diff --git a/Cargo.toml b/Cargo.toml index 8c60b1530..a66d05392 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ cairo-lang-filesystem = { git = "https://github.com/starkware-libs/cairo", rev = cairo-lang-formatter = { git = "https://github.com/starkware-libs/cairo", rev = "5b5cb8943283472a44f09319a0cc3a84491b4037" } cairo-lang-language-server = { git = "https://github.com/starkware-libs/cairo", rev = "5b5cb8943283472a44f09319a0cc3a84491b4037" } cairo-lang-lowering = { git = "https://github.com/starkware-libs/cairo", rev = "5b5cb8943283472a44f09319a0cc3a84491b4037" } +cairo-lang-parser = { git = "https://github.com/starkware-libs/cairo", rev = "5b5cb8943283472a44f09319a0cc3a84491b4037" } cairo-lang-project = { git = "https://github.com/starkware-libs/cairo", rev = "5b5cb8943283472a44f09319a0cc3a84491b4037" } cairo-lang-runner = { git = "https://github.com/starkware-libs/cairo", rev = "5b5cb8943283472a44f09319a0cc3a84491b4037" } cairo-lang-semantic = { git = "https://github.com/starkware-libs/cairo", rev = "5b5cb8943283472a44f09319a0cc3a84491b4037" } diff --git a/scarb/Cargo.toml b/scarb/Cargo.toml index a691a5645..074a52757 100644 --- a/scarb/Cargo.toml +++ b/scarb/Cargo.toml @@ -23,6 +23,7 @@ cairo-lang-filesystem.workspace = true cairo-lang-formatter.workspace = true cairo-lang-macro = { path = "../plugins/cairo-lang-macro" } cairo-lang-macro-stable = { path = "../plugins/cairo-lang-macro-stable" } +cairo-lang-parser.workspace = true cairo-lang-semantic.workspace = true cairo-lang-sierra-to-casm.workspace = true cairo-lang-sierra.workspace = true diff --git a/scarb/src/bin/scarb/args.rs b/scarb/src/bin/scarb/args.rs index 48cbca81b..4b007d799 100644 --- a/scarb/src/bin/scarb/args.rs +++ b/scarb/src/bin/scarb/args.rs @@ -149,6 +149,8 @@ pub enum Command { Remove(RemoveArgs), /// Compile current project. Build(BuildArgs), + /// Expand macros. + Expand(ExpandArgs), /// Manipulate packages cache. #[clap(subcommand)] Cache(CacheSubcommand), @@ -216,6 +218,21 @@ pub struct BuildArgs { pub features: FeaturesSpec, } +/// Arguments accepted by the `expand` command. +#[derive(Parser, Clone, Debug)] +pub struct ExpandArgs { + #[command(flatten)] + pub packages_filter: PackagesFilter, + + /// Specify features to enable. + #[command(flatten)] + pub features: FeaturesSpec, + + /// Do not attempt formatting. + #[arg(long, default_value_t = false)] + pub ugly: bool, +} + /// Arguments accepted by the `run` command. #[derive(Parser, Clone, Debug)] #[clap(trailing_var_arg = true)] diff --git a/scarb/src/bin/scarb/commands/expand.rs b/scarb/src/bin/scarb/commands/expand.rs new file mode 100644 index 000000000..67939ab71 --- /dev/null +++ b/scarb/src/bin/scarb/commands/expand.rs @@ -0,0 +1,17 @@ +use anyhow::Result; + +use crate::args::ExpandArgs; +use scarb::core::Config; +use scarb::ops; +use scarb::ops::ExpandOpts; + +#[tracing::instrument(skip_all, level = "info")] +pub fn run(args: ExpandArgs, config: &Config) -> Result<()> { + let ws = ops::read_workspace(config.manifest_path(), config)?; + let package = args.packages_filter.match_one(&ws)?; + let opts = ExpandOpts { + features: args.features.try_into()?, + ugly: args.ugly, + }; + ops::expand(package, opts, &ws) +} diff --git a/scarb/src/bin/scarb/commands/mod.rs b/scarb/src/bin/scarb/commands/mod.rs index 58c765e01..936976a9d 100644 --- a/scarb/src/bin/scarb/commands/mod.rs +++ b/scarb/src/bin/scarb/commands/mod.rs @@ -13,6 +13,7 @@ pub mod cache_path; pub mod check; pub mod clean; pub mod commands; +mod expand; pub mod external; pub mod fetch; pub mod fmt; @@ -34,6 +35,7 @@ pub fn run(command: Command, config: &mut Config) -> Result<()> { // Keep these sorted alphabetically. Add(args) => add::run(args, config), Build(args) => build::run(args, config), + Expand(args) => expand::run(args, config), Cache(CacheSubcommand::Clean) => cache_clean::run(config), Cache(CacheSubcommand::Path) => cache_path::run(config), Check(args) => check::run(args, config), diff --git a/scarb/src/ops/compile.rs b/scarb/src/ops/compile.rs index 39b9891f2..63252fbfe 100644 --- a/scarb/src/ops/compile.rs +++ b/scarb/src/ops/compile.rs @@ -140,7 +140,7 @@ where Ok(()) } -fn compile_unit(unit: CompilationUnit, ws: &Workspace<'_>) -> Result<()> { +pub fn compile_unit(unit: CompilationUnit, ws: &Workspace<'_>) -> Result<()> { let package_name = unit.main_package_id().name.clone(); ws.config() diff --git a/scarb/src/ops/expand.rs b/scarb/src/ops/expand.rs new file mode 100644 index 000000000..a15226f51 --- /dev/null +++ b/scarb/src/ops/expand.rs @@ -0,0 +1,214 @@ +use crate::compiler::db::{build_scarb_root_database, ScarbDatabase}; +use crate::compiler::helpers::{build_compiler_config, write_string}; +use crate::compiler::{CairoCompilationUnit, CompilationUnit, CompilationUnitAttributes}; +use crate::core::{Package, TargetKind, Workspace}; +use crate::ops; +use crate::ops::FeaturesOpts; +use anyhow::{anyhow, bail, Context, Result}; +use cairo_lang_compiler::db::RootDatabase; +use cairo_lang_compiler::diagnostics::DiagnosticsError; +use cairo_lang_defs::db::DefsGroup; +use cairo_lang_defs::ids::{LanguageElementId, ModuleId, ModuleItemId}; +use cairo_lang_defs::patcher::PatchBuilder; +use cairo_lang_diagnostics::ToOption; +use cairo_lang_filesystem::db::FilesGroup; +use cairo_lang_filesystem::ids::CrateLongId; +use cairo_lang_formatter::{CairoFormatter, FormatOutcome, FormatterConfig}; +use cairo_lang_parser::db::ParserGroup; +use cairo_lang_syntax::node::helpers::UsePathEx; +use cairo_lang_syntax::node::{ast, TypedStablePtr, TypedSyntaxNode}; +use cairo_lang_utils::Upcast; +use std::collections::HashSet; + +#[derive(Debug)] +pub struct ExpandOpts { + pub features: FeaturesOpts, + pub ugly: bool, +} + +pub fn expand(package: Package, opts: ExpandOpts, ws: &Workspace<'_>) -> Result<()> { + let package_name = package.id.name.to_string(); + let resolve = ops::resolve_workspace(ws)?; + let compilation_units = ops::generate_compilation_units(&resolve, &opts.features, ws)?; + + // Compile procedural macros. + compilation_units + .iter() + .filter(|unit| matches!(unit, CompilationUnit::ProcMacro(_))) + .map(|unit| ops::compile::compile_unit(unit.clone(), ws)) + .collect::>>()?; + + let Some(compilation_unit) = compilation_units.into_iter().find(|unit| { + unit.main_package_id() == package.id + && unit.main_component().target_kind() == TargetKind::LIB + }) else { + bail!("compilation unit not found for `{package_name}`") + }; + let CompilationUnit::Cairo(compilation_unit) = compilation_unit else { + bail!("only cairo compilation units can be expanded") + }; + let ScarbDatabase { db, .. } = build_scarb_root_database(&compilation_unit, ws)?; + let mut compiler_config = build_compiler_config(&compilation_unit, ws); + compiler_config + .diagnostics_reporter + .ensure(&db) + .map_err(|err| err.into()) + .map_err(|err| { + if !suppress_error(&err) { + ws.config().ui().anyhow(&err); + } + + anyhow!("could not check `{package_name}` due to previous error") + })?; + + do_expand(&db, &compilation_unit, opts, ws)?; + + Ok(()) +} + +/// Memorize opened modules for adding appropriate bracketing to the code. +struct ModuleStack(Vec); + +impl ModuleStack { + fn new() -> Self { + Self(Vec::new()) + } + + /// Register a module path in the stack, opening new module blocks if necessary. + fn register(&mut self, module_path: String) -> String { + let open_module = |builder: &mut Vec| { + let module = module_path + .split("::") + .last() + .expect("module full path cannot be empty"); + builder.push(format!("\nmod {module} {{\n")); + }; + let close_module = |builder: &mut Vec| { + builder.push(" }\n".to_string()); + }; + let mut builder: Vec = Vec::new(); + while !self.0.is_empty() { + // Can safely unwrap, as the stack is not empty. + let current_module = self.0.last().unwrap(); + if current_module.clone() != module_path { + if module_path.starts_with(current_module) { + self.0.push(module_path.clone()); + open_module(&mut builder); + break; + } else { + close_module(&mut builder); + self.0.pop(); + continue; + } + } else { + break; + } + } + if self.0.is_empty() { + self.0.push(module_path.clone()); + open_module(&mut builder); + } + builder.concat() + } + + /// Pop all module paths from the stack, closing all module blocks. + fn drain(&mut self) -> String { + let mut builder = String::new(); + while !self.0.is_empty() { + builder = format!("{builder}}}\n"); + self.0.pop(); + } + builder + } +} + +fn do_expand( + db: &RootDatabase, + compilation_unit: &CairoCompilationUnit, + opts: ExpandOpts, + ws: &Workspace<'_>, +) -> Result<()> { + let main_crate_id = db.intern_crate(CrateLongId::Real( + compilation_unit.main_component().cairo_package_name(), + )); + let main_module = ModuleId::CrateRoot(main_crate_id); + let module_file = db + .module_main_file(main_module) + .to_option() + .context("failed to retrieve module main file")?; + let file_syntax = db + .file_module_syntax(module_file) + .to_option() + .context("failed to retrieve module main file syntax")?; + + let crate_modules = db.crate_modules(main_crate_id); + let item_asts = file_syntax.items(db); + + let mut builder = PatchBuilder::new(db, &item_asts); + let mut module_stack = ModuleStack::new(); + + for module_id in crate_modules.iter() { + builder.add_str(module_stack.register(module_id.full_path(db)).as_str()); + let Some(module_items) = db.module_items(*module_id).to_option() else { + continue; + }; + let mut seen_uses = HashSet::new(); + for item_id in module_items.iter() { + // We need to handle uses manually, as module data only includes use leaf instead of path. + if let ModuleItemId::Use(use_id) = item_id { + let use_item = use_id.stable_ptr(db).lookup(db.upcast()); + let item = ast::UsePath::Leaf(use_item.clone()).get_item(db.upcast()); + let item = item.use_path(db.upcast()); + // We need to deduplicate multi-uses (`a::{b, c}`), which are split into multiple leaves. + if !seen_uses.insert(item.stable_ptr()) { + continue; + } + builder.add_str("use "); + builder.add_node(item.as_syntax_node()); + builder.add_str(";\n"); + continue; + } + // We can skip submodules, as they will be printed as part of `crate_modules`. + if let ModuleItemId::Submodule(_) = item_id { + continue; + } + let node = item_id.stable_location(db).syntax_node(db); + builder.add_node(node); + } + } + + builder.add_str(module_stack.drain().as_str()); + let (content, _) = builder.build(); + let content = if opts.ugly { + content + } else { + // Ignores formatting errors. + format_cairo(content.clone()).unwrap_or(content) + }; + + let file_name = format!( + "{}.expanded.cairo", + compilation_unit + .main_component() + .first_target() + .name + .clone() + ); + let target_dir = compilation_unit.target_dir(ws); + write_string(file_name.as_str(), "output file", &target_dir, ws, content)?; + Ok(()) +} + +fn format_cairo(content: String) -> Option { + let formatter = CairoFormatter::new(FormatterConfig::default()); + let content = formatter.format_to_string(&content).ok()?; + // Get formatted string, whether any changes have been made, or not. + Some(match content { + FormatOutcome::Identical(value) => value, + FormatOutcome::DiffFound(diff) => diff.formatted, + }) +} + +fn suppress_error(err: &anyhow::Error) -> bool { + matches!(err.downcast_ref(), Some(&DiagnosticsError)) +} diff --git a/scarb/src/ops/mod.rs b/scarb/src/ops/mod.rs index a2edc3fa3..7dbab6b74 100644 --- a/scarb/src/ops/mod.rs +++ b/scarb/src/ops/mod.rs @@ -5,6 +5,7 @@ pub use cache::*; pub use clean::*; pub use compile::*; +pub use expand::*; pub use fmt::*; pub use manifest::*; pub use metadata::*; @@ -19,6 +20,7 @@ pub use workspace::*; mod cache; mod clean; mod compile; +mod expand; mod fmt; mod lockfile; mod manifest;