From 5a1c6f08588d4ebe29d1fc5b83f6ee6cf640f248 Mon Sep 17 00:00:00 2001 From: Pi Lanningham Date: Fri, 27 Dec 2024 02:05:47 -0500 Subject: [PATCH 1/3] Add a panic hook Given that a block producing node is considered mission critical, a panic should almost certainly be considered a bug, as the node should catch and respond to any errors in a more appropriate way. If we ever *do* panic, we show a nice message, and ask the user to open a ticket --- Cargo.lock | 120 ++++++++++++++++++++++++++++ crates/amaru/Cargo.toml | 13 ++- crates/amaru/build.rs | 3 + crates/amaru/src/bin/amaru/main.rs | 4 + crates/amaru/src/bin/amaru/panic.rs | 113 ++++++++++++++++++++++++++ 5 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 crates/amaru/build.rs create mode 100644 crates/amaru/src/bin/amaru/panic.rs diff --git a/Cargo.lock b/Cargo.lock index 4d5e286..cfeee19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,11 +44,13 @@ version = "0.1.0" dependencies = [ "async-trait", "bech32 0.11.0", + "built", "clap", "envpath", "gasket", "hex", "indicatif", + "indoc", "insta", "miette 7.2.0", "opentelemetry", @@ -56,6 +58,7 @@ dependencies = [ "opentelemetry_sdk", "ouroboros", "ouroboros-praos", + "owo-colors", "pallas-addresses", "pallas-codec", "pallas-crypto", @@ -66,6 +69,7 @@ dependencies = [ "proptest", "rocksdb", "serde", + "strip-ansi-escapes", "thiserror 2.0.3", "tokio", "tokio-util", @@ -327,6 +331,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -524,6 +539,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "built" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b" +dependencies = [ + "git2", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -1226,6 +1250,19 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "git2" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url", +] + [[package]] name = "glob" version = "0.3.1" @@ -1317,6 +1354,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.3.9" @@ -1615,6 +1661,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + [[package]] name = "insta" version = "1.41.1" @@ -1634,6 +1686,12 @@ version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -1744,6 +1802,18 @@ version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +[[package]] +name = "libgit2-sys" +version = "0.17.0+1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + [[package]] name = "libloading" version = "0.8.5" @@ -1793,6 +1863,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", + "libc", "pkg-config", "vcpkg", ] @@ -2253,6 +2324,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +dependencies = [ + "supports-color", +] + [[package]] name = "pallas-addresses" version = "0.31.0" @@ -3033,6 +3113,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strip-ansi-escapes" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "011cbb39cf7c1f62871aea3cc46e5817b0937b49e9447370c93cacbe93a766d8" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3064,6 +3153,16 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "supports-color" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ba6faf2ca7ee42fdd458f4347ae0a9bd6bcc445ad7cb57ad82b383f18870d6f" +dependencies = [ + "atty", + "is_ci", +] + [[package]] name = "syn" version = "1.0.109" @@ -3558,6 +3657,27 @@ dependencies = [ "thiserror 1.0.64", ] +[[package]] +name = "vte" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbce692ab4ca2f1f3047fcf732430249c0e971bfdd2b234cf2c47ad93af5983" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "wait-timeout" version = "0.2.0" diff --git a/crates/amaru/Cargo.toml b/crates/amaru/Cargo.toml index 1b7886e..b64da37 100644 --- a/crates/amaru/Cargo.toml +++ b/crates/amaru/Cargo.toml @@ -10,6 +10,7 @@ homepage = "https://github.com/pragma-org/amaru" documentation = "https://docs.rs/amaru" readme = "README.md" rust-version = "1.81.0" +build = "build.rs" [dependencies] async-trait = "0.1.83" @@ -17,6 +18,9 @@ clap = { version = "4.5.20", features = ["derive"] } gasket = { version = "0.8.0", features = ["derive"] } hex = "0.4.3" miette = "7.2.0" +indoc = "2.0" +strip-ansi-escapes = "0.1.1" +owo-colors = { version = "3.5.0", features = ["supports-colors"] } ouroboros = { git = "https://github.com/pragma-org/ouroboros", rev = "ca1d447a6c106e421e6c2b1c7d9d59abf5ca9589" } ouroboros-praos = { git = "https://github.com/pragma-org/ouroboros", rev = "ca1d447a6c106e421e6c2b1c7d9d59abf5ca9589" } pallas-addresses = "0.31.0" @@ -39,10 +43,17 @@ serde = "1.0.215" bech32 = "0.11.0" opentelemetry = { version = "0.27.1" } opentelemetry_sdk = { version = "0.27.1", features = ["async-std", "rt-tokio"] } -opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "http-proto", "reqwest-client"] } +opentelemetry-otlp = { version = "0.27.0", features = [ + "grpc-tonic", + "http-proto", + "reqwest-client", +] } tracing-opentelemetry = { version = "0.28.0" } [dev-dependencies] envpath = { version = "0.0.1-beta.3", features = ["rand"] } insta = { version = "1.41.1", features = ["json"] } proptest = "1.5.0" + +[build-dependencies] +built = { version = "0.7.1", features = ["git2"] } diff --git a/crates/amaru/build.rs b/crates/amaru/build.rs new file mode 100644 index 0000000..d8f91cb --- /dev/null +++ b/crates/amaru/build.rs @@ -0,0 +1,3 @@ +fn main() { + built::write_built_file().expect("Failed to acquire build-time information"); +} diff --git a/crates/amaru/src/bin/amaru/main.rs b/crates/amaru/src/bin/amaru/main.rs index 79e278f..f64706f 100644 --- a/crates/amaru/src/bin/amaru/main.rs +++ b/crates/amaru/src/bin/amaru/main.rs @@ -1,10 +1,12 @@ use clap::{Parser, Subcommand}; use opentelemetry::metrics::Counter; +use panic::panic_handler; use std::env; mod cmd; mod config; mod exit; +mod panic; pub const SERVICE_NAME: &str = "amaru"; @@ -28,6 +30,8 @@ struct Cli { #[tokio::main] async fn main() -> miette::Result<()> { + panic_handler(); + let counter = setup_tracing(); let args = Cli::parse(); diff --git a/crates/amaru/src/bin/amaru/panic.rs b/crates/amaru/src/bin/amaru/panic.rs new file mode 100644 index 0000000..4673af7 --- /dev/null +++ b/crates/amaru/src/bin/amaru/panic.rs @@ -0,0 +1,113 @@ +use owo_colors::OwoColorize; + +/// Installs a panic handler that prints some useful diagnostics and +/// asks the user to report the issue. +pub fn panic_handler() { + std::panic::set_hook(Box::new(move |info| { + let message = info + .payload() + .downcast_ref::<&str>() + .map(|s| (*s).to_string()) + .or_else(|| { + info.payload() + .downcast_ref::() + .map(|s| s.to_string()) + }) + .unwrap_or_else(|| "unknown error".to_string()); + + let location = info.location().map_or_else( + || "".into(), + |location| { + format!( + "{}:{}:{}\n\n ", + location.file(), + location.line(), + location.column(), + ) + }, + ); + + // We present the user with a helpful and welcoming error message; + // Block producing nodes should be considered mission critical software, and so + // They should endeavor *never* to crash, and should always handle and recover from errors. + // So if the process panics, it should be treated as a bug, and we very much want the user to report it. + // TODO(pi): We could go a step further, and prefill some issue details like title, body, labels, etc. + // using query parameters: https://github.com/sindresorhus/new-github-issue-url?tab=readme-ov-file#api + let error_message = indoc::formatdoc! { + r#"{fatal} + Whoops! The Amaru process panicked, rather than handling the error it encountered gracefully. + + This is almost certainly a bug, and we'd appreciate a report so we can improve Amaru. + + Please report this error at https://github.com/pragma-org/amaru/issues/new. + + In your bug report please provide the information below and if possible the code + that produced it. + {info} + + {location}{message}"#, + info = node_info(), + fatal = "amaru::fatal::error".red().bold(), + location = location.purple(), + }; + + println!("\n{}", indent(&error_message, 3)); + })); +} + +// TODO: pulled from aiken; should we have our own utility crate for pretty printing? +// https://github.com/aiken-lang/aiken/blob/main/crates/aiken-project/src/pretty.rs#L126C1-L134C2 +pub fn indent(lines: &str, n: usize) -> String { + let tab = pad_left(String::new(), n, " "); + lines + .lines() + .map(|line| format!("{tab}{line}")) + .collect::>() + .join("\n") +} + +pub fn pad_left(mut text: String, n: usize, delimiter: &str) -> String { + let diff = n as i32 - ansi_len(&text) as i32; + if diff.is_positive() { + for _ in 0..diff { + text.insert_str(0, delimiter); + } + } + text +} + +pub fn ansi_len(s: &str) -> usize { + String::from_utf8(strip_ansi_escapes::strip(s).unwrap()) + .unwrap() + .chars() + .count() +} + +// TODO: pulled from aiken; should we have our own config utility crate? +// https://github.com/aiken-lang/aiken/blob/main/crates/aiken-project/src/config.rs#L382C1-L393C2 +mod built_info { + include!(concat!(env!("OUT_DIR"), "/built.rs")); +} +pub fn node_info() -> String { + format!( + r#" +Operating System: {} +Architecture: {} +Version: {}"#, + built_info::CFG_OS, + built_info::CFG_TARGET_ARCH, + node_version(true), + ) +} +pub fn node_version(include_commit_hash: bool) -> String { + let version = built_info::PKG_VERSION; + let suffix = if include_commit_hash { + format!( + "+{}", + built_info::GIT_COMMIT_HASH_SHORT.unwrap_or("unknown") + ) + } else { + "".to_string() + }; + format!("v{version}{suffix}") +} From e9bfdf5931ff6b093e830bf0252c6be29eaa7372 Mon Sep 17 00:00:00 2001 From: Pi Lanningham Date: Fri, 27 Dec 2024 02:06:54 -0500 Subject: [PATCH 2/3] Add a sample panic For code reviewers to test if they'd like, just `cargo run daemon --peer-address 127.0.0.1`; will remove in the next commit --- crates/amaru/src/bin/amaru/cmd/daemon.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/amaru/src/bin/amaru/cmd/daemon.rs b/crates/amaru/src/bin/amaru/cmd/daemon.rs index b093d8c..5391a72 100644 --- a/crates/amaru/src/bin/amaru/cmd/daemon.rs +++ b/crates/amaru/src/bin/amaru/cmd/daemon.rs @@ -34,6 +34,7 @@ pub struct Args { } pub async fn run(args: Args, counter: Counter) -> miette::Result<()> { + panic!("sample panic for testing"); let config = parse_args(args, counter)?; let client = Arc::new(Mutex::new( From 91c0b8b13c83fbe627786169980a1ae7800b12f0 Mon Sep 17 00:00:00 2001 From: Pi Lanningham Date: Fri, 27 Dec 2024 02:07:14 -0500 Subject: [PATCH 3/3] Remove example panic --- crates/amaru/src/bin/amaru/cmd/daemon.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/amaru/src/bin/amaru/cmd/daemon.rs b/crates/amaru/src/bin/amaru/cmd/daemon.rs index 5391a72..b093d8c 100644 --- a/crates/amaru/src/bin/amaru/cmd/daemon.rs +++ b/crates/amaru/src/bin/amaru/cmd/daemon.rs @@ -34,7 +34,6 @@ pub struct Args { } pub async fn run(args: Args, counter: Counter) -> miette::Result<()> { - panic!("sample panic for testing"); let config = parse_args(args, counter)?; let client = Arc::new(Mutex::new(