From d9e7b898a303df232f89832cac828a841160a31c Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 16 Oct 2023 19:26:06 +0800 Subject: [PATCH] feat: add walconfig dir back (#2606) Signed-off-by: ZhangJian He --- config/standalone.example.toml | 2 ++ src/cmd/src/datanode.rs | 11 +++++++++++ src/cmd/src/options.rs | 3 +++ src/cmd/src/standalone.rs | 2 ++ src/common/config/src/lib.rs | 5 ++++- src/datanode/src/datanode.rs | 5 ++++- 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/config/standalone.example.toml b/config/standalone.example.toml index c274d11fa376..e4bd41a616c4 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -82,6 +82,8 @@ enable = true # WAL options. [wal] +# WAL data directory +# dir = "/tmp/greptimedb/wal" # WAL file size in bytes. file_size = "256MB" # WAL purge threshold. diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs index fbf4d2cb7333..da2466497bbc 100644 --- a/src/cmd/src/datanode.rs +++ b/src/cmd/src/datanode.rs @@ -96,6 +96,8 @@ struct StartCommand { #[clap(long)] data_home: Option, #[clap(long)] + wal_dir: Option, + #[clap(long)] http_addr: Option, #[clap(long)] http_timeout: Option, @@ -149,6 +151,10 @@ impl StartCommand { opts.storage.data_home = data_home.clone(); } + if let Some(wal_dir) = &self.wal_dir { + opts.wal.dir = Some(wal_dir.clone()); + } + if let Some(http_addr) = &self.http_addr { opts.http.addr = http_addr.clone(); } @@ -255,6 +261,7 @@ mod tests { assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr); assert_eq!(Some(42), options.node_id); + assert_eq!("/other/wal", options.wal.dir.unwrap()); assert_eq!(Duration::from_secs(600), options.wal.purge_interval); assert_eq!(1024 * 1024 * 1024, options.wal.file_size.0); @@ -439,6 +446,7 @@ mod tests { || { let command = StartCommand { config_file: Some(file.path().to_str().unwrap().to_string()), + wal_dir: Some("/other/wal/dir".to_string()), env_prefix: env_prefix.to_string(), ..Default::default() }; @@ -466,6 +474,9 @@ mod tests { // Should be read from config file, config file > env > default values. assert_eq!(opts.storage.compaction.max_purge_tasks, 32); + // Should be read from cli, cli > config file > env > default values. + assert_eq!(opts.wal.dir.unwrap(), "/other/wal/dir"); + // Should be default value. assert_eq!( opts.storage.manifest.checkpoint_margin, diff --git a/src/cmd/src/options.rs b/src/cmd/src/options.rs index dbd260405d43..9bd6fc07e46d 100644 --- a/src/cmd/src/options.rs +++ b/src/cmd/src/options.rs @@ -263,6 +263,9 @@ mod tests { ] ); + // Should be the values from config file, not environment variables. + assert_eq!(opts.wal.dir.unwrap(), "/tmp/greptimedb/wal"); + // Should be default values. assert_eq!(opts.node_id, None); }, diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 4a87294d3e7d..86bafc546348 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -493,6 +493,8 @@ mod tests { assert_eq!(None, fe_opts.mysql.reject_no_database); assert!(fe_opts.influxdb.enable); + assert_eq!("/tmp/greptimedb/test/wal", dn_opts.wal.dir.unwrap()); + match &dn_opts.storage.store { datanode::config::ObjectStoreConfig::S3(s3_config) => { assert_eq!( diff --git a/src/common/config/src/lib.rs b/src/common/config/src/lib.rs index fd232735873e..34418c147cf1 100644 --- a/src/common/config/src/lib.rs +++ b/src/common/config/src/lib.rs @@ -20,6 +20,8 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(default)] pub struct WalConfig { + // wal directory + pub dir: Option, // wal file size in bytes pub file_size: ReadableSize, // wal purge threshold in bytes @@ -36,7 +38,8 @@ pub struct WalConfig { impl Default for WalConfig { fn default() -> Self { Self { - file_size: ReadableSize::mb(256), // log file size 256MB + dir: None, + file_size: ReadableSize::mb(256), // log file size 256MB purge_threshold: ReadableSize::gb(4), // purge threshold 4GB purge_interval: Duration::from_secs(600), read_batch_size: 128, diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs index d1ad159ffa9c..8a6b437de602 100644 --- a/src/datanode/src/datanode.rs +++ b/src/datanode/src/datanode.rs @@ -367,7 +367,10 @@ impl DatanodeBuilder { /// Build [RaftEngineLogStore] async fn build_log_store(opts: &DatanodeOptions) -> Result> { let data_home = normalize_dir(&opts.storage.data_home); - let wal_dir = format!("{}{WAL_DIR}", data_home); + let wal_dir = match &opts.wal.dir { + Some(dir) => dir.clone(), + None => format!("{}{WAL_DIR}", data_home), + }; let wal_config = opts.wal.clone(); // create WAL directory