From a9cfa5c05d35778fe065a84565bdf3e4720e903e Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 7 Feb 2024 16:47:46 +0800 Subject: [PATCH] refactor(configs): Move configs defination and declearation to global namespace (#1876) Move the uses of DSN_DECLARE_* and DSN_DEFINE_* to global namespace because they are all global variables, we can access them without any namespace scopes. --- src/block_service/hdfs/hdfs_service.cpp | 24 ++++----- src/block_service/test/hdfs_service_test.cpp | 6 +-- src/common/common.cpp | 3 +- src/common/duplication_common.cpp | 6 +-- src/common/duplication_common.h | 4 +- src/common/fs_manager.cpp | 28 +++++----- src/common/fs_manager.h | 4 +- src/common/replication_common.cpp | 6 +-- .../test/failure_detector.cpp | 3 +- src/geo/lib/geo_client.cpp | 5 +- src/geo/test/geo_test.cpp | 4 +- src/http/http_client.cpp | 4 +- src/http/http_server.cpp | 4 +- src/http/http_server.h | 4 +- src/meta/app_balance_policy.cpp | 9 ++-- src/meta/cluster_balance_policy.cpp | 8 +-- src/meta/greedy_load_balancer.cpp | 10 ++-- src/meta/load_balance_policy.cpp | 3 +- src/meta/meta_backup_service.cpp | 7 ++- src/meta/meta_bulk_load_ingestion_context.cpp | 6 +-- src/meta/meta_bulk_load_ingestion_context.h | 6 +-- src/meta/meta_bulk_load_service.cpp | 6 +-- src/meta/meta_bulk_load_service.h | 6 +-- src/meta/meta_data.cpp | 6 +-- src/meta/meta_http_service.cpp | 7 ++- src/meta/meta_options.cpp | 5 +- src/meta/meta_service.cpp | 40 +++++++-------- src/meta/meta_state_service_zookeeper.cpp | 4 +- src/meta/partition_guardian.cpp | 6 +-- src/meta/server_state.cpp | 5 +- src/meta/test/backup_test.cpp | 8 +-- src/meta/test/main.cpp | 4 +- src/meta/test/meta_app_operation_test.cpp | 8 +-- src/meta/test/meta_test_base.cpp | 6 +-- src/meta/test/server_state_test.cpp | 5 +- src/meta/test/state_sync_test.cpp | 6 +-- src/meta/test/update_configuration_test.cpp | 6 +-- src/nfs/nfs_client_impl.cpp | 50 +++++++++--------- src/nfs/nfs_server_impl.cpp | 10 ++-- src/perf_counter/perf_counter_atomic.cpp | 3 +- src/perf_counter/test/perf_counter_test.cpp | 10 ++-- src/ranger/ranger_resource_policy_manager.cpp | 6 +-- .../ranger_resource_policy_manager_test.cpp | 3 +- src/replica/backup/replica_backup_manager.cpp | 6 +-- src/replica/backup/replica_backup_server.cpp | 3 +- src/replica/disk_cleaner.cpp | 5 +- src/replica/disk_cleaner.h | 8 +-- .../test/load_from_private_log_test.cpp | 3 +- src/replica/log_file.cpp | 2 +- src/replica/mutation.cpp | 5 +- src/replica/mutation_log.cpp | 9 ++-- src/replica/replica.cpp | 51 ++++++++++--------- src/replica/replica.h | 5 +- src/replica/replica_2pc.cpp | 5 +- src/replica/replica_backup.cpp | 15 +++--- src/replica/replica_check.cpp | 8 +-- src/replica/replica_chkpt.cpp | 8 +-- src/replica/replica_config.cpp | 4 +- src/replica/replica_init.cpp | 5 +- src/replica/replica_learn.cpp | 8 +-- src/replica/replica_stub.cpp | 27 +++++----- src/replica/replica_stub.h | 4 +- src/replica/split/replica_split_manager.cpp | 6 +-- .../storage/simple_kv/simple_kv.main.cpp | 2 +- .../storage/simple_kv/test/checker.cpp | 4 +- .../storage/simple_kv/test/simple_kv.main.cpp | 2 +- src/replica/test/mock_utils.h | 4 +- src/replica/test/replica_disk_test.cpp | 3 +- .../test/replica_http_service_test.cpp | 13 +++-- src/replica/test/replica_test.cpp | 5 +- src/runtime/env.sim.cpp | 6 +-- src/runtime/nativerun.cpp | 4 +- src/runtime/profiler.cpp | 14 ++--- src/runtime/providers.common.cpp | 4 +- src/runtime/rpc/asio_net_provider.cpp | 10 ++-- src/runtime/rpc/network.cpp | 2 +- src/runtime/rpc/network.sim.cpp | 10 ++-- src/runtime/rpc/rpc_engine.cpp | 2 +- src/runtime/rpc/rpc_message.cpp | 7 +-- src/runtime/service_api_c.cpp | 9 ++-- src/runtime/simulator.cpp | 4 +- src/runtime/task/task_spec.cpp | 7 ++- src/runtime/test/netprovider.cpp | 3 +- src/runtime/tracer.cpp | 4 +- src/security/access_controller.cpp | 5 +- src/security/init.cpp | 5 +- src/security/kinit_context.cpp | 17 +++---- src/security/meta_access_controller.cpp | 5 +- src/security/negotiation.cpp | 6 +-- src/security/negotiation_manager.cpp | 5 +- src/security/replica_access_controller.cpp | 5 +- src/security/sasl_client_wrapper.cpp | 5 +- src/security/sasl_init.cpp | 3 +- src/security/sasl_server_wrapper.cpp | 5 +- src/security/server_negotiation.cpp | 5 +- .../test/meta_access_controller_test.cpp | 3 +- .../test/negotiation_manager_test.cpp | 5 +- .../test/replica_access_controller_test.cpp | 3 +- src/server/available_detector.cpp | 10 ++-- src/server/capacity_unit_calculator.cpp | 6 +-- src/server/hotkey_collector.cpp | 6 +-- src/server/hotspot_partition_calculator.cpp | 10 ++-- src/server/info_collector.cpp | 22 ++++---- src/server/pegasus_event_listener.cpp | 8 +-- src/server/pegasus_manual_compact_service.cpp | 9 ++-- src/server/pegasus_mutation_duplicator.cpp | 3 +- src/server/pegasus_server_impl.cpp | 34 ++++++------- src/server/pegasus_server_impl.h | 12 ++--- src/server/pegasus_server_impl_init.cpp | 16 +++--- src/server/pegasus_server_write.cpp | 3 +- src/server/pegasus_write_service.cpp | 15 +++--- src/server/result_writer.cpp | 6 +-- src/server/rocksdb_wrapper.cpp | 10 ++-- .../test/capacity_unit_calculator_test.cpp | 6 +-- src/server/test/hotkey_collector_test.cpp | 4 +- src/server/test/hotspot_partition_test.cpp | 6 +-- .../test/manual_compact_service_test.cpp | 4 +- src/test/bench_test/benchmark.cpp | 6 +-- .../security/test_kms_client.cpp | 13 ++--- .../function_test/throttle/test_throttle.cpp | 14 ++--- src/test/kill_test/data_verifier.cpp | 32 ++++++------ src/test/kill_test/kill_testor.cpp | 5 +- src/test/kill_test/killer_handler_shell.cpp | 6 +-- src/test/kill_test/partition_kill_testor.cpp | 4 +- src/test/kill_test/process_kill_testor.cpp | 6 +-- src/test/pressure_test/config-pressure.ini | 2 +- src/test/pressure_test/main.cpp | 16 +++--- src/test_util/test_util.h | 4 +- src/tools/mutation_log_tool.cpp | 4 +- src/utils/builtin_metrics.cpp | 4 +- src/utils/latency_tracer.cpp | 5 +- src/utils/latency_tracer.h | 5 +- src/utils/logging.cpp | 3 +- src/utils/metrics.cpp | 16 +++--- src/utils/shared_io_service.cpp | 8 +-- src/utils/simple_logger.cpp | 11 ++-- src/utils/test/flag_test.cpp | 6 +-- src/utils/test/logger.cpp | 4 +- src/utils/test/metrics_test.cpp | 4 +- .../distributed_lock_service_zookeeper.cpp | 4 +- src/zookeeper/zookeeper_session.cpp | 20 +++----- src/zookeeper/zookeeper_session_mgr.cpp | 4 +- 142 files changed, 568 insertions(+), 575 deletions(-) diff --git a/src/block_service/hdfs/hdfs_service.cpp b/src/block_service/hdfs/hdfs_service.cpp index e50a8efe25..b8e9dee4ee 100644 --- a/src/block_service/hdfs/hdfs_service.cpp +++ b/src/block_service/hdfs/hdfs_service.cpp @@ -39,18 +39,6 @@ #include "utils/safe_strerror_posix.h" #include "utils/strings.h" -DSN_DECLARE_bool(enable_direct_io); - -struct hdfsBuilder; - -namespace dsn { -class task_tracker; - -namespace dist { -namespace block_service { - -DEFINE_TASK_CODE(LPC_HDFS_SERVICE_CALL, TASK_PRIORITY_COMMON, THREAD_POOL_BLOCK_SERVICE) - DSN_DEFINE_uint64(replication, hdfs_read_batch_size_bytes, 64 << 20, @@ -69,6 +57,18 @@ DSN_DEFINE_uint64(replication, "hdfs write batch size, the default value is 64MB"); DSN_TAG_VARIABLE(hdfs_write_batch_size_bytes, FT_MUTABLE); +DSN_DECLARE_bool(enable_direct_io); + +struct hdfsBuilder; + +namespace dsn { +class task_tracker; + +namespace dist { +namespace block_service { + +DEFINE_TASK_CODE(LPC_HDFS_SERVICE_CALL, TASK_PRIORITY_COMMON, THREAD_POOL_BLOCK_SERVICE) + hdfs_service::hdfs_service() { _read_token_bucket.reset(new folly::DynamicTokenBucket()); diff --git a/src/block_service/test/hdfs_service_test.cpp b/src/block_service/test/hdfs_service_test.cpp index 54a1337b3e..f96549ec98 100644 --- a/src/block_service/test/hdfs_service_test.cpp +++ b/src/block_service/test/hdfs_service_test.cpp @@ -48,9 +48,6 @@ #include "utils/test_macros.h" #include "utils/threadpool_code.h" -using namespace dsn; -using namespace dsn::dist::block_service; - DSN_DEFINE_string(hdfs_test, test_name_node, "", "hdfs name node"); DSN_DEFINE_string(hdfs_test, test_backup_path, "", "path for uploading and downloading test files"); @@ -60,6 +57,9 @@ DSN_DEFINE_uint32(hdfs_test, 64, "number of total files for hdfs concurrent test"); +using namespace dsn; +using namespace dsn::dist::block_service; + DEFINE_TASK_CODE(LPC_TEST_HDFS, TASK_PRIORITY_HIGH, dsn::THREAD_POOL_DEFAULT) class HDFSClientTest : public pegasus::encrypt_data_test_base diff --git a/src/common/common.cpp b/src/common/common.cpp index f32a86aefc..9ea1f558d0 100644 --- a/src/common/common.cpp +++ b/src/common/common.cpp @@ -21,9 +21,10 @@ #include "utils/fmt_logging.h" #include "utils/strings.h" -namespace dsn { DSN_DEFINE_string(replication, cluster_name, "", "name of this cluster"); +namespace dsn { + /*extern*/ const char *get_current_cluster_name() { CHECK(!utils::is_empty(FLAGS_cluster_name), "cluster_name is not set"); diff --git a/src/common/duplication_common.cpp b/src/common/duplication_common.cpp index cb2fb02984..e46a7556b0 100644 --- a/src/common/duplication_common.cpp +++ b/src/common/duplication_common.cpp @@ -31,15 +31,15 @@ #include "utils/singleton.h" #include "utils/time_utils.h" -namespace dsn { -namespace replication { - DSN_DEFINE_uint32(replication, duplicate_log_batch_bytes, 4096, "send mutation log batch bytes size per rpc"); DSN_TAG_VARIABLE(duplicate_log_batch_bytes, FT_MUTABLE); +namespace dsn { +namespace replication { + const std::string duplication_constants::kDuplicationCheckpointRootDir /*NOLINT*/ = "duplication"; const std::string duplication_constants::kClustersSectionName /*NOLINT*/ = "pegasus.clusters"; const std::string duplication_constants::kDuplicationEnvMasterClusterKey /*NOLINT*/ = diff --git a/src/common/duplication_common.h b/src/common/duplication_common.h index 6b953f6e96..fcc43d3b20 100644 --- a/src/common/duplication_common.h +++ b/src/common/duplication_common.h @@ -30,11 +30,11 @@ #include "utils/flags.h" #include "utils/fmt_utils.h" +DSN_DECLARE_uint32(duplicate_log_batch_bytes); + namespace dsn { namespace replication { -DSN_DECLARE_uint32(duplicate_log_batch_bytes); - typedef rpc_holder duplication_modify_rpc; typedef rpc_holder duplication_add_rpc; typedef rpc_holder duplication_query_rpc; diff --git a/src/common/fs_manager.cpp b/src/common/fs_manager.cpp index 89487b0c63..2aaa953c1b 100644 --- a/src/common/fs_manager.cpp +++ b/src/common/fs_manager.cpp @@ -43,6 +43,20 @@ #include "utils/math.h" #include "utils/ports.h" +DSN_DEFINE_int32(replication, + disk_min_available_space_ratio, + 10, + "if disk available space ratio " + "is below this value, this " + "disk will be considered as " + "space insufficient"); +DSN_TAG_VARIABLE(disk_min_available_space_ratio, FT_MUTABLE); + +DSN_DEFINE_bool(replication, + ignore_broken_disk, + true, + "true means ignore broken data disk when initialize"); + METRIC_DEFINE_entity(disk); METRIC_DEFINE_gauge_int64(disk, @@ -58,20 +72,6 @@ METRIC_DEFINE_gauge_int64(disk, namespace dsn { namespace replication { -DSN_DEFINE_int32(replication, - disk_min_available_space_ratio, - 10, - "if disk available space ratio " - "is below this value, this " - "disk will be considered as " - "space insufficient"); -DSN_TAG_VARIABLE(disk_min_available_space_ratio, FT_MUTABLE); - -DSN_DEFINE_bool(replication, - ignore_broken_disk, - true, - "true means ignore broken data disk when initialize"); - error_code disk_status_to_error_code(disk_status::type ds) { switch (ds) { diff --git a/src/common/fs_manager.h b/src/common/fs_manager.h index b4217323eb..4a9d1f69fc 100644 --- a/src/common/fs_manager.h +++ b/src/common/fs_manager.h @@ -36,14 +36,14 @@ #include "utils/ports.h" #include "utils/zlocks.h" +DSN_DECLARE_int32(disk_min_available_space_ratio); + namespace dsn { class gpid; namespace replication { class disk_info; -DSN_DECLARE_int32(disk_min_available_space_ratio); - error_code disk_status_to_error_code(disk_status::type ds); class disk_capacity_metrics diff --git a/src/common/replication_common.cpp b/src/common/replication_common.cpp index b5d25713fe..c656beb4d2 100644 --- a/src/common/replication_common.cpp +++ b/src/common/replication_common.cpp @@ -44,9 +44,6 @@ #include "utils/string_conv.h" #include "utils/strings.h" -namespace dsn { -namespace replication { - DSN_DEFINE_bool(replication, duplication_enabled, true, "is duplication enabled"); DSN_DEFINE_int32(replication, @@ -104,6 +101,9 @@ DSN_DEFINE_string(replication, "", "The prefix of cold backup data path on remote storage"); +namespace dsn { +namespace replication { + replication_options::~replication_options() {} void replication_options::initialize() diff --git a/src/failure_detector/test/failure_detector.cpp b/src/failure_detector/test/failure_detector.cpp index 42b3cbb783..1a726a82dc 100644 --- a/src/failure_detector/test/failure_detector.cpp +++ b/src/failure_detector/test/failure_detector.cpp @@ -65,12 +65,11 @@ #include "utils/zlocks.h" DSN_DECLARE_int32(max_succssive_unstable_restart); +DSN_DECLARE_uint64(stable_rs_min_running_seconds); using namespace dsn; using namespace dsn::fd; -DSN_DECLARE_uint64(stable_rs_min_running_seconds); - #define MPORT_START 30001 #define WPORT 40001 #define MCOUNT 3 diff --git a/src/geo/lib/geo_client.cpp b/src/geo/lib/geo_client.cpp index 1cffa9fd06..bc415eeb34 100644 --- a/src/geo/lib/geo_client.cpp +++ b/src/geo/lib/geo_client.cpp @@ -50,8 +50,6 @@ #include "utils/fmt_logging.h" #include "utils/synchronize.h" -namespace pegasus { -namespace geo { DSN_DEFINE_int32(geo_client.lib, min_level, 12, @@ -79,6 +77,9 @@ DSN_DEFINE_group_validator(min_max_level, [](std::string &message) -> bool { DSN_DEFINE_uint32(geo_client.lib, latitude_index, 5, "latitude index in value"); DSN_DEFINE_uint32(geo_client.lib, longitude_index, 4, "longitude index in value"); +namespace pegasus { +namespace geo { + struct SearchResultNearer { inline bool operator()(const SearchResult &l, const SearchResult &r) diff --git a/src/geo/test/geo_test.cpp b/src/geo/test/geo_test.cpp index aacd668e14..bb4cc92da6 100644 --- a/src/geo/test/geo_test.cpp +++ b/src/geo/test/geo_test.cpp @@ -47,11 +47,11 @@ #include "utils/fmt_logging.h" #include "utils/string_conv.h" +DSN_DECLARE_int32(min_level); + namespace pegasus { namespace geo { -DSN_DECLARE_int32(min_level); - // TODO(yingchun): it doesn't make sense to derive from pegasus::encrypt_data_test_base to test // encryption or non-encryption senarios, because the Pegasus cluster has been started with a // fixed value of FLAGS_encrypt_data_at_rest. diff --git a/src/http/http_client.cpp b/src/http/http_client.cpp index 612d71a63f..24dd575f5c 100644 --- a/src/http/http_client.cpp +++ b/src/http/http_client.cpp @@ -25,14 +25,14 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" -namespace dsn { - DSN_DEFINE_uint32(http, curl_timeout_ms, 10000, "The maximum time in milliseconds that you allow the libcurl transfer operation " "to complete"); +namespace dsn { + #define RETURN_IF_CURL_NOT_OK(expr, ok, ...) \ do { \ const auto &code = (expr); \ diff --git a/src/http/http_server.cpp b/src/http/http_server.cpp index 94de34bb7b..d8e38cf53b 100644 --- a/src/http/http_server.cpp +++ b/src/http/http_server.cpp @@ -41,10 +41,10 @@ #include "utils/output_utils.h" #include "utils/strings.h" -namespace dsn { - DSN_DEFINE_bool(http, enable_http_server, true, "whether to enable the embedded HTTP server"); +namespace dsn { + namespace { error_s update_config(const http_request &req) { diff --git a/src/http/http_server.h b/src/http/http_server.h index 71fd8bde2f..0f600ec4fe 100644 --- a/src/http/http_server.h +++ b/src/http/http_server.h @@ -33,10 +33,10 @@ #include "utils/flags.h" #include "utils/threadpool_code.h" -namespace dsn { - DSN_DECLARE_bool(enable_http_server); +namespace dsn { + /// The rpc code for all the HTTP RPCs. DEFINE_TASK_CODE_RPC(RPC_HTTP_SERVICE, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT); diff --git a/src/meta/app_balance_policy.cpp b/src/meta/app_balance_policy.cpp index 00ebbc6cf0..cafeed9adc 100644 --- a/src/meta/app_balance_policy.cpp +++ b/src/meta/app_balance_policy.cpp @@ -28,16 +28,17 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" -namespace dsn { -class rpc_address; - -namespace replication { DSN_DEFINE_bool(meta_server, balancer_in_turn, false, "balance the apps one-by-one/concurrently"); DSN_DEFINE_bool(meta_server, only_primary_balancer, false, "only try to make the primary balanced"); DSN_DEFINE_bool(meta_server, only_move_primary, false, "only try to make the primary balanced by move"); + +namespace dsn { +class rpc_address; + +namespace replication { app_balance_policy::app_balance_policy(meta_service *svc) : load_balance_policy(svc) { if (_svc != nullptr) { diff --git a/src/meta/cluster_balance_policy.cpp b/src/meta/cluster_balance_policy.cpp index febfff5355..6a33cd30a7 100644 --- a/src/meta/cluster_balance_policy.cpp +++ b/src/meta/cluster_balance_policy.cpp @@ -30,16 +30,16 @@ #include "utils/fmt_logging.h" #include "utils/utils.h" -namespace dsn { -namespace replication { -class meta_service; - DSN_DEFINE_uint32(meta_server, balance_op_count_per_round, 10, "balance operation count per round for cluster balancer"); DSN_TAG_VARIABLE(balance_op_count_per_round, FT_MUTABLE); +namespace dsn { +namespace replication { +class meta_service; + uint32_t get_partition_count(const node_state &ns, balance_type type, int32_t app_id) { unsigned count = 0; diff --git a/src/meta/greedy_load_balancer.cpp b/src/meta/greedy_load_balancer.cpp index dc8ef4f9d9..48f79258aa 100644 --- a/src/meta/greedy_load_balancer.cpp +++ b/src/meta/greedy_load_balancer.cpp @@ -49,16 +49,16 @@ #include "utils/math.h" #include "utils/metrics.h" -namespace dsn { -class gpid; - -namespace replication { - DSN_DEFINE_bool(meta_server, balance_cluster, false, "whether to enable cluster balancer"); DSN_TAG_VARIABLE(balance_cluster, FT_MUTABLE); DSN_DECLARE_uint64(min_live_node_count_for_unfreeze); +namespace dsn { +class gpid; + +namespace replication { + greedy_load_balancer::greedy_load_balancer(meta_service *_svc) : server_load_balancer(_svc) { _app_balance_policy = std::make_unique(_svc); diff --git a/src/meta/load_balance_policy.cpp b/src/meta/load_balance_policy.cpp index 59d30dd2e4..8cff520c94 100644 --- a/src/meta/load_balance_policy.cpp +++ b/src/meta/load_balance_policy.cpp @@ -37,9 +37,10 @@ #include "absl/strings/string_view.h" #include "utils/strings.h" +DSN_DECLARE_uint64(min_live_node_count_for_unfreeze); + namespace dsn { namespace replication { -DSN_DECLARE_uint64(min_live_node_count_for_unfreeze); void dump_disk_load(app_id id, const rpc_address &node, bool only_primary, const disk_load &load) { diff --git a/src/meta/meta_backup_service.cpp b/src/meta/meta_backup_service.cpp index 93c4ac739f..5c8a941ae9 100644 --- a/src/meta/meta_backup_service.cpp +++ b/src/meta/meta_backup_service.cpp @@ -53,8 +53,10 @@ #include "utils/fmt_logging.h" #include "utils/time_utils.h" -METRIC_DEFINE_entity(backup_policy); +DSN_DECLARE_int32(cold_backup_checkpoint_reserve_minutes); +DSN_DECLARE_int32(fd_lease_seconds); +METRIC_DEFINE_entity(backup_policy); METRIC_DEFINE_gauge_int64(backup_policy, backup_recent_duration_ms, dsn::metric_unit::kMilliSeconds, @@ -63,9 +65,6 @@ METRIC_DEFINE_gauge_int64(backup_policy, namespace dsn { namespace replication { -DSN_DECLARE_int32(cold_backup_checkpoint_reserve_minutes); -DSN_DECLARE_int32(fd_lease_seconds); - namespace { metric_entity_ptr instantiate_backup_policy_metric_entity(const std::string &policy_name) diff --git a/src/meta/meta_bulk_load_ingestion_context.cpp b/src/meta/meta_bulk_load_ingestion_context.cpp index fbb2c775e7..d6b9de80eb 100644 --- a/src/meta/meta_bulk_load_ingestion_context.cpp +++ b/src/meta/meta_bulk_load_ingestion_context.cpp @@ -28,9 +28,6 @@ #include "utils/string_conv.h" #include "absl/strings/string_view.h" -namespace dsn { -namespace replication { - DSN_DEFINE_uint32(meta_server, bulk_load_node_max_ingesting_count, 4, @@ -40,6 +37,9 @@ DSN_TAG_VARIABLE(bulk_load_node_max_ingesting_count, FT_MUTABLE); DSN_DEFINE_uint32(meta_server, bulk_load_node_min_disk_count, 1, "min disk count of one node"); DSN_TAG_VARIABLE(bulk_load_node_min_disk_count, FT_MUTABLE); +namespace dsn { +namespace replication { + ingestion_context::ingestion_context() { reset_all(); } ingestion_context::~ingestion_context() { reset_all(); } diff --git a/src/meta/meta_bulk_load_ingestion_context.h b/src/meta/meta_bulk_load_ingestion_context.h index 50e7872d5f..3b18bce273 100644 --- a/src/meta/meta_bulk_load_ingestion_context.h +++ b/src/meta/meta_bulk_load_ingestion_context.h @@ -25,15 +25,15 @@ #include "runtime/rpc/rpc_address.h" #include "utils/flags.h" +DSN_DECLARE_uint32(bulk_load_node_max_ingesting_count); +DSN_DECLARE_uint32(bulk_load_node_min_disk_count); + namespace dsn { class partition_configuration; namespace replication { class config_context; -DSN_DECLARE_uint32(bulk_load_node_max_ingesting_count); -DSN_DECLARE_uint32(bulk_load_node_min_disk_count); - // Meta bulk load helper class, used to manage ingesting partitions class ingestion_context { diff --git a/src/meta/meta_bulk_load_service.cpp b/src/meta/meta_bulk_load_service.cpp index 8648cf9892..082a635b3f 100644 --- a/src/meta/meta_bulk_load_service.cpp +++ b/src/meta/meta_bulk_load_service.cpp @@ -51,9 +51,6 @@ #include "utils/string_conv.h" #include "absl/strings/string_view.h" -namespace dsn { -namespace replication { - DSN_DEFINE_uint32(meta_server, bulk_load_max_rollback_times, 10, @@ -76,6 +73,9 @@ DSN_DEFINE_bool(meta_server, "whether to enable different apps to execute bulk load at the same time"); DSN_TAG_VARIABLE(enable_concurrent_bulk_load, FT_MUTABLE); +namespace dsn { +namespace replication { + bulk_load_service::bulk_load_service(meta_service *meta_svc, const std::string &bulk_load_dir) : _meta_svc(meta_svc), _state(meta_svc->get_server_state()), _bulk_load_root(bulk_load_dir) { diff --git a/src/meta/meta_bulk_load_service.h b/src/meta/meta_bulk_load_service.h index c411d87f44..216a4b9911 100644 --- a/src/meta/meta_bulk_load_service.h +++ b/src/meta/meta_bulk_load_service.h @@ -42,6 +42,9 @@ #include "utils/flags.h" #include "utils/zlocks.h" +DSN_DECLARE_uint32(bulk_load_max_rollback_times); +DSN_DECLARE_bool(enable_concurrent_bulk_load); + namespace dsn { class partition_configuration; @@ -50,9 +53,6 @@ class app_state; class config_context; class meta_service; -DSN_DECLARE_uint32(bulk_load_max_rollback_times); -DSN_DECLARE_bool(enable_concurrent_bulk_load); - /// /// bulk load path on remote storage: /// /bulk_load/ -> app_bulk_load_info diff --git a/src/meta/meta_data.cpp b/src/meta/meta_data.cpp index a44180d4a0..46cd8ac5cd 100644 --- a/src/meta/meta_data.cpp +++ b/src/meta/meta_data.cpp @@ -36,9 +36,6 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" -namespace dsn { -namespace replication { - // There is an option FLAGS_max_replicas_in_group which restricts the max replica count of the whole // cluster. It's a cluster-level option. However, now that it's allowed to update the replication // factor of each table, this cluster-level option should be replaced. @@ -75,6 +72,9 @@ DSN_DEFINE_uint32(meta_server, "max reserved number allowed for dropped replicas"); DSN_TAG_VARIABLE(max_reserved_dropped_replicas, FT_MUTABLE); +namespace dsn { +namespace replication { + void when_update_replicas(config_type::type t, const std::function &func) { switch (t) { diff --git a/src/meta/meta_http_service.cpp b/src/meta/meta_http_service.cpp index 19f3e53451..8ccbc82299 100644 --- a/src/meta/meta_http_service.cpp +++ b/src/meta/meta_http_service.cpp @@ -56,10 +56,9 @@ #include "utils/output_utils.h" #include "utils/time_utils.h" -namespace dsn { -namespace dist { DSN_DECLARE_string(hosts_list); -} // namespace dist + +namespace dsn { namespace replication { struct list_nodes_helper @@ -477,7 +476,7 @@ void meta_http_service::get_cluster_info_handler(const http_request &req, http_r } tp.add_row_name_and_data("meta_servers", meta_servers_str); tp.add_row_name_and_data("primary_meta_server", dsn_primary_address().to_std_string()); - tp.add_row_name_and_data("zookeeper_hosts", dsn::dist::FLAGS_hosts_list); + tp.add_row_name_and_data("zookeeper_hosts", FLAGS_hosts_list); tp.add_row_name_and_data("zookeeper_root", _service->_cluster_root); tp.add_row_name_and_data( "meta_function_level", diff --git a/src/meta/meta_options.cpp b/src/meta/meta_options.cpp index cb1e770beb..fa23894762 100644 --- a/src/meta/meta_options.cpp +++ b/src/meta/meta_options.cpp @@ -34,8 +34,6 @@ #include "utils/fmt_logging.h" #include "utils/strings.h" -namespace dsn { -namespace replication { // TODO(yingchun): add more description for string configs, and add validators DSN_DEFINE_string(meta_server, meta_state_service_parameters, @@ -54,6 +52,9 @@ DSN_DEFINE_string(meta_server, "", "white list of replica-servers in meta-server"); +namespace dsn { +namespace replication { + void meta_options::initialize() { utils::split_args(FLAGS_meta_state_service_parameters, meta_state_service_args); diff --git a/src/meta/meta_service.cpp b/src/meta/meta_service.cpp index 952b33a687..844de1be8a 100644 --- a/src/meta/meta_service.cpp +++ b/src/meta/meta_service.cpp @@ -69,27 +69,7 @@ #include "utils/fmt_logging.h" #include "utils/strings.h" -METRIC_DEFINE_counter(server, - replica_server_disconnections, - dsn::metric_unit::kDisconnections, - "The number of disconnections with replica servers"); - -METRIC_DEFINE_gauge_int64(server, - unalive_replica_servers, - dsn::metric_unit::kServers, - "The number of unalive replica servers"); - -METRIC_DEFINE_gauge_int64(server, - alive_replica_servers, - dsn::metric_unit::kServers, - "The number of alive replica servers"); - -namespace dsn { -namespace dist { DSN_DECLARE_string(hosts_list); -} // namespace dist - -namespace replication { DSN_DEFINE_bool(meta_server, recover_from_replica_server, false, @@ -142,6 +122,24 @@ DSN_DECLARE_int32(fd_grace_seconds); DSN_DECLARE_int32(fd_lease_seconds); DSN_DECLARE_string(cold_backup_root); +METRIC_DEFINE_counter(server, + replica_server_disconnections, + dsn::metric_unit::kDisconnections, + "The number of disconnections with replica servers"); + +METRIC_DEFINE_gauge_int64(server, + unalive_replica_servers, + dsn::metric_unit::kServers, + "The number of unalive replica servers"); + +METRIC_DEFINE_gauge_int64(server, + alive_replica_servers, + dsn::metric_unit::kServers, + "The number of alive replica servers"); + +namespace dsn { +namespace replication { + #define CHECK_APP_ID_STATUS_AND_AUTHZ(app_id) \ do { \ const auto &_app_id = (app_id); \ @@ -720,7 +718,7 @@ void meta_service::on_query_cluster_info(configuration_cluster_info_rpc rpc) response.keys.push_back("primary_meta_server"); response.values.push_back(dsn_primary_address().to_std_string()); response.keys.push_back("zookeeper_hosts"); - response.values.push_back(dsn::dist::FLAGS_hosts_list); + response.values.push_back(FLAGS_hosts_list); response.keys.push_back("zookeeper_root"); response.values.push_back(_cluster_root); response.keys.push_back("meta_function_level"); diff --git a/src/meta/meta_state_service_zookeeper.cpp b/src/meta/meta_state_service_zookeeper.cpp index a367c1897e..c300d8f50a 100644 --- a/src/meta/meta_state_service_zookeeper.cpp +++ b/src/meta/meta_state_service_zookeeper.cpp @@ -42,11 +42,11 @@ #include "zookeeper/zookeeper_session.h" #include "zookeeper/zookeeper_session_mgr.h" +DSN_DECLARE_int32(timeout_ms); + namespace dsn { namespace dist { -DSN_DECLARE_int32(timeout_ms); - class zoo_transaction : public meta_state_service::transaction_entries { public: diff --git a/src/meta/partition_guardian.cpp b/src/meta/partition_guardian.cpp index d99ce226ce..ebe26b2274 100644 --- a/src/meta/partition_guardian.cpp +++ b/src/meta/partition_guardian.cpp @@ -38,15 +38,15 @@ #include "utils/strings.h" #include "utils/time_utils.h" -namespace dsn { -namespace replication { - DSN_DEFINE_int32(meta_server, max_replicas_in_group, 4, "max replicas(alive & dead) in a group"); DSN_DEFINE_int64(meta_server, replica_assign_delay_ms_for_dropouts, 300000, "The delay milliseconds to dropout replicas assign"); +namespace dsn { +namespace replication { + partition_guardian::partition_guardian(meta_service *svc) : _svc(svc) { if (svc != nullptr) { diff --git a/src/meta/server_state.cpp b/src/meta/server_state.cpp index aef6d03183..997641e619 100644 --- a/src/meta/server_state.cpp +++ b/src/meta/server_state.cpp @@ -81,8 +81,6 @@ #include "utils/string_conv.h" #include "utils/strings.h" -namespace dsn { -namespace replication { DSN_DEFINE_bool(meta_server, add_secondary_enable_flow_control, false, @@ -127,6 +125,9 @@ DSN_DEFINE_int32(meta_server, DSN_DECLARE_bool(recover_from_replica_server); +namespace dsn { +namespace replication { + static const char *lock_state = "lock"; static const char *unlock_state = "unlock"; diff --git a/src/meta/test/backup_test.cpp b/src/meta/test/backup_test.cpp index 7d2500bf84..e2ff7d574d 100644 --- a/src/meta/test/backup_test.cpp +++ b/src/meta/test/backup_test.cpp @@ -60,15 +60,15 @@ #include "utils/time_utils.h" #include "utils/zlocks.h" +DSN_DECLARE_int32(cold_backup_checkpoint_reserve_minutes); +DSN_DECLARE_string(cluster_root); +DSN_DECLARE_string(meta_state_service_type); + namespace dsn { namespace replication { class meta_options; class mock_policy; -DSN_DECLARE_int32(cold_backup_checkpoint_reserve_minutes); -DSN_DECLARE_string(cluster_root); -DSN_DECLARE_string(meta_state_service_type); - struct method_record { dsn::utils::notify_event event; diff --git a/src/meta/test/main.cpp b/src/meta/test/main.cpp index e410572bcc..283282e2de 100644 --- a/src/meta/test/main.cpp +++ b/src/meta/test/main.cpp @@ -35,6 +35,8 @@ #include "utils/fmt_logging.h" #include "utils/threadpool_code.h" +DSN_DEFINE_uint32(tools.simulator, random_seed, 0, "random seed"); + int gtest_flags = 0; int gtest_ret = 0; @@ -46,8 +48,6 @@ DEFINE_TASK_CODE(TASK_META_TEST, TASK_PRIORITY_COMMON, THREAD_POOL_META_TEST) meta_service_test_app *g_app; -DSN_DEFINE_uint32(tools.simulator, random_seed, 0, "random seed"); - // as it is not easy to clean test environment in some cases, we simply run these tests in several // commands, // please check the script "run.sh" to modify the GTEST_FILTER diff --git a/src/meta/test/meta_app_operation_test.cpp b/src/meta/test/meta_app_operation_test.cpp index a6f3cce2c1..ae8a4da358 100644 --- a/src/meta/test/meta_app_operation_test.cpp +++ b/src/meta/test/meta_app_operation_test.cpp @@ -50,15 +50,15 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" +DSN_DECLARE_int32(max_allowed_replica_count); +DSN_DECLARE_int32(min_allowed_replica_count); +DSN_DECLARE_uint64(min_live_node_count_for_unfreeze); + namespace dsn { class blob; namespace replication { -DSN_DECLARE_int32(max_allowed_replica_count); -DSN_DECLARE_int32(min_allowed_replica_count); -DSN_DECLARE_uint64(min_live_node_count_for_unfreeze); - class meta_app_operation_test : public meta_test_base { public: diff --git a/src/meta/test/meta_test_base.cpp b/src/meta/test/meta_test_base.cpp index 6fc5371f29..0a9d440fc3 100644 --- a/src/meta/test/meta_test_base.cpp +++ b/src/meta/test/meta_test_base.cpp @@ -48,13 +48,13 @@ #include "utils/fmt_logging.h" #include "utils/zlocks.h" -namespace dsn { -namespace replication { - DSN_DECLARE_uint64(min_live_node_count_for_unfreeze); DSN_DECLARE_string(partition_guardian_type); DSN_DECLARE_string(server_load_balancer_type); +namespace dsn { +namespace replication { + meta_test_base::~meta_test_base() {} void meta_test_base::SetUp() diff --git a/src/meta/test/server_state_test.cpp b/src/meta/test/server_state_test.cpp index 25f5ac2ad7..0c6295b95d 100644 --- a/src/meta/test/server_state_test.cpp +++ b/src/meta/test/server_state_test.cpp @@ -46,11 +46,12 @@ #include "utils/error_code.h" #include "utils/flags.h" -namespace dsn { -namespace replication { DSN_DECLARE_string(cluster_root); DSN_DECLARE_string(meta_state_service_type); +namespace dsn { +namespace replication { + static const std::vector keys = {"manual_compact.once.trigger_time", "manual_compact.once.target_level", "manual_compact.once.bottommost_level_compaction", diff --git a/src/meta/test/state_sync_test.cpp b/src/meta/test/state_sync_test.cpp index ccd83ee744..6ebc610d50 100644 --- a/src/meta/test/state_sync_test.cpp +++ b/src/meta/test/state_sync_test.cpp @@ -54,13 +54,13 @@ #include "utils/strings.h" #include "utils/utils.h" +DSN_DECLARE_string(cluster_root); +DSN_DECLARE_string(meta_state_service_type); + namespace dsn { namespace replication { class meta_options; -DSN_DECLARE_string(cluster_root); -DSN_DECLARE_string(meta_state_service_type); - static void random_assign_partition_config(std::shared_ptr &app, const std::vector &server_list, int max_replica_count) diff --git a/src/meta/test/update_configuration_test.cpp b/src/meta/test/update_configuration_test.cpp index d41d47e37f..7db2a96268 100644 --- a/src/meta/test/update_configuration_test.cpp +++ b/src/meta/test/update_configuration_test.cpp @@ -67,13 +67,13 @@ #include "utils/fmt_logging.h" #include "utils/zlocks.h" -namespace dsn { -namespace replication { - DSN_DECLARE_int32(node_live_percentage_threshold_for_update); DSN_DECLARE_int64(replica_assign_delay_ms_for_dropouts); DSN_DECLARE_uint64(min_live_node_count_for_unfreeze); +namespace dsn { +namespace replication { + class fake_sender_meta_service : public dsn::replication::meta_service { private: diff --git a/src/nfs/nfs_client_impl.cpp b/src/nfs/nfs_client_impl.cpp index 63cb1b8b35..1f2656ddd2 100644 --- a/src/nfs/nfs_client_impl.cpp +++ b/src/nfs/nfs_client_impl.cpp @@ -40,31 +40,6 @@ #include "utils/fmt_logging.h" #include "utils/token_buckets.h" -METRIC_DEFINE_counter(server, - nfs_client_copy_bytes, - dsn::metric_unit::kBytes, - "The accumulated data size in bytes requested by client during nfs copy"); - -METRIC_DEFINE_counter(server, - nfs_client_copy_failed_requests, - dsn::metric_unit::kRequests, - "The number of failed nfs copy requests (requested by client)"); - -METRIC_DEFINE_counter( - server, - nfs_client_write_bytes, - dsn::metric_unit::kBytes, - "The accumulated data size in bytes that are written to local file in client"); - -METRIC_DEFINE_counter(server, - nfs_client_failed_writes, - dsn::metric_unit::kWrites, - "The number of failed writes to local file in client"); - -namespace dsn { -namespace service { -static uint32_t current_max_copy_rate_megabytes = 0; - DSN_DEFINE_uint32(nfs, nfs_copy_block_bytes, 4 * 1024 * 1024, @@ -113,6 +88,31 @@ DSN_DEFINE_int32(nfs, "rpc timeout in milliseconds for nfs copy, " "0 means use default timeout of rpc engine"); +METRIC_DEFINE_counter(server, + nfs_client_copy_bytes, + dsn::metric_unit::kBytes, + "The accumulated data size in bytes requested by client during nfs copy"); + +METRIC_DEFINE_counter(server, + nfs_client_copy_failed_requests, + dsn::metric_unit::kRequests, + "The number of failed nfs copy requests (requested by client)"); + +METRIC_DEFINE_counter( + server, + nfs_client_write_bytes, + dsn::metric_unit::kBytes, + "The accumulated data size in bytes that are written to local file in client"); + +METRIC_DEFINE_counter(server, + nfs_client_failed_writes, + dsn::metric_unit::kWrites, + "The number of failed writes to local file in client"); + +namespace dsn { +namespace service { +static uint32_t current_max_copy_rate_megabytes = 0; + nfs_client_impl::nfs_client_impl() : _concurrent_copy_request_count(0), _concurrent_local_write_count(0), diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index 80fbea5c04..df1418822d 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -56,11 +56,6 @@ METRIC_DEFINE_counter( dsn::metric_unit::kRequests, "The number of nfs copy requests (received by server) that fail to read local file in server"); -namespace dsn { -class disk_file; - -namespace service { - static const char *kMaxSendRateMegaBytesPerDiskDesc = "The maximum bandwidth (MB/s) of reading data per local disk " "when transferring data to remote node, 0 means no limit"; @@ -70,6 +65,11 @@ DSN_TAG_VARIABLE(max_send_rate_megabytes_per_disk, FT_MUTABLE); DSN_DECLARE_int32(file_close_timer_interval_ms_on_server); DSN_DECLARE_int32(file_close_expire_time_ms); +namespace dsn { +class disk_file; + +namespace service { + nfs_service_impl::nfs_service_impl() : ::dsn::serverlet("nfs"), METRIC_VAR_INIT_server(nfs_server_copy_bytes), diff --git a/src/perf_counter/perf_counter_atomic.cpp b/src/perf_counter/perf_counter_atomic.cpp index 0f88ccd476..a27b5ee957 100644 --- a/src/perf_counter/perf_counter_atomic.cpp +++ b/src/perf_counter/perf_counter_atomic.cpp @@ -30,14 +30,13 @@ #include "utils/flags.h" #include "utils/shared_io_service.h" -namespace dsn { - DSN_DEFINE_int32(components.pegasus_perf_counter_number_percentile_atomic, counter_computation_interval_seconds, 10, "The interval seconds of the system to compute the percentiles of the " "pegasus_perf_counter_number_percentile_atomic counters"); +namespace dsn { perf_counter_number_percentile_atomic::perf_counter_number_percentile_atomic( const char *app, const char *section, diff --git a/src/perf_counter/test/perf_counter_test.cpp b/src/perf_counter/test/perf_counter_test.cpp index 001ba464e6..2c8ba2a123 100644 --- a/src/perf_counter/test/perf_counter_test.cpp +++ b/src/perf_counter/test/perf_counter_test.cpp @@ -41,16 +41,16 @@ #include "utils/fmt_logging.h" #include "utils/shared_io_service.h" -using namespace dsn; -using namespace dsn::tools; - -const int count_times = 10000; - DSN_DEFINE_int32(components.simple_perf_counter, counter_computation_interval_seconds_for_testing, 3, "period"); +using namespace dsn; +using namespace dsn::tools; + +const int count_times = 10000; + static void adder_function(perf_counter_ptr pc, int id, const std::vector &vec) { for (int i = id; i < 10000; i += 10) diff --git a/src/ranger/ranger_resource_policy_manager.cpp b/src/ranger/ranger_resource_policy_manager.cpp index fe719a8987..4de545da83 100644 --- a/src/ranger/ranger_resource_policy_manager.cpp +++ b/src/ranger/ranger_resource_policy_manager.cpp @@ -56,9 +56,6 @@ #include "utils/process_utils.h" #include "utils/strings.h" -namespace dsn { -namespace ranger { - DSN_DEFINE_uint32(security, update_ranger_policy_interval_sec, 5, @@ -77,6 +74,9 @@ DSN_DEFINE_string(ranger, "The name of the Ranger database policy matched by the legacy table(The table " "name does not follow the naming rules of {database_name}.{table_name})"); +namespace dsn { +namespace ranger { + #define RETURN_ERR_IF_MISSING_MEMBER(obj, member) \ do { \ if (!obj.IsObject() || !obj.HasMember(member)) { \ diff --git a/src/ranger/test/ranger_resource_policy_manager_test.cpp b/src/ranger/test/ranger_resource_policy_manager_test.cpp index ba355970a4..a326dad48a 100644 --- a/src/ranger/test/ranger_resource_policy_manager_test.cpp +++ b/src/ranger/test/ranger_resource_policy_manager_test.cpp @@ -34,9 +34,10 @@ #include "utils/blob.h" #include "utils/flags.h" +DSN_DECLARE_string(legacy_table_database_mapping_policy_name); + namespace dsn { namespace ranger { -DSN_DECLARE_string(legacy_table_database_mapping_policy_name); TEST(ranger_resource_policy_manager_test, parse_policies_from_json_for_test) { diff --git a/src/replica/backup/replica_backup_manager.cpp b/src/replica/backup/replica_backup_manager.cpp index 390860883a..2fb70b7664 100644 --- a/src/replica/backup/replica_backup_manager.cpp +++ b/src/replica/backup/replica_backup_manager.cpp @@ -59,12 +59,12 @@ METRIC_DEFINE_gauge_int64(replica, dsn::metric_unit::kBytes, "The max size of uploaded files among backups"); -namespace dsn { -namespace replication { - DSN_DECLARE_int32(cold_backup_checkpoint_reserve_minutes); DSN_DECLARE_int32(gc_interval_ms); +namespace dsn { +namespace replication { + // returns true if this checkpoint dir belongs to the policy static bool is_policy_checkpoint(const std::string &chkpt_dirname, const std::string &policy_name) { diff --git a/src/replica/backup/replica_backup_server.cpp b/src/replica/backup/replica_backup_server.cpp index fba320e94e..ef7ddc4fcf 100644 --- a/src/replica/backup/replica_backup_server.cpp +++ b/src/replica/backup/replica_backup_server.cpp @@ -33,11 +33,12 @@ #include "utils/fmt_logging.h" #include "utils/strings.h" +DSN_DECLARE_string(cold_backup_root); + namespace dsn { class message_ex; namespace replication { -DSN_DECLARE_string(cold_backup_root); replica_backup_server::replica_backup_server(const replica_stub *rs) : _stub(rs) { diff --git a/src/replica/disk_cleaner.cpp b/src/replica/disk_cleaner.cpp index a4cc0d9f55..1e9a360c5c 100644 --- a/src/replica/disk_cleaner.cpp +++ b/src/replica/disk_cleaner.cpp @@ -38,9 +38,6 @@ #include "utils/string_conv.h" #include "absl/strings/string_view.h" -namespace dsn { -namespace replication { - DSN_DEFINE_uint64( replication, gc_disk_error_replica_interval_seconds, @@ -69,6 +66,8 @@ DSN_DEFINE_uint64(replication, "directory with '.ori' suffixed"); DSN_TAG_VARIABLE(gc_disk_migration_origin_replica_interval_seconds, FT_MUTABLE); +namespace dsn { +namespace replication { const std::string kFolderSuffixErr = ".err"; const std::string kFolderSuffixGar = ".gar"; const std::string kFolderSuffixBak = ".bak"; diff --git a/src/replica/disk_cleaner.h b/src/replica/disk_cleaner.h index 54e37db491..9815035eb0 100644 --- a/src/replica/disk_cleaner.h +++ b/src/replica/disk_cleaner.h @@ -26,15 +26,15 @@ #include "utils/errors.h" #include "utils/flags.h" -namespace dsn { -namespace replication { -struct dir_node; - DSN_DECLARE_uint64(gc_disk_error_replica_interval_seconds); DSN_DECLARE_uint64(gc_disk_garbage_replica_interval_seconds); DSN_DECLARE_uint64(gc_disk_migration_tmp_replica_interval_seconds); DSN_DECLARE_uint64(gc_disk_migration_origin_replica_interval_seconds); +namespace dsn { +namespace replication { +struct dir_node; + // the invalid folder suffix, server will check disk folder and deal with them extern const std::string kFolderSuffixErr; // replica error dir extern const std::string kFolderSuffixGar; // replica closed and assign garbage dir diff --git a/src/replica/duplication/test/load_from_private_log_test.cpp b/src/replica/duplication/test/load_from_private_log_test.cpp index 5953d07c2d..0d7141745d 100644 --- a/src/replica/duplication/test/load_from_private_log_test.cpp +++ b/src/replica/duplication/test/load_from_private_log_test.cpp @@ -68,6 +68,8 @@ #include "replica/mutation_log_utils.h" #include "test_util/test_util.h" +DSN_DECLARE_bool(plog_force_flush); + namespace dsn { namespace replication { @@ -144,7 +146,6 @@ class load_from_private_log_test : public duplication_test_base int last_commit_decree_start = 5; int decree_start = 10; { - DSN_DECLARE_bool(plog_force_flush); auto reserved_plog_force_flush = FLAGS_plog_force_flush; FLAGS_plog_force_flush = true; for (int i = decree_start; i <= num_entries + decree_start; i++) { diff --git a/src/replica/log_file.cpp b/src/replica/log_file.cpp index 22f2be1e8e..ab88a3e2e6 100644 --- a/src/replica/log_file.cpp +++ b/src/replica/log_file.cpp @@ -336,7 +336,7 @@ aio_task_ptr log_file::commit_log_blocks(log_appender &pending, hash); } - if (dsn_unlikely(utils::FLAGS_enable_latency_tracer)) { + if (dsn_unlikely(FLAGS_enable_latency_tracer)) { tsk->_tracer->set_parent_point_name("commit_pending_mutations"); tsk->_tracer->set_description("log"); for (const auto &mutation : pending.mutations()) { diff --git a/src/replica/mutation.cpp b/src/replica/mutation.cpp index 7814bf299a..2f097e420e 100644 --- a/src/replica/mutation.cpp +++ b/src/replica/mutation.cpp @@ -45,15 +45,14 @@ #include "utils/latency_tracer.h" #include "utils/ports.h" -namespace dsn { -namespace replication { - DSN_DEFINE_uint64(replication, abnormal_write_trace_latency_threshold, 1000 * 1000 * 1000, // 1s "latency trace will be logged when exceed the write latency threshold"); DSN_TAG_VARIABLE(abnormal_write_trace_latency_threshold, FT_MUTABLE); +namespace dsn { +namespace replication { std::atomic mutation::s_tid(0); mutation::mutation() diff --git a/src/replica/mutation_log.cpp b/src/replica/mutation_log.cpp index 7597bc9e49..84a1e5be37 100644 --- a/src/replica/mutation_log.cpp +++ b/src/replica/mutation_log.cpp @@ -54,13 +54,14 @@ #include "utils/latency_tracer.h" #include "utils/ports.h" -namespace dsn { -namespace replication { DSN_DEFINE_bool(replication, plog_force_flush, false, "when write private log, whether to flush file after write done"); +namespace dsn { +namespace replication { + mutation_log_private::mutation_log_private(const std::string &dir, int32_t max_log_file_mb, gpid gpid, @@ -256,7 +257,7 @@ void mutation_log_private::commit_pending_mutations(log_file_ptr &lf, std::shared_ptr &pending, decree max_commit) { - if (dsn_unlikely(utils::FLAGS_enable_latency_tracer)) { + if (dsn_unlikely(FLAGS_enable_latency_tracer)) { for (const auto &mu : pending->mutations()) { ADD_POINT(mu->_tracer); } @@ -273,7 +274,7 @@ void mutation_log_private::commit_pending_mutations(log_file_ptr &lf, CHECK_EQ(hdr->magic, 0xdeadbeef); } - if (dsn_unlikely(utils::FLAGS_enable_latency_tracer)) { + if (dsn_unlikely(FLAGS_enable_latency_tracer)) { for (const auto &mu : pending->mutations()) { ADD_CUSTOM_POINT(mu->_tracer, "commit_pending_completed"); } diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp index 1286d74526..fc84e1a95a 100644 --- a/src/replica/replica.cpp +++ b/src/replica/replica.cpp @@ -54,11 +54,37 @@ #include "security/access_controller.h" #include "split/replica_split_manager.h" #include "utils/filesystem.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/latency_tracer.h" #include "utils/ports.h" #include "utils/rand.h" +DSN_DEFINE_bool(replication, + batch_write_disabled, + false, + "whether to disable auto-batch of replicated write requests"); +DSN_DEFINE_int32(replication, + staleness_for_commit, + 10, + "how many concurrent two phase commit rounds are allowed"); +DSN_DEFINE_int32(replication, + max_mutation_count_in_prepare_list, + 110, + "maximum number of mutations in prepare list"); +DSN_DEFINE_group_validator(max_mutation_count_in_prepare_list, [](std::string &message) -> bool { + if (FLAGS_max_mutation_count_in_prepare_list < FLAGS_staleness_for_commit) { + message = fmt::format("replication.max_mutation_count_in_prepare_list({}) should be >= " + "replication.staleness_for_commit({})", + FLAGS_max_mutation_count_in_prepare_list, + FLAGS_staleness_for_commit); + return false; + } + return true; +}); + +DSN_DECLARE_int32(checkpoint_max_interval_hours); + METRIC_DEFINE_gauge_int64(replica, private_log_size_mb, dsn::metric_unit::kMegaBytes, @@ -239,31 +265,6 @@ METRIC_DEFINE_counter(replica, namespace dsn { namespace replication { -DSN_DEFINE_bool(replication, - batch_write_disabled, - false, - "whether to disable auto-batch of replicated write requests"); -DSN_DEFINE_int32(replication, - staleness_for_commit, - 10, - "how many concurrent two phase commit rounds are allowed"); -DSN_DEFINE_int32(replication, - max_mutation_count_in_prepare_list, - 110, - "maximum number of mutations in prepare list"); -DSN_DEFINE_group_validator(max_mutation_count_in_prepare_list, [](std::string &message) -> bool { - if (FLAGS_max_mutation_count_in_prepare_list < FLAGS_staleness_for_commit) { - message = fmt::format("replication.max_mutation_count_in_prepare_list({}) should be >= " - "replication.staleness_for_commit({})", - FLAGS_max_mutation_count_in_prepare_list, - FLAGS_staleness_for_commit); - return false; - } - return true; -}); - -DSN_DECLARE_int32(checkpoint_max_interval_hours); - const std::string replica::kAppInfo = ".app-info"; replica::replica(replica_stub *stub, diff --git a/src/replica/replica.h b/src/replica/replica.h index e552a33b0b..83e78105aa 100644 --- a/src/replica/replica.h +++ b/src/replica/replica.h @@ -42,18 +42,17 @@ #include "mutation.h" #include "mutation_log.h" #include "prepare_list.h" +#include "ranger/access_type.h" #include "replica/backup/cold_backup_context.h" #include "replica/replica_base.h" #include "replica_context.h" #include "runtime/api_layer1.h" -#include "ranger/access_type.h" #include "runtime/rpc/rpc_message.h" #include "runtime/serverlet.h" #include "runtime/task/task.h" #include "runtime/task/task_tracker.h" #include "utils/autoref_ptr.h" #include "utils/error_code.h" -#include "utils/flags.h" #include "utils/metrics.h" #include "utils/thread_access_checker.h" #include "utils/throttling_controller.h" @@ -128,8 +127,6 @@ class test_checker; } \ } while (0) -DSN_DECLARE_bool(reject_write_when_disk_insufficient); - // get bool envs[name], return false if value is not bool bool get_bool_envs(const std::map &envs, const std::string &name, diff --git a/src/replica/replica_2pc.cpp b/src/replica/replica_2pc.cpp index 0e335d49b1..3fe8eb3a61 100644 --- a/src/replica/replica_2pc.cpp +++ b/src/replica/replica_2pc.cpp @@ -77,9 +77,6 @@ #include "utils/thread_access_checker.h" #include "utils/uniq_timestamp_us.h" -namespace dsn { -namespace replication { - DSN_DEFINE_bool(replication, reject_write_when_disk_insufficient, true, @@ -107,6 +104,8 @@ DSN_DEFINE_uint64( DSN_DECLARE_int32(max_mutation_count_in_prepare_list); DSN_DECLARE_int32(staleness_for_commit); +namespace dsn { +namespace replication { void replica::on_client_write(dsn::message_ex *request, bool ignore_throttling) { _checker.only_one_thread_access(); diff --git a/src/replica/replica_backup.cpp b/src/replica/replica_backup.cpp index f51c20ade3..427eabc822 100644 --- a/src/replica/replica_backup.cpp +++ b/src/replica/replica_backup.cpp @@ -59,6 +59,13 @@ #include "utils/thread_access_checker.h" #include "utils/time_utils.h" +DSN_DEFINE_uint64(replication, + max_concurrent_uploading_file_count, + 10, + "concurrent uploading file count to block service"); + +DSN_DECLARE_string(cold_backup_root); + namespace dsn { namespace dist { namespace block_service { @@ -67,14 +74,6 @@ class block_filesystem; } // namespace dist namespace replication { - -DSN_DEFINE_uint64(replication, - max_concurrent_uploading_file_count, - 10, - "concurrent uploading file count to block service"); - -DSN_DECLARE_string(cold_backup_root); - void replica::on_cold_backup(const backup_request &request, /*out*/ backup_response &response) { _checker.only_one_thread_access(); diff --git a/src/replica/replica_check.cpp b/src/replica/replica_check.cpp index ae12c2bff6..05023bda3f 100644 --- a/src/replica/replica_check.cpp +++ b/src/replica/replica_check.cpp @@ -60,10 +60,7 @@ #include "utils/metrics.h" #include "utils/thread_access_checker.h" -namespace dsn { -namespace replication { - -// The replica membership state periodical checking part of replica. +/// The replica membership state periodical checking part of replica. DSN_DEFINE_bool(replication, group_check_disabled, false, "whether group check is disabled"); DSN_DEFINE_int32(replication, @@ -73,6 +70,9 @@ DSN_DEFINE_int32(replication, DSN_DECLARE_bool(empty_write_disabled); +namespace dsn { +namespace replication { + void replica::init_group_check() { FAIL_POINT_INJECT_F("replica_init_group_check", [](absl::string_view) {}); diff --git a/src/replica/replica_chkpt.cpp b/src/replica/replica_chkpt.cpp index d255573dd9..68413e5298 100644 --- a/src/replica/replica_chkpt.cpp +++ b/src/replica/replica_chkpt.cpp @@ -62,10 +62,7 @@ #include "utils/metrics.h" #include "utils/thread_access_checker.h" -namespace dsn { -namespace replication { - -// The checkpoint of the replicated app part of replica. +/// The checkpoint of the replicated app part of replica. DSN_DEFINE_int32(replication, checkpoint_max_interval_hours, @@ -86,6 +83,9 @@ DSN_DEFINE_int32(replication, "FLAGS_log_private_reserve_max_time_seconds are both satisfied, the useless logs " "can be reserved."); +namespace dsn { +namespace replication { + const std::string kCheckpointFolderPrefix /*NOLINT*/ = "checkpoint"; static std::string checkpoint_folder(int64_t decree) diff --git a/src/replica/replica_config.cpp b/src/replica/replica_config.cpp index bf2e07f503..0c5536ca88 100644 --- a/src/replica/replica_config.cpp +++ b/src/replica/replica_config.cpp @@ -74,11 +74,11 @@ #include "utils/strings.h" #include "utils/thread_access_checker.h" +/// The configuration management part of replica. + namespace dsn { namespace replication { -// The configuration management part of replica. - bool get_bool_envs(const std::map &envs, const std::string &name, bool &value) diff --git a/src/replica/replica_init.cpp b/src/replica/replica_init.cpp index 0948a3685c..811a5cbad4 100644 --- a/src/replica/replica_init.cpp +++ b/src/replica/replica_init.cpp @@ -52,8 +52,6 @@ #include "utils/fmt_logging.h" #include "utils/uniq_timestamp_us.h" -namespace dsn { -namespace replication { DSN_DEFINE_bool(replication, checkpoint_disabled, false, "whether checkpoint is disabled"); DSN_DEFINE_int32(replication, checkpoint_interval_seconds, @@ -65,6 +63,9 @@ DSN_DEFINE_int32(replication, 32, "private log maximum segment file size (MB)"); +namespace dsn { +namespace replication { + error_code replica::initialize_on_new() { // TODO: check if _dir contain other file or directory except for diff --git a/src/replica/replica_learn.cpp b/src/replica/replica_learn.cpp index 1fc4a7ada8..b2116422f0 100644 --- a/src/replica/replica_learn.cpp +++ b/src/replica/replica_learn.cpp @@ -73,10 +73,7 @@ #include "utils/metrics.h" #include "utils/thread_access_checker.h" -namespace dsn { -namespace replication { - -// The replication learning process part of replica. +/// The replication learning process part of replica. DSN_DEFINE_int32(replication, learn_app_max_concurrent_count, @@ -85,6 +82,9 @@ DSN_DEFINE_int32(replication, DSN_DECLARE_int32(max_mutation_count_in_prepare_list); +namespace dsn { +namespace replication { + void replica::init_learn(uint64_t signature) { _checker.only_one_thread_access(); diff --git a/src/replica/replica_stub.cpp b/src/replica/replica_stub.cpp index 6a2ca4de2a..90ceb4dac8 100644 --- a/src/replica/replica_stub.cpp +++ b/src/replica/replica_stub.cpp @@ -226,16 +226,18 @@ METRIC_DEFINE_gauge_int64(server, dsn::metric_unit::kBytes, "The max size of copied files among all splitting replicas"); +DSN_DECLARE_bool(duplication_enabled); +DSN_DECLARE_bool(enable_acl); DSN_DECLARE_bool(encrypt_data_at_rest); -DSN_DECLARE_string(server_key); - -namespace dsn { +DSN_DECLARE_int32(fd_beacon_interval_seconds); +DSN_DECLARE_int32(fd_check_interval_seconds); +DSN_DECLARE_int32(fd_grace_seconds); +DSN_DECLARE_int32(fd_lease_seconds); +DSN_DECLARE_int32(gc_interval_ms); DSN_DECLARE_string(cluster_name); +DSN_DECLARE_string(data_dirs); +DSN_DECLARE_string(server_key); -namespace security { -DSN_DECLARE_bool(enable_acl); -} -namespace replication { DSN_DEFINE_bool(replication, deny_client_on_start, false, @@ -316,15 +318,8 @@ DSN_DEFINE_string( "Provide the comma-separated list of URLs from which to retrieve the " "file system's server key. Example format: 'hostname1:1234/kms,hostname2:1234/kms'."); -DSN_DECLARE_bool(duplication_enabled); -DSN_DECLARE_int32(fd_beacon_interval_seconds); -DSN_DECLARE_int32(fd_check_interval_seconds); -DSN_DECLARE_int32(fd_grace_seconds); -DSN_DECLARE_int32(fd_lease_seconds); -DSN_DECLARE_int32(gc_interval_ms); -DSN_DECLARE_string(data_dirs); DSN_DEFINE_group_validator(encrypt_data_at_rest_pre_check, [](std::string &message) -> bool { - if (!dsn::security::FLAGS_enable_acl && FLAGS_encrypt_data_at_rest) { + if (!FLAGS_enable_acl && FLAGS_encrypt_data_at_rest) { message = fmt::format("[pegasus.server] encrypt_data_at_rest should be enabled only if " "[security] enable_acl is enabled."); return false; @@ -343,6 +338,8 @@ DSN_DEFINE_group_validator(encrypt_data_at_rest_with_kms_url, [](std::string &me return true; }); +namespace dsn { +namespace replication { bool replica_stub::s_not_exit_on_log_failure = false; replica_stub::replica_stub(replica_state_subscriber subscriber /*= nullptr*/, diff --git a/src/replica/replica_stub.h b/src/replica/replica_stub.h index 6fdc8ec7e1..ed4ffe3220 100644 --- a/src/replica/replica_stub.h +++ b/src/replica/replica_stub.h @@ -68,6 +68,8 @@ #include "utils/metrics.h" #include "utils/zlocks.h" +DSN_DECLARE_uint32(max_concurrent_manual_emergency_checkpointing_count); + namespace dsn { class command_deregister; class message_ex; @@ -88,8 +90,6 @@ class configuration_query_by_node_response; class configuration_update_request; class potential_secondary_context; -DSN_DECLARE_uint32(max_concurrent_manual_emergency_checkpointing_count); - typedef rpc_holder learn_completion_notification_rpc; typedef rpc_holder group_check_rpc; typedef rpc_holder diff --git a/src/replica/split/replica_split_manager.cpp b/src/replica/split/replica_split_manager.cpp index 8df996e5d6..9ea5d41806 100644 --- a/src/replica/split/replica_split_manager.cpp +++ b/src/replica/split/replica_split_manager.cpp @@ -79,12 +79,12 @@ METRIC_DEFINE_counter(replica, dsn::metric_unit::kPartitionSplittings, "The number of successful splittings"); -namespace dsn { -namespace replication { - DSN_DECLARE_bool(empty_write_disabled); DSN_DECLARE_int32(max_mutation_count_in_prepare_list); +namespace dsn { +namespace replication { + replica_split_manager::replica_split_manager(replica *r) : replica_base(r), _replica(r), diff --git a/src/replica/storage/simple_kv/simple_kv.main.cpp b/src/replica/storage/simple_kv/simple_kv.main.cpp index 5b2ac4ff95..e96ca72d07 100644 --- a/src/replica/storage/simple_kv/simple_kv.main.cpp +++ b/src/replica/storage/simple_kv/simple_kv.main.cpp @@ -36,7 +36,7 @@ static void dsn_app_registration_simple_kv() { - dsn::FLAGS_enable_http_server = false; // disable http server + FLAGS_enable_http_server = false; // disable http server dsn::replication::application::simple_kv_service_impl::register_service(); diff --git a/src/replica/storage/simple_kv/test/checker.cpp b/src/replica/storage/simple_kv/test/checker.cpp index 427126ad3c..dfa86299db 100644 --- a/src/replica/storage/simple_kv/test/checker.cpp +++ b/src/replica/storage/simple_kv/test/checker.cpp @@ -58,12 +58,12 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" +DSN_DECLARE_string(partition_guardian_type); + namespace dsn { class gpid; namespace replication { -DSN_DECLARE_string(partition_guardian_type); - namespace test { class checker_partition_guardian : public partition_guardian diff --git a/src/replica/storage/simple_kv/test/simple_kv.main.cpp b/src/replica/storage/simple_kv/test/simple_kv.main.cpp index 3ea189a203..20cd004637 100644 --- a/src/replica/storage/simple_kv/test/simple_kv.main.cpp +++ b/src/replica/storage/simple_kv/test/simple_kv.main.cpp @@ -44,7 +44,7 @@ void dsn_app_registration_simple_kv() { - dsn::FLAGS_enable_http_server = false; + FLAGS_enable_http_server = false; dsn::replication::test::simple_kv_service_impl::register_service(); dsn::service::meta_service_app::register_all(); diff --git a/src/replica/test/mock_utils.h b/src/replica/test/mock_utils.h index bf2e894002..878df25598 100644 --- a/src/replica/test/mock_utils.h +++ b/src/replica/test/mock_utils.h @@ -35,11 +35,11 @@ #include "replica/replica_stub.h" #include "replica/backup/cold_backup_context.h" +DSN_DECLARE_int32(log_private_file_size_mb); + namespace dsn { namespace replication { -DSN_DECLARE_int32(log_private_file_size_mb); - class mock_replication_app_base : public replication_app_base { public: diff --git a/src/replica/test/replica_disk_test.cpp b/src/replica/test/replica_disk_test.cpp index f9b557d30f..c072556d31 100644 --- a/src/replica/test/replica_disk_test.cpp +++ b/src/replica/test/replica_disk_test.cpp @@ -49,11 +49,12 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" +DSN_DECLARE_bool(fd_disabled); + using pegasus::AssertEventually; namespace dsn { namespace replication { -DSN_DECLARE_bool(fd_disabled); using query_disk_info_rpc = rpc_holder; diff --git a/src/replica/test/replica_http_service_test.cpp b/src/replica/test/replica_http_service_test.cpp index 8de287862f..9e7598c03b 100644 --- a/src/replica/test/replica_http_service_test.cpp +++ b/src/replica/test/replica_http_service_test.cpp @@ -33,17 +33,16 @@ #include "utils/flags.h" #include "utils/test_macros.h" +DSN_DECLARE_bool(duplication_enabled); +DSN_DECLARE_bool(enable_acl); +DSN_DECLARE_bool(fd_disabled); +DSN_DECLARE_uint32(config_sync_interval_ms); + using std::map; using std::string; namespace dsn { -namespace security { -DSN_DECLARE_bool(enable_acl); -} // namespace security namespace replication { -DSN_DECLARE_bool(duplication_enabled); -DSN_DECLARE_bool(fd_disabled); -DSN_DECLARE_uint32(config_sync_interval_ms); class replica_http_service_test : public replica_test_base { @@ -58,7 +57,7 @@ class replica_http_service_test : public replica_test_base // is successful when encrypt_data_at_rest is also true. // TODO(jingwei): It's a trick for test, it should set together at class // pegasus::encrypt_data_at_rest. - dsn::security::FLAGS_enable_acl = true; + FLAGS_enable_acl = true; stub->initialize_start(); http_call_registry::instance().clear_paths(); diff --git a/src/replica/test/replica_test.cpp b/src/replica/test/replica_test.cpp index eafbe9517a..fa6373c265 100644 --- a/src/replica/test/replica_test.cpp +++ b/src/replica/test/replica_test.cpp @@ -66,11 +66,12 @@ #include "utils/string_conv.h" #include "utils/test_macros.h" -namespace dsn { -namespace replication { DSN_DECLARE_bool(fd_disabled); DSN_DECLARE_string(cold_backup_root); +namespace dsn { +namespace replication { + class replica_test : public replica_test_base { public: diff --git a/src/runtime/env.sim.cpp b/src/runtime/env.sim.cpp index 25135e32bc..3b29d56624 100644 --- a/src/runtime/env.sim.cpp +++ b/src/runtime/env.sim.cpp @@ -36,14 +36,14 @@ #include "utils/threadpool_code.h" #include "utils/threadpool_spec.h" -namespace dsn { -namespace tools { - DSN_DEFINE_int32(tools.simulator, random_seed, 0, "random seed for the simulator, 0 for random seed"); +namespace dsn { +namespace tools { + void sim_env_provider::on_worker_start(task_worker *worker) { rand::reseed_thread_local_rng( diff --git a/src/runtime/nativerun.cpp b/src/runtime/nativerun.cpp index ea629dfec0..93eb9e967f 100644 --- a/src/runtime/nativerun.cpp +++ b/src/runtime/nativerun.cpp @@ -35,11 +35,11 @@ #include "utils/flags.h" #include "utils/threadpool_spec.h" +DSN_DECLARE_bool(enable_udp); + namespace dsn { namespace tools { -DSN_DECLARE_bool(enable_udp); - void nativerun::install(service_spec &spec) { if (spec.env_factory_name == "") diff --git a/src/runtime/profiler.cpp b/src/runtime/profiler.cpp index 0105f413b6..0113a11789 100644 --- a/src/runtime/profiler.cpp +++ b/src/runtime/profiler.cpp @@ -73,6 +73,13 @@ START<== queue(server) == ENQUEUE <===== net(reply) ======= REPLY <============= #include "utils/join_point.h" #include "utils/metrics.h" +DSN_DEFINE_bool(task..default, is_profile, false, "whether to profile this kind of task"); +DSN_DEFINE_bool(task..default, + collect_call_count, + true, + "whether to collect how many time this kind of tasks invoke each of other kinds " + "tasks"); + METRIC_DEFINE_entity(profiler); METRIC_DEFINE_gauge_int64(profiler, @@ -147,13 +154,6 @@ struct service_spec; namespace tools { -DSN_DEFINE_bool(task..default, is_profile, false, "whether to profile this kind of task"); -DSN_DEFINE_bool(task..default, - collect_call_count, - true, - "whether to collect how many time this kind of tasks invoke each of other kinds " - "tasks"); - typedef uint64_extension_helper task_ext_for_profiler; typedef uint64_extension_helper message_ext_for_profiler; diff --git a/src/runtime/providers.common.cpp b/src/runtime/providers.common.cpp index 966ecf3e83..17a8b195d7 100644 --- a/src/runtime/providers.common.cpp +++ b/src/runtime/providers.common.cpp @@ -42,11 +42,11 @@ #include "utils/lockp.std.h" #include "utils/zlock_provider.h" +DSN_DEFINE_bool(network, enable_udp, true, "whether to enable udp rpc engine"); + namespace dsn { namespace tools { -DSN_DEFINE_bool(network, enable_udp, true, "whether to enable udp rpc engine"); - void register_std_lock_providers() { lock_provider::register_component("dsn::tools::std_lock_provider"); diff --git a/src/runtime/rpc/asio_net_provider.cpp b/src/runtime/rpc/asio_net_provider.cpp index f06166d5ec..e8ec85b9eb 100644 --- a/src/runtime/rpc/asio_net_provider.cpp +++ b/src/runtime/rpc/asio_net_provider.cpp @@ -81,16 +81,16 @@ #include "utils/fmt_logging.h" #include "utils/rand.h" -namespace dsn { -class rpc_engine; - -namespace tools { - DSN_DEFINE_uint32(network, io_service_worker_count, 1, "thread number for io service (timer and boost network)"); +namespace dsn { +class rpc_engine; + +namespace tools { + const int threads_per_event_loop = 1; asio_network_provider::asio_network_provider(rpc_engine *srv, network *inner_provider) diff --git a/src/runtime/rpc/network.cpp b/src/runtime/rpc/network.cpp index c55826449c..7749b62d21 100644 --- a/src/runtime/rpc/network.cpp +++ b/src/runtime/rpc/network.cpp @@ -58,7 +58,6 @@ METRIC_DEFINE_gauge_int64(server, dsn::metric_unit::kSessions, "The number of sessions from server side"); -namespace dsn { DSN_DEFINE_uint32(network, conn_threshold_per_ip, 0, @@ -75,6 +74,7 @@ DSN_DEFINE_string(network, "network interface name used to init primary ipv4 address, " "if empty, means using a site local address"); +namespace dsn { /*static*/ join_point rpc_session::on_rpc_session_connected("rpc.session.connected"); /*static*/ join_point diff --git a/src/runtime/rpc/network.sim.cpp b/src/runtime/rpc/network.sim.cpp index a2c185dbc4..1fe00ad8a1 100644 --- a/src/runtime/rpc/network.sim.cpp +++ b/src/runtime/rpc/network.sim.cpp @@ -42,17 +42,17 @@ #include "utils/singleton_store.h" #include "utils/utils.h" -namespace dsn { -class rpc_engine; - -namespace tools { - DSN_DEFINE_uint32(tools.simulator, min_message_delay_microseconds, 1, "min message delay (us)"); DSN_DEFINE_uint32(tools.simulator, max_message_delay_microseconds, 100000, "max message delay (us)"); +namespace dsn { +class rpc_engine; + +namespace tools { + // switch[channel][header_format] // multiple machines connect to the same switch // 10 should be >= than rpc_channel::max_value() + 1 diff --git a/src/runtime/rpc/rpc_engine.cpp b/src/runtime/rpc/rpc_engine.cpp index 97394d7f32..48774a0a12 100644 --- a/src/runtime/rpc/rpc_engine.cpp +++ b/src/runtime/rpc/rpc_engine.cpp @@ -47,9 +47,9 @@ #include "utils/rand.h" #include "utils/threadpool_code.h" -namespace dsn { DSN_DECLARE_uint32(local_hash); +namespace dsn { DEFINE_TASK_CODE(LPC_RPC_TIMEOUT, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT) class rpc_timeout_task : public task diff --git a/src/runtime/rpc/rpc_message.cpp b/src/runtime/rpc/rpc_message.cpp index 7f34b6b857..e691ac3cd4 100644 --- a/src/runtime/rpc/rpc_message.cpp +++ b/src/runtime/rpc/rpc_message.cpp @@ -43,9 +43,6 @@ #include "utils/strings.h" #include "utils/utils.h" -using namespace dsn::utils; - -namespace dsn { // init common for all per-node providers DSN_DEFINE_uint32(core, local_hash, @@ -54,6 +51,10 @@ DSN_DEFINE_uint32(core, "the same order, and therefore the mapping between rpc code string and integer " "is the same, which we leverage for fast rpc handler lookup optimization"); +using namespace dsn::utils; + +namespace dsn { + std::atomic message_ex::_id(0); message_ex::message_ex() diff --git a/src/runtime/service_api_c.cpp b/src/runtime/service_api_c.cpp index 271037aaf3..2a7964caf7 100644 --- a/src/runtime/service_api_c.cpp +++ b/src/runtime/service_api_c.cpp @@ -91,12 +91,9 @@ DSN_DEFINE_double(core, "[0.0, 10.0]"); #endif -namespace dsn { -namespace security { DSN_DECLARE_bool(enable_auth); DSN_DECLARE_bool(enable_zookeeper_kerberos); -} // namespace security -} // namespace dsn + // // global state // @@ -485,7 +482,7 @@ bool run(const char *config_file, dsn_all.engine_ready = true; // init security if FLAGS_enable_auth == true - if (dsn::security::FLAGS_enable_auth) { + if (FLAGS_enable_auth) { if (!dsn::security::init(is_server)) { return false; } @@ -494,7 +491,7 @@ bool run(const char *config_file, // include two steps: // 1) apply kerberos ticket and keep it valid // 2) complete sasl init for client(use FLAGS_sasl_plugin_path) - } else if (dsn::security::FLAGS_enable_zookeeper_kerberos && app_list == "meta") { + } else if (FLAGS_enable_zookeeper_kerberos && app_list == "meta") { if (!dsn::security::init_for_zookeeper_client()) { return false; } diff --git a/src/runtime/simulator.cpp b/src/runtime/simulator.cpp index 87c242c905..ab1225c171 100644 --- a/src/runtime/simulator.cpp +++ b/src/runtime/simulator.cpp @@ -42,11 +42,11 @@ #include "utils/threadpool_spec.h" #include "utils/zlock_provider.h" +DSN_DECLARE_int32(random_seed); + namespace dsn { namespace tools { -DSN_DECLARE_int32(random_seed); - /*static*/ void simulator::register_checker(const std::string &name, checker::factory f) { diff --git a/src/runtime/task/task_spec.cpp b/src/runtime/task/task_spec.cpp index 8786a2c41d..044fbe5c1b 100644 --- a/src/runtime/task/task_spec.cpp +++ b/src/runtime/task/task_spec.cpp @@ -36,10 +36,9 @@ #include "utils/fmt_logging.h" #include "utils/threadpool_spec.h" -namespace dsn { -namespace tools { DSN_DECLARE_bool(enable_udp); -} + +namespace dsn { constexpr int TASK_SPEC_STORE_CAPACITY = 512; @@ -231,7 +230,7 @@ bool task_spec::init() } } - if (spec->rpc_call_channel == RPC_CHANNEL_UDP && !dsn::tools::FLAGS_enable_udp) { + if (spec->rpc_call_channel == RPC_CHANNEL_UDP && !FLAGS_enable_udp) { LOG_ERROR("task rpc_call_channel RPC_CHANNEL_UCP need udp service, make sure " "[network].enable_udp"); return false; diff --git a/src/runtime/test/netprovider.cpp b/src/runtime/test/netprovider.cpp index cef7f40ec9..820cd3b349 100644 --- a/src/runtime/test/netprovider.cpp +++ b/src/runtime/test/netprovider.cpp @@ -52,9 +52,10 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" -namespace dsn { DSN_DECLARE_uint32(conn_threshold_per_ip); +namespace dsn { + class asio_network_provider_test : public tools::asio_network_provider { public: diff --git a/src/runtime/tracer.cpp b/src/runtime/tracer.cpp index 3de063e088..c3a7e9154b 100644 --- a/src/runtime/tracer.cpp +++ b/src/runtime/tracer.cpp @@ -49,11 +49,11 @@ #include "utils/fmt_logging.h" #include "utils/join_point.h" +DSN_DEFINE_bool(task..default, is_trace, false, "whether to trace tasks by default"); + namespace dsn { namespace tools { -DSN_DEFINE_bool(task..default, is_trace, false, "whether to trace tasks by default"); - static void tracer_on_task_create(task *caller, task *callee) { dsn_task_type_t type = callee->spec().type; diff --git a/src/security/access_controller.cpp b/src/security/access_controller.cpp index 2a2f230f11..1cc708d081 100644 --- a/src/security/access_controller.cpp +++ b/src/security/access_controller.cpp @@ -23,8 +23,6 @@ #include "utils/fmt_logging.h" #include "utils/strings.h" -namespace dsn { -namespace security { DSN_DEFINE_bool(security, enable_acl, false, "whether enable access controller or not"); DSN_DEFINE_bool(security, enable_ranger_acl, @@ -35,6 +33,9 @@ DSN_DEFINE_string(security, "", "super users for access controller, comma-separated list of user names"); +namespace dsn { +namespace security { + access_controller::access_controller() { // when FLAGS_enable_ranger_acl is true, FLAGS_enable_acl must be true. diff --git a/src/security/init.cpp b/src/security/init.cpp index 870dd32204..06f716505f 100644 --- a/src/security/init.cpp +++ b/src/security/init.cpp @@ -25,13 +25,14 @@ #include "utils/fmt_logging.h" #include "utils/strings.h" -namespace dsn { -namespace security { DSN_DECLARE_bool(enable_auth); DSN_DECLARE_string(krb5_config); DSN_DECLARE_string(krb5_keytab); DSN_DECLARE_string(krb5_principal); +namespace dsn { +namespace security { + /*** * set kerberos envs(for more details: * https://web.mit.edu/kerberos/krb5-1.12/doc/admin/env_variables.html) diff --git a/src/security/kinit_context.cpp b/src/security/kinit_context.cpp index 7b281d5d3a..2347e7abb5 100644 --- a/src/security/kinit_context.cpp +++ b/src/security/kinit_context.cpp @@ -41,13 +41,18 @@ #include "utils/strings.h" #include "utils/time_utils.h" +DSN_DECLARE_bool(enable_auth); +DSN_DECLARE_bool(enable_zookeeper_kerberos); +DSN_DEFINE_string(security, krb5_keytab, "", "absolute path of keytab file"); +DSN_DEFINE_string(security, krb5_config, "", "absolute path of krb5_config file"); +DSN_DEFINE_string(security, krb5_principal, "", "kerberos principal"); +DSN_DEFINE_string(security, service_fqdn, "", "the fully qualified domain name of the server"); +DSN_DEFINE_string(security, service_name, "", "service name"); + namespace dsn { namespace security { class kinit_context; -DSN_DECLARE_bool(enable_auth); -DSN_DECLARE_bool(enable_zookeeper_kerberos); - #define KRB5_RETURN_NOT_OK(err, msg) \ do { \ krb5_error_code __err_code__ = (err); \ @@ -56,12 +61,6 @@ DSN_DECLARE_bool(enable_zookeeper_kerberos); } \ } while (0); -DSN_DEFINE_string(security, krb5_keytab, "", "absolute path of keytab file"); -DSN_DEFINE_string(security, krb5_config, "", "absolute path of krb5_config file"); -DSN_DEFINE_string(security, krb5_principal, "", "kerberos principal"); -DSN_DEFINE_string(security, service_fqdn, "", "the fully qualified domain name of the server"); -DSN_DEFINE_string(security, service_name, "", "service name"); - // Attention: we can't do these check work by `DSN_DEFINE_validator`, because somebody may don't // want to use security, so these configuration may not setted. In this situation, these checks // will not pass. diff --git a/src/security/meta_access_controller.cpp b/src/security/meta_access_controller.cpp index 8133d7533d..204e34cb12 100644 --- a/src/security/meta_access_controller.cpp +++ b/src/security/meta_access_controller.cpp @@ -29,8 +29,6 @@ #include "utils/fmt_logging.h" #include "utils/strings.h" -namespace dsn { -namespace security { DSN_DEFINE_string(security, meta_acl_rpc_allow_list, "", @@ -39,6 +37,9 @@ DSN_DEFINE_string(security, DSN_DECLARE_bool(enable_acl); DSN_DECLARE_bool(enable_ranger_acl); +namespace dsn { +namespace security { + meta_access_controller::meta_access_controller( const std::shared_ptr &policy_manager) : _ranger_resource_policy_manager(policy_manager) diff --git a/src/security/negotiation.cpp b/src/security/negotiation.cpp index e8a615b6e1..0b9e270dee 100644 --- a/src/security/negotiation.cpp +++ b/src/security/negotiation.cpp @@ -26,9 +26,6 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" -namespace dsn { -namespace security { - DSN_DEFINE_bool(security, enable_auth, false, "whether open auth or not"); DSN_DEFINE_bool(security, enable_zookeeper_kerberos, @@ -37,6 +34,9 @@ DSN_DEFINE_bool(security, DSN_DEFINE_bool(security, mandatory_auth, false, "wheter to do authertication mandatorily"); DSN_TAG_VARIABLE(mandatory_auth, FT_MUTABLE); +namespace dsn { +namespace security { + const std::set negotiation::kSupportedMechanisms{"GSSAPI"}; negotiation::~negotiation() {} diff --git a/src/security/negotiation_manager.cpp b/src/security/negotiation_manager.cpp index 26c60a3374..22a4098f52 100644 --- a/src/security/negotiation_manager.cpp +++ b/src/security/negotiation_manager.cpp @@ -36,11 +36,12 @@ #include "utils/ports.h" #include "utils/synchronize.h" -namespace dsn { -namespace security { DSN_DECLARE_bool(enable_auth); DSN_DECLARE_bool(mandatory_auth); +namespace dsn { +namespace security { + inline bool is_negotiation_message(dsn::task_code code) { return code == RPC_NEGOTIATION || code == RPC_NEGOTIATION_ACK; diff --git a/src/security/replica_access_controller.cpp b/src/security/replica_access_controller.cpp index c50a60ea34..0f480c77b4 100644 --- a/src/security/replica_access_controller.cpp +++ b/src/security/replica_access_controller.cpp @@ -36,10 +36,11 @@ #include "utils/fmt_logging.h" #include "utils/strings.h" -namespace dsn { -namespace security { DSN_DECLARE_bool(enable_acl); DSN_DECLARE_bool(enable_ranger_acl); + +namespace dsn { +namespace security { replica_access_controller::replica_access_controller(const std::string &replica_name) { _name = replica_name; diff --git a/src/security/sasl_client_wrapper.cpp b/src/security/sasl_client_wrapper.cpp index 462f8db1be..50296826da 100644 --- a/src/security/sasl_client_wrapper.cpp +++ b/src/security/sasl_client_wrapper.cpp @@ -25,11 +25,12 @@ #include "utils/flags.h" #include "absl/strings/string_view.h" -namespace dsn { -namespace security { DSN_DECLARE_string(service_fqdn); DSN_DECLARE_string(service_name); +namespace dsn { +namespace security { + error_s sasl_client_wrapper::init() { FAIL_POINT_INJECT_F("sasl_client_wrapper_init", [](absl::string_view str) { diff --git a/src/security/sasl_init.cpp b/src/security/sasl_init.cpp index 651491ff02..1f9712c34c 100644 --- a/src/security/sasl_init.cpp +++ b/src/security/sasl_init.cpp @@ -28,9 +28,10 @@ #include "utils/fmt_logging.h" #include "utils/synchronize.h" +DSN_DEFINE_string(security, sasl_plugin_path, "/usr/lib/sasl2", "path to search sasl plugins"); + namespace dsn { namespace security { -DSN_DEFINE_string(security, sasl_plugin_path, "/usr/lib/sasl2", "path to search sasl plugins"); log_level_t get_log_level(int level) { diff --git a/src/security/sasl_server_wrapper.cpp b/src/security/sasl_server_wrapper.cpp index c8bed00f82..4cda84241b 100644 --- a/src/security/sasl_server_wrapper.cpp +++ b/src/security/sasl_server_wrapper.cpp @@ -25,11 +25,12 @@ #include "utils/flags.h" #include "absl/strings/string_view.h" -namespace dsn { -namespace security { DSN_DECLARE_string(service_fqdn); DSN_DECLARE_string(service_name); +namespace dsn { +namespace security { + error_s sasl_server_wrapper::init() { FAIL_POINT_INJECT_F("sasl_server_wrapper_init", [](absl::string_view str) { diff --git a/src/security/server_negotiation.cpp b/src/security/server_negotiation.cpp index 2785ff84f2..174b418677 100644 --- a/src/security/server_negotiation.cpp +++ b/src/security/server_negotiation.cpp @@ -34,11 +34,12 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" -namespace dsn { -namespace security { DSN_DECLARE_string(service_fqdn); DSN_DECLARE_string(service_name); +namespace dsn { +namespace security { + server_negotiation::server_negotiation(rpc_session_ptr session) : negotiation(session) { _name = fmt::format("SERVER_NEGOTIATION(CLIENT={})", _session->remote_address()); diff --git a/src/security/test/meta_access_controller_test.cpp b/src/security/test/meta_access_controller_test.cpp index 31d1368a00..2816d359df 100644 --- a/src/security/test/meta_access_controller_test.cpp +++ b/src/security/test/meta_access_controller_test.cpp @@ -30,9 +30,10 @@ #include "utils/autoref_ptr.h" #include "utils/flags.h" +DSN_DECLARE_bool(enable_acl); + namespace dsn { namespace security { -DSN_DECLARE_bool(enable_acl); class meta_access_controller_test : public testing::Test { diff --git a/src/security/test/negotiation_manager_test.cpp b/src/security/test/negotiation_manager_test.cpp index eafb3edc31..7030e87110 100644 --- a/src/security/test/negotiation_manager_test.cpp +++ b/src/security/test/negotiation_manager_test.cpp @@ -32,11 +32,12 @@ #include "utils/autoref_ptr.h" #include "utils/flags.h" -namespace dsn { -namespace security { DSN_DECLARE_bool(enable_auth); DSN_DECLARE_bool(mandatory_auth); +namespace dsn { +namespace security { + class negotiation_manager_test : public testing::Test { public: diff --git a/src/security/test/replica_access_controller_test.cpp b/src/security/test/replica_access_controller_test.cpp index 37f186b00b..1b2d45c5a4 100644 --- a/src/security/test/replica_access_controller_test.cpp +++ b/src/security/test/replica_access_controller_test.cpp @@ -31,9 +31,10 @@ #include "utils/autoref_ptr.h" #include "utils/flags.h" +DSN_DECLARE_bool(enable_acl); + namespace dsn { namespace security { -DSN_DECLARE_bool(enable_acl); class replica_access_controller_test : public testing::Test { diff --git a/src/server/available_detector.cpp b/src/server/available_detector.cpp index ca90170fdf..168481a7b4 100644 --- a/src/server/available_detector.cpp +++ b/src/server/available_detector.cpp @@ -51,11 +51,6 @@ #include "utils/threadpool_code.h" #include "utils/time_utils.h" -namespace pegasus { -namespace server { - -DEFINE_TASK_CODE(LPC_DETECT_AVAILABLE, TASK_PRIORITY_COMMON, ::dsn::THREAD_POOL_DEFAULT) - DSN_DEFINE_int32(pegasus.collector, available_detect_alert_fail_count, 30, @@ -82,6 +77,11 @@ DSN_DEFINE_string(pegasus.collector, "", "available detect alert email address, empty means not send email"); +namespace pegasus { +namespace server { + +DEFINE_TASK_CODE(LPC_DETECT_AVAILABLE, TASK_PRIORITY_COMMON, ::dsn::THREAD_POOL_DEFAULT) + available_detector::available_detector() : _client(nullptr), _ddl_client(nullptr), diff --git a/src/server/capacity_unit_calculator.cpp b/src/server/capacity_unit_calculator.cpp index 49bd751e4c..cfbc086f41 100644 --- a/src/server/capacity_unit_calculator.cpp +++ b/src/server/capacity_unit_calculator.cpp @@ -89,9 +89,6 @@ METRIC_DEFINE_counter(replica, dsn::metric_unit::kBytes, "The number of bytes for backup requests"); -namespace pegasus { -namespace server { - DSN_DEFINE_uint64(pegasus.server, perf_counter_read_capacity_unit_size, 4 * 1024, @@ -106,6 +103,9 @@ DSN_DEFINE_uint64(pegasus.server, DSN_DEFINE_validator(perf_counter_write_capacity_unit_size, [](const uint64_t value) -> bool { return powerof2(value); }); +namespace pegasus { +namespace server { + capacity_unit_calculator::capacity_unit_calculator( replica_base *r, std::shared_ptr read_hotkey_collector, diff --git a/src/server/hotkey_collector.cpp b/src/server/hotkey_collector.cpp index 5fb5ad397a..a05c3012d8 100644 --- a/src/server/hotkey_collector.cpp +++ b/src/server/hotkey_collector.cpp @@ -35,9 +35,6 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" -namespace pegasus { -namespace server { - DSN_DEFINE_uint32( pegasus.server, hot_bucket_variance_threshold, @@ -77,6 +74,9 @@ DSN_DEFINE_uint32( "the max time (in seconds) allowed to capture hotkey, will stop if hotkey's not found"); DSN_TAG_VARIABLE(max_seconds_to_detect_hotkey, FT_MUTABLE); +namespace pegasus { +namespace server { + // 68–95–99.7 rule, same algorithm as hotspot_partition_calculator::stat_histories_analyse /*extern*/ bool find_outlier_index(const std::vector &captured_keys, int threshold, int &hot_index) diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index 2d028c828c..b3d481ccf0 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -34,11 +34,6 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" -struct row_data; - -namespace pegasus { -namespace server { - DSN_DEFINE_int64(pegasus.collector, max_hotspot_store_size, 100, @@ -67,6 +62,11 @@ DSN_DEFINE_uint32(pegasus.collector, "hot paritiotion occurrence times' threshold to send rpc to detect hotkey"); DSN_TAG_VARIABLE(occurrence_threshold, FT_MUTABLE); +struct row_data; + +namespace pegasus { +namespace server { + void hotspot_partition_calculator::data_aggregate(const std::vector &partition_stats) { while (_partitions_stat_histories.size() >= FLAGS_max_hotspot_store_size) { diff --git a/src/server/info_collector.cpp b/src/server/info_collector.cpp index 1c7f362288..6523cad4d9 100644 --- a/src/server/info_collector.cpp +++ b/src/server/info_collector.cpp @@ -41,17 +41,6 @@ #include "utils/strings.h" #include "utils/threadpool_code.h" -namespace pegasus { -namespace server { - -DEFINE_TASK_CODE(LPC_PEGASUS_APP_STAT_TIMER, TASK_PRIORITY_COMMON, ::dsn::THREAD_POOL_DEFAULT) -DEFINE_TASK_CODE(LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, - TASK_PRIORITY_COMMON, - ::dsn::THREAD_POOL_DEFAULT) -DEFINE_TASK_CODE(LPC_PEGASUS_STORAGE_SIZE_STAT_TIMER, - TASK_PRIORITY_COMMON, - ::dsn::THREAD_POOL_DEFAULT) - DSN_DEFINE_uint32(pegasus.collector, app_stat_interval_seconds, 10, "app stat interval seconds"); DSN_DEFINE_uint32(pegasus.collector, capacity_unit_fetch_interval_seconds, @@ -69,6 +58,17 @@ DSN_DEFINE_string(pegasus.collector, DSN_DEFINE_validator(usage_stat_app, [](const char *value) -> bool { return !dsn::utils::is_empty(value); }); +namespace pegasus { +namespace server { + +DEFINE_TASK_CODE(LPC_PEGASUS_APP_STAT_TIMER, TASK_PRIORITY_COMMON, ::dsn::THREAD_POOL_DEFAULT) +DEFINE_TASK_CODE(LPC_PEGASUS_CAPACITY_UNIT_STAT_TIMER, + TASK_PRIORITY_COMMON, + ::dsn::THREAD_POOL_DEFAULT) +DEFINE_TASK_CODE(LPC_PEGASUS_STORAGE_SIZE_STAT_TIMER, + TASK_PRIORITY_COMMON, + ::dsn::THREAD_POOL_DEFAULT) + info_collector::info_collector() { std::vector<::dsn::rpc_address> meta_servers; diff --git a/src/server/pegasus_event_listener.cpp b/src/server/pegasus_event_listener.cpp index e3e3f2d772..1c4ebbfd66 100644 --- a/src/server/pegasus_event_listener.cpp +++ b/src/server/pegasus_event_listener.cpp @@ -27,10 +27,6 @@ #include "utils/autoref_ptr.h" #include "utils/fmt_logging.h" -namespace rocksdb { -class DB; -} // namespace rocksdb - METRIC_DEFINE_counter(replica, rdb_flush_completed_count, dsn::metric_unit::kFlushes, @@ -68,6 +64,10 @@ METRIC_DEFINE_counter( dsn::metric_unit::kWrites, "The number of rocksdb stopped writes changed from another write stall condition"); +namespace rocksdb { +class DB; +} // namespace rocksdb + namespace pegasus { namespace server { diff --git a/src/server/pegasus_manual_compact_service.cpp b/src/server/pegasus_manual_compact_service.cpp index 131c7e2e58..a2cdf4812e 100644 --- a/src/server/pegasus_manual_compact_service.cpp +++ b/src/server/pegasus_manual_compact_service.cpp @@ -50,16 +50,15 @@ METRIC_DEFINE_gauge_int64(replica, dsn::metric_unit::kTasks, "The number of current running tasks of rocksdb manual compaction"); -namespace pegasus { -namespace server { - -DEFINE_TASK_CODE(LPC_MANUAL_COMPACT, TASK_PRIORITY_COMMON, THREAD_POOL_COMPACT) - DSN_DEFINE_int32(pegasus.server, manual_compact_min_interval_seconds, 0, "minimal interval time in seconds to start a new manual compaction, <= 0 " "means no interval limit"); +namespace pegasus { +namespace server { + +DEFINE_TASK_CODE(LPC_MANUAL_COMPACT, TASK_PRIORITY_COMMON, THREAD_POOL_COMPACT) const std::string pegasus_manual_compact_service::MANUAL_COMPACT_BOTTOMMOST_LEVEL_COMPACTION_FORCE("force"); diff --git a/src/server/pegasus_mutation_duplicator.cpp b/src/server/pegasus_mutation_duplicator.cpp index 3553bcfc79..d30f70400c 100644 --- a/src/server/pegasus_mutation_duplicator.cpp +++ b/src/server/pegasus_mutation_duplicator.cpp @@ -234,8 +234,7 @@ void pegasus_mutation_duplicator::duplicate(mutation_tuple_set muts, callback cb batch_bytes += raw_message.length(); } - if (batch_count == muts.size() || - batch_bytes >= dsn::replication::FLAGS_duplicate_log_batch_bytes) { + if (batch_count == muts.size() || batch_bytes >= FLAGS_duplicate_log_batch_bytes) { // since all the plog's mutations of replica belong to same gpid though the hash of // mutation is different, use the last mutation of one batch to get and represents the // current hash value, it will still send to remote correct replica diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index f4a95e3a61..99551043f0 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -86,23 +86,6 @@ #include "utils/token_bucket_throttling_controller.h" #include "utils/utils.h" -namespace rocksdb { -class WriteBufferManager; -} // namespace rocksdb - -using namespace dsn::literals::chrono_literals; - -namespace pegasus { -namespace server { - -DEFINE_TASK_CODE(LPC_PEGASUS_SERVER_DELAY, TASK_PRIORITY_COMMON, ::dsn::THREAD_POOL_DEFAULT) - -DSN_DECLARE_int32(read_amp_bytes_per_bit); -DSN_DECLARE_uint32(checkpoint_reserve_min_count); -DSN_DECLARE_uint32(checkpoint_reserve_time_seconds); -DSN_DECLARE_uint64(rocksdb_iteration_threshold_time_ms); -DSN_DECLARE_uint64(rocksdb_slow_query_threshold_ns); - DSN_DEFINE_bool(pegasus.server, rocksdb_verbose_log, false, @@ -121,6 +104,23 @@ DSN_DEFINE_int32(pegasus.server, "Which error code to inject in read path, 0 means no error. Only for test."); DSN_TAG_VARIABLE(inject_read_error_for_test, FT_MUTABLE); +DSN_DECLARE_int32(read_amp_bytes_per_bit); +DSN_DECLARE_uint32(checkpoint_reserve_min_count); +DSN_DECLARE_uint32(checkpoint_reserve_time_seconds); +DSN_DECLARE_uint64(rocksdb_iteration_threshold_time_ms); +DSN_DECLARE_uint64(rocksdb_slow_query_threshold_ns); + +namespace rocksdb { +class WriteBufferManager; +} // namespace rocksdb + +using namespace dsn::literals::chrono_literals; + +namespace pegasus { +namespace server { + +DEFINE_TASK_CODE(LPC_PEGASUS_SERVER_DELAY, TASK_PRIORITY_COMMON, ::dsn::THREAD_POOL_DEFAULT) + static std::string chkpt_get_dir_name(int64_t decree) { char buffer[256]; diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index 4a2b8b522b..7cac634402 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -54,6 +54,12 @@ #include "utils/rand.h" #include "utils/synchronize.h" +DSN_DECLARE_uint64(rocksdb_abnormal_batch_get_bytes_threshold); +DSN_DECLARE_uint64(rocksdb_abnormal_batch_get_count_threshold); +DSN_DECLARE_uint64(rocksdb_abnormal_get_size_threshold); +DSN_DECLARE_uint64(rocksdb_abnormal_multi_get_iterate_count_threshold); +DSN_DECLARE_uint64(rocksdb_abnormal_multi_get_size_threshold); + namespace pegasus { namespace server { class KeyWithTTLCompactionFilterFactory; @@ -89,12 +95,6 @@ typedef dsn::utils::token_bucket_throttling_controller throttling_controller; namespace pegasus { namespace server { -DSN_DECLARE_uint64(rocksdb_abnormal_batch_get_bytes_threshold); -DSN_DECLARE_uint64(rocksdb_abnormal_batch_get_count_threshold); -DSN_DECLARE_uint64(rocksdb_abnormal_get_size_threshold); -DSN_DECLARE_uint64(rocksdb_abnormal_multi_get_iterate_count_threshold); -DSN_DECLARE_uint64(rocksdb_abnormal_multi_get_size_threshold); - class capacity_unit_calculator; class hotkey_collector; class meta_store; diff --git a/src/server/pegasus_server_impl_init.cpp b/src/server/pegasus_server_impl_init.cpp index 8cbd2214c3..961245e1af 100644 --- a/src/server/pegasus_server_impl_init.cpp +++ b/src/server/pegasus_server_impl_init.cpp @@ -59,12 +59,6 @@ #include "utils/strings.h" #include "utils/token_bucket_throttling_controller.h" -namespace dsn { -namespace replication { -class replica; -} // namespace replication -} // namespace dsn - METRIC_DEFINE_counter(replica, get_requests, dsn::metric_unit::kRequests, @@ -250,9 +244,6 @@ METRIC_DEFINE_gauge_int64(server, "The through bytes per second that go through the rate limiter which " "takes control of the write rate of flush and compaction of rocksdb"); -namespace pegasus { -namespace server { - DSN_DEFINE_int64( pegasus.server, rocksdb_limiter_max_write_megabytes_per_sec, @@ -574,6 +565,13 @@ DSN_DEFINE_validator(rocksdb_filter_type, [](const char *value) -> bool { return dsn::utils::equals(value, "common") || dsn::utils::equals(value, "prefix"); }); +namespace dsn { +namespace replication { +class replica; +} // namespace replication +} // namespace dsn +namespace pegasus { +namespace server { static const std::unordered_map INDEX_TYPE_STRING_MAP = { {"binary_search", rocksdb::BlockBasedTableOptions::IndexType::kBinarySearch}, diff --git a/src/server/pegasus_server_write.cpp b/src/server/pegasus_server_write.cpp index 4cc1039aba..1bb6c968ad 100644 --- a/src/server/pegasus_server_write.cpp +++ b/src/server/pegasus_server_write.cpp @@ -47,9 +47,10 @@ METRIC_DEFINE_counter(replica, dsn::metric_unit::kRequests, "The number of corrupt writes for each replica"); +DSN_DECLARE_bool(rocksdb_verbose_log); + namespace pegasus { namespace server { -DSN_DECLARE_bool(rocksdb_verbose_log); pegasus_server_write::pegasus_server_write(pegasus_server_impl *server) : replica_base(server), diff --git a/src/server/pegasus_write_service.cpp b/src/server/pegasus_write_service.cpp index 09f05ae577..a7e9d93420 100644 --- a/src/server/pegasus_write_service.cpp +++ b/src/server/pegasus_write_service.cpp @@ -46,11 +46,6 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" -namespace dsn { -class blob; -class message_ex; -} // namespace dsn - METRIC_DEFINE_counter(replica, put_requests, dsn::metric_unit::kRequests, @@ -137,15 +132,19 @@ METRIC_DEFINE_counter( dsn::metric_unit::kRequests, "the number of lagging writes (time lag larger than `dup_lagging_write_threshold_ms`)"); -namespace pegasus { -namespace server { - DSN_DEFINE_int64(pegasus.server, dup_lagging_write_threshold_ms, 10 * 1000, "If the duration that a write flows from master to slave is larger than this " "threshold, the write is defined a lagging write."); +namespace dsn { +class blob; +class message_ex; +} // namespace dsn +namespace pegasus { +namespace server { + DEFINE_TASK_CODE(LPC_INGESTION, TASK_PRIORITY_COMMON, THREAD_POOL_INGESTION) pegasus_write_service::pegasus_write_service(pegasus_server_impl *server) diff --git a/src/server/result_writer.cpp b/src/server/result_writer.cpp index d4e9ebeb74..5629106b90 100644 --- a/src/server/result_writer.cpp +++ b/src/server/result_writer.cpp @@ -30,14 +30,14 @@ #include "utils/fmt_logging.h" #include "utils/threadpool_code.h" -namespace pegasus { -namespace server { - DSN_DEFINE_int32(pegasus.collector, capacity_unit_saving_ttl_days, 90, "the ttl of the CU data, 0 if no ttl"); +namespace pegasus { +namespace server { + DEFINE_TASK_CODE(LPC_WRITE_RESULT, TASK_PRIORITY_COMMON, ::dsn::THREAD_POOL_DEFAULT) result_writer::result_writer(pegasus_client *client) : _client(client) {} diff --git a/src/server/rocksdb_wrapper.cpp b/src/server/rocksdb_wrapper.cpp index 794d1d6332..8331dbfd92 100644 --- a/src/server/rocksdb_wrapper.cpp +++ b/src/server/rocksdb_wrapper.cpp @@ -39,11 +39,6 @@ #include "utils/fmt_logging.h" #include "utils/ports.h" -METRIC_DECLARE_counter(read_expired_values); - -namespace pegasus { -namespace server { - DSN_DEFINE_int32(pegasus.server, inject_write_error_for_test, 0, @@ -56,6 +51,11 @@ DSN_DEFINE_bool(pegasus.server, "'rocksdb.external_sst_file.global_seqno' of ssttable file during ingest process. " "If false, it will not be modified."); +METRIC_DECLARE_counter(read_expired_values); + +namespace pegasus { +namespace server { + rocksdb_wrapper::rocksdb_wrapper(pegasus_server_impl *server) : replica_base(server), _db(server->_db), diff --git a/src/server/test/capacity_unit_calculator_test.cpp b/src/server/test/capacity_unit_calculator_test.cpp index 2fe66a756c..b093cb0586 100644 --- a/src/server/test/capacity_unit_calculator_test.cpp +++ b/src/server/test/capacity_unit_calculator_test.cpp @@ -39,6 +39,9 @@ #include "utils/flags.h" #include "utils/token_bucket_throttling_controller.h" +DSN_DECLARE_uint64(perf_counter_read_capacity_unit_size); +DSN_DECLARE_uint64(perf_counter_write_capacity_unit_size); + namespace dsn { namespace replication { struct replica_base; @@ -48,9 +51,6 @@ struct replica_base; namespace pegasus { namespace server { -DSN_DECLARE_uint64(perf_counter_read_capacity_unit_size); -DSN_DECLARE_uint64(perf_counter_write_capacity_unit_size); - class mock_capacity_unit_calculator : public capacity_unit_calculator { public: diff --git a/src/server/test/hotkey_collector_test.cpp b/src/server/test/hotkey_collector_test.cpp index 90d9d2dafc..c8eb49cadb 100644 --- a/src/server/test/hotkey_collector_test.cpp +++ b/src/server/test/hotkey_collector_test.cpp @@ -40,6 +40,8 @@ #include "utils/fmt_logging.h" #include "utils/rand.h" +DSN_DECLARE_uint32(hotkey_buckets_num); + namespace dsn { class message_ex; } // namespace dsn @@ -47,8 +49,6 @@ class message_ex; namespace pegasus { namespace server { -DSN_DECLARE_uint32(hotkey_buckets_num); - static std::string generate_hash_key_by_random(bool is_hotkey, int probability = 100) { if (is_hotkey && (dsn::rand::next_u32(100) < probability)) { diff --git a/src/server/test/hotspot_partition_test.cpp b/src/server/test/hotspot_partition_test.cpp index 4bba7e4ca3..60e0648673 100644 --- a/src/server/test/hotspot_partition_test.cpp +++ b/src/server/test/hotspot_partition_test.cpp @@ -32,12 +32,12 @@ #include "utils/fail_point.h" #include "utils/flags.h" -namespace pegasus { -namespace server { - DSN_DECLARE_int32(occurrence_threshold); DSN_DECLARE_bool(enable_detect_hotkey); +namespace pegasus { +namespace server { + class hotspot_partition_test : public pegasus_server_test_base { public: diff --git a/src/server/test/manual_compact_service_test.cpp b/src/server/test/manual_compact_service_test.cpp index 9dc2acaea2..e6972e784e 100644 --- a/src/server/test/manual_compact_service_test.cpp +++ b/src/server/test/manual_compact_service_test.cpp @@ -33,11 +33,11 @@ #include "utils/strings.h" #include "utils/time_utils.h" +DSN_DECLARE_int32(manual_compact_min_interval_seconds); + namespace pegasus { namespace server { -DSN_DECLARE_int32(manual_compact_min_interval_seconds); - class manual_compact_service_test : public pegasus_server_test_base { public: diff --git a/src/test/bench_test/benchmark.cpp b/src/test/bench_test/benchmark.cpp index b0420ccbd5..321c56f86a 100644 --- a/src/test/bench_test/benchmark.cpp +++ b/src/test/bench_test/benchmark.cpp @@ -40,9 +40,6 @@ #include "utils/fmt_logging.h" #include "utils/strings.h" -namespace pegasus { -namespace test { - DSN_DEFINE_uint64(pegasus.benchmark, benchmark_num, 10000, @@ -95,6 +92,9 @@ DSN_DEFINE_group_validator(multi_count, [](std::string &message) -> bool { return true; }); +namespace pegasus { +namespace test { + benchmark::benchmark() { _client = diff --git a/src/test/function_test/security/test_kms_client.cpp b/src/test/function_test/security/test_kms_client.cpp index 33b2ac0ecc..20cf20b0c1 100644 --- a/src/test/function_test/security/test_kms_client.cpp +++ b/src/test/function_test/security/test_kms_client.cpp @@ -27,15 +27,9 @@ #include "utils/errors.h" #include "utils/flags.h" -namespace dsn { -DSN_DECLARE_string(cluster_name); -namespace security { DSN_DECLARE_bool(enable_acl); -} // namespace security -namespace replication { +DSN_DECLARE_string(cluster_name); DSN_DECLARE_string(hadoop_kms_url); -} // namespace replication -} // namespace dsn class kms_client_test : public testing::Test { @@ -43,13 +37,12 @@ class kms_client_test : public testing::Test TEST_F(kms_client_test, test_generate_and_decrypt_encryption_key) { - if (strlen(dsn::replication::FLAGS_hadoop_kms_url) == 0) { + if (strlen(FLAGS_hadoop_kms_url) == 0) { GTEST_SKIP() << "Set a proper 'hadoop_kms_url' in config.ini to enable this test."; } auto _key_provider = std::make_unique( - ::absl::StrSplit(dsn::replication::FLAGS_hadoop_kms_url, ",", ::absl::SkipEmpty()), - dsn::FLAGS_cluster_name); + ::absl::StrSplit(FLAGS_hadoop_kms_url, ",", ::absl::SkipEmpty()), FLAGS_cluster_name); dsn::replication::kms_info info; // 1. generate encryption key. diff --git a/src/test/function_test/throttle/test_throttle.cpp b/src/test/function_test/throttle/test_throttle.cpp index d7e5f61b51..4eaa92728c 100644 --- a/src/test/function_test/throttle/test_throttle.cpp +++ b/src/test/function_test/throttle/test_throttle.cpp @@ -42,13 +42,6 @@ #include "utils/rand.h" #include "utils/test_macros.h" -using namespace dsn; -using namespace dsn::replication; -using namespace pegasus; -using std::string; - -static const uint64_t kLimitDurationMs = 10 * 1000; - DSN_DEFINE_int32(function_test.throttle_test, throttle_test_medium_value_kb, 20, @@ -59,6 +52,13 @@ DSN_DEFINE_int32(function_test.throttle_test, 50, "The size of generated large value for test"); +using namespace dsn; +using namespace dsn::replication; +using namespace pegasus; +using std::string; + +static const uint64_t kLimitDurationMs = 10 * 1000; + enum class throttle_type { read_by_qps, diff --git a/src/test/kill_test/data_verifier.cpp b/src/test/kill_test/data_verifier.cpp index 257af5e04f..9f1e06ae38 100644 --- a/src/test/kill_test/data_verifier.cpp +++ b/src/test/kill_test/data_verifier.cpp @@ -34,6 +34,22 @@ #include "utils/fmt_logging.h" #include "utils/strings.h" +DSN_DEFINE_uint32(pegasus.killtest, + set_and_get_timeout_milliseconds, + 3000, + "set() and get() timeout in milliseconds."); +DSN_DEFINE_uint32(pegasus.killtest, set_thread_count, 5, "Thread count of the setter."); +DSN_DEFINE_uint32(pegasus.killtest, + get_thread_count, + FLAGS_set_thread_count * 4, + "Thread count of the getter."); +DSN_DEFINE_string(pegasus.killtest, pegasus_cluster_name, "onebox", "The Pegasus cluster name"); +DSN_DEFINE_validator(pegasus_cluster_name, + [](const char *value) -> bool { return !dsn::utils::is_empty(value); }); +DSN_DEFINE_string(pegasus.killtest, verify_app_name, "temp", "verify app name"); +DSN_DEFINE_validator(verify_app_name, + [](const char *value) -> bool { return !dsn::utils::is_empty(value); }); + namespace pegasus { namespace test { @@ -55,22 +71,6 @@ static const long stat_p999_pos = stat_batch - stat_batch / 1000 - 1; static const long stat_p9999_pos = stat_batch - stat_batch / 10000 - 1; static const long stat_max_pos = stat_batch - 1; -DSN_DEFINE_uint32(pegasus.killtest, - set_and_get_timeout_milliseconds, - 3000, - "set() and get() timeout in milliseconds."); -DSN_DEFINE_uint32(pegasus.killtest, set_thread_count, 5, "Thread count of the setter."); -DSN_DEFINE_uint32(pegasus.killtest, - get_thread_count, - FLAGS_set_thread_count * 4, - "Thread count of the getter."); -DSN_DEFINE_string(pegasus.killtest, pegasus_cluster_name, "onebox", "The Pegasus cluster name"); -DSN_DEFINE_validator(pegasus_cluster_name, - [](const char *value) -> bool { return !dsn::utils::is_empty(value); }); -DSN_DEFINE_string(pegasus.killtest, verify_app_name, "temp", "verify app name"); -DSN_DEFINE_validator(verify_app_name, - [](const char *value) -> bool { return !dsn::utils::is_empty(value); }); - // return time in us. long get_time() { diff --git a/src/test/kill_test/kill_testor.cpp b/src/test/kill_test/kill_testor.cpp index 4c9b23a279..7c01eb1422 100644 --- a/src/test/kill_test/kill_testor.cpp +++ b/src/test/kill_test/kill_testor.cpp @@ -37,14 +37,15 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" -namespace pegasus { -namespace test { DSN_DEFINE_uint32(pegasus.killtest, kill_interval_seconds, 30, ""); DSN_DEFINE_uint32(pegasus.killtest, max_seconds_for_all_partitions_to_recover, 600, ""); DSN_DECLARE_string(pegasus_cluster_name); DSN_DECLARE_string(verify_app_name); +namespace pegasus { +namespace test { + kill_testor::kill_testor(const char *config_file) { // initialize the _client. diff --git a/src/test/kill_test/killer_handler_shell.cpp b/src/test/kill_test/killer_handler_shell.cpp index 8ba0adcd58..fa386109c7 100644 --- a/src/test/kill_test/killer_handler_shell.cpp +++ b/src/test/kill_test/killer_handler_shell.cpp @@ -30,13 +30,13 @@ #include "utils/safe_strerror_posix.h" #include "utils/strings.h" -namespace pegasus { -namespace test { - DSN_DEFINE_string(killer.handler.shell, onebox_run_path, "~/pegasus/run.sh", "onebox run path"); DSN_DEFINE_validator(onebox_run_path, [](const char *value) -> bool { return !dsn::utils::is_empty(value); }); +namespace pegasus { +namespace test { + bool killer_handler_shell::has_meta_dumped_core(int index) { char find_core[1024]; diff --git a/src/test/kill_test/partition_kill_testor.cpp b/src/test/kill_test/partition_kill_testor.cpp index 271cc64c5b..d6f0054755 100644 --- a/src/test/kill_test/partition_kill_testor.cpp +++ b/src/test/kill_test/partition_kill_testor.cpp @@ -36,11 +36,11 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" +DSN_DECLARE_uint32(kill_interval_seconds); + namespace pegasus { namespace test { -DSN_DECLARE_uint32(kill_interval_seconds); - partition_kill_testor::partition_kill_testor(const char *config_file) : kill_testor(config_file) {} void partition_kill_testor::Run() diff --git a/src/test/kill_test/process_kill_testor.cpp b/src/test/kill_test/process_kill_testor.cpp index 400a1a02fb..9e52c1c4d1 100644 --- a/src/test/kill_test/process_kill_testor.cpp +++ b/src/test/kill_test/process_kill_testor.cpp @@ -35,9 +35,6 @@ #include "utils/process_utils.h" #include "utils/strings.h" -namespace pegasus { -namespace test { - DSN_DEFINE_int32(pegasus.killtest, total_meta_count, 0, "total meta count"); DSN_DEFINE_int32(pegasus.killtest, total_replica_count, 0, "total replica count"); DSN_DEFINE_int32(pegasus.killtest, total_zookeeper_count, 0, "total zookeeper count"); @@ -75,6 +72,9 @@ DSN_DEFINE_validator(killer_handler, DSN_DECLARE_uint32(kill_interval_seconds); +namespace pegasus { +namespace test { + process_kill_testor::process_kill_testor(const char *config_file) : kill_testor(config_file) { register_kill_handlers(); diff --git a/src/test/pressure_test/config-pressure.ini b/src/test/pressure_test/config-pressure.ini index 94bcd495ca..f1468ec525 100644 --- a/src/test/pressure_test/config-pressure.ini +++ b/src/test/pressure_test/config-pressure.ini @@ -98,7 +98,7 @@ sortkey_limit = 10000 hashkey_len = 64 sortkey_len = 64 value_len = 100 -cluster_name = onebox +test_cluster_name = onebox app_name = temp ;; operation name : set/get/del/scan operation_name = set diff --git a/src/test/pressure_test/main.cpp b/src/test/pressure_test/main.cpp index dc73d567eb..876df3789d 100644 --- a/src/test/pressure_test/main.cpp +++ b/src/test/pressure_test/main.cpp @@ -39,11 +39,6 @@ #include "utils/strings.h" #include "utils/threadpool_code.h" -using namespace std; -using namespace ::pegasus; - -DEFINE_TASK_CODE(LPC_DEFAUT_TASK, TASK_PRIORITY_COMMON, dsn::THREAD_POOL_DEFAULT) - DSN_DEFINE_int32(pressureclient, qps, 0, "qps of pressure client"); DSN_DEFINE_int32(pressureclient, hashkey_len, 64, "hashkey length"); DSN_DEFINE_int32(pressureclient, sortkey_len, 64, "sortkey length"); @@ -57,8 +52,8 @@ DSN_DEFINE_int64(pressureclient, sortkey_limit, 0, "The sortkey range to generate, in format [0, ****key_limit]."); -DSN_DEFINE_string(pressureclient, cluster_name, "onebox", "cluster name"); -DSN_DEFINE_validator(cluster_name, +DSN_DEFINE_string(pressureclient, test_cluster_name, "onebox", "cluster name"); +DSN_DEFINE_validator(test_cluster_name, [](const char *value) -> bool { return !dsn::utils::is_empty(value); }); DSN_DEFINE_string(pressureclient, app_name, "temp", "app name"); @@ -69,6 +64,11 @@ DSN_DEFINE_string(pressureclient, operation_name, "", "operation name"); DSN_DEFINE_validator(operation_name, [](const char *value) -> bool { return !dsn::utils::is_empty(value); }); +using namespace std; +using namespace ::pegasus; + +DEFINE_TASK_CODE(LPC_DEFAUT_TASK, TASK_PRIORITY_COMMON, dsn::THREAD_POOL_DEFAULT) + // for app static pegasus_client *pg_client = nullptr; static string op_name; // set/get/scan/del @@ -248,7 +248,7 @@ int main(int argc, const char **argv) LOG_INFO("pressureclient {} qps = {}", FLAGS_operation_name, FLAGS_qps); - pg_client = pegasus_client_factory::get_client(FLAGS_cluster_name, FLAGS_app_name); + pg_client = pegasus_client_factory::get_client(FLAGS_test_cluster_name, FLAGS_app_name); CHECK_NOTNULL(pg_client, "initialize pg_client failed"); auto it = _all_funcs.find(FLAGS_operation_name); diff --git a/src/test_util/test_util.h b/src/test_util/test_util.h index 90930c226c..46be616c90 100644 --- a/src/test_util/test_util.h +++ b/src/test_util/test_util.h @@ -36,14 +36,14 @@ #include "utils/flags.h" #include "utils/test_macros.h" +DSN_DECLARE_bool(encrypt_data_at_rest); + namespace dsn { namespace replication { class file_meta; } // namespace replication } // namespace dsn -DSN_DECLARE_bool(encrypt_data_at_rest); - // Save the current value of a flag and restore it at the end of the function. #define PRESERVE_FLAG(name) \ auto PRESERVED_FLAGS_##name = FLAGS_##name; \ diff --git a/src/tools/mutation_log_tool.cpp b/src/tools/mutation_log_tool.cpp index b68a4979b5..0e88d06f4a 100644 --- a/src/tools/mutation_log_tool.cpp +++ b/src/tools/mutation_log_tool.cpp @@ -49,11 +49,11 @@ #include "utils/flags.h" #include "utils/time_utils.h" +DSN_DECLARE_int32(log_private_file_size_mb); + namespace dsn { namespace replication { -DSN_DECLARE_int32(log_private_file_size_mb); - bool mutation_log_tool::dump( const std::string &log_dir, gpid pid, diff --git a/src/utils/builtin_metrics.cpp b/src/utils/builtin_metrics.cpp index 1690e966be..3ffb8f1474 100644 --- a/src/utils/builtin_metrics.cpp +++ b/src/utils/builtin_metrics.cpp @@ -36,13 +36,13 @@ METRIC_DEFINE_gauge_int64(server, dsn::metric_unit::kMegaBytes, "The total amount of physical memory usage in MB"); -namespace dsn { - DSN_DEFINE_uint64(metrics, builtin_metrics_update_interval_ms, 10 * 1000, "The interval (milliseconds) at which builtin metrics are updated."); +namespace dsn { + builtin_metrics::builtin_metrics() : METRIC_VAR_INIT_server(virtual_mem_usage_mb), METRIC_VAR_INIT_server(resident_mem_usage_mb) { diff --git a/src/utils/latency_tracer.cpp b/src/utils/latency_tracer.cpp index ded91db3c9..e8408dd938 100644 --- a/src/utils/latency_tracer.cpp +++ b/src/utils/latency_tracer.cpp @@ -38,9 +38,6 @@ METRIC_DEFINE_percentile_int64(latency_tracer, dsn::metric_unit::kNanoSeconds, "The duration between two points(stages)"); -namespace dsn { -namespace utils { - DSN_DEFINE_bool(replication, enable_latency_tracer, false, @@ -53,6 +50,8 @@ DSN_DEFINE_bool(replication, "whether open the latency tracer report for metrics"); DSN_TAG_VARIABLE(enable_latency_tracer_report, FT_MUTABLE); +namespace dsn { +namespace utils { namespace { #define LATENCY_TRACER_METRIC_ENTITY_ID(description, starting_point, end_point) \ diff --git a/src/utils/latency_tracer.h b/src/utils/latency_tracer.h index 4597866459..e1136d475a 100644 --- a/src/utils/latency_tracer.h +++ b/src/utils/latency_tracer.h @@ -28,6 +28,9 @@ #include "utils/ports.h" #include "utils/synchronize.h" +DSN_DECLARE_bool(enable_latency_tracer); +DSN_DECLARE_bool(enable_latency_tracer_report); + namespace dsn { namespace utils { @@ -87,8 +90,6 @@ namespace utils { * * "request.tracer" will record the time duration among all trace points. **/ -DSN_DECLARE_bool(enable_latency_tracer); -DSN_DECLARE_bool(enable_latency_tracer_report); class latency_tracer { diff --git a/src/utils/logging.cpp b/src/utils/logging.cpp index 5b55b9d4c6..be25695991 100644 --- a/src/utils/logging.cpp +++ b/src/utils/logging.cpp @@ -39,7 +39,6 @@ #include "utils/logging_provider.h" #include "utils/sys_exit_hook.h" -log_level_t log_start_level = LOG_LEVEL_INFO; DSN_DEFINE_string(core, logging_start_level, "LOG_LEVEL_INFO", @@ -47,6 +46,8 @@ DSN_DEFINE_string(core, DSN_DEFINE_bool(core, logging_flush_on_exit, true, "flush log when exit system"); +log_level_t log_start_level = LOG_LEVEL_INFO; + namespace dsn { using namespace tools; diff --git a/src/utils/metrics.cpp b/src/utils/metrics.cpp index 09d08c4c9a..e948f77eb5 100644 --- a/src/utils/metrics.cpp +++ b/src/utils/metrics.cpp @@ -40,6 +40,13 @@ #include "utils/string_conv.h" #include "utils/strings.h" +DSN_DEFINE_uint64(metrics, + entity_retirement_delay_ms, + 10 * 60 * 1000, + "The retention interval (milliseconds) for an entity after it becomes stale."); + +DSN_DECLARE_string(cluster_name); + METRIC_DEFINE_entity(server); dsn::metric_entity_ptr server_metric_entity() @@ -50,11 +57,6 @@ dsn::metric_entity_ptr server_metric_entity() namespace dsn { -DSN_DEFINE_uint64(metrics, - entity_retirement_delay_ms, - 10 * 60 * 1000, - "The retention interval (milliseconds) for an entity after it becomes stale."); - metric_entity::metric_entity(const metric_entity_prototype *prototype, const std::string &id, const attr_map &attrs) @@ -476,8 +478,6 @@ metric_entity_ptr metric_registry::find_or_create_entity(const metric_entity_pro return entity; } -DSN_DECLARE_string(cluster_name); - namespace { #define ENCODE_OBJ_VAL(cond, val) \ @@ -493,7 +493,7 @@ void encode_cluster(dsn::metric_json_writer &writer) { writer.Key(dsn::kMetricClusterField.c_str()); - ENCODE_OBJ_VAL(!utils::is_empty(dsn::FLAGS_cluster_name), dsn::FLAGS_cluster_name); + ENCODE_OBJ_VAL(!utils::is_empty(FLAGS_cluster_name), FLAGS_cluster_name); } void encode_role(dsn::metric_json_writer &writer) diff --git a/src/utils/shared_io_service.cpp b/src/utils/shared_io_service.cpp index 523b043c12..285ef23e7a 100644 --- a/src/utils/shared_io_service.cpp +++ b/src/utils/shared_io_service.cpp @@ -24,10 +24,7 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" -namespace dsn { -namespace tools { - -const uint32_t kMinTimerServiceWorkerCount = 3; +static const uint32_t kMinTimerServiceWorkerCount = 3; DSN_DEFINE_uint32(core, timer_service_worker_count, kMinTimerServiceWorkerCount, @@ -43,6 +40,9 @@ DSN_DEFINE_validator(timer_service_worker_count, [](uint32_t worker_count) -> bo return true; }); +namespace dsn { +namespace tools { + shared_io_service::shared_io_service() { _workers.reserve(FLAGS_timer_service_worker_count); diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index f1464aa26b..5e580ccbba 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -47,11 +47,6 @@ #include "utils/strings.h" #include "utils/time_utils.h" -DSN_DECLARE_string(logging_start_level); - -namespace dsn { -namespace tools { - DSN_DEFINE_bool(tools.simple_logger, fast_flush, false, "whether to flush immediately"); DSN_DEFINE_bool(tools.simple_logger, short_header, @@ -68,9 +63,13 @@ DSN_DEFINE_string(tools.simple_logger, "LOG_LEVEL_WARNING", "copy log messages at or above this level to stderr in addition to logfiles"); DSN_DEFINE_validator(stderr_start_level, [](const char *level) -> bool { - return !utils::equals(level, "LOG_LEVEL_INVALID"); + return !dsn::utils::equals(level, "LOG_LEVEL_INVALID"); }); +DSN_DECLARE_string(logging_start_level); + +namespace dsn { +namespace tools { static void print_header(FILE *fp, log_level_t log_level) { // The leading character of each log lines, corresponding to the log level diff --git a/src/utils/test/flag_test.cpp b/src/utils/test/flag_test.cpp index e3b601991d..235fe0e927 100644 --- a/src/utils/test/flag_test.cpp +++ b/src/utils/test/flag_test.cpp @@ -25,9 +25,6 @@ #include "utils/errors.h" #include "utils/flags.h" -namespace dsn { -namespace utils { - DSN_DEFINE_int32(flag_test, test_int32, 5, ""); DSN_TAG_VARIABLE(test_int32, FT_MUTABLE); @@ -137,6 +134,9 @@ DSN_DEFINE_group_validator(lesser_greater_1, [](std::string &message) -> bool { return true; }); +namespace dsn { +namespace utils { + TEST(flag_test, update_config) { auto res = update_flag("test_int32", "3"); diff --git a/src/utils/test/logger.cpp b/src/utils/test/logger.cpp index 2b6e8a6973..70d6ef1e2b 100644 --- a/src/utils/test/logger.cpp +++ b/src/utils/test/logger.cpp @@ -43,11 +43,11 @@ #include "utils/safe_strerror_posix.h" #include "utils/simple_logger.h" +DSN_DECLARE_uint64(max_number_of_log_files_on_disk); + namespace dsn { namespace tools { -DSN_DECLARE_uint64(max_number_of_log_files_on_disk); - namespace { void get_log_file_index(std::vector &log_index) diff --git a/src/utils/test/metrics_test.cpp b/src/utils/test/metrics_test.cpp index 640f8267e0..9f3a4d1909 100644 --- a/src/utils/test/metrics_test.cpp +++ b/src/utils/test/metrics_test.cpp @@ -43,10 +43,10 @@ #include "utils/strings.h" #include "utils/test/nth_element_utils.h" -namespace dsn { - DSN_DECLARE_uint64(entity_retirement_delay_ms); +namespace dsn { + class my_gauge : public metric { public: diff --git a/src/zookeeper/distributed_lock_service_zookeeper.cpp b/src/zookeeper/distributed_lock_service_zookeeper.cpp index 6d57ef4bd0..52d1f98f62 100644 --- a/src/zookeeper/distributed_lock_service_zookeeper.cpp +++ b/src/zookeeper/distributed_lock_service_zookeeper.cpp @@ -42,11 +42,11 @@ #include "zookeeper_error.h" #include "zookeeper_session.h" +DSN_DECLARE_int32(timeout_ms); + namespace dsn { namespace dist { -DSN_DECLARE_int32(timeout_ms); - std::string distributed_lock_service_zookeeper::LOCK_NODE_PREFIX = "LOCKNODE"; distributed_lock_service_zookeeper::distributed_lock_service_zookeeper() : ref_counter() diff --git a/src/zookeeper/zookeeper_session.cpp b/src/zookeeper/zookeeper_session.cpp index c2c868cd78..18ed6ea838 100644 --- a/src/zookeeper/zookeeper_session.cpp +++ b/src/zookeeper/zookeeper_session.cpp @@ -37,8 +37,6 @@ #include "zookeeper/zookeeper.jute.h" #include "zookeeper_session.h" -namespace dsn { -namespace security { DSN_DECLARE_bool(enable_zookeeper_kerberos); DSN_DEFINE_string(security, zookeeper_kerberos_service_name, @@ -48,11 +46,6 @@ DSN_DEFINE_string(security, zookeeper_sasl_service_fqdn, "", "The FQDN of a Zookeeper server, used in Kerberos Principal"); -} // namespace security -} // namespace dsn - -namespace dsn { -namespace dist { // TODO(yingchun): to keep compatibility, the global name is FLAGS_timeout_ms. The name is not very // suitable, maybe improve the macro to us another global name. DSN_DEFINE_int32(zookeeper, @@ -61,6 +54,9 @@ DSN_DEFINE_int32(zookeeper, "The timeout of accessing ZooKeeper, in milliseconds"); DSN_DEFINE_string(zookeeper, hosts_list, "", "Zookeeper hosts list"); +namespace dsn { +namespace dist { + zookeeper_session::zoo_atomic_packet::zoo_atomic_packet(unsigned int size) { _capacity = size; @@ -162,15 +158,15 @@ int zookeeper_session::attach(void *callback_owner, const state_callback &cb) { utils::auto_write_lock l(_watcher_lock); if (nullptr == _handle) { - if (dsn::security::FLAGS_enable_zookeeper_kerberos) { + if (FLAGS_enable_zookeeper_kerberos) { zoo_sasl_params_t sasl_params = {0}; - sasl_params.service = dsn::security::FLAGS_zookeeper_kerberos_service_name; + sasl_params.service = FLAGS_zookeeper_kerberos_service_name; sasl_params.mechlist = "GSSAPI"; rpc_address addr; - CHECK(addr.from_string_ipv4(dsn::security::FLAGS_zookeeper_sasl_service_fqdn), + CHECK(addr.from_string_ipv4(FLAGS_zookeeper_sasl_service_fqdn), "zookeeper_sasl_service_fqdn {} is invalid", - dsn::security::FLAGS_zookeeper_sasl_service_fqdn); - sasl_params.host = dsn::security::FLAGS_zookeeper_sasl_service_fqdn; + FLAGS_zookeeper_sasl_service_fqdn); + sasl_params.host = FLAGS_zookeeper_sasl_service_fqdn; _handle = zookeeper_init_sasl(FLAGS_hosts_list, global_watcher, FLAGS_timeout_ms, diff --git a/src/zookeeper/zookeeper_session_mgr.cpp b/src/zookeeper/zookeeper_session_mgr.cpp index 8f4750df88..64e23d2260 100644 --- a/src/zookeeper/zookeeper_session_mgr.cpp +++ b/src/zookeeper/zookeeper_session_mgr.cpp @@ -35,11 +35,11 @@ #include "utils/singleton_store.h" #include "zookeeper_session.h" +DSN_DEFINE_string(zookeeper, logfile, "", "The Zookeeper logfile"); + namespace dsn { namespace dist { -DSN_DEFINE_string(zookeeper, logfile, "", "The Zookeeper logfile"); - zookeeper_session_mgr::zookeeper_session_mgr() { FILE *fp = fopen(FLAGS_logfile, "a");