From c05e3f65c790671ea09e9fa0908ce2cc7c59645e Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 17 Jul 2024 20:01:26 +0800 Subject: [PATCH 01/15] refactor(shell): Refactor on 'remote_command' command (#2052) This patch refactors the code for the `remote_command` shell CLI tool, now uses `argh::parser` instead of `getopt_long`. And now the error output is colored to notice the administors. The effected commands are `server_info`, `server_stat`, `flush_log` and `remote_command`. To keep compatible, `remote_command server-info`, `remote_command server_info` and `server_info` are equal. The same to `server_stat` and `flush_log`. Behavior changes: - If there are some errors, the tool return false instead of true, then the USAGE can be output for hint. - The output is organized as JSON format, and the embeded structures are also in JSON format. (This is very useful for thirdparty tools to parse the output.) Example: ``` >>> remote_command server-info -l b334667ddf87:34801 { "command": "server-info ", "details": { "b334667ddf87:34801": { "acked": true, "message": { "build_type": "Debug", "git_SHA": "cbdca43302c36f5e03b0f96d0dbd6a59149f2ce6", "start_time": "2024-07-15 07:25:50", "version": "2.6.0-SNAPSHOT" }, "role": "user-specified" } }, "failed_count": 0, "succeed_count": 1 } >>> server_stat -l b334667ddf87:34801 { "command": "server-stat ", "details": { "b334667ddf87:34801": { "acked": true, "message": "replica*app.pegasus*manual.compact.enqueue.count=not_found, replica*app.pegasus*manual.compact.running.count=not_found, replica*app.pegasus*rdb.block_cache.memory_usage=not_found, replica*eon.replica_stub*closing.replica(Count)=not_found, replica*eon.replica_stub*disk.available.max.ratio=not_found, replica*eon.replica_stub*disk.available.min.ratio=not_found, replica*eon.replica_stub*disk.available.total.ratio=not_found, replica*eon.replica_stub*disk.capacity.total(MB)=not_found, replica*eon.replica_stub*opening.replica(Count)=not_found, replica*eon.replica_stub*replica(Count)=not_found, replica*eon.replica_stub*replicas.commit.qps=not_found, replica*eon.replica_stub*replicas.learning.count=not_found, replica*server*memused.res(MB)=not_found, replica*server*memused.virt(MB)=not_found, zion*profiler*RPC_RRDB_RRDB_BATCH_GET.latency.server=not_found, zion*profiler*RPC_RRDB_RRDB_BATCH_GET.qps=not_found, zion*profiler*RPC_RRDB_RRDB_GET.latency.server=not_found, zion*profiler*RPC_RRDB_RRDB_GET.qps=not_found, zion*profiler*RPC_RRDB_RRDB_MULTI_GET.latency.server=not_found, zion*profiler*RPC_RRDB_RRDB_MULTI_GET.qps=not_found, zion*profiler*RPC_RRDB_RRDB_MULTI_PUT.latency.server=not_found, zion*profiler*RPC_RRDB_RRDB_MULTI_PUT.qps=not_found, zion*profiler*RPC_RRDB_RRDB_PUT.latency.server=not_found, zion*profiler*RPC_RRDB_RRDB_PUT.qps=not_found", "role": "user-specified" } }, "failed_count": 0, "succeed_count": 1 } >>> remote_command meta.lb.add_secondary_max_count_for_one_node -l b334667ddf87:34601 { "command": "meta.lb.add_secondary_max_count_for_one_node ", "details": { "b334667ddf87:34601": { "acked": true, "message": { "error": "ok", "meta.lb.add_secondary_max_count_for_one_node": "10" }, "role": "user-specified" } }, "failed_count": 0, "succeed_count": 1 } ``` --- src/shell/commands/node_management.cpp | 212 ++++++++++++------------- src/utils/command_manager.h | 8 +- 2 files changed, 109 insertions(+), 111 deletions(-) diff --git a/src/shell/commands/node_management.cpp b/src/shell/commands/node_management.cpp index 803d2ee2af..32cdc091b3 100644 --- a/src/shell/commands/node_management.cpp +++ b/src/shell/commands/node_management.cpp @@ -17,17 +17,22 @@ * under the License. */ +#include +#include #include +#include +#include #include #include -#include #include // IWYU pragma: no_include #include #include +#include #include #include #include +#include #include #include #include @@ -39,19 +44,19 @@ #include "dsn.layer2_types.h" #include "meta_admin_types.h" #include "runtime/rpc/rpc_host_port.h" +#include "shell/argh.h" #include "shell/command_executor.h" #include "shell/command_helper.h" #include "shell/command_utils.h" #include "shell/commands.h" -#include "shell/sds/sds.h" #include "utils/error_code.h" #include "utils/errors.h" #include "utils/flags.h" +#include "utils/fmt_logging.h" #include "utils/math.h" #include "utils/metrics.h" #include "utils/output_utils.h" #include "utils/ports.h" -#include "utils/strings.h" DSN_DEFINE_uint32(shell, nodes_sample_interval_ms, 1000, "The interval between sampling metrics."); DSN_DEFINE_validator(nodes_sample_interval_ms, [](uint32_t value) -> bool { return value > 0; }); @@ -544,145 +549,134 @@ bool ls_nodes(command_executor *e, shell_context *sc, arguments args) bool server_info(command_executor *e, shell_context *sc, arguments args) { - char *argv[args.argc + 1]; - memcpy(argv, args.argv, sizeof(char *) * args.argc); - argv[args.argc] = (char *)"server-info"; - arguments new_args; - new_args.argc = args.argc + 1; - new_args.argv = argv; - return remote_command(e, sc, new_args); + return remote_command(e, sc, args); } bool server_stat(command_executor *e, shell_context *sc, arguments args) { - char *argv[args.argc + 1]; - memcpy(argv, args.argv, sizeof(char *) * args.argc); - argv[args.argc] = (char *)"server-stat"; - arguments new_args; - new_args.argc = args.argc + 1; - new_args.argv = argv; - return remote_command(e, sc, new_args); + return remote_command(e, sc, args); } -bool remote_command(command_executor *e, shell_context *sc, arguments args) +bool flush_log(command_executor *e, shell_context *sc, arguments args) { - static struct option long_options[] = {{"node_type", required_argument, 0, 't'}, - {"node_list", required_argument, 0, 'l'}, - {"resolve_ip", no_argument, 0, 'r'}, - {0, 0, 0, 0}}; + return remote_command(e, sc, args); +} - std::string type; - std::string nodes; - optind = 0; - bool resolve_ip = false; - while (true) { - int option_index = 0; - int c; - c = getopt_long(args.argc, args.argv, "t:l:r", long_options, &option_index); - if (c == -1) - break; - switch (c) { - case 't': - type = optarg; - break; - case 'l': - nodes = optarg; - break; - case 'r': - resolve_ip = true; +bool remote_command(command_executor *e, shell_context *sc, arguments args) +{ + // Command format: [remote_command] [arguments...] + // [-t all|meta-server|replica-server] + // [-r|--resolve_ip] + // [-l host:port,host:port...] + argh::parser cmd(args.argc, args.argv, argh::parser::PREFER_PARAM_FOR_UNREG_OPTION); + + std::string command; + std::vector pos_args; + int pos = 0; + do { + // Try to parse the positional args. + const auto &pos_arg = cmd(pos++); + if (!pos_arg) { break; - default: - return false; } - } - if (!type.empty() && !nodes.empty()) { - fprintf(stderr, "can not specify both node_type and node_list\n"); - return false; - } + // Ignore the args that are useless to the command. + static const std::set kIgnoreArgs({"remote_command"}); + if (kIgnoreArgs.count(pos_arg.str()) == 1) { + continue; + } - if (type.empty() && nodes.empty()) { - type = "all"; - } + // Collect the positional args following by the command. + if (!command.empty()) { + pos_args.emplace_back(pos_arg.str()); + continue; + } - if (!type.empty() && type != "all" && type != "meta-server" && type != "replica-server") { - fprintf(stderr, "invalid type, should be: all | meta-server | replica-server\n"); + // Initialize the command. + const std::map kCmdsMapping({{"server_info", "server-info"}, + {"server_stat", "server-stat"}, + {"flush_log", "flush-log"}}); + const auto &it = kCmdsMapping.find(pos_arg.str()); + if (it != kCmdsMapping.end()) { + // Use the mapped command. + command = it->second; + } else { + command = pos_arg.str(); + } + } while (true); + + if (command.empty()) { + SHELL_PRINTLN_ERROR("missing "); return false; } + const auto resolve_ip = cmd[{"-r", "--resolve_ip"}]; + auto node_type = cmd({"-t"}).str(); + std::vector nodes_str; + PARSE_OPT_STRS(nodes_str, "", {"-l"}); - if (optind == args.argc) { - fprintf(stderr, "command not specified\n"); + if (!node_type.empty() && !nodes_str.empty()) { + SHELL_PRINTLN_ERROR("can not specify both node_type and nodes_str"); return false; } - std::string cmd = args.argv[optind]; - std::vector arguments; - for (int i = optind + 1; i < args.argc; i++) { - arguments.push_back(args.argv[i]); + if (node_type.empty() && nodes_str.empty()) { + node_type = "all"; } - std::vector node_list; - if (!type.empty()) { - if (!fill_nodes(sc, type, node_list)) { - fprintf(stderr, "prepare nodes failed, type = %s\n", type.c_str()); - return true; - } - } else { - std::vector tokens; - dsn::utils::split_args(nodes.c_str(), tokens, ','); - if (tokens.empty()) { - fprintf(stderr, "can't parse node from node_list\n"); - return true; - } + static const std::set kValidNodeTypes({"all", "meta-server", "replica-server"}); + if (!node_type.empty() && kValidNodeTypes.count(node_type) == 0) { + SHELL_PRINTLN_ERROR("invalid node_type, should be in [{}]", + fmt::join(kValidNodeTypes, ", ")); + return false; + } - for (std::string &token : tokens) { - const auto node = dsn::host_port::from_string(token); - if (!node) { - fprintf(stderr, "parse %s as a host:port node failed\n", token.c_str()); - return true; + std::vector nodes; + do { + if (node_type.empty()) { + for (const auto &node_str : nodes_str) { + const auto node = dsn::host_port::from_string(node_str); + if (!node) { + SHELL_PRINTLN_ERROR("parse '{}' as host:port failed", node_str); + return false; + } + nodes.emplace_back("user-specified", node); } - node_list.emplace_back("user-specified", node); + break; } - } - - fprintf(stderr, "COMMAND: %s", cmd.c_str()); - for (auto &s : arguments) { - fprintf(stderr, " %s", s.c_str()); - } - fprintf(stderr, "\n\n"); - std::vector> results = - call_remote_command(sc, node_list, cmd, arguments); + if (!fill_nodes(sc, node_type, nodes)) { + SHELL_PRINTLN_ERROR("prepare nodes failed, node_type = {}", node_type); + return false; + } + } while (false); + nlohmann::json info; + info["command"] = fmt::format("{} {}", command, fmt::join(pos_args, " ")); + const auto results = call_remote_command(sc, nodes, command, pos_args); int succeed = 0; int failed = 0; - // TODO (yingchun) output is hard to read, need do some refactor - for (int i = 0; i < node_list.size(); ++i) { - const auto &node = node_list[i]; - const auto hostname = replication_ddl_client::node_name(node.hp, resolve_ip); - fprintf(stderr, "CALL [%s] [%s] ", node.desc.c_str(), hostname.c_str()); + CHECK_EQ(results.size(), nodes.size()); + for (int i = 0; i < nodes.size(); ++i) { + nlohmann::json node_info; + node_info["role"] = nodes[i].desc; + node_info["acked"] = results[i].first; + try { + // Treat the message as a JSON object by default. + node_info["message"] = nlohmann::json::parse(results[i].second); + } catch (nlohmann::json::exception &exp) { + // Treat it as a string if failed to parse as a JSON object. + node_info["message"] = results[i].second; + } if (results[i].first) { - fprintf(stderr, "succeed:\n%s\n", results[i].second.c_str()); succeed++; } else { - fprintf(stderr, "failed:\n%s\n", results[i].second.c_str()); failed++; } + info["details"].emplace(replication_ddl_client::node_name(nodes[i].hp, resolve_ip), + node_info); } - - fprintf(stderr, "\nSucceed count: %d\n", succeed); - fprintf(stderr, "Failed count: %d\n", failed); - + info["succeed_count"] = succeed; + info["failed_count"] = failed; + fmt::println(stdout, "{}", info.dump(2)); return true; } - -bool flush_log(command_executor *e, shell_context *sc, arguments args) -{ - char *argv[args.argc + 1]; - memcpy(argv, args.argv, sizeof(char *) * args.argc); - argv[args.argc] = (char *)"flush-log"; - arguments new_args; - new_args.argc = args.argc + 1; - new_args.argv = argv; - return remote_command(e, sc, new_args); -} diff --git a/src/utils/command_manager.h b/src/utils/command_manager.h index b124388fd8..0a34dd4c16 100644 --- a/src/utils/command_manager.h +++ b/src/utils/command_manager.h @@ -27,6 +27,7 @@ #pragma once #include +#include // IWYU pragma: no_include #include #include @@ -147,7 +148,9 @@ class command_manager : public ::dsn::utils::singleton // Invalid arguments size. if (args.size() > 1) { - msg["error"] = "ERR: invalid arguments, only one integer argument is acceptable"; + msg["error"] = + fmt::format("ERR: invalid arguments '{}', only one argument is acceptable", + fmt::join(args, " ")); return msg.dump(2); } @@ -162,7 +165,8 @@ class command_manager : public ::dsn::utils::singleton T new_value = 0; if (!internal::buf2signed(args[0], new_value) || !validator(static_cast(new_value))) { - msg["error"] = "ERR: invalid arguments"; + msg["error"] = + fmt::format("ERR: invalid argument '{}', the value is not acceptable", args[0]); return msg.dump(2); } From 013e2e16b4c1926d7e00d93e7961ccd350bf4fd5 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 22 Jul 2024 11:23:22 +0800 Subject: [PATCH 02/15] feat(logging): Make it possible to print logs to stderr when using screen_logger (#2079) If using `screen_logger`, the logs are only printed to stdout. This patch introduces a new config `[tools.screen_logger]stderr_start_level_on_stdout` to define the lowest level of log messages to be copied to stderr in addition to stdout. This is useful for testing. A test using `ASSERT_DEATH` is added to check the logs are printed to stderr indeed. --- src/utils/logging_provider.h | 3 +++ src/utils/simple_logger.cpp | 27 +++++++++++++++++++++------ src/utils/simple_logger.h | 7 +++---- src/utils/test/fmt_logging_test.cpp | 4 +++- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/utils/logging_provider.h b/src/utils/logging_provider.h index 8adec47433..ec6bec53d0 100644 --- a/src/utils/logging_provider.h +++ b/src/utils/logging_provider.h @@ -71,7 +71,10 @@ class logging_provider static logging_provider *create_default_instance(); + logging_provider(log_level_t stderr_start_level) : _stderr_start_level(stderr_start_level) {} + std::vector> _cmds; + const log_level_t _stderr_start_level; }; void set_log_prefixed_message_func(std::function func); diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index 86145d4a8e..6e56cbbca9 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -79,6 +79,15 @@ DSN_DEFINE_uint64( DSN_DEFINE_validator(max_number_of_log_files_on_disk, [](int32_t value) -> bool { return value > 0; }); +DSN_DEFINE_string(tools.screen_logger, + stderr_start_level_on_stdout, + "LOG_LEVEL_WARNING", + "The lowest level of log messages to be copied to stderr in addition to stdout"); +DSN_DEFINE_validator(stderr_start_level_on_stdout, [](const char *value) -> bool { + const auto level = enum_from_string(value, LOG_LEVEL_INVALID); + return LOG_LEVEL_DEBUG <= level && level <= LOG_LEVEL_FATAL; +}); + DSN_DEFINE_string( tools.simple_logger, stderr_start_level, @@ -169,9 +178,15 @@ inline void process_fatal_log(log_level_t log_level) } // anonymous namespace +screen_logger::screen_logger(const char *, const char *) + : logging_provider(enum_from_string(FLAGS_stderr_start_level_on_stdout, LOG_LEVEL_INVALID)), + _short_header(true) +{ +} + void screen_logger::print_header(log_level_t log_level) { - ::dsn::tools::print_header(stdout, LOG_LEVEL_COUNT, log_level); + ::dsn::tools::print_header(stdout, _stderr_start_level, log_level); } void screen_logger::print_long_header(const char *file, @@ -180,12 +195,12 @@ void screen_logger::print_long_header(const char *file, log_level_t log_level) { ::dsn::tools::print_long_header( - stdout, file, function, line, _short_header, LOG_LEVEL_COUNT, log_level); + stdout, file, function, line, _short_header, _stderr_start_level, log_level); } void screen_logger::print_body(const char *body, log_level_t log_level) { - ::dsn::tools::print_body(stdout, body, LOG_LEVEL_COUNT, log_level); + ::dsn::tools::print_body(stdout, body, _stderr_start_level, log_level); } void screen_logger::log( @@ -206,10 +221,10 @@ void screen_logger::log( void screen_logger::flush() { ::fflush(stdout); } simple_logger::simple_logger(const char *log_dir, const char *role_name) - : _log_dir(std::string(log_dir)), + : logging_provider(enum_from_string(FLAGS_stderr_start_level, LOG_LEVEL_INVALID)), + _log_dir(std::string(log_dir)), _log(nullptr), - _file_bytes(0), - _stderr_start_level(enum_from_string(FLAGS_stderr_start_level, LOG_LEVEL_INVALID)) + _file_bytes(0) { // Use 'role_name' if it is specified, otherwise use 'base_name'. const std::string symlink_name( diff --git a/src/utils/simple_logger.h b/src/utils/simple_logger.h index 7fe8d79f8a..5437c0fe6a 100644 --- a/src/utils/simple_logger.h +++ b/src/utils/simple_logger.h @@ -44,7 +44,7 @@ namespace tools { class screen_logger : public logging_provider { public: - explicit screen_logger(const char *, const char *) : _short_header(true) {} + explicit screen_logger(const char *, const char *); ~screen_logger() override = default; void log(const char *file, @@ -56,12 +56,12 @@ class screen_logger : public logging_provider virtual void flush(); private: - static void print_header(log_level_t log_level); + void print_header(log_level_t log_level); void print_long_header(const char *file, const char *function, const int line, log_level_t log_level); - static void print_body(const char *body, log_level_t log_level); + void print_body(const char *body, log_level_t log_level); ::dsn::utils::ex_lock_nr _lock; const bool _short_header; @@ -119,7 +119,6 @@ class simple_logger : public logging_provider FILE *_log; // The byte size of the current log file. uint64_t _file_bytes; - const log_level_t _stderr_start_level; }; } // namespace tools } // namespace dsn diff --git a/src/utils/test/fmt_logging_test.cpp b/src/utils/test/fmt_logging_test.cpp index 95e7d0dfd9..20017ddecd 100644 --- a/src/utils/test/fmt_logging_test.cpp +++ b/src/utils/test/fmt_logging_test.cpp @@ -27,13 +27,14 @@ #include #include +#include "absl/strings/string_view.h" #include "common/gpid.h" #include "common/replication.codes.h" #include "gtest/gtest.h" #include "runtime/task/task_code.h" #include "utils/error_code.h" #include "utils/errors.h" -#include "absl/strings/string_view.h" +#include "utils/fmt_logging.h" namespace dsn { namespace replication { @@ -47,6 +48,7 @@ TEST(fmt_logging, basic) ASSERT_EQ(fmt::format("{}", LPC_REPLICATION_LOW), "LPC_REPLICATION_LOW"); ASSERT_EQ(absl::string_view("yes"), "yes"); ASSERT_EQ(fmt::format("{}", absl::string_view("yes\0yes")), "yes\0yes"); + ASSERT_DEATH(CHECK(false, "CHECK false in test"), "CHECK false in test"); } } // namespace replication From 722af2e407a530ff71f7e919bb48a1ce8069f8a8 Mon Sep 17 00:00:00 2001 From: "shalk(xiao kun)" Date: Fri, 26 Jul 2024 11:24:28 +0800 Subject: [PATCH 03/15] feat(docker): support arm64 (#2081) Resolve https://github.com/apache/incubator-pegasus/issues/2080. --- .github/workflows/build-push-env-docker.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-push-env-docker.yml b/.github/workflows/build-push-env-docker.yml index f72af174a6..bc6bf447da 100644 --- a/.github/workflows/build-push-env-docker.yml +++ b/.github/workflows/build-push-env-docker.yml @@ -52,19 +52,20 @@ jobs: - name: Checkout # The glibc version on ubuntu1804 and centos7 is lower than the actions/checkout@v4 required, so # we need to force to use actions/checkout@v3. - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2.10.0 + uses: docker/build-push-action@v6 with: + platforms: linux/amd64,linux/arm64 context: . file: ./docker/pegasus-build-env/${{ matrix.dockertag }}/Dockerfile push: true From 34ab9cbea06afd4d42d18906e5f3c45545e14ad6 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 31 Jul 2024 12:06:21 +0800 Subject: [PATCH 04/15] feat(util): Introduce map utilities (#2076) When using C++ standard map-like containers, it's inconvenient to use the find*() functions, we have to compare the found iterator to the `end()` element, using the `second` to access its value, etc. It will be convenient to introduce some map utilities. The facebook/folly implementation [1] is limited, it lacks of some useful functions (for example, ContainsKey(), InsertIfNotPresent()). The original "gutil" implementation maybe from google/supersonic [2], but it has been archived and not hasn't been well maintainanced for years. The apache/kudu and impala/kudu implementation [3][4] is copied from some google implementation, but they lack of meaningful licenses. Another evoluted implementation is google/zetasql [5], and it contains clear license. It's a SQL analyzer framework but not a lightweight infrastructure library, it's too heavy to introduce the whole project. This patch introduces the necessary source code of map-util and its unit tests (under src/gutil/*), and give a few examples to show how to use. 1. https://github.com/facebook/folly/blob/main/folly/container/MapUtil.h 2. https://github.com/google/supersonic/blob/master/supersonic/utils/map_util.h 3. https://github.com/apache/kudu/blob/master/src/kudu/gutil/map-util.h 4. https://github.com/apache/impala/blob/master/be/src/gutil/map-util.h 5. https://github.com/google/zetasql/blob/master/zetasql/base/map_util.h --- .licenserc.yaml | 8 + LICENSE | 23 + src/CMakeLists.txt | 1 + src/base/idl_utils.h | 9 +- src/common/duplication_common.cpp | 1 + src/gutil/CMakeLists.txt | 21 + src/gutil/map_traits.h | 80 ++ src/gutil/map_util.h | 717 ++++++++++++++++++ src/gutil/no_destructor.h | 114 +++ src/gutil/test/CMakeLists.txt | 36 + src/gutil/test/main.cpp | 26 + src/gutil/test/map_traits_test.cpp | 78 ++ src/gutil/test/map_util_test.h | 475 ++++++++++++ src/gutil/test/map_util_unittest.cpp | 523 +++++++++++++ src/gutil/test/no_destructor_test.cpp | 184 +++++ src/gutil/test/run.sh | 19 + src/http/http_call_registry.h | 8 +- src/http/http_server.cpp | 10 +- src/meta/app_balance_policy.cpp | 4 +- src/meta/app_env_validator.cpp | 32 +- src/meta/cluster_balance_policy.cpp | 11 +- .../duplication/meta_duplication_service.cpp | 6 +- src/meta/test/cluster_balance_policy_test.cpp | 8 +- .../test/meta_bulk_load_ingestion_test.cpp | 9 +- src/meta/test/meta_bulk_load_service_test.cpp | 15 +- src/nfs/nfs_server_impl.cpp | 23 +- src/replica/replica_config.cpp | 35 +- src/utils/metrics.h | 9 +- src/utils/test/logger.cpp | 4 +- src/utils/test/metrics_test.cpp | 60 +- 30 files changed, 2424 insertions(+), 125 deletions(-) create mode 100644 src/gutil/CMakeLists.txt create mode 100644 src/gutil/map_traits.h create mode 100644 src/gutil/map_util.h create mode 100644 src/gutil/no_destructor.h create mode 100644 src/gutil/test/CMakeLists.txt create mode 100644 src/gutil/test/main.cpp create mode 100644 src/gutil/test/map_traits_test.cpp create mode 100644 src/gutil/test/map_util_test.h create mode 100644 src/gutil/test/map_util_unittest.cpp create mode 100644 src/gutil/test/no_destructor_test.cpp create mode 100755 src/gutil/test/run.sh diff --git a/.licenserc.yaml b/.licenserc.yaml index 6128a75b7c..b8e45b5ec4 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -667,5 +667,13 @@ header: - 'src/zookeeper/zookeeper_session.h' - 'src/zookeeper/zookeeper_session_mgr.cpp' - 'src/zookeeper/zookeeper_session_mgr.h' + # Apache License, Version 2.0, Copyright 2018 Google LLC + - 'src/gutil/test/map_traits_test.cpp' + - 'src/gutil/test/map_util_test.h' + - 'src/gutil/test/map_util_unittest.cpp' + - 'src/gutil/test/no_destructor_test.cpp' + - 'src/gutil/map_traits.h' + - 'src/gutil/map_util.h' + - 'src/gutil/no_destructor.h' comment: on-failure diff --git a/LICENSE b/LICENSE index c89dd134d9..a8e975f469 100644 --- a/LICENSE +++ b/LICENSE @@ -570,3 +570,26 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +src/gutil/test/map_traits_test.cpp +src/gutil/test/map_util_test.h +src/gutil/test/map_util_unittest.cpp +src/gutil/test/no_destructor_test.cpp +src/gutil/map_traits.h +src/gutil/map_util.h +src/gutil/no_destructor.h + +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a90098773..0bd9f65395 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -41,6 +41,7 @@ add_subdirectory(client_lib) add_subdirectory(common) add_subdirectory(failure_detector) add_subdirectory(geo) +add_subdirectory(gutil) add_subdirectory(http) add_subdirectory(meta) add_subdirectory(nfs) diff --git a/src/base/idl_utils.h b/src/base/idl_utils.h index bb04018102..88324929b0 100644 --- a/src/base/idl_utils.h +++ b/src/base/idl_utils.h @@ -23,16 +23,17 @@ #include "rrdb/rrdb_types.h" #include "utils/fmt_utils.h" +#include "gutil/map_util.h" namespace pegasus { inline std::string cas_check_type_to_string(dsn::apps::cas_check_type::type type) { - auto it = dsn::apps::_cas_check_type_VALUES_TO_NAMES.find(type); - if (it == dsn::apps::_cas_check_type_VALUES_TO_NAMES.end()) { - return std::string("INVALID=") + std::to_string(int(type)); + const auto *name = gutil::FindOrNull(dsn::apps::_cas_check_type_VALUES_TO_NAMES, type); + if (dsn_unlikely(name == nullptr)) { + return fmt::format("INVALID={}", type); } - return it->second; + return *name; } inline bool cas_is_check_operand_needed(dsn::apps::cas_check_type::type type) diff --git a/src/common/duplication_common.cpp b/src/common/duplication_common.cpp index 0aea933473..4dc5323c65 100644 --- a/src/common/duplication_common.cpp +++ b/src/common/duplication_common.cpp @@ -122,6 +122,7 @@ class duplication_group_registry : public utils::singleton + +namespace gutil { +namespace subtle { +namespace internal_map_traits { +struct Rank1 +{ +}; +struct Rank0 : Rank1 +{ +}; + +template +auto GetKey(V &&v, Rank0) -> decltype((std::forward(v).first)) +{ + return std::forward(v).first; +} +template +auto GetKey(V &&v, Rank1) -> decltype(std::forward(v).key()) +{ + return std::forward(v).key(); +} + +template +auto GetMapped(V &&v, Rank0) -> decltype((std::forward(v).second)) +{ + return std::forward(v).second; +} +template +auto GetMapped(V &&v, Rank1) -> decltype(std::forward(v).value()) +{ + return std::forward(v).value(); +} + +} // namespace internal_map_traits + +// Accesses the `key_type` from a `value_type`. +template +auto GetKey(V &&v) + -> decltype(internal_map_traits::GetKey(std::forward(v), internal_map_traits::Rank0())) +{ + return internal_map_traits::GetKey(std::forward(v), internal_map_traits::Rank0()); +} + +// Accesses the `mapped_type` from a `value_type`. +template +auto GetMapped(V &&v) + -> decltype(internal_map_traits::GetMapped(std::forward(v), internal_map_traits::Rank0())) +{ + return internal_map_traits::GetMapped(std::forward(v), internal_map_traits::Rank0()); +} + +} // namespace subtle +} // namespace gutil diff --git a/src/gutil/map_util.h b/src/gutil/map_util.h new file mode 100644 index 0000000000..e8788e4abd --- /dev/null +++ b/src/gutil/map_util.h @@ -0,0 +1,717 @@ +// +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +// This file provides utility functions for use with STL map-like data +// structures, such as std::map and hash_map. Some functions will also work with +// sets, such as ContainsKey(). +// +// The main functions in this file fall into the following categories: +// +// - Find*() +// - Contains*() +// - Insert*() +// - Lookup*() +// +// These functions often have "...OrDie" or "...OrDieNoPrint" variants. These +// variants will crash the process with a CHECK() failure on error, including +// the offending key/data in the log message. The NoPrint variants will not +// include the key/data in the log output under the assumption that it's not a +// printable type. +// +// Most functions are fairly self explanatory from their names, with the +// exception of Find*() vs Lookup*(). The Find functions typically use the map's +// .find() member function to locate and return the map's value type. The +// Lookup*() functions typically use the map's .insert() (yes, insert) member +// function to insert the given value if necessary and returns (usually a +// reference to) the map's value type for the found item. +// +// See the per-function comments for specifics. +// +// There are also a handful of functions for doing other miscellaneous things. +// +// A note on terminology: +// +// In this file, `m` and `M` represent a map and its type. +// +// Map-like containers are collections of pairs. Like all STL containers they +// contain a few standard typedefs identifying the types of data they contain. +// Given the following map declaration: +// +// std::map my_map; +// +// the notable typedefs would be as follows: +// +// - key_type -- string +// - value_type -- std::pair +// - mapped_type -- int +// +// Note that the map above contains two types of "values": the key-value pairs +// themselves (value_type) and the values within the key-value pairs +// (mapped_type). A value_type consists of a key_type and a mapped_type. +// +// The documentation below is written for programmers thinking in terms of keys +// and the (mapped_type) values associated with a given key. For example, the +// statement +// +// my_map["foo"] = 3; +// +// has a key of "foo" (type: string) with a value of 3 (type: int). +// + +#include +#include +#include +#include + +#include "absl/meta/type_traits.h" +#include "gutil/map_traits.h" +#include "gutil/no_destructor.h" +#include "utils/fmt_logging.h" + +namespace gutil { + +// These helper template aliases are implementation details of map_util, +// provided for notational convenience. Despite the file-level documentation +// about map typedefs, map_util doesn't actually use them. +// It uses the Map's value_type, and the value_type's first_type and +// second_type. This can matter for nonconformant containers. +template +using MapUtilValueT = typename M::value_type; +template +using MapUtilKeyT = typename MapUtilValueT::first_type; +template +using MapUtilMappedT = typename MapUtilValueT::second_type; + +namespace internal_map_util { + +template +struct HasTryEmplace : std::false_type +{ +}; + +template +struct HasTryEmplace< + M, + absl::void_t().try_emplace(std::declval &>()))>> + : std::true_type +{ +}; + +template +struct InitType +{ + using type = MapUtilValueT; +}; + +template +struct InitType>::value>::type> +{ + using type = typename M::init_type; +}; + +template +const V &ValueInitializedDefault() +{ + static const gutil::NoDestructor value_initialized_default{}; + return *value_initialized_default; +} + +} // namespace internal_map_util + +template +using MapUtilInitT = typename internal_map_util::InitType::type; + +// +// Find*() +// + +// Returns a const reference to the value associated with the given key if it +// exists. Crashes otherwise. +// +// This is intended as a replacement for operator[] as an rvalue (for reading) +// when the key is guaranteed to exist. +// +// operator[] for lookup is discouraged for several reasons (note that these +// reasons may apply to only some map types): +// * It has a side-effect of inserting missing keys +// * It is not thread-safe (even when it is not inserting, it can still +// choose to resize the underlying storage) +// * It invalidates iterators (when it chooses to resize) +// * It default constructs a value object even if it doesn't need to +// +// This version assumes the key is printable, and includes it in the fatal log +// message. +template > +const MapUtilMappedT &FindOrDie(const M &m, const KeyType &key) +{ + auto it = m.find(key); + CHECK(it != m.end(), "Map key not found: {}", key); + return gutil::subtle::GetMapped(*it); +} + +// Same as above, but returns a non-const reference. +template > +MapUtilMappedT &FindOrDie(M &m, // NOLINT + const KeyType &key) +{ + auto it = m.find(key); + CHECK(it != m.end(), "Map key not found: {}", key); + return gutil::subtle::GetMapped(*it); +} + +// Same as FindOrDie above, but doesn't log the key on failure. +template > +const MapUtilMappedT &FindOrDieNoPrint(const M &m, const KeyType &key) +{ + auto it = m.find(key); + CHECK(it != m.end(), "Map key not found"); + return gutil::subtle::GetMapped(*it); +} + +// Same as above, but returns a non-const reference. +template > +MapUtilMappedT &FindOrDieNoPrint(M &m, // NOLINT + const KeyType &key) +{ + auto it = m.find(key); + CHECK(it != m.end(), "Map key not found"); + return gutil::subtle::GetMapped(*it); +} + +// Returns a const reference to the value associated with the given key if it +// exists, otherwise returns a const reference to a value-initialized object +// that is never destroyed. +template > +const MapUtilMappedT &FindWithDefault(const M &m, const KeyType &key) +{ + auto it = m.find(key); + if (it != m.end()) + return gutil::subtle::GetMapped(*it); + return internal_map_util::ValueInitializedDefault>(); +} + +// Returns a const reference to the value associated with the given key if it +// exists, otherwise returns a const reference to the provided default value. +// +// Prefer the two-argument form unless you need to specify a custom default +// value (i.e., one that is not equal to a value-initialized instance). +// +// WARNING: If a temporary object is passed as the default "value," +// this function will return a reference to that temporary object, +// which will be destroyed at the end of the statement. A common +// example: if you have a map with string values, and you pass a char* +// as the default "value," either use the returned value immediately +// or store it in a string (not string&). +// +// TODO: Stop using this. +template +const MapUtilMappedT & +FindWithDefault(const M &m, const MapUtilKeyT &key, const MapUtilMappedT &value) +{ + auto it = m.find(key); + if (it != m.end()) + return gutil::subtle::GetMapped(*it); + return value; +} + +// Returns a pointer to the const value associated with the given key if it +// exists, or null otherwise. +template > +const MapUtilMappedT *FindOrNull(const M &m, const KeyType &key) +{ + auto it = m.find(key); + if (it == m.end()) + return nullptr; + return &gutil::subtle::GetMapped(*it); +} + +// Returns a pointer to the non-const value associated with the given key if it +// exists, or null otherwise. +template > +MapUtilMappedT *FindOrNull(M &m, // NOLINT + const KeyType &key) +{ + auto it = m.find(key); + if (it == m.end()) + return nullptr; + return &gutil::subtle::GetMapped(*it); +} + +// Returns the pointer value associated with the given key. If none is found, +// null is returned. The function is designed to be used with a map of keys +// to pointers. +// +// This function does not distinguish between a missing key and a key mapped +// to a null value. +template > +MapUtilMappedT FindPtrOrNull(const M &m, const KeyType &key) +{ + auto it = m.find(key); + if (it == m.end()) + return MapUtilMappedT(); + return gutil::subtle::GetMapped(*it); +} + +// Same as above, except takes non-const reference to m. +// +// This function is needed for containers that propagate constness to the +// pointee, such as boost::ptr_map. +template > +MapUtilMappedT FindPtrOrNull(M &m, // NOLINT + const KeyType &key) +{ + auto it = m.find(key); + if (it == m.end()) + return MapUtilMappedT(); + return gutil::subtle::GetMapped(*it); +} + +// Finds the value associated with the given key and copies it to *value (if +// non-null). Returns false if the key was not found, true otherwise. +template +bool FindCopy(const M &m, const Key &key, Value *value) +{ + auto it = m.find(key); + if (it == m.end()) + return false; + if (value) + *value = gutil::subtle::GetMapped(*it); + return true; +} + +// +// Contains*() +// + +// Returns true if and only if the given m contains the given key. +template +bool ContainsKey(const M &m, const Key &key) +{ + return m.find(key) != m.end(); +} + +// Returns true if and only if the given m contains the given key-value +// pair. +template +bool ContainsKeyValuePair(const M &m, const Key &key, const Value &value) +{ + auto range = m.equal_range(key); + for (auto it = range.first; it != range.second; ++it) { + if (gutil::subtle::GetMapped(*it) == value) { + return true; + } + } + return false; +} + +// +// Insert*() +// + +// Inserts the given key-value pair into the m. Returns true if and +// only if the key from the given pair didn't previously exist. Otherwise, the +// value in the map is replaced with the value from the given pair. +template +bool InsertOrUpdate(M *m, const MapUtilInitT &vt) +{ + auto ret = m->insert(vt); + if (ret.second) + return true; + subtle::GetMapped(*ret.first) = subtle::GetMapped(vt); // update + return false; +} + +// Same as above, except that the key and value are passed separately. +template +bool InsertOrUpdate(M *m, const MapUtilKeyT &key, const MapUtilMappedT &value) +{ + return InsertOrUpdate(m, {key, value}); +} + +// Inserts/updates all the key-value pairs from the range defined by the +// iterators "first" and "last" into the given m. +template +void InsertOrUpdateMany(M *m, InputIterator first, InputIterator last) +{ + for (; first != last; ++first) { + InsertOrUpdate(m, *first); + } +} + +// Change the value associated with a particular key in a map or hash_map +// of the form std::map which owns the objects pointed to by the +// value pointers. If there was an existing value for the key, it is deleted. +// True indicates an insert took place, false indicates an update + delete. +template +bool InsertAndDeleteExisting(M *m, const MapUtilKeyT &key, const MapUtilMappedT &value) +{ + auto ret = m->insert(MapUtilValueT(key, value)); + if (ret.second) + return true; + delete ret.first->second; + ret.first->second = value; + return false; +} + +// Inserts the given key and value into the given m if and only if the +// given key did NOT already exist in the m. If the key previously +// existed in the m, the value is not changed. Returns true if the +// key-value pair was inserted; returns false if the key was already present. +template +bool InsertIfNotPresent(M *m, const MapUtilInitT &vt) +{ + return m->insert(vt).second; +} + +// Same as above except the key and value are passed separately. +template +bool InsertIfNotPresent(M *m, const MapUtilKeyT &key, const MapUtilMappedT &value) +{ + return InsertIfNotPresent(m, {key, value}); +} + +// Same as above except dies if the key already exists in the m. +template +void InsertOrDie(M *m, const MapUtilInitT &value) +{ + CHECK(InsertIfNotPresent(m, value), "duplicate value: {}", value); +} + +// Same as above except doesn't log the value on error. +template +void InsertOrDieNoPrint(M *m, const MapUtilInitT &value) +{ + CHECK(InsertIfNotPresent(m, value), "duplicate value."); +} + +// Inserts the key-value pair into the m. Dies if key was already +// present. +template +void InsertOrDie(M *m, const MapUtilKeyT &key, const MapUtilMappedT &data) +{ + CHECK(InsertIfNotPresent(m, key, data), "duplicate key: ", key); +} + +// Same as above except doesn't log the key on error. +template +void InsertOrDieNoPrint(M *m, const MapUtilKeyT &key, const MapUtilMappedT &data) +{ + CHECK(InsertIfNotPresent(m, key, data), "duplicate key."); +} + +// Inserts a new key and default-initialized value. Dies if the key was already +// present. Returns a reference to the value. Example usage: +// +// std::map m; +// SomeProto& proto = InsertKeyOrDie(&m, 3); +// proto.set_field("foo"); +template +auto InsertKeyOrDie(M *m, const MapUtilKeyT &key) -> + typename std::enable_if::value, MapUtilMappedT &>::type +{ + auto res = m->try_emplace(key); + CHECK(res.second, "duplicate key: ", key); + return gutil::subtle::GetMapped(*res.first); +} + +// Anything without try_emplace, we support with the legacy code path. +template +auto InsertKeyOrDie(M *m, const MapUtilKeyT &key) -> + typename std::enable_if::value, MapUtilMappedT &>::type +{ + auto res = m->insert(MapUtilValueT(key, MapUtilMappedT())); + CHECK(res.second, "duplicate key: ", key); + return res.first->second; +} + +// +// Lookup*() +// + +// Looks up a given key and value pair in m and inserts the key-value pair if +// it's not already present. Returns a reference to the value associated with +// the key. +template +MapUtilMappedT &LookupOrInsert(M *m, const MapUtilInitT &vt) +{ + return subtle::GetMapped(*m->insert(vt).first); +} + +// Same as above except the key-value are passed separately. +template +MapUtilMappedT &LookupOrInsert(M *m, const MapUtilKeyT &key, const MapUtilMappedT &value) +{ + return LookupOrInsert(m, {key, value}); +} + +// Returns a reference to the pointer associated with key. If not found, a +// pointee is constructed and added to the map. In that case, the new pointee is +// forwarded constructor arguments; when no arguments are provided the default +// constructor is used. +// +// Useful for containers of the form Map, where Ptr is pointer-like. +template +MapUtilMappedT &LookupOrInsertNew(M *m, const MapUtilKeyT &key, Args &&...args) +{ + using Mapped = MapUtilMappedT; + using MappedDeref = decltype(*std::declval()); + using Element = typename std::decay::type; + auto ret = m->insert(MapUtilValueT(key, Mapped())); + if (ret.second) { + ret.first->second = Mapped(new Element(std::forward(args)...)); + } + return ret.first->second; +} + +// +// Misc Utility Functions +// + +// Updates the value associated with the given key. If the key was not already +// present, then the key-value pair are inserted and "previous" is unchanged. If +// the key was already present, the value is updated and "*previous" will +// contain a copy of the old value. +// +// Returns true if and only if there was an already existing value. +// +// InsertOrReturnExisting has complementary behavior that returns the +// address of an already existing value, rather than updating it. + +template +bool UpdateReturnCopy(M *m, const MapUtilValueT &vt, MapUtilMappedT *previous) +{ + auto ret = m->insert(vt); + if (ret.second) + return false; + if (previous) + *previous = ret.first->second; + ret.first->second = vt.second; // update + return true; +} + +// Same as above except that the key and mapped value are passed separately. +template +bool UpdateReturnCopy(M *m, + const MapUtilKeyT &key, + const MapUtilMappedT &value, + MapUtilMappedT *previous) +{ + return UpdateReturnCopy(m, MapUtilValueT(key, value), previous); +} + +// Tries to insert the given key-value pair into the m. Returns null +// if the insert succeeds. Otherwise, returns a pointer to the existing value. +// +// This complements UpdateReturnCopy in that it allows to update only after +// verifying the old value and still insert quickly without having to look up +// twice. Unlike UpdateReturnCopy this also does not come with the issue of an +// undefined previous* in case new data was inserted. +template +MapUtilMappedT *InsertOrReturnExisting(M *m, const MapUtilValueT &vt) +{ + auto ret = m->insert(vt); + if (ret.second) + return nullptr; // Inserted, no previous value. + return &ret.first->second; // Return address of previous value. +} + +// Same as above, except for explicit key and data. +template +MapUtilMappedT * +InsertOrReturnExisting(M *m, const MapUtilKeyT &key, const MapUtilMappedT &data) +{ + return InsertOrReturnExisting(m, MapUtilValueT(key, data)); +} + +// Saves the reverse mapping into reverse. Returns true if values could all be +// inserted. +template +bool ReverseMap(const M &m, ReverseM *reverse) +{ + CHECK_NOTNULL(reverse, ""); + bool all_unique = true; + for (const auto &kv : m) { + if (!InsertOrUpdate(reverse, kv.second, kv.first)) { + all_unique = false; + } + } + return all_unique; +} + +// Like ReverseMap above, but returns its output m. Return type has to +// be specified explicitly. Example: +// M::M(...) : m_(...), r_(ReverseMap(m_)) {} +template +ReverseM ReverseMap(const M &m) +{ + typename std::remove_const::type reverse; + ReverseMap(m, &reverse); + return reverse; +} + +// Erases the m item identified by the given key, and returns the value +// associated with that key. It is assumed that the value (i.e., the +// mapped_type) is a pointer. Returns null if the key was not found in the +// m. +// +// Examples: +// std::map my_map; +// +// One line cleanup: +// delete EraseKeyReturnValuePtr(&my_map, "abc"); +// +// Use returned value: +// std::unique_ptr value_ptr( +// EraseKeyReturnValuePtr(&my_map, "abc")); +// if (value_ptr.get()) +// value_ptr->DoSomething(); +// +template +MapUtilMappedT EraseKeyReturnValuePtr(M *m, const MapUtilKeyT &key) +{ + auto it = m->find(key); + if (it == m->end()) + return nullptr; + MapUtilMappedT v = std::move(gutil::subtle::GetMapped(*it)); + m->erase(it); + return v; +} + +// Inserts all the keys from m into key_container, which must +// support insert(M::key_type). +// +// Note: any initial contents of the key_container are not cleared. +template +void InsertKeysFromMap(const M &m, KeyContainer *key_container) +{ + CHECK_NOTNULL(key_container, ""); + for (const auto &kv : m) { + key_container->insert(kv.first); + } +} + +// Appends all the keys from m into key_container, which must +// support push_back(M::key_type). +// +// Note: any initial contents of the key_container are not cleared. +template +void AppendKeysFromMap(const M &m, KeyContainer *key_container) +{ + CHECK_NOTNULL(key_container, ""); + for (const auto &kv : m) { + key_container->push_back(kv.first); + } +} + +// A more specialized overload of AppendKeysFromMap to optimize reallocations +// for the common case in which we're appending keys to a vector and hence can +// (and sometimes should) call reserve() first. +// +// (It would be possible to play SFINAE games to call reserve() for any +// m that supports it, but this seems to get us 99% of what we need +// without the complexity of a SFINAE-based solution.) +template +void AppendKeysFromMap(const M &m, std::vector *key_container) +{ + CHECK_NOTNULL(key_container, ""); + // We now have the opportunity to call reserve(). Calling reserve() every + // time is a bad idea for some use cases: libstdc++'s implementation of + // std::vector<>::reserve() resizes the vector's backing store to exactly the + // given size (unless it's already at least that big). Because of this, + // the use case that involves appending a lot of small maps (total size + // N) one by one to a vector would be O(N^2). But never calling reserve() + // loses the opportunity to improve the use case of adding from a large + // map to an empty vector (this improves performance by up to 33%). A + // number of heuristics are possible. Here we use the simplest one. + if (key_container->empty()) { + key_container->reserve(m.size()); + } + for (const auto &kv : m) { + key_container->push_back(kv.first); + } +} + +// Inserts all the values from m into value_container, which must +// support push_back(M::mapped_type). +// +// Note: any initial contents of the value_container are not cleared. +template +void AppendValuesFromMap(const M &m, ValueContainer *value_container) +{ + CHECK_NOTNULL(value_container, ""); + for (const auto &kv : m) { + value_container->push_back(kv.second); + } +} + +// A more specialized overload of AppendValuesFromMap to optimize reallocations +// for the common case in which we're appending values to a vector and hence +// can (and sometimes should) call reserve() first. +// +// (It would be possible to play SFINAE games to call reserve() for any +// m that supports it, but this seems to get us 99% of what we need +// without the complexity of a SFINAE-based solution.) +template +void AppendValuesFromMap(const M &m, std::vector *value_container) +{ + CHECK_NOTNULL(value_container, ""); + // See AppendKeysFromMap for why this is done. + if (value_container->empty()) { + value_container->reserve(m.size()); + } + for (const auto &kv : m) { + value_container->push_back(kv.second); + } +} + +// Erases all elements of m where predicate evaluates to true. +// Note: To avoid unnecessary temporary copies of map elements passed to the +// predicate, the predicate must accept 'const M::value_type&'. +// In particular, the value type for a map is 'std::pair', and so a +// predicate accepting 'std::pair' will result in temporary copies. +template +auto AssociativeEraseIf(M *m, Predicate predicate) -> + typename std::enable_iferase(m->begin()))>::value>::type +{ + CHECK_NOTNULL(m, ""); + for (auto it = m->begin(); it != m->end();) { + if (predicate(*it)) { + m->erase(it++); + } else { + ++it; + } + } +} + +template +auto AssociativeEraseIf(M *m, Predicate predicate) -> typename std::enable_if< + std::is_samebegin()), decltype(m->erase(m->begin()))>::value>::type +{ + CHECK_NOTNULL(m, ""); + for (auto it = m->begin(); it != m->end();) { + if (predicate(*it)) { + it = m->erase(it); + } else { + ++it; + } + } +} + +} // namespace gutil diff --git a/src/gutil/no_destructor.h b/src/gutil/no_destructor.h new file mode 100644 index 0000000000..d693fe98d1 --- /dev/null +++ b/src/gutil/no_destructor.h @@ -0,0 +1,114 @@ +// +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include +#include +#include + +namespace gutil { + +// NoDestructor is a wrapper around an object of type T that +// * stores the object of type T inline inside NoDestructor +// * eagerly forwards constructor arguments to it (i.e. acts like T in terms +// of construction) +// * provides access to the object of type T like a pointer via ->, *, and get() +// (note that const NoDestructor works like a pointer to const T) +// * never calls T's destructor for the object +// (hence NoDestructor objects created on the stack or as member variables +// will lead to memory and/or resource leaks) +// +// One key use case of NoDestructor (which in itself is not lazy) is optimizing +// the following pattern of safe on-demand construction of an object with +// non-trivial constructor in static storage without destruction ever happening: +// const string& MyString() { +// static string* x = new string("foo"); // note the "static" +// return *x; +// } +// By using NoDestructor we do not need to involve heap allocation and +// corresponding pointer following (and hence extra CPU cache usage/needs) +// on each access: +// const string& MyString() { +// static NoDestructor x("foo"); +// return *x; +// } +// Since C++11 this static-in-a-function pattern results in exactly-once, +// thread-safe, on-demand construction of an object, and very fast access +// thereafter (the cost is a few extra cycles). +// NoDestructor makes accesses even faster by storing the object inline in +// static storage. +// +// Note that: +// * Since destructor is never called, the object lives on during program exit +// and can be safely accessed by any threads that have not been joined. +// +// Also note that +// static NoDestructor ptr(whatever); +// can safely replace +// static NonPOD* ptr = new NonPOD(whatever); +// or +// static NonPOD obj(whatever); +// at file-level scope when the safe static-in-a-function pattern is infeasible +// to use for some good reason. +// All three of the NonPOD patterns above suffer from the same issue that +// initialization of that object happens non-thread-safely at +// a globally-undefined point during initialization of static-storage objects, +// but NoDestructor<> usage provides both the safety of having the object alive +// during program exit sequence and the performance of not doing extra memory +// dereference on access. +// +template +class NoDestructor +{ +public: + typedef T element_type; + + // Forwards arguments to the T's constructor: calls T(args...). + template ::type...), void(NoDestructor)>::value, + int>::type = 0> + explicit NoDestructor(Ts &&...args) + { + new (&space_) T(std::forward(args)...); + } + + // Forwards copy and move construction for T. Enables usage like this: + // static NoDestructor> x{{{"1", "2", "3"}}}; + // static NoDestructor> x{{1, 2, 3}}; + explicit NoDestructor(const T &x) { new (&space_) T(x); } + explicit NoDestructor(T &&x) { new (&space_) T(std::move(x)); } + + // No copying. + NoDestructor(const NoDestructor &) = delete; + NoDestructor &operator=(const NoDestructor &) = delete; + + // Pretend to be a smart pointer to T with deep constness. + // Never returns a null pointer. + T &operator*() { return *get(); } + T *operator->() { return get(); } + T *get() { return reinterpret_cast(&space_); } + const T &operator*() const { return *get(); } + const T *operator->() const { return get(); } + const T *get() const { return reinterpret_cast(&space_); } + +private: + typename std::aligned_storage::type space_; +}; + +} // namespace gutil diff --git a/src/gutil/test/CMakeLists.txt b/src/gutil/test/CMakeLists.txt new file mode 100644 index 0000000000..75e489cd73 --- /dev/null +++ b/src/gutil/test/CMakeLists.txt @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set(MY_PROJ_NAME pgs_gutil_test) +set(MY_PROJ_SRC "") +set(MY_SRC_SEARCH_MODE "GLOB") +set(MY_PROJ_LIBS + absl::btree + absl::flat_hash_map + absl::node_hash_map + dsn_runtime + dsn_utils + rocksdb + lz4 + zstd + snappy + gmock + gtest) +set(MY_BOOST_LIBS Boost::system Boost::filesystem) +set(MY_BINPLACES + run.sh) +dsn_add_test() diff --git a/src/gutil/test/main.cpp b/src/gutil/test/main.cpp new file mode 100644 index 0000000000..90a402ea3a --- /dev/null +++ b/src/gutil/test/main.cpp @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "runtime/app_model.h" + +GTEST_API_ int main(int argc, char **argv) +{ + testing::InitGoogleTest(&argc, argv); + dsn_exit(RUN_ALL_TESTS()); +} diff --git a/src/gutil/test/map_traits_test.cpp b/src/gutil/test/map_traits_test.cpp new file mode 100644 index 0000000000..9da1bdced5 --- /dev/null +++ b/src/gutil/test/map_traits_test.cpp @@ -0,0 +1,78 @@ +// +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "gutil/map_traits.h" + +#include +#include + +#include "absl/container/node_hash_map.h" +#include "gtest/gtest.h" + +namespace gutil { +namespace subtle { +namespace { + +TEST(MapTraitsTest, UnorderedMap) +{ + absl::node_hash_map m = {{1, 2}}; + EXPECT_EQ(1, GetKey(*m.begin())); + EXPECT_EQ(2, GetMapped(*m.begin())); +} + +TEST(MapTraitsTest, UnorderedMapReferences) +{ + absl::node_hash_map m = {{1, 2}}; + auto it = m.begin(); + const int *k = &it->first; + int *v = &it->second; + EXPECT_EQ(k, &GetKey(*it)); + EXPECT_EQ(v, &GetMapped(*it)); + GetMapped(*it) = 3; + EXPECT_EQ(3, m[1]); +} + +TEST(MapTraitsTest, UnorderedMapConstReferences) +{ + const absl::node_hash_map m = {{1, 2}}; + auto it = m.begin(); + const int *k = &it->first; + const int *v = &it->second; + EXPECT_EQ(k, &GetKey(*it)); + EXPECT_EQ(v, &GetMapped(*it)); +} + +struct CustomMapValueType +{ + int first; + int second; + + // Intentionally add 1 to the result to verify that first/second are preferred + // to key()/value(). + int key() const { return first + 1; } + int value() const { return second + 1; } +}; + +TEST(MapTraitsTest, ValueTypeHasBothFieldsAndGetters) +{ + CustomMapValueType entry = {100, 1000}; + EXPECT_EQ(100, GetKey(entry)); + EXPECT_EQ(1000, GetMapped(entry)); +} + +} // namespace +} // namespace subtle +} // namespace gutil diff --git a/src/gutil/test/map_util_test.h b/src/gutil/test/map_util_test.h new file mode 100644 index 0000000000..655dad0f11 --- /dev/null +++ b/src/gutil/test/map_util_test.h @@ -0,0 +1,475 @@ +// +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +// Contains map_util tests templated on STL std::map-like types. + +#include +#include +#include + +#include "gtest/gtest.h" +#include "gutil/map_util.h" +#include "gutil/test/map_util_test.h" + +namespace gutil { + +template +class MapUtilIntIntTest : public ::testing::Test +{ +}; +TYPED_TEST_SUITE_P(MapUtilIntIntTest); + +template +class MapUtilIntIntPtrTest : public ::testing::Test +{ +}; +TYPED_TEST_SUITE_P(MapUtilIntIntPtrTest); + +template +class MapUtilIntIntSharedPtrTest : public ::testing::Test +{ +}; +TYPED_TEST_SUITE_P(MapUtilIntIntSharedPtrTest); + +template +class MapUtilIntIntSharedPtrOnlyTest : public ::testing::Test +{ +}; +TYPED_TEST_SUITE_P(MapUtilIntIntSharedPtrOnlyTest); + +template +class MultiMapUtilIntIntTest : public ::testing::Test +{ +}; +TYPED_TEST_SUITE_P(MultiMapUtilIntIntTest); + +TYPED_TEST_P(MapUtilIntIntTest, ValueMapTests) +{ + using Map = TypeParam; + Map m; + + // Check that I get a default when the key is not present. + EXPECT_EQ(0, FindWithDefault(m, 0, 0)); + EXPECT_EQ(0, FindWithDefault(m, 0)); + // Check that I can insert a value. + EXPECT_TRUE(InsertOrUpdate(&m, 0, 1)); + // .. and get that value back. + EXPECT_EQ(1, FindWithDefault(m, 0, 0)); + EXPECT_EQ(1, FindWithDefault(m, 0)); + // Check that I can update a value. + EXPECT_FALSE(InsertOrUpdate(&m, 0, 2)); + // .. and get that value back. + EXPECT_EQ(2, FindWithDefault(m, 0, 0)); + EXPECT_EQ(2, FindWithDefault(m, 0)); + // Check that FindOrDie works when the value exists. + EXPECT_EQ(2, FindOrDie(m, 0)); + EXPECT_EQ(2, FindOrDieNoPrint(m, 0)); + + // Check FindCopy + int i = 0; + EXPECT_TRUE(FindCopy(m, 0, &i)); + EXPECT_EQ(i, 2); + EXPECT_FALSE(FindCopy(m, 1, &i)); + EXPECT_TRUE(FindCopy(m, 0, static_cast(nullptr))); + EXPECT_FALSE(FindCopy(m, 1, static_cast(nullptr))); + + // Check FindOrNull + int *p1 = FindOrNull(m, 0); + ASSERT_EQ(*p1, 2); + ++(*p1); + const int *p2 = FindOrNull(const_cast(m), 0); + ASSERT_EQ(*p2, 3); + ASSERT_TRUE(FindOrNull(m, 1) == nullptr); + + // Check contains + EXPECT_TRUE(ContainsKey(m, 0)); + EXPECT_FALSE(ContainsKey(m, 1)); + + // Check ContainsKeyValuePair + EXPECT_TRUE(ContainsKeyValuePair(m, 0, 3)); + EXPECT_FALSE(ContainsKeyValuePair(m, 0, 4)); + EXPECT_FALSE(ContainsKeyValuePair(m, 1, 0)); + + // Check insert if not present + EXPECT_FALSE(InsertIfNotPresent(&m, 0, 2)); + EXPECT_TRUE(InsertIfNotPresent(&m, 1, 3)); + + // Check lookup or insert + EXPECT_EQ(3, LookupOrInsert(&m, 0, 2)); + EXPECT_EQ(4, LookupOrInsert(&m, 2, 4)); + EXPECT_EQ(4, FindWithDefault(m, 2, 0)); + EXPECT_EQ(4, FindWithDefault(m, 2)); + + EXPECT_FALSE(InsertOrUpdate(&m, typename Map::value_type(0, 2))); + EXPECT_EQ(2, FindWithDefault(m, 0, 0)); + EXPECT_EQ(2, FindWithDefault(m, 0)); + + // Check InsertOrUpdateMany + std::vector> entries; + entries.push_back(std::make_pair(0, 100)); + entries.push_back(std::make_pair(100, 101)); + entries.push_back(std::make_pair(200, 102)); + + InsertOrUpdateMany(&m, entries.begin(), entries.end()); + EXPECT_EQ(100, FindWithDefault(m, 0, 0)); + EXPECT_EQ(100, FindWithDefault(m, 0)); + EXPECT_EQ(101, FindWithDefault(m, 100, 0)); + EXPECT_EQ(101, FindWithDefault(m, 100)); + EXPECT_EQ(102, FindWithDefault(m, 200, 0)); + EXPECT_EQ(102, FindWithDefault(m, 200)); +} + +TYPED_TEST_P(MapUtilIntIntPtrTest, LookupOrInsertNewTest) +{ + using PtrMap = TypeParam; + PtrMap m; + int *v1, *v2, *v3, *v4; + + // Check inserting one item. + v1 = LookupOrInsertNew(&m, 7); + EXPECT_EQ(0, *v1); + ASSERT_TRUE(v1 != nullptr); + EXPECT_TRUE(ContainsKey(m, 7)); + EXPECT_EQ(m.size(), 1); + + // Check inserting the same item. + v2 = LookupOrInsertNew(&m, 7); + ASSERT_TRUE(v2 != nullptr); + EXPECT_EQ(v1, v2); + EXPECT_EQ(m.size(), 1); + + // Check a couple more items. + v1 = LookupOrInsertNew(&m, 8); + ASSERT_TRUE(v1 != nullptr); + EXPECT_NE(v1, v2); + EXPECT_TRUE(ContainsKey(m, 8)); + EXPECT_EQ(m.size(), 2); + + v2 = LookupOrInsertNew(&m, 8); + EXPECT_EQ(v1, v2); + EXPECT_EQ(m.size(), 2); + + v3 = LookupOrInsertNew(&m, 8, 88); + EXPECT_NE(88, *v3); + EXPECT_EQ(v3, v2); + EXPECT_EQ(m.size(), 2); + + v4 = LookupOrInsertNew(&m, 9, 99); + EXPECT_EQ(99, *v4); + EXPECT_NE(v1, v4); + EXPECT_NE(v2, v4); + EXPECT_NE(v3, v4); + EXPECT_EQ(m.size(), 3); + + // Return by reference, so that the stored value can be modified in the map. + // We check this by verifying the address of the returned value is identical. + EXPECT_EQ(&LookupOrInsertNew(&m, 9), &LookupOrInsertNew(&m, 9, 999)); + + for (auto &kv : m) + delete kv.second; +} + +TYPED_TEST_P(MapUtilIntIntSharedPtrTest, LookupOrInsertNewSharedPtrTest) +{ + using SharedPtrMap = TypeParam; + using SharedPtr = typename TypeParam::value_type::second_type; + SharedPtrMap m; + SharedPtr v1, v2, v3, v4, v5; + + // Check inserting one item. + v1 = LookupOrInsertNew(&m, 7); + ASSERT_TRUE(v1.get() != nullptr); + EXPECT_TRUE(ContainsKey(m, 7)); + EXPECT_EQ(m.size(), 1); + *v1 = 25; + + // Check inserting the same item. + v2 = LookupOrInsertNew(&m, 7); + ASSERT_TRUE(v2.get() != nullptr); + EXPECT_EQ(v1.get(), v2.get()); + EXPECT_EQ(m.size(), 1); + EXPECT_EQ(25, *v2.get()); + + // Check a couple more items. + v2 = LookupOrInsertNew(&m, 8); + ASSERT_TRUE(v2.get() != nullptr); + EXPECT_NE(v1.get(), v2.get()); + EXPECT_TRUE(ContainsKey(m, 8)); + EXPECT_EQ(m.size(), 2); + *v2 = 42; + + v3 = LookupOrInsertNew(&m, 8); + EXPECT_NE(v1.get(), v2.get()); + EXPECT_EQ(v2.get(), v3.get()); + EXPECT_EQ(m.size(), 2); + EXPECT_EQ(25, *v1.get()); + EXPECT_EQ(42, *v2.get()); + EXPECT_EQ(42, *v3.get()); + + m.clear(); + // Since the container does not own the elements and because we still have the + // shared pointers we can still access the old values. + v3 = LookupOrInsertNew(&m, 7); + EXPECT_NE(v1.get(), v3.get()); + EXPECT_NE(v2.get(), v3.get()); + EXPECT_EQ(m.size(), 1); + EXPECT_EQ(25, *v1.get()); + EXPECT_EQ(42, *v2.get()); + EXPECT_EQ(0, *v3.get()); // Also checks for default init of POD elements + + v4 = LookupOrInsertNew(&m, 7, 77); + EXPECT_NE(v1.get(), v4.get()); + EXPECT_NE(v2.get(), v4.get()); + EXPECT_EQ(v3.get(), v4.get()); + EXPECT_EQ(m.size(), 1); + EXPECT_EQ(25, *v1.get()); + EXPECT_EQ(42, *v2.get()); + EXPECT_EQ(0, *v3.get()); + EXPECT_EQ(0, *v4.get()); + + v5 = LookupOrInsertNew(&m, 8, 88); + EXPECT_NE(v1.get(), v5.get()); + EXPECT_NE(v2.get(), v5.get()); + EXPECT_NE(v3.get(), v5.get()); + EXPECT_NE(v4.get(), v5.get()); + EXPECT_EQ(m.size(), 2); + EXPECT_EQ(25, *v1.get()); + EXPECT_EQ(42, *v2.get()); + EXPECT_EQ(0, *v3.get()); + EXPECT_EQ(0, *v4.get()); + EXPECT_EQ(88, *v5.get()); +} + +TYPED_TEST_P(MapUtilIntIntSharedPtrOnlyTest, LookupOrInsertNewSharedPtrSwapTest) +{ + using SharedPtrMap = TypeParam; + using SharedPtr = typename TypeParam::value_type::second_type; + SharedPtrMap m; + SharedPtr v1, v2, v3, v4; + + v1.reset(new int(1)); + LookupOrInsertNew(&m, 11).swap(v1); + EXPECT_TRUE(v1.get() != nullptr); + EXPECT_EQ(0, *v1.get()); // The element created by LookupOrInsertNew + EXPECT_TRUE(ContainsKey(m, 11)); + EXPECT_EQ(1, m.size()); + // If the functions does not correctly return by ref then v2 will contain 0 + // instead of 1 even though v2 still points to the held entry. The tests that + // depend on return by ref use ASSERT_*(). + v2 = LookupOrInsertNew(&m, 11); + ASSERT_EQ(1, *v2.get()); + EXPECT_EQ(v2.get(), LookupOrInsertNew(&m, 11).get()); + + *v2 = 2; + v3 = LookupOrInsertNew(&m, 11); + EXPECT_EQ(2, *v2.get()); + EXPECT_EQ(2, *v3.get()); + ASSERT_NE(v1.get(), v2.get()); + EXPECT_EQ(v2.get(), v3.get()); + ASSERT_NE(v1.get(), LookupOrInsertNew(&m, 11).get()); + EXPECT_EQ(v2.get(), LookupOrInsertNew(&m, 11).get()); + EXPECT_EQ(v3.get(), LookupOrInsertNew(&m, 11).get()); + + v4.reset(new int(4)); + LookupOrInsertNew(&m, 11).swap(v4); + EXPECT_EQ(2, *v4.get()); + ASSERT_EQ(4, *LookupOrInsertNew(&m, 11).get()); + ASSERT_EQ(v3.get(), v4.get()); +} + +TYPED_TEST_P(MapUtilIntIntPtrTest, InsertAndDeleteExistingTest) +{ + using PtrMap = TypeParam; + PtrMap m; + + // Add a few items. + int *v1 = new int; + int *v2 = new int; + int *v3 = new int; + EXPECT_TRUE(InsertAndDeleteExisting(&m, 1, v1)); + EXPECT_TRUE(InsertAndDeleteExisting(&m, 2, v2)); + EXPECT_TRUE(InsertAndDeleteExisting(&m, 3, v3)); + EXPECT_EQ(v1, FindPtrOrNull(m, 1)); + EXPECT_EQ(v2, FindPtrOrNull(m, 2)); + EXPECT_EQ(v3, FindPtrOrNull(m, 3)); + + // Replace a couple. + int *v4 = new int; + int *v5 = new int; + EXPECT_FALSE(InsertAndDeleteExisting(&m, 1, v4)); + EXPECT_FALSE(InsertAndDeleteExisting(&m, 2, v5)); + EXPECT_EQ(v4, FindPtrOrNull(m, 1)); + EXPECT_EQ(v5, FindPtrOrNull(m, 2)); + EXPECT_EQ(v3, FindPtrOrNull(m, 3)); + + // Add one more item. + int *v6 = new int; + EXPECT_TRUE(InsertAndDeleteExisting(&m, 6, v6)); + EXPECT_EQ(v4, FindPtrOrNull(m, 1)); + EXPECT_EQ(v5, FindPtrOrNull(m, 2)); + EXPECT_EQ(v3, FindPtrOrNull(m, 3)); + EXPECT_EQ(v6, FindPtrOrNull(m, 6)); + + // 6 total allocations, this will only delete 4. Heap-check will fail + // here if the existing entries weren't properly deleted. + EXPECT_EQ(4, m.size()); + for (auto &kv : m) + delete kv.second; +} + +TYPED_TEST_P(MapUtilIntIntTest, UpdateReturnCopyTest) +{ + using Map = TypeParam; + Map m; + + int p = 10; + EXPECT_FALSE(UpdateReturnCopy(&m, 0, 5, &p)); + EXPECT_EQ(10, p); + + EXPECT_TRUE(UpdateReturnCopy(&m, 0, 7, &p)); + EXPECT_EQ(5, p); + + // Check UpdateReturnCopy using value_type + p = 10; + EXPECT_FALSE(UpdateReturnCopy(&m, typename Map::value_type(1, 4), &p)); + EXPECT_EQ(10, p); + + EXPECT_TRUE(UpdateReturnCopy(&m, typename Map::value_type(1, 8), &p)); + EXPECT_EQ(4, p); +} + +TYPED_TEST_P(MapUtilIntIntTest, InsertOrReturnExistingTest) +{ + using Map = TypeParam; + Map m; + + EXPECT_EQ(nullptr, InsertOrReturnExisting(&m, 25, 42)); + EXPECT_EQ(42, m[25]); + + int *previous = InsertOrReturnExisting(&m, 25, 666); + EXPECT_EQ(42, *previous); + EXPECT_EQ(42, m[25]); +} + +TYPED_TEST_P(MapUtilIntIntPtrTest, FindPtrOrNullTest) +{ + // Check FindPtrOrNull + using PtrMap = TypeParam; + PtrMap ptr_map; + InsertOrUpdate(&ptr_map, 35, new int(35)); + int *p1 = FindPtrOrNull(ptr_map, 3); + EXPECT_TRUE(nullptr == p1); + const int *p2 = FindPtrOrNull(const_cast(ptr_map), 3); + EXPECT_TRUE(nullptr == p2); + EXPECT_EQ(35, *FindPtrOrNull(ptr_map, 35)); + + for (auto &kv : ptr_map) + delete kv.second; +} + +TYPED_TEST_P(MapUtilIntIntSharedPtrTest, FindPtrOrNullTest) +{ + using SharedPtrMap = TypeParam; + using SharedPtr = typename TypeParam::value_type::second_type; + SharedPtrMap shared_ptr_map; + InsertOrUpdate(&shared_ptr_map, 35, SharedPtr(new int(35))); + const SharedPtr p1 = FindPtrOrNull(shared_ptr_map, 3); + EXPECT_TRUE(nullptr == p1.get()); + const SharedPtr p2 = FindPtrOrNull(const_cast(shared_ptr_map), 3); + EXPECT_TRUE(nullptr == p2.get()); + const SharedPtr p3 = FindPtrOrNull(shared_ptr_map, 35); + const SharedPtr p4 = FindPtrOrNull(shared_ptr_map, 35); + EXPECT_EQ(35, *p3.get()); + EXPECT_EQ(35, *p4.get()); +} + +TYPED_TEST_P(MapUtilIntIntTest, FindOrDieTest) +{ + using Map = TypeParam; + Map m; + m[10] = 15; + EXPECT_EQ(15, FindOrDie(m, 10)); + ASSERT_DEATH(FindOrDie(m, 8), "Map key not found: 8"); + EXPECT_EQ(15, FindOrDieNoPrint(m, 10)); + ASSERT_DEATH(FindOrDieNoPrint(m, 8), "Map key not found"); + + // Make sure the non-const reference returning version works. + FindOrDie(m, 10) = 20; + EXPECT_EQ(20, FindOrDie(m, 10)); + + // Make sure we can lookup values in a const map. + const Map &const_m = m; + EXPECT_EQ(20, FindOrDie(const_m, 10)); +} + +TYPED_TEST_P(MapUtilIntIntTest, InsertOrDieTest) +{ + using Map = TypeParam; + Map m; + InsertOrDie(&m, 1, 2); + EXPECT_EQ(m[1], 2); + ASSERT_DEATH(InsertOrDie(&m, 1, 3), "duplicate"); +} + +TYPED_TEST_P(MapUtilIntIntTest, InsertKeyOrDieTest) +{ + using Map = TypeParam; + Map m; + int &v = InsertKeyOrDie(&m, 1); + EXPECT_EQ(m[1], 0); + v = 2; + EXPECT_EQ(m[1], 2); + ASSERT_DEATH(InsertKeyOrDie(&m, 1), "duplicate"); +} + +TYPED_TEST_P(MapUtilIntIntPtrTest, EraseKeyReturnValuePtrTest) +{ + using PtrMap = TypeParam; + PtrMap ptr_map; + int *v = new int(35); + InsertOrUpdate(&ptr_map, 35, v); + EXPECT_TRUE(EraseKeyReturnValuePtr(&ptr_map, 0) == nullptr); // Test no-op. + EXPECT_EQ(ptr_map.size(), 1); + int *retv = EraseKeyReturnValuePtr(&ptr_map, 35); // Successful operation + EXPECT_EQ(ptr_map.size(), 0); + EXPECT_EQ(v, retv); + delete v; + EXPECT_TRUE(EraseKeyReturnValuePtr(&ptr_map, 35) == nullptr); // Empty map. +} + +TYPED_TEST_P(MultiMapUtilIntIntTest, ContainsKeyValuePairTest) +{ + using Map = TypeParam; + + Map m; + + m.insert(std::make_pair(1, 10)); + m.insert(std::make_pair(1, 11)); + m.insert(std::make_pair(1, 12)); + + m.insert(std::make_pair(3, 13)); + + EXPECT_FALSE(ContainsKeyValuePair(m, 0, 0)); + EXPECT_FALSE(ContainsKeyValuePair(m, 1, 0)); + EXPECT_TRUE(ContainsKeyValuePair(m, 1, 10)); + EXPECT_TRUE(ContainsKeyValuePair(m, 1, 11)); + EXPECT_TRUE(ContainsKeyValuePair(m, 1, 12)); + EXPECT_FALSE(ContainsKeyValuePair(m, 1, 13)); +} + +} // namespace gutil diff --git a/src/gutil/test/map_util_unittest.cpp b/src/gutil/test/map_util_unittest.cpp new file mode 100644 index 0000000000..0c3e70011f --- /dev/null +++ b/src/gutil/test/map_util_unittest.cpp @@ -0,0 +1,523 @@ +// +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "gutil/map_util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// All of the templates for the tests are defined here. +// This file is critical to understand what is tested. +#include "absl/container/btree_set.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/node_hash_map.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "map_util_test.h" + +namespace gutil { + +using ::testing::ElementsAre; +using ::testing::IsEmpty; +using ::testing::Pair; +using ::testing::Pointee; +using ::testing::UnorderedElementsAre; + +TEST(MapUtil, ImplicitTypeConversion) +{ + using Map = std::map; + Map m; + + // Check that I can use a type that's implicitly convertible to the + // key or value type, such as const char* -> string. + EXPECT_EQ("", FindWithDefault(m, "foo", "")); + EXPECT_EQ("", FindWithDefault(m, "foo")); + EXPECT_TRUE(InsertOrUpdate(&m, "foo", "bar")); + EXPECT_EQ("bar", FindWithDefault(m, "foo", "")); + EXPECT_EQ("bar", FindWithDefault(m, "foo")); + EXPECT_EQ("bar", *FindOrNull(m, "foo")); + std::string str; + EXPECT_TRUE(FindCopy(m, "foo", &str)); + EXPECT_EQ("bar", str); + EXPECT_TRUE(ContainsKey(m, "foo")); +} + +TEST(MapUtil, HeterogeneousLookup) +{ + absl::flat_hash_map m; + const auto &const_m = m; + + // Verify that I can use a key type that's appropriate for heterogeneous + // lookup, such as string_view -> string. + constexpr absl::string_view kLookupKey = "foo"; + EXPECT_EQ(FindWithDefault(m, kLookupKey), ""); + EXPECT_EQ(FindWithDefault(const_m, kLookupKey), ""); + + m["foo"] = "bar"; + + EXPECT_EQ(FindOrDie(m, kLookupKey), "bar"); + EXPECT_EQ(FindOrDie(const_m, kLookupKey), "bar"); + + EXPECT_EQ(FindOrDieNoPrint(m, kLookupKey), "bar"); + EXPECT_EQ(FindOrDieNoPrint(const_m, kLookupKey), "bar"); + + EXPECT_EQ(FindWithDefault(m, kLookupKey), "bar"); + EXPECT_EQ(FindWithDefault(const_m, kLookupKey), "bar"); + + EXPECT_THAT(FindOrNull(m, kLookupKey), Pointee(testing::Eq("bar"))); + EXPECT_THAT(FindOrNull(const_m, kLookupKey), Pointee(testing::Eq("bar"))); + + std::string str; + EXPECT_TRUE(FindCopy(m, kLookupKey, &str)); + EXPECT_EQ(str, "bar"); + + std::string str_from_const; + EXPECT_TRUE(FindCopy(const_m, kLookupKey, &str_from_const)); + EXPECT_EQ(str_from_const, "bar"); + + absl::flat_hash_map ptr_m; + const auto &const_ptr_m = ptr_m; + + // Insert an arbitrary non-null pointer into the map. + ASSERT_TRUE(InsertOrUpdate(&ptr_m, "foo", &ptr_m)); + + EXPECT_EQ(FindPtrOrNull(ptr_m, kLookupKey), &ptr_m); + EXPECT_EQ(FindPtrOrNull(const_ptr_m, kLookupKey), &ptr_m); +} + +TEST(MapUtil, SetOperations) +{ + // Set operations + using Set = std::set; + Set s; + EXPECT_TRUE(InsertIfNotPresent(&s, 0)); + EXPECT_FALSE(InsertIfNotPresent(&s, 0)); + EXPECT_TRUE(ContainsKey(s, 0)); +} + +TEST(MapUtil, ReverseMapWithoutDups) +{ + std::map forward; + forward["1"] = 1; + forward["2"] = 2; + forward["3"] = 3; + forward["4"] = 4; + forward["5"] = 5; + std::map reverse; + EXPECT_TRUE(ReverseMap(forward, &reverse)); + EXPECT_THAT(reverse, + ElementsAre(Pair(1, "1"), Pair(2, "2"), Pair(3, "3"), Pair(4, "4"), Pair(5, "5"))); +} + +TEST(MapUtil, ReverseMapWithDups) +{ + std::map forward; + forward["1"] = 1; + forward["2"] = 2; + forward["3"] = 3; + forward["4"] = 4; + forward["5"] = 5; + forward["6"] = 1; + forward["7"] = 2; + std::map reverse; + EXPECT_FALSE(ReverseMap(forward, &reverse)); + // There are 5 distinct values in forward. + EXPECT_THAT(reverse, + ElementsAre(Pair(1, "6"), Pair(2, "7"), Pair(3, "3"), Pair(4, "4"), Pair(5, "5"))); +} + +TEST(MapUtil, SingleArgumentReverseMapWithoutDups) +{ + std::map forward; + forward["1"] = 1; + forward["2"] = 2; + forward["3"] = 3; + forward["4"] = 4; + forward["5"] = 5; + const std::map reverse = ReverseMap>(forward); + EXPECT_THAT(reverse, + ElementsAre(Pair(1, "1"), Pair(2, "2"), Pair(3, "3"), Pair(4, "4"), Pair(5, "5"))); +} + +TEST(MapUtil, SingleArgumentReverseMapWithDups) +{ + std::map forward; + forward["1"] = 1; + forward["2"] = 2; + forward["3"] = 3; + forward["4"] = 4; + forward["5"] = 5; + forward["6"] = 1; + forward["7"] = 2; + const std::map reverse = ReverseMap>(forward); + // There are 5 distinct values in forward. + EXPECT_THAT(reverse, + ElementsAre(Pair(1, "6"), Pair(2, "7"), Pair(3, "3"), Pair(4, "4"), Pair(5, "5"))); +} + +// Wrapper around an int that we can use to test a key without operator<<. +struct Unprintable +{ + int a; + explicit Unprintable(int a) : a(a) {} + bool operator<(const Unprintable &other) const { return a < other.a; } + bool operator==(const Unprintable &other) const { return a == other.a; } +}; + +TEST(MapUtilDeathTest, FindOrDieNoPrint) +{ + // Test FindOrDieNoPrint with a value with no operator<<. + std::map m; + m[Unprintable(1)] = 8; + EXPECT_EQ(8, FindOrDieNoPrint(m, Unprintable(1))); + ASSERT_DEATH(FindOrDieNoPrint(m, Unprintable(2)), "Map key not found"); + + // Make sure the non-const reference returning version works. + FindOrDieNoPrint(m, Unprintable(1)) = 20; + EXPECT_EQ(20, FindOrDieNoPrint(m, Unprintable(1))); + + // Make sure we can lookup values in a const std::map. + const std::map &const_m = m; + EXPECT_EQ(20, FindOrDieNoPrint(const_m, Unprintable(1))); +} + +TEST(MapUtilDeathTest, SetInsertOrDieTest) +{ + std::set s; + InsertOrDie(&s, 1); + EXPECT_TRUE(ContainsKey(s, 1)); + ASSERT_DEATH(InsertOrDie(&s, 1), "duplicate"); +} + +TEST(MapUtilDeathTest, InsertOrDieNoPrint) +{ + std::pair key = std::make_pair(1, 1); + + std::map, int> m; + InsertOrDieNoPrint(&m, key, 2); + EXPECT_EQ(m[key], 2); + ASSERT_DEATH(InsertOrDieNoPrint(&m, key, 3), "duplicate"); + + std::set> s; + InsertOrDieNoPrint(&s, key); + EXPECT_TRUE(ContainsKey(s, key)); + ASSERT_DEATH(InsertOrDieNoPrint(&s, key), "duplicate"); +} + +TEST(MapUtil, InsertKeysFromMap) +{ + const std::map empty_map; + std::set keys_as_ints; + InsertKeysFromMap(empty_map, &keys_as_ints); + EXPECT_TRUE(keys_as_ints.empty()); + + std::set keys_as_longs; // NOLINT + InsertKeysFromMap(empty_map, &keys_as_longs); + EXPECT_TRUE(keys_as_longs.empty()); + + const std::pair number_names_array[] = { + std::make_pair("one", 1), std::make_pair("two", 2), std::make_pair("three", 3)}; + std::map number_names_map( + number_names_array, + number_names_array + sizeof number_names_array / sizeof *number_names_array); + absl::btree_set names; + InsertKeysFromMap(number_names_map, &names); + // No two numbers have the same name, so the container sizes must match. + EXPECT_EQ(names.size(), number_names_map.size()); + EXPECT_EQ(names.count("one"), 1); + EXPECT_EQ(names.count("two"), 1); + EXPECT_EQ(names.count("three"), 1); +} + +TEST(MapUtil, AppendKeysFromMap) +{ + const std::map empty_map; + std::vector keys_as_ints; + AppendKeysFromMap(empty_map, &keys_as_ints); + EXPECT_TRUE(keys_as_ints.empty()); + + std::list keys_as_longs; // NOLINT + AppendKeysFromMap(empty_map, &keys_as_longs); + EXPECT_TRUE(keys_as_longs.empty()); + + const std::pair number_names_array[] = { + std::make_pair("one", 1), std::make_pair("two", 2), std::make_pair("three", 3)}; + std::map number_names_map( + number_names_array, + number_names_array + sizeof number_names_array / sizeof *number_names_array); + std::deque names; + AppendKeysFromMap(number_names_map, &names); + // No two numbers have the same name, so the container sizes must match. + EXPECT_EQ(names.size(), number_names_map.size()); + // The names are appended in the order in which they are found in the + // map, i.e., lexicographical order. + EXPECT_EQ(names[0], "one"); + EXPECT_EQ(names[1], "three"); + EXPECT_EQ(names[2], "two"); + + // Appending again should double the size of the std::deque + AppendKeysFromMap(number_names_map, &names); + EXPECT_EQ(names.size(), 2 * number_names_map.size()); +} + +// Vector is a special case. +TEST(MapUtil, AppendKeysFromMapIntoVector) +{ + const std::map empty_map; + std::vector keys_as_ints; + AppendKeysFromMap(empty_map, &keys_as_ints); + EXPECT_TRUE(keys_as_ints.empty()); + + std::vector keys_as_longs; // NOLINT + AppendKeysFromMap(empty_map, &keys_as_longs); + EXPECT_TRUE(keys_as_longs.empty()); + + const std::pair number_names_array[] = { + std::make_pair("one", 1), std::make_pair("two", 2), std::make_pair("three", 3)}; + std::map number_names_map( + number_names_array, + number_names_array + sizeof number_names_array / sizeof *number_names_array); + std::vector names; + AppendKeysFromMap(number_names_map, &names); + // No two numbers have the same name, so the container sizes must match. + EXPECT_EQ(names.size(), number_names_map.size()); + // The names are appended in the order in which they are found in the + // map, i.e., lexicographical order. + EXPECT_EQ(names[0], "one"); + EXPECT_EQ(names[1], "three"); + EXPECT_EQ(names[2], "two"); + + // Appending again should double the size of the std::deque + AppendKeysFromMap(number_names_map, &names); + EXPECT_EQ(names.size(), 2 * number_names_map.size()); +} + +TEST(MapUtil, AppendValuesFromMap) +{ + const std::map empty_map; + std::vector values_as_ints; + AppendValuesFromMap(empty_map, &values_as_ints); + EXPECT_TRUE(values_as_ints.empty()); + + std::list values_as_longs; // NOLINT + AppendValuesFromMap(empty_map, &values_as_longs); + EXPECT_TRUE(values_as_longs.empty()); + + const std::pair number_names_array[] = { + std::make_pair("one", 1), std::make_pair("two", 2), std::make_pair("three", 3)}; + std::map number_names_map( + number_names_array, + number_names_array + sizeof number_names_array / sizeof *number_names_array); + std::deque numbers; + AppendValuesFromMap(number_names_map, &numbers); + // No two numbers have the same name, so the container sizes must match. + EXPECT_EQ(numbers.size(), number_names_map.size()); + // The numbers are appended in the order in which they are found in the + // map, i.e., lexicographical order. + EXPECT_EQ(numbers[0], 1); + EXPECT_EQ(numbers[1], 3); + EXPECT_EQ(numbers[2], 2); + + // Appending again should double the size of the std::deque + AppendValuesFromMap(number_names_map, &numbers); + EXPECT_EQ(numbers.size(), 2 * number_names_map.size()); +} + +TEST(MapUtil, AppendValuesFromMapIntoVector) +{ + const std::map empty_map; + std::vector values_as_ints; + AppendValuesFromMap(empty_map, &values_as_ints); + EXPECT_TRUE(values_as_ints.empty()); + + std::list values_as_longs; // NOLINT + AppendValuesFromMap(empty_map, &values_as_longs); + EXPECT_TRUE(values_as_longs.empty()); + + const std::pair number_names_array[] = { + std::make_pair("one", 1), std::make_pair("two", 2), std::make_pair("three", 3)}; + std::map number_names_map( + number_names_array, + number_names_array + sizeof number_names_array / sizeof *number_names_array); + std::vector numbers; + AppendValuesFromMap(number_names_map, &numbers); + // No two numbers have the same name, so the container sizes must match. + EXPECT_EQ(numbers.size(), number_names_map.size()); + // The numbers are appended in the order in which they are found in the + // map, i.e., lexicographical order. + EXPECT_EQ(numbers[0], 1); + EXPECT_EQ(numbers[1], 3); + EXPECT_EQ(numbers[2], 2); + + // Appending again should double the size of the std::deque + AppendValuesFromMap(number_names_map, &numbers); + EXPECT_EQ(numbers.size(), 2 * number_names_map.size()); +} + +//////////////////////////////////////////////////////////////////////////////// +// Instantiate tests for std::map and std::unordered_map. +//////////////////////////////////////////////////////////////////////////////// + +// Finish setup for MapType +REGISTER_TYPED_TEST_SUITE_P(MapUtilIntIntTest, + ValueMapTests, + UpdateReturnCopyTest, + InsertOrReturnExistingTest, + FindOrDieTest, + InsertOrDieTest, + InsertKeyOrDieTest); +using MapIntIntTypes = ::testing:: + Types, absl::node_hash_map, absl::node_hash_map>; +INSTANTIATE_TYPED_TEST_SUITE_P(MapUtilTest, MapUtilIntIntTest, MapIntIntTypes); + +// Finish setup for MapType +REGISTER_TYPED_TEST_SUITE_P(MapUtilIntIntPtrTest, + LookupOrInsertNewTest, + InsertAndDeleteExistingTest, + FindPtrOrNullTest, + EraseKeyReturnValuePtrTest); +using MapIntIntPtrTypes = ::testing::Types, absl::node_hash_map>; +INSTANTIATE_TYPED_TEST_SUITE_P(MapUtilTest, MapUtilIntIntPtrTest, MapIntIntPtrTypes); + +// Finish setup for MapType > +REGISTER_TYPED_TEST_SUITE_P(MapUtilIntIntSharedPtrTest, + FindPtrOrNullTest, + LookupOrInsertNewSharedPtrTest); +using MapIntIntSharedPtrTypes = ::testing::Types>, + absl::node_hash_map>>; +INSTANTIATE_TYPED_TEST_SUITE_P(MapUtilTest, MapUtilIntIntSharedPtrTest, MapIntIntSharedPtrTypes); + +REGISTER_TYPED_TEST_SUITE_P(MapUtilIntIntSharedPtrOnlyTest, LookupOrInsertNewSharedPtrSwapTest); +typedef ::testing::Types>, + absl::node_hash_map>> + MapIntIntSharedPtrOnlyTypes; +INSTANTIATE_TYPED_TEST_SUITE_P(MapUtilTest, + MapUtilIntIntSharedPtrOnlyTest, + MapIntIntSharedPtrOnlyTypes); + +using AssociateEraseMapTypes = + ::testing::Types, absl::node_hash_map>; + +template +class AssociativeEraseIfTest : public ::testing::Test +{ +}; +TYPED_TEST_SUITE_P(AssociativeEraseIfTest); + +TYPED_TEST_P(AssociativeEraseIfTest, Basic) +{ + using ValueType = std::pair; + TypeParam m; + m["a"] = 1; + m["b"] = 2; + m["c"] = 3; + m["d"] = 4; + + // Test that none of the elements are removed when the predicate always + // returns false. + struct FalseFunc + { + bool operator()(const ValueType &unused) const { return 0; } + }; + AssociativeEraseIf(&m, FalseFunc()); + EXPECT_THAT(m, UnorderedElementsAre(Pair("a", 1), Pair("b", 2), Pair("c", 3), Pair("d", 4))); + + // Test removing a single element. + struct KeyEqualsA + { + bool operator()(const ValueType &pair) const { return pair.first == "a"; } + }; + AssociativeEraseIf(&m, KeyEqualsA()); + EXPECT_THAT(m, UnorderedElementsAre(Pair("b", 2), Pair("c", 3), Pair("d", 4))); + + // Put the element back and test removing a couple elements, + m["a"] = 1; + struct ValueGreaterThanTwo + { + bool operator()(const ValueType &pair) const { return pair.second > 2; } + }; + AssociativeEraseIf(&m, ValueGreaterThanTwo()); + EXPECT_THAT(m, UnorderedElementsAre(Pair("a", 1), Pair("b", 2))); + + // Put the elements back and test removing all of them. + m["c"] = 3; + m["d"] = 4; + struct TrueFunc + { + bool operator()(const ValueType &unused) const { return 1; } + }; + AssociativeEraseIf(&m, TrueFunc()); + EXPECT_THAT(m, IsEmpty()); +} +REGISTER_TYPED_TEST_SUITE_P(AssociativeEraseIfTest, Basic); + +INSTANTIATE_TYPED_TEST_SUITE_P(MapUtilTest, AssociativeEraseIfTest, AssociateEraseMapTypes); + +TEST(MapUtil, InsertKeyOrDie_SmartPtrTest) +{ + absl::node_hash_map> m; + m[1].reset(new int(10)); + m[2].reset(new int(20)); + + EXPECT_THAT( + m, UnorderedElementsAre(Pair(1, ::testing::Pointee(10)), Pair(2, ::testing::Pointee(20)))); + InsertKeyOrDie(&m, 3).reset(new int(30)); + EXPECT_THAT(m, + UnorderedElementsAre(Pair(1, ::testing::Pointee(10)), + Pair(2, ::testing::Pointee(20)), + Pair(3, ::testing::Pointee(30)))); +} + +TEST(MapUtil, EraseKeyReturnValuePtr_SmartPtrTest) +{ + std::map> m; + m[1] = std::unique_ptr(new int(10)); + m[2] = std::unique_ptr(new int(20)); + + std::unique_ptr val1 = EraseKeyReturnValuePtr(&m, 1); + EXPECT_EQ(10, *val1); + EXPECT_THAT(m, ElementsAre(Pair(2, ::testing::Pointee(20)))); + auto val2 = EraseKeyReturnValuePtr(&m, 2); + EXPECT_EQ(20, *val2); +} + +TEST(MapUtil, LookupOrInsertNewVariadicTest) +{ + struct TwoArg + { + TwoArg(int one_in, int two_in) : one(one_in), two(two_in) {} + int one; + int two; + }; + + std::map> m; + TwoArg *val = LookupOrInsertNew(&m, 1, 100, 200).get(); + EXPECT_EQ(100, val->one); + EXPECT_EQ(200, val->two); +} + +} // namespace gutil diff --git a/src/gutil/test/no_destructor_test.cpp b/src/gutil/test/no_destructor_test.cpp new file mode 100644 index 0000000000..a0f5885877 --- /dev/null +++ b/src/gutil/test/no_destructor_test.cpp @@ -0,0 +1,184 @@ +// +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "gutil/no_destructor.h" + +#include +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "utils/fmt_logging.h" + +namespace gutil { +namespace { + +struct Blob +{ + Blob() : val(42) {} + Blob(int x, int y) : val(x + y) {} + Blob(std::initializer_list xs) + { + val = 0; + for (auto &x : xs) + val += x; + } + + Blob(const Blob & /*b*/) = delete; + Blob(Blob &&b) noexcept : val(b.val) { b.moved_out = true; } // moving is fine + + // no crash: NoDestructor indeed does not destruct (the moved-out Blob + // temporaries do get destroyed though) + ~Blob() { CHECK(moved_out, "~Blob"); } + + int val; + bool moved_out = false; +}; + +struct TypeWithDeletedDestructor +{ + ~TypeWithDeletedDestructor() = delete; +}; + +TEST(NoDestructorTest, DestructorNeverCalled) +{ + NoDestructor a; + (void)a; +} + +TEST(NoDestructorTest, Noncopyable) +{ + using T = NoDestructor; + + EXPECT_FALSE((std::is_constructible::value)); + EXPECT_FALSE((std::is_constructible::value)); + EXPECT_FALSE((std::is_constructible::value)); + EXPECT_FALSE((std::is_constructible::value)); + + EXPECT_FALSE((std::is_assignable::value)); + EXPECT_FALSE((std::is_assignable::value)); + EXPECT_FALSE((std::is_assignable::value)); + EXPECT_FALSE((std::is_assignable::value)); +} + +TEST(NoDestructorTest, Interface) +{ + EXPECT_TRUE(std::is_trivially_destructible>::value); + EXPECT_TRUE(std::is_trivially_destructible>::value); + { + NoDestructor b; // default c-tor + // access: *, ->, get() + EXPECT_EQ(42, (*b).val); + (*b).val = 55; + EXPECT_EQ(55, b->val); + b->val = 66; + EXPECT_EQ(66, b.get()->val); + b.get()->val = 42; + EXPECT_EQ(42, (*b).val); + } + { + NoDestructor b(70, 7); // regular c-tor, const + EXPECT_EQ(77, (*b).val); + EXPECT_EQ(77, b->val); + EXPECT_EQ(77, b.get()->val); + } + { + const NoDestructor b{{20, 28, 40}}; // init-list c-tor, deep const + // This only works in clang, not in gcc: + // const NoDestructor b({20, 28, 40}); + EXPECT_EQ(88, (*b).val); + EXPECT_EQ(88, b->val); + EXPECT_EQ(88, b.get()->val); + } +} + +// ========================================================================= // + +std::string *Str0() +{ + static NoDestructor x; + return x.get(); +} + +extern const std::string &Str2(); + +const char *Str1() +{ + static NoDestructor x(Str2() + "_Str1"); + return x->c_str(); +} + +const std::string &Str2() +{ + static NoDestructor x("Str2"); + return *x; +} + +const std::string &Str2Copy() +{ + static NoDestructor x(Str2()); // exercise copy construction + return *x; +} + +typedef std::array MyArray; +const MyArray &Array() +{ + static NoDestructor x{{{"foo", "bar", "baz"}}}; + // This only works in clang, not in gcc: + // static NoDestructor x({{"foo", "bar", "baz"}}); + return *x; +} + +typedef std::vector MyVector; +const MyVector &Vector() +{ + static NoDestructor x{{1, 2, 3}}; + return *x; +} + +const int &Int() +{ + static NoDestructor x; + return *x; +} + +TEST(NoDestructorTest, StaticPattern) +{ + EXPECT_TRUE(std::is_trivially_destructible>::value); + EXPECT_TRUE(std::is_trivially_destructible>::value); + EXPECT_TRUE(std::is_trivially_destructible>::value); + EXPECT_TRUE(std::is_trivially_destructible>::value); + + EXPECT_EQ(*Str0(), ""); + Str0()->append("foo"); + EXPECT_EQ(*Str0(), "foo"); + + EXPECT_EQ(std::string(Str1()), "Str2_Str1"); + + EXPECT_EQ(Str2(), "Str2"); + EXPECT_EQ(Str2Copy(), "Str2"); + + EXPECT_THAT(Array(), testing::ElementsAre("foo", "bar", "baz")); + + EXPECT_THAT(Vector(), testing::ElementsAre(1, 2, 3)); + + EXPECT_EQ(0, Int()); // should get zero-initialized +} + +} // namespace +} // namespace gutil diff --git a/src/gutil/test/run.sh b/src/gutil/test/run.sh new file mode 100755 index 0000000000..798d1ac202 --- /dev/null +++ b/src/gutil/test/run.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +./pgs_gutil_test diff --git a/src/http/http_call_registry.h b/src/http/http_call_registry.h index 69eb80fc1c..7decdc9ff3 100644 --- a/src/http/http_call_registry.h +++ b/src/http/http_call_registry.h @@ -20,6 +20,7 @@ #include "utils/fmt_logging.h" #include "http_server.h" #include "utils/errors.h" +#include "gutil/map_util.h" #include "utils/singleton.h" namespace dsn { @@ -35,11 +36,8 @@ class http_call_registry : public utils::singleton std::shared_ptr find(const std::string &path) const { std::lock_guard guard(_mu); - const auto &iter = _call_map.find(path); - if (iter == _call_map.end()) { - return nullptr; - } - return iter->second; + const auto *hc = gutil::FindOrNull(_call_map, path); + return hc == nullptr ? nullptr : *hc; } void remove(const std::string &path) diff --git a/src/http/http_server.cpp b/src/http/http_server.cpp index ce2178bfde..d5575f580e 100644 --- a/src/http/http_server.cpp +++ b/src/http/http_server.cpp @@ -23,19 +23,20 @@ #include #include -#include "http/builtin_http_calls.h" #include "fmt/core.h" -#include "http/http_method.h" +#include "gutil/map_util.h" +#include "http/builtin_http_calls.h" #include "http/http_call_registry.h" #include "http/http_message_parser.h" +#include "http/http_method.h" #include "http/http_server_impl.h" +#include "http/uri_decoder.h" #include "nodejs/http_parser.h" #include "runtime/api_layer1.h" #include "runtime/rpc/rpc_message.h" #include "runtime/rpc/rpc_stream.h" #include "runtime/serverlet.h" #include "runtime/tool_api.h" -#include "http/uri_decoder.h" #include "utils/error_code.h" #include "utils/fmt_logging.h" #include "utils/output_utils.h" @@ -250,8 +251,7 @@ void http_server::serve(message_ex *msg) if (sep + 1 < arg_val.size()) { value = arg_val.substr(sep + 1, arg_val.size() - sep); } - auto iter = ret.query_args.find(name); - if (iter != ret.query_args.end()) { + if (gutil::ContainsKey(ret.query_args, name)) { return FMT_ERR(ERR_INVALID_PARAMETERS, "duplicate parameter: {}", name); } ret.query_args.emplace(std::move(name), std::move(value)); diff --git a/src/meta/app_balance_policy.cpp b/src/meta/app_balance_policy.cpp index 3f7a0a985e..76ce2fba76 100644 --- a/src/meta/app_balance_policy.cpp +++ b/src/meta/app_balance_policy.cpp @@ -18,11 +18,11 @@ #include #include #include -#include #include #include "app_balance_policy.h" #include "common/gpid.h" +#include "gutil/map_util.h" #include "meta/load_balance_policy.h" #include "metadata_types.h" #include "utils/flags.h" @@ -162,7 +162,7 @@ bool copy_secondary_operation::can_select(gpid pid, migration_list *result) } // if the pid have been used - if (result->find(pid) != result->end()) { + if (gutil::ContainsKey(*result, pid)) { LOG_DEBUG("{}: skip gpid({}.{}) coz it is already copyed", _app->get_logname(), pid.get_app_id(), diff --git a/src/meta/app_env_validator.cpp b/src/meta/app_env_validator.cpp index c27bdb2cd5..06288cb611 100644 --- a/src/meta/app_env_validator.cpp +++ b/src/meta/app_env_validator.cpp @@ -29,6 +29,7 @@ #include "common/replica_envs.h" #include "http/http_status_code.h" #include "utils/fmt_logging.h" +#include "gutil/map_util.h" #include "utils/string_conv.h" #include "utils/strings.h" #include "utils/throttling_controller.h" @@ -52,10 +53,8 @@ bool app_env_validator::validate_app_envs(const std::mapsecond.type) { + switch (func->type) { case ValueType::kBool: { // Check by the default boolean validator. bool result = false; @@ -197,8 +196,7 @@ bool app_env_validator::validate_app_env(const std::string &env_name, } case ValueType::kString: { // Check by the self defined validator. - if (nullptr != func_iter->second.string_validator && - !func_iter->second.string_validator(env_value, hint_message)) { + if (nullptr != func->string_validator && !func->string_validator(env_value, hint_message)) { return false; } break; @@ -208,13 +206,11 @@ bool app_env_validator::validate_app_env(const std::string &env_name, __builtin_unreachable(); } - if (func_iter->second.type == ValueType::kInt32 || - func_iter->second.type == ValueType::kInt64) { + if (func->type == ValueType::kInt32 || func->type == ValueType::kInt64) { // Check by the self defined validator. - if (nullptr != func_iter->second.int_validator && - !func_iter->second.int_validator(int_result)) { - hint_message = fmt::format( - "invalid value '{}', should be '{}'", env_value, func_iter->second.limit_desc); + if (nullptr != func->int_validator && !func->int_validator(int_result)) { + hint_message = + fmt::format("invalid value '{}', should be '{}'", env_value, func->limit_desc); return false; } } @@ -354,10 +350,10 @@ const std::unordered_map nlohmann::json app_env_validator::EnvInfo::to_json() const { - const auto &type_str = ValueType2String.find(type); - CHECK_TRUE(type_str != ValueType2String.end()); + const auto *type_str = gutil::FindOrNull(ValueType2String, type); + CHECK_NOTNULL(type_str, ""); nlohmann::json info; - info["type"] = type_str->second; + info["type"] = *type_str; info["limitation"] = limit_desc; info["sample"] = sample; return info; diff --git a/src/meta/cluster_balance_policy.cpp b/src/meta/cluster_balance_policy.cpp index 2b10e39ece..8415803af9 100644 --- a/src/meta/cluster_balance_policy.cpp +++ b/src/meta/cluster_balance_policy.cpp @@ -25,6 +25,7 @@ #include #include "dsn.layer2_types.h" +#include "gutil/map_util.h" #include "meta/load_balance_policy.h" #include "runtime/rpc/dns_resolver.h" // IWYU pragma: keep #include "runtime/rpc/rpc_address.h" @@ -258,14 +259,8 @@ void cluster_balance_policy::get_node_migration_info(const node_state &ns, if (!context.get_disk_tag(ns.host_port(), disk_tag)) { continue; } - auto pid = context.pc->pid; - if (info.partitions.find(disk_tag) != info.partitions.end()) { - info.partitions[disk_tag].insert(pid); - } else { - partition_set pset; - pset.insert(pid); - info.partitions.emplace(disk_tag, pset); - } + auto &partitions_of_disk = gutil::LookupOrInsert(&info.partitions, disk_tag, {}); + partitions_of_disk.insert(context.pc->pid); } } } diff --git a/src/meta/duplication/meta_duplication_service.cpp b/src/meta/duplication/meta_duplication_service.cpp index 02562afeb0..4f0d7f816e 100644 --- a/src/meta/duplication/meta_duplication_service.cpp +++ b/src/meta/duplication/meta_duplication_service.cpp @@ -30,6 +30,7 @@ #include "common/replication_other_types.h" #include "dsn.layer2_types.h" #include "duplication_types.h" +#include "gutil/map_util.h" #include "meta/meta_service.h" #include "meta/meta_state_service_utils.h" #include "meta_admin_types.h" @@ -104,13 +105,12 @@ void meta_duplication_service::modify_duplication(duplication_modify_rpc rpc) return; } - auto it = app->duplications.find(dupid); - if (it == app->duplications.end()) { + auto dup = gutil::FindPtrOrNull(app->duplications, dupid); + if (!dup) { response.err = ERR_OBJECT_NOT_FOUND; return; } - duplication_info_s_ptr dup = it->second; auto to_status = request.__isset.status ? request.status : dup->status(); auto to_fail_mode = request.__isset.fail_mode ? request.fail_mode : dup->fail_mode(); response.err = dup->alter_status(to_status, to_fail_mode); diff --git a/src/meta/test/cluster_balance_policy_test.cpp b/src/meta/test/cluster_balance_policy_test.cpp index f8da5a3a14..7d18fd3fdd 100644 --- a/src/meta/test/cluster_balance_policy_test.cpp +++ b/src/meta/test/cluster_balance_policy_test.cpp @@ -30,6 +30,7 @@ #include "common/gpid.h" #include "dsn.layer2_types.h" #include "gtest/gtest.h" +#include "gutil/map_util.h" #include "meta/cluster_balance_policy.h" #include "meta/load_balance_policy.h" #include "meta/meta_data.h" @@ -186,9 +187,10 @@ TEST(cluster_balance_policy, get_node_migration_info) policy.get_node_migration_info(ns, all_apps, migration_info); ASSERT_EQ(migration_info.hp, hp); - ASSERT_NE(migration_info.partitions.find(disk_tag), migration_info.partitions.end()); - ASSERT_EQ(migration_info.partitions.at(disk_tag).size(), 1); - ASSERT_EQ(*migration_info.partitions.at(disk_tag).begin(), pid); + const auto *ps = gutil::FindOrNull(migration_info.partitions, disk_tag); + ASSERT_NE(ps, nullptr); + ASSERT_EQ(1, ps->size()); + ASSERT_EQ(pid, *ps->begin()); } TEST(cluster_balance_policy, get_min_max_set) diff --git a/src/meta/test/meta_bulk_load_ingestion_test.cpp b/src/meta/test/meta_bulk_load_ingestion_test.cpp index f6ee7e9fc6..b482be5124 100644 --- a/src/meta/test/meta_bulk_load_ingestion_test.cpp +++ b/src/meta/test/meta_bulk_load_ingestion_test.cpp @@ -30,6 +30,7 @@ #include "runtime/rpc/rpc_address.h" #include "runtime/rpc/rpc_host_port.h" #include "utils/fail_point.h" +#include "gutil/map_util.h" namespace dsn { namespace replication { @@ -64,10 +65,11 @@ class node_context_test : public meta_test_base uint32_t get_disk_count(const std::string &disk_tag) { - if (_context.disk_ingesting_counts.find(disk_tag) == _context.disk_ingesting_counts.end()) { + const auto *count = gutil::FindOrNull(_context.disk_ingesting_counts, disk_tag); + if (count == nullptr) { return -1; } - return _context.disk_ingesting_counts[disk_tag]; + return *count; } void mock_get_max_disk_ingestion_count(const uint32_t node_min_disk_count, @@ -255,8 +257,7 @@ class ingestion_context_test : public meta_test_base bool is_partition_ingesting(const uint32_t pidx) const { - return _context->_running_partitions.find(gpid(APP_ID, pidx)) != - _context->_running_partitions.end(); + return gutil::ContainsKey(_context->_running_partitions, gpid(APP_ID, pidx)); } uint32_t get_app_ingesting_count() const { return _context->get_app_ingesting_count(APP_ID); } diff --git a/src/meta/test/meta_bulk_load_service_test.cpp b/src/meta/test/meta_bulk_load_service_test.cpp index 48477b361c..53f67a5bf6 100644 --- a/src/meta/test/meta_bulk_load_service_test.cpp +++ b/src/meta/test/meta_bulk_load_service_test.cpp @@ -40,6 +40,7 @@ #include "common/replication_other_types.h" #include "dsn.layer2_types.h" #include "gtest/gtest.h" +#include "gutil/map_util.h" #include "meta/meta_bulk_load_service.h" #include "meta/meta_data.h" #include "meta/meta_server_failure_detector.h" @@ -382,14 +383,12 @@ class bulk_load_service_test : public meta_test_base &partition_bulk_load_info_map, &pinfo_map]() { for (const auto app_id : app_id_set) { - auto app_iter = app_bulk_load_info_map.find(app_id); - auto partition_iter = partition_bulk_load_info_map.find(app_id); - if (app_iter != app_bulk_load_info_map.end()) { + const auto *app = gutil::FindOrNull(app_bulk_load_info_map, app_id); + if (app != nullptr) { + const auto *partition = + gutil::FindOrNull(partition_bulk_load_info_map, app_id); mock_app_bulk_load_info_on_remote_storage( - app_iter->second, - partition_iter == partition_bulk_load_info_map.end() - ? pinfo_map - : partition_iter->second); + *app, partition == nullptr ? pinfo_map : *partition); } } }); @@ -492,7 +491,7 @@ class bulk_load_service_test : public meta_test_base bool is_app_bulk_load_states_reset(int32_t app_id) { - return bulk_svc()._bulk_load_app_id.find(app_id) == bulk_svc()._bulk_load_app_id.end(); + return !gutil::ContainsKey(bulk_svc()._bulk_load_app_id, app_id); } meta_op_status get_op_status() { return _ms->get_op_status(); } diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index 21a7f3a8af..001090d095 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -30,10 +30,11 @@ #include #include #include -#include #include #include "absl/strings/string_view.h" +#include "fmt/core.h" // IWYU pragma: keep +#include "gutil/map_util.h" #include "nfs/nfs_code_definition.h" #include "nlohmann/json.hpp" #include "runtime/api_layer1.h" @@ -43,6 +44,7 @@ #include "utils/env.h" #include "utils/filesystem.h" #include "utils/flags.h" +#include "utils/ports.h" #include "utils/utils.h" METRIC_DEFINE_counter( @@ -95,24 +97,25 @@ void nfs_service_impl::on_copy(const ::dsn::service::copy_request &request, do { zauto_lock l(_handles_map_lock); - auto it = _handles_map.find(file_path); // find file handle cache first - if (it == _handles_map.end()) { + auto &fh = gutil::LookupOrInsert(&_handles_map, file_path, {}); + if (!fh) { dfile = file::open(file_path, file::FileOpenType::kReadOnly); - if (dfile == nullptr) { + if (dsn_unlikely(dfile == nullptr)) { LOG_ERROR("[nfs_service] open file {} failed", file_path); + gutil::EraseKeyReturnValuePtr(&_handles_map, file_path); ::dsn::service::copy_response resp; resp.error = ERR_OBJECT_NOT_FOUND; reply(resp); return; } - - auto fh = std::make_shared(); + fh = std::make_shared(); fh->file_handle = dfile; - it = _handles_map.insert(std::make_pair(file_path, std::move(fh))).first; + } else { + dfile = fh->file_handle; } - dfile = it->second->file_handle; - it->second->file_access_count++; - it->second->last_access_time = dsn_now_ms(); + DCHECK(fh, ""); + fh->file_access_count++; + fh->last_access_time = dsn_now_ms(); } while (false); CHECK_NOTNULL(dfile, ""); diff --git a/src/replica/replica_config.cpp b/src/replica/replica_config.cpp index 23767986b6..7abeaf4549 100644 --- a/src/replica/replica_config.cpp +++ b/src/replica/replica_config.cpp @@ -74,6 +74,7 @@ #include "utils/fail_point.h" #include "utils/flags.h" #include "utils/fmt_logging.h" +#include "gutil/map_util.h" #include "utils/string_conv.h" #include "utils/strings.h" #include "utils/thread_access_checker.h" @@ -89,9 +90,9 @@ bool get_bool_envs(const std::map &envs, const std::string &name, bool &value) { - auto iter = envs.find(name); - if (iter != envs.end()) { - if (!buf2bool(iter->second, value)) { + const auto *value_ptr = gutil::FindOrNull(envs, name); + if (value_ptr != nullptr) { + if (!buf2bool(*value_ptr, value)) { return false; } } @@ -202,7 +203,7 @@ void replica::add_potential_secondary(const configuration_update_request &propos _primary_states.pc.hp_secondaries.size() + _primary_states.learners.size(); if (potential_secondaries_count >= _primary_states.pc.max_replica_count - 1) { if (proposal.type == config_type::CT_ADD_SECONDARY) { - if (_primary_states.learners.find(node) == _primary_states.learners.end()) { + if (!gutil::ContainsKey(_primary_states.learners, node)) { LOG_INFO_PREFIX( "already have enough secondaries or potential secondaries, ignore new " "potential secondary proposal"); @@ -225,9 +226,9 @@ void replica::add_potential_secondary(const configuration_update_request &propos state.prepare_start_decree = invalid_decree; state.timeout_task = nullptr; // TODO: add timer for learner task - auto it = _primary_states.learners.find(node); - if (it != _primary_states.learners.end()) { - state.signature = it->second.signature; + const auto *rls = gutil::FindOrNull(_primary_states.learners, node); + if (rls != nullptr) { + state.signature = rls->signature; } else { state.signature = ++_primary_states.next_learning_version; _primary_states.learners[node] = state; @@ -585,9 +586,10 @@ void replica::update_bool_envs(const std::map &envs, void replica::update_ac_allowed_users(const std::map &envs) { std::string allowed_users; - auto iter = envs.find(replica_envs::REPLICA_ACCESS_CONTROLLER_ALLOWED_USERS); - if (iter != envs.end()) { - allowed_users = iter->second; + const auto *env = + gutil::FindOrNull(envs, replica_envs::REPLICA_ACCESS_CONTROLLER_ALLOWED_USERS); + if (env != nullptr) { + allowed_users = *env; } _access_controller->update_allowed_users(allowed_users); @@ -595,9 +597,10 @@ void replica::update_ac_allowed_users(const std::map & void replica::update_ac_ranger_policies(const std::map &envs) { - auto iter = envs.find(replica_envs::REPLICA_ACCESS_CONTROLLER_RANGER_POLICIES); - if (iter != envs.end()) { - _access_controller->update_ranger_policies(iter->second); + const auto *env = + gutil::FindOrNull(envs, replica_envs::REPLICA_ACCESS_CONTROLLER_RANGER_POLICIES); + if (env != nullptr) { + _access_controller->update_ranger_policies(*env); } } @@ -623,14 +626,14 @@ void replica::update_allow_ingest_behind(const std::map &envs) { - auto env_iter = envs.find(replica_envs::DENY_CLIENT_REQUEST); - if (env_iter == envs.end()) { + const auto *env = gutil::FindOrNull(envs, replica_envs::DENY_CLIENT_REQUEST); + if (env == nullptr) { _deny_client.reset(); return; } std::vector sub_sargs; - utils::split_args(env_iter->second.c_str(), sub_sargs, '*', true); + utils::split_args(env->c_str(), sub_sargs, '*', true); CHECK_EQ_PREFIX(sub_sargs.size(), 2); _deny_client.reconfig = (sub_sargs[0] == "reconfig"); diff --git a/src/utils/metrics.h b/src/utils/metrics.h index 2990537203..199d11d483 100644 --- a/src/utils/metrics.h +++ b/src/utils/metrics.h @@ -52,6 +52,7 @@ #include "utils/fmt_logging.h" #include "utils/long_adder.h" #include "utils/macros.h" +#include "gutil/map_util.h" #include "utils/nth_element.h" #include "utils/ports.h" #include "utils/singleton.h" @@ -1746,13 +1747,13 @@ inline error_s parse_metric_attribute(const metric_entity::attr_map &attrs, const std::string &name, TAttrValue &value) { - const auto &iter = attrs.find(name); - if (dsn_unlikely(iter == attrs.end())) { + const auto *value_ptr = gutil::FindOrNull(attrs, name); + if (dsn_unlikely(value_ptr == nullptr)) { return FMT_ERR(dsn::ERR_INVALID_DATA, "{} field was not found", name); } - if (dsn_unlikely(!dsn::buf2numeric(iter->second, value))) { - return FMT_ERR(dsn::ERR_INVALID_DATA, "invalid {}: {}", name, iter->second); + if (dsn_unlikely(!dsn::buf2numeric(*value_ptr, value))) { + return FMT_ERR(dsn::ERR_INVALID_DATA, "invalid {}: {}", name, *value_ptr); } return dsn::error_s::ok(); diff --git a/src/utils/test/logger.cpp b/src/utils/test/logger.cpp index 0ef7bcb220..e60e8fe5d4 100644 --- a/src/utils/test/logger.cpp +++ b/src/utils/test/logger.cpp @@ -31,10 +31,10 @@ #include #include #include -#include #include #include "gtest/gtest.h" +#include "gutil/map_util.h" #include "utils/api_utilities.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -85,7 +85,7 @@ class simple_logger_test : public logger_test for (const auto &path : sub_list) { std::string name(utils::filesystem::get_file_name(path)); if (std::regex_match(name, pattern)) { - ASSERT_TRUE(file_names.insert(name).second); + ASSERT_TRUE(gutil::InsertIfNotPresent(&file_names, name)); } } } diff --git a/src/utils/test/metrics_test.cpp b/src/utils/test/metrics_test.cpp index 9f3a4d1909..fcd9cf227c 100644 --- a/src/utils/test/metrics_test.cpp +++ b/src/utils/test/metrics_test.cpp @@ -39,6 +39,7 @@ #include "runtime/rpc/rpc_message.h" #include "utils/errors.h" #include "utils/flags.h" +#include "gutil/map_util.h" #include "utils/rand.h" #include "utils/strings.h" #include "utils/test/nth_element_utils.h" @@ -246,7 +247,7 @@ TEST(metrics_test, create_entity) auto attrs = entity->attributes(); ASSERT_EQ(attrs, test.entity_attrs); - ASSERT_EQ(entities.find(test.entity_id), entities.end()); + ASSERT_TRUE(!gutil::ContainsKey(entities, test.entity_id)); entities[test.entity_id] = entity; } @@ -313,25 +314,21 @@ TEST(metrics_test, create_metric) my_metric = test.prototype->instantiate(test.entity, test.value); } - ASSERT_EQ(my_metric->value(), test.value); + ASSERT_EQ(test.value, my_metric->value()); - auto iter = expected_entities.find(test.entity.get()); - if (iter == expected_entities.end()) { - expected_entities[test.entity.get()] = {{test.prototype, my_metric}}; - } else { - iter->second[test.prototype] = my_metric; - } + auto &iter = gutil::LookupOrInsert(&expected_entities, test.entity.get(), {}); + iter.emplace(test.prototype, my_metric); } entity_map actual_entities; - auto entities = metric_registry::instance().entities(); - for (const auto &entity : entities) { - if (expected_entities.find(entity.second.get()) != expected_entities.end()) { - actual_entities[entity.second.get()] = entity.second->metrics(); + const auto entities = metric_registry::instance().entities(); + for (const auto &[_, entity] : entities) { + if (gutil::ContainsKey(expected_entities, entity.get())) { + actual_entities[entity.get()] = entity->metrics(); } } - ASSERT_EQ(actual_entities, expected_entities); + ASSERT_EQ(expected_entities, actual_entities); } TEST(metrics_test, recreate_metric) @@ -778,7 +775,7 @@ void run_percentile(const metric_entity_ptr &my_entity, std::vector actual_elements; for (const auto &kth : kAllKthPercentileTypes) { T value; - if (kth_percentiles.find(kth) == kth_percentiles.end()) { + if (!gutil::ContainsKey(kth_percentiles, kth)) { ASSERT_FALSE(my_metric->get(kth, value)); checker(value, 0); } else { @@ -1099,8 +1096,7 @@ void compare_floating_metric_value_map(const metric_value_map &actual_value_m filters.with_metric_fields = metric_fields; \ \ metric_value_map expected_value_map; \ - if (expected_metric_fields.find(kMetricSingleValueField) != \ - expected_metric_fields.end()) { \ + if (gutil::ContainsKey(expected_metric_fields, kMetricSingleValueField)) { \ expected_value_map[kMetricSingleValueField] = test.expected_value; \ } \ \ @@ -1283,7 +1279,7 @@ void generate_metric_value_map(MetricType *my_metric, for (const auto &type : kth_percentiles) { auto name = kth_percentile_to_name(type); // Only add the chosen fields to the expected value map. - if (expected_metric_fields.find(name) != expected_metric_fields.end()) { + if (gutil::ContainsKey(expected_metric_fields, name)) { value_map[name] = *value; } ++value; @@ -2871,13 +2867,11 @@ TEST(metrics_test, http_get_metrics) for (const auto &test : tests) { entity_container expected_entities; for (const auto &entity_pair : test.expected_entity_metrics) { - const auto &iter = entities.find(entity_pair.first); - ASSERT_NE(entities.end(), iter); - - const auto &entity = iter->second; - expected_entities.emplace(entity->id(), - entity_properties{entity->prototype()->name(), - entity->attributes(), + const auto *entity = gutil::FindOrNull(entities, entity_pair.first); + ASSERT_NE(entity, nullptr); + expected_entities.emplace((*entity)->id(), + entity_properties{(*entity)->prototype()->name(), + (*entity)->attributes(), entity_pair.second}); } @@ -3131,11 +3125,11 @@ void scoped_entity::test_survival_immediately_after_initialization() const // Use internal member directly instead of calling entities(). We don't want to have // any reference which may affect the test results. const auto &entities = metric_registry::instance()._entities; - const auto &iter = entities.find(_my_entity_id); - ASSERT_NE(entities.end(), iter); - ASSERT_EQ(_expected_my_entity_raw_ptr, iter->second.get()); + const auto *entity = gutil::FindOrNull(entities, _my_entity_id); + ASSERT_NE(entity, nullptr); + ASSERT_EQ(_expected_my_entity_raw_ptr, entity->get()); - const auto &actual_surviving_metrics = get_actual_surviving_metrics(iter->second); + const auto &actual_surviving_metrics = get_actual_surviving_metrics(*entity); ASSERT_EQ(_expected_all_metrics, actual_surviving_metrics); } @@ -3149,18 +3143,18 @@ void scoped_entity::test_survival_after_retirement() const // Use internal member directly instead of calling entities(). We don't want to have // any reference which may affect the test results. const auto &entities = metric_registry::instance()._entities; - const auto &iter = entities.find(_my_entity_id); + const auto *iter = gutil::FindOrNull(entities, _my_entity_id); if (_my_entity == nullptr) { // The entity has been retired. - ASSERT_EQ(entities.end(), iter); + ASSERT_EQ(iter, nullptr); ASSERT_TRUE(_expected_surviving_metrics.empty()); return; } - ASSERT_NE(entities.end(), iter); - ASSERT_EQ(_expected_my_entity_raw_ptr, iter->second.get()); + ASSERT_NE(iter, nullptr); + ASSERT_EQ(_expected_my_entity_raw_ptr, iter->get()); - const auto &actual_surviving_metrics = get_actual_surviving_metrics(iter->second); + const auto &actual_surviving_metrics = get_actual_surviving_metrics(*iter); ASSERT_EQ(_expected_surviving_metrics, actual_surviving_metrics); } From 097068d2fa6dfd9ff61c4f51cdc1c33d3b1f5d29 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Thu, 1 Aug 2024 10:52:52 +0800 Subject: [PATCH 05/15] feat(logging): Introduce spdlog to thirdparty (#2084) This is a previous work of https://github.com/apache/incubator-pegasus/pull/2083. This patch introduces [spdlog](https://github.com/gabime/spdlog) as a thirdparty to prepare the docker image, then the #2083 can use it to run CI. spdlog is a very fast C++ logging library, you can see the benchmarks: https://github.com/gabime/spdlog?tab=readme-ov-file#benchmarks. spdlog using `libfmt` to format message content, it is fast and compatible with the current logging practice in Pegasus. It's a good choice to replace the current `simple_logger` or `screen_logger` implementation in Pegasus. --- thirdparty/CMakeLists.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 7105983542..d197e26b50 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -473,3 +473,19 @@ ExternalProject_Add(http-parser DOWNLOAD_NO_PROGRESS true ) +ExternalProject_Add(spdlog + URL https://github.com/gabime/spdlog/archive/refs/tags/v1.14.1.tar.gz + URL_MD5 f2c3f15c20e67b261836ff7bfda302cf + PATCH_COMMAND "" + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DSPDLOG_ENABLE_PCH=ON + -DSPDLOG_BUILD_PIC=ON + -DSPDLOG_FMT_EXTERNAL=ON + -DSPDLOG_NO_EXCEPTIONS=ON + -Dfmt_DIR=${TP_OUTPUT}/lib/cmake/fmt + BUILD_COMMAND make -j${PARALLEL} + INSTALL_COMMAND make install + DEPENDS fmt + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) From 581a811256cc33d300b5f8e0e3a1e1ae2fe17756 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Fri, 2 Aug 2024 15:52:58 +0800 Subject: [PATCH 06/15] chore(CI): Reduce the target size when build on GitHub actions (#2086) The GitHub action runners for free plan is very limit, it's easy to exceed the disk space. This patch adds some build and link flags to reduce the target size to ensure the actions can be completed without disk exhaust issues. Now the release packge is reduced from about 2,600,468,480 bytes to 686,325,578 bytes, and the ASAN package can be built completly now. --- cmake_modules/BaseFunctions.cmake | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 33c819fbb5..9b784dc441 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -193,12 +193,20 @@ endfunction() function(dsn_setup_compiler_flags) if(CMAKE_BUILD_TYPE STREQUAL "Debug") add_definitions(-DDSN_BUILD_TYPE=Debug) - add_definitions(-g) else() - add_definitions(-g) add_definitions(-O2) add_definitions(-DDSN_BUILD_TYPE=Release) endif() + + if("$ENV{GITHUB_ACTION}" STREQUAL "" OR APPLE) + add_definitions(-g) + else() + # Reduce the target size when build on GitHub actions and non-macOS. + message(WARNING "Running GitHub actions, the target size will be reduced!") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Os -ffunction-sections -fdata-sections -fno-unwind-tables -fno-asynchronous-unwind-tables") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-s -Wl,--gc-sections") + endif() + cmake_host_system_information(RESULT BUILD_HOSTNAME QUERY HOSTNAME) add_definitions(-DDSN_BUILD_HOSTNAME=${BUILD_HOSTNAME}) From 5057aadab70531fac64ab16f01aaaa1e536618d7 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 7 Aug 2024 14:05:31 +0800 Subject: [PATCH 07/15] fix(util): Fix the key doesn't print when check failed (#2090) --- src/gutil/map_util.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gutil/map_util.h b/src/gutil/map_util.h index e8788e4abd..4ce286fc07 100644 --- a/src/gutil/map_util.h +++ b/src/gutil/map_util.h @@ -406,7 +406,7 @@ void InsertOrDieNoPrint(M *m, const MapUtilInitT &value) template void InsertOrDie(M *m, const MapUtilKeyT &key, const MapUtilMappedT &data) { - CHECK(InsertIfNotPresent(m, key, data), "duplicate key: ", key); + CHECK(InsertIfNotPresent(m, key, data), "duplicate key: {}", key); } // Same as above except doesn't log the key on error. @@ -427,7 +427,7 @@ auto InsertKeyOrDie(M *m, const MapUtilKeyT &key) -> typename std::enable_if::value, MapUtilMappedT &>::type { auto res = m->try_emplace(key); - CHECK(res.second, "duplicate key: ", key); + CHECK(res.second, "duplicate key: {}", key); return gutil::subtle::GetMapped(*res.first); } @@ -437,7 +437,7 @@ auto InsertKeyOrDie(M *m, const MapUtilKeyT &key) -> typename std::enable_if::value, MapUtilMappedT &>::type { auto res = m->insert(MapUtilValueT(key, MapUtilMappedT())); - CHECK(res.second, "duplicate key: ", key); + CHECK(res.second, "duplicate key: {}", key); return res.first->second; } From af3677e3f4f183895e5aa7692b156da7d7c2cef3 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 7 Aug 2024 14:06:06 +0800 Subject: [PATCH 08/15] feat(logging): Move the logger remote_command handler to command_manager (#2087) --- src/utils/command_manager.cpp | 112 ++++++++++++++++++---------------- src/utils/command_manager.h | 20 +++--- src/utils/logging_provider.h | 3 - src/utils/simple_logger.cpp | 66 ++++++++++---------- src/utils/test/logger.cpp | 17 +----- 5 files changed, 105 insertions(+), 113 deletions(-) diff --git a/src/utils/command_manager.cpp b/src/utils/command_manager.cpp index 35b678b81b..b5d56e2afb 100644 --- a/src/utils/command_manager.cpp +++ b/src/utils/command_manager.cpp @@ -29,13 +29,17 @@ #include // IWYU pragma: no_include #include +#include #include #include #include // IWYU pragma: keep #include +#include #include #include +#include "gutil/map_util.h" + namespace dsn { std::unique_ptr @@ -44,19 +48,19 @@ command_manager::register_command(const std::vector &commands, const std::string &args, command_handler handler) { - auto *c = new command_instance(); - c->commands = commands; - c->help = help; - c->args = args; - c->handler = std::move(handler); + auto ch = std::make_shared(); + ch->commands = commands; + ch->help = help; + ch->args = args; + ch->handler = std::move(handler); utils::auto_write_lock l(_lock); for (const auto &cmd : commands) { CHECK(!cmd.empty(), "should not register empty command"); - CHECK(_handlers.emplace(cmd, c).second, "command '{}' already registered", cmd); + gutil::InsertOrDie(&_handler_by_cmd, cmd, ch); } - return std::make_unique(reinterpret_cast(c)); + return std::make_unique(reinterpret_cast(ch.get())); } std::unique_ptr command_manager::register_bool_command( @@ -94,35 +98,39 @@ command_manager::register_multiple_commands(const std::vector &comm handler); } -void command_manager::deregister_command(uintptr_t handle) +void command_manager::deregister_command(uintptr_t cmd_id) { - auto c = reinterpret_cast(handle); - CHECK_NOTNULL(c, "cannot deregister a null handle"); + const auto ch = reinterpret_cast(cmd_id); + CHECK_NOTNULL(ch, "cannot deregister a null command id"); utils::auto_write_lock l(_lock); - for (const std::string &cmd : c->commands) { - _handlers.erase(cmd); + for (const auto &cmd : ch->commands) { + _handler_by_cmd.erase(cmd); } } +void command_manager::add_global_cmd(std::unique_ptr cmd) +{ + utils::auto_write_lock l(_lock); + _cmds.push_back(std::move(cmd)); +} + bool command_manager::run_command(const std::string &cmd, const std::vector &args, /*out*/ std::string &output) { - command_instance *h = nullptr; + std::shared_ptr ch; { utils::auto_read_lock l(_lock); - auto it = _handlers.find(cmd); - if (it != _handlers.end()) - h = it->second; + ch = gutil::FindPtrOrNull(_handler_by_cmd, cmd); } - if (h == nullptr) { - output = std::string("unknown command '") + cmd + "'"; + if (!ch) { + output = fmt::format("unknown command '{}'", cmd); return false; - } else { - output = h->handler(args); - return true; } + + output = ch->handler(args); + return true; } std::string command_manager::set_bool(bool &value, @@ -159,36 +167,34 @@ std::string command_manager::set_bool(bool &value, command_manager::command_manager() { - _cmds.emplace_back( - register_multiple_commands({"help", "h", "H", "Help"}, - "Display help information", - "[command]", - [this](const std::vector &args) { - std::stringstream ss; - if (args.empty()) { - std::unordered_set cmds; - utils::auto_read_lock l(_lock); - for (const auto &c : this->_handlers) { - // Multiple commands with the same handler are print - // only once. - if (cmds.insert(c.second.get()).second) { - ss << c.second->help << std::endl; - } - } - } else { - utils::auto_read_lock l(_lock); - auto it = _handlers.find(args[0]); - if (it == _handlers.end()) { - ss << "cannot find command '" << args[0] << "'"; - } else { - ss.width(6); - ss << std::left << it->second->help << std::endl - << it->second->args << std::endl; - } - } - - return ss.str(); - })); + _cmds.emplace_back(register_multiple_commands( + {"help", "h", "H", "Help"}, + "Display help information", + "[command]", + [this](const std::vector &args) { + std::stringstream ss; + if (args.empty()) { + std::unordered_set chs; + utils::auto_read_lock l(_lock); + for (const auto &[_, ch] : _handler_by_cmd) { + // Multiple commands with the same handler are print only once. + if (gutil::InsertIfNotPresent(&chs, ch.get())) { + ss << ch->help << std::endl; + } + } + } else { + utils::auto_read_lock l(_lock); + const auto ch = gutil::FindPtrOrNull(_handler_by_cmd, args[0]); + if (!ch) { + ss << "cannot find command '" << args[0] << "'"; + } else { + ss.width(6); + ss << std::left << ch->help << std::endl << ch->args << std::endl; + } + } + + return ss.str(); + })); _cmds.emplace_back(register_multiple_commands( {"repeat", "r", "R", "Repeat"}, @@ -241,10 +247,10 @@ command_manager::command_manager() command_manager::~command_manager() { _cmds.clear(); - CHECK(_handlers.empty(), + CHECK(_handler_by_cmd.empty(), "All commands must be deregistered before command_manager is destroyed, however '{}' is " "still registered", - _handlers.begin()->first); + _handler_by_cmd.begin()->first); } } // namespace dsn diff --git a/src/utils/command_manager.h b/src/utils/command_manager.h index 0a34dd4c16..b971522d01 100644 --- a/src/utils/command_manager.h +++ b/src/utils/command_manager.h @@ -28,17 +28,16 @@ #include #include +#include +#include // IWYU pragma: no_include #include #include #include #include -#include -#include #include #include -#include "utils/autoref_ptr.h" #include "utils/fmt_logging.h" #include "utils/ports.h" #include "utils/singleton.h" @@ -47,7 +46,6 @@ #include "utils/synchronize.h" namespace dsn { - class command_deregister; class command_manager : public ::dsn::utils::singleton @@ -85,7 +83,7 @@ class command_manager : public ::dsn::utils::singleton }); } - // Register a single 'command' with the 'help' description, its arguments are describe in + // Register a single 'command' with the 'help' description, its arguments are described in // 'args'. std::unique_ptr register_single_command(const std::string &command, @@ -93,7 +91,7 @@ class command_manager : public ::dsn::utils::singleton const std::string &args, command_handler handler) WARN_UNUSED_RESULT; - // Register multiple 'commands' with the 'help' description, their arguments are describe in + // Register multiple 'commands' with the 'help' description, their arguments are described in // 'args'. std::unique_ptr register_multiple_commands(const std::vector &commands, @@ -101,6 +99,9 @@ class command_manager : public ::dsn::utils::singleton const std::string &args, command_handler handler) WARN_UNUSED_RESULT; + // Register a global command which is not associated with any objects. + void add_global_cmd(std::unique_ptr cmd); + bool run_command(const std::string &cmd, const std::vector &args, /*out*/ std::string &output); @@ -112,7 +113,7 @@ class command_manager : public ::dsn::utils::singleton command_manager(); ~command_manager(); - struct command_instance : public ref_counter + struct commands_handler { std::vector commands; std::string help; @@ -126,7 +127,7 @@ class command_manager : public ::dsn::utils::singleton const std::string &args, command_handler handler) WARN_UNUSED_RESULT; - void deregister_command(uintptr_t handle); + void deregister_command(uintptr_t cmd_id); static std::string set_bool(bool &value, const std::string &name, const std::vector &args); @@ -177,9 +178,8 @@ class command_manager : public ::dsn::utils::singleton return msg.dump(2); } - typedef ref_ptr command_instance_ptr; utils::rw_lock_nr _lock; - std::map _handlers; + std::map> _handler_by_cmd; std::vector> _cmds; }; diff --git a/src/utils/logging_provider.h b/src/utils/logging_provider.h index ec6bec53d0..264d721ce8 100644 --- a/src/utils/logging_provider.h +++ b/src/utils/logging_provider.h @@ -64,8 +64,6 @@ class logging_provider virtual void flush() = 0; - void deregister_commands() { _cmds.clear(); } - protected: static std::unique_ptr _logger; @@ -73,7 +71,6 @@ class logging_provider logging_provider(log_level_t stderr_start_level) : _stderr_start_level(stderr_start_level) {} - std::vector> _cmds; const log_level_t _stderr_start_level; }; diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index 6e56cbbca9..f67132408e 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include #include @@ -234,37 +234,39 @@ simple_logger::simple_logger(const char *log_dir, const char *role_name) create_log_file(); - // TODO(yingchun): simple_logger is destroyed after command_manager, so will cause crash like - // "assertion expression: [_handlers.empty()] All commands must be deregistered before - // command_manager is destroyed, however 'flush-log' is still registered". - // We need to fix it. - _cmds.emplace_back(::dsn::command_manager::instance().register_single_command( - "flush-log", - "Flush log to stderr or file", - "", - [this](const std::vector &args) { - this->flush(); - return "Flush done."; - })); - - _cmds.emplace_back(::dsn::command_manager::instance().register_single_command( - "reset-log-start-level", - "Reset the log start level", - "[DEBUG | INFO | WARNING | ERROR | FATAL]", - [](const std::vector &args) { - log_level_t start_level; - if (args.size() == 0) { - start_level = enum_from_string(FLAGS_logging_start_level, LOG_LEVEL_INVALID); - } else { - std::string level_str = "LOG_LEVEL_" + args[0]; - start_level = enum_from_string(level_str.c_str(), LOG_LEVEL_INVALID); - if (start_level == LOG_LEVEL_INVALID) { - return "ERROR: invalid level '" + args[0] + "'"; - } - } - set_log_start_level(start_level); - return std::string("OK, current level is ") + enum_to_string(start_level); - })); + static std::once_flag flag; + std::call_once(flag, [&]() { + ::dsn::command_manager::instance().add_global_cmd( + ::dsn::command_manager::instance().register_single_command( + "flush-log", + "Flush log to stderr or file", + "", + [this](const std::vector &args) { + this->flush(); + return "Flush done."; + })); + + ::dsn::command_manager::instance().add_global_cmd( + ::dsn::command_manager::instance().register_single_command( + "reset-log-start-level", + "Reset the log start level", + "[DEBUG | INFO | WARNING | ERROR | FATAL]", + [](const std::vector &args) { + log_level_t start_level; + if (args.size() == 0) { + start_level = + enum_from_string(FLAGS_logging_start_level, LOG_LEVEL_INVALID); + } else { + std::string level_str = "LOG_LEVEL_" + args[0]; + start_level = enum_from_string(level_str.c_str(), LOG_LEVEL_INVALID); + if (start_level == LOG_LEVEL_INVALID) { + return "ERROR: invalid level '" + args[0] + "'"; + } + } + set_log_start_level(start_level); + return std::string("OK, current level is ") + enum_to_string(start_level); + })); + }); } void simple_logger::create_log_file() diff --git a/src/utils/test/logger.cpp b/src/utils/test/logger.cpp index e60e8fe5d4..a06d85ef18 100644 --- a/src/utils/test/logger.cpp +++ b/src/utils/test/logger.cpp @@ -38,7 +38,6 @@ #include "utils/api_utilities.h" #include "utils/filesystem.h" #include "utils/flags.h" -#include "utils/logging_provider.h" #include "utils/simple_logger.h" #include "utils/test_macros.h" @@ -51,23 +50,11 @@ class logger_test : public testing::Test public: void SetUp() override { - // Deregister commands to avoid re-register error. - dsn::logging_provider::instance()->deregister_commands(); - } -}; - -class simple_logger_test : public logger_test -{ -public: - void SetUp() override - { - logger_test::SetUp(); - std::string cwd; ASSERT_TRUE(dsn::utils::filesystem::get_current_directory(cwd)); // NOTE: Don't name the dir with "test", otherwise the whole utils test dir would be // removed. - test_dir = dsn::utils::filesystem::path_combine(cwd, "simple_logger_test"); + test_dir = dsn::utils::filesystem::path_combine(cwd, "logger_test"); NO_FATALS(prepare_test_dir()); std::set files; @@ -154,7 +141,7 @@ TEST_F(logger_test, screen_logger_test) logger->flush(); } -TEST_F(simple_logger_test, redundant_log_test) +TEST_F(logger_test, redundant_log_test) { // Create redundant log files to test if their number could be restricted. for (unsigned int i = 0; i < FLAGS_max_number_of_log_files_on_disk + 10; ++i) { From ad7b18b3c6660056b1bcff0fcc171600af831bec Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 7 Aug 2024 14:09:56 +0800 Subject: [PATCH 09/15] refactor(string_view): Use std::string_view instead of absl::string_view (#2091) --- src/base/pegasus_utils.h | 6 +- src/base/pegasus_value_schema.h | 18 ++--- src/base/test/value_manager_test.cpp | 4 +- src/base/test/value_schema_test.cpp | 4 +- src/base/value_field.h | 4 +- src/base/value_schema_manager.cpp | 4 +- src/base/value_schema_manager.h | 4 +- src/base/value_schema_v0.cpp | 8 +-- src/base/value_schema_v0.h | 6 +- src/base/value_schema_v1.cpp | 10 +-- src/base/value_schema_v1.h | 8 +-- src/base/value_schema_v2.cpp | 10 +-- src/base/value_schema_v2.h | 8 +-- src/block_service/local/local_service.cpp | 4 +- src/client/replication_ddl_client.cpp | 2 +- src/client/replication_ddl_client.h | 6 +- src/client_lib/pegasus_client_impl.cpp | 2 +- src/common/fs_manager.cpp | 14 ++-- src/common/fs_manager.h | 12 ++-- .../serialization_helper/thrift_helper.h | 6 +- src/failure_detector/failure_detector.cpp | 2 +- src/gutil/test/map_util_unittest.cpp | 4 +- src/http/http_client.cpp | 6 +- src/http/http_client.h | 8 +-- src/http/uri_decoder.cpp | 6 +- src/http/uri_decoder.h | 4 +- .../duplication/meta_duplication_service.cpp | 66 +++++++++---------- src/meta/load_balance_policy.cpp | 5 +- src/meta/meta_backup_service.cpp | 2 +- src/meta/meta_bulk_load_ingestion_context.cpp | 8 +-- src/meta/meta_bulk_load_service.cpp | 18 ++--- src/meta/meta_server_failure_detector.cpp | 4 +- src/meta/meta_service.cpp | 2 +- src/meta/table_metrics.cpp | 2 +- src/nfs/nfs_client_impl.cpp | 2 +- src/nfs/nfs_server_impl.cpp | 2 +- src/redis_protocol/proxy_lib/redis_parser.cpp | 6 +- src/replica/backup/replica_backup_manager.cpp | 2 +- src/replica/bulk_load/replica_bulk_loader.cpp | 4 +- src/replica/disk_cleaner.cpp | 4 +- .../duplication/duplication_pipeline.cpp | 4 +- .../duplication/load_from_private_log.cpp | 4 +- src/replica/duplication/mutation_batch.cpp | 2 +- src/replica/duplication/mutation_duplicator.h | 4 +- .../duplication/replica_duplicator.cpp | 2 +- .../replica_duplicator_manager.cpp | 2 +- src/replica/duplication/replica_follower.cpp | 8 +-- .../duplication/test/duplication_test_base.h | 2 +- src/replica/log_file_stream.h | 2 +- src/replica/mutation_log_replay.cpp | 4 +- src/replica/mutation_log_utils.cpp | 8 +-- src/replica/mutation_log_utils.h | 4 +- src/replica/replica.cpp | 2 +- src/replica/replica_base.cpp | 4 +- src/replica/replica_base.h | 4 +- src/replica/replica_check.cpp | 6 +- src/replica/replica_config.cpp | 4 +- src/replica/replica_disk_migrator.cpp | 8 +-- src/replica/replica_stub.cpp | 11 ++-- src/replica/replication_app_base.cpp | 4 +- src/replica/split/replica_split_manager.cpp | 28 ++++---- src/runtime/profiler.cpp | 2 +- src/runtime/rpc/dns_resolver.cpp | 2 +- src/runtime/rpc/network.cpp | 2 +- src/runtime/rpc/rpc_address.cpp | 4 +- src/runtime/rpc/thrift_message_parser.cpp | 4 +- src/runtime/task/task_queue.cpp | 2 +- src/security/sasl_client_wrapper.cpp | 8 +-- src/security/sasl_server_wrapper.cpp | 8 +-- src/security/sasl_wrapper.cpp | 4 +- src/server/capacity_unit_calculator.cpp | 2 +- src/server/compaction_filter_rule.cpp | 26 ++++---- src/server/compaction_filter_rule.h | 26 ++++---- src/server/compaction_operation.cpp | 20 +++--- src/server/compaction_operation.h | 27 ++++---- src/server/hotkey_collector.cpp | 10 +-- src/server/hotkey_collector.h | 4 +- src/server/hotspot_partition_calculator.cpp | 4 +- src/server/key_ttl_compaction_filter.h | 4 +- src/server/pegasus_event_listener.cpp | 2 +- src/server/pegasus_manual_compact_service.cpp | 2 +- src/server/pegasus_mutation_duplicator.cpp | 10 +-- src/server/pegasus_mutation_duplicator.h | 6 +- src/server/pegasus_server_impl.cpp | 4 +- src/server/pegasus_server_impl_init.cpp | 3 +- src/server/pegasus_server_write.cpp | 2 +- src/server/pegasus_write_service.cpp | 2 +- src/server/pegasus_write_service_impl.h | 16 ++--- src/server/rocksdb_wrapper.cpp | 24 +++---- src/server/rocksdb_wrapper.h | 14 ++-- src/server/test/hotkey_collector_test.cpp | 4 +- .../test/pegasus_mutation_duplicator_test.cpp | 3 +- .../test/pegasus_write_service_impl_test.cpp | 6 +- src/server/test/rocksdb_wrapper_test.cpp | 8 +-- src/shell/command_helper.h | 4 +- src/shell/commands.h | 2 +- src/utils/blob.h | 8 +-- src/utils/builtin_metrics.cpp | 2 +- src/utils/endians.h | 6 +- src/utils/errors.h | 8 +-- src/utils/fail_point.cpp | 10 +-- src/utils/fail_point.h | 6 +- src/utils/fail_point_impl.h | 10 +-- src/utils/filesystem.cpp | 14 ++-- src/utils/latency_tracer.cpp | 2 +- src/utils/metrics.cpp | 2 +- src/utils/metrics.h | 14 ++-- src/utils/simple_logger.cpp | 4 +- src/utils/string_conv.h | 22 +++---- src/utils/test/fail_point_test.cpp | 8 +-- src/utils/test/fmt_logging_test.cpp | 6 +- src/utils/test/string_conv_test.cpp | 46 ++++++------- src/utils/time_utils.h | 6 +- 113 files changed, 427 insertions(+), 432 deletions(-) diff --git a/src/base/pegasus_utils.h b/src/base/pegasus_utils.h index a6ea381f74..adb0788884 100644 --- a/src/base/pegasus_utils.h +++ b/src/base/pegasus_utils.h @@ -28,7 +28,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "utils/flags.h" DSN_DECLARE_bool(encrypt_data_at_rest); @@ -122,9 +122,9 @@ const std::string &redact_sensitive_string(const T &src) } } -inline absl::string_view to_string_view(rocksdb::Slice s) { return {s.data(), s.size()}; } +inline std::string_view to_string_view(rocksdb::Slice s) { return {s.data(), s.size()}; } -inline rocksdb::Slice to_rocksdb_slice(absl::string_view s) { return {s.data(), s.size()}; } +inline rocksdb::Slice to_rocksdb_slice(std::string_view s) { return {s.data(), s.size()}; } } // namespace utils } // namespace pegasus diff --git a/src/base/pegasus_value_schema.h b/src/base/pegasus_value_schema.h index d8f651b163..c95ceedbd3 100644 --- a/src/base/pegasus_value_schema.h +++ b/src/base/pegasus_value_schema.h @@ -32,7 +32,7 @@ #include "utils/blob.h" #include "utils/endians.h" #include "utils/fmt_logging.h" -#include "absl/strings/string_view.h" +#include #include "value_field.h" namespace pegasus { @@ -55,7 +55,7 @@ inline uint64_t extract_timestamp_from_timetag(uint64_t timetag) /// Extracts expire_ts from rocksdb value with given version. /// The value schema must be in v0 or v1. /// \return expire_ts in host endian -inline uint32_t pegasus_extract_expire_ts(uint32_t version, absl::string_view value) +inline uint32_t pegasus_extract_expire_ts(uint32_t version, std::string_view value) { CHECK_LE(version, PEGASUS_DATA_VERSION_MAX); return dsn::data_input(value).read_u32(); @@ -76,7 +76,7 @@ pegasus_extract_user_data(uint32_t version, std::string &&raw_value, ::dsn::blob if (version == 1) { input.skip(sizeof(uint64_t)); } - absl::string_view view = input.read_str(); + std::string_view view = input.read_str(); // tricky code to avoid memory copy std::shared_ptr buf(const_cast(view.data()), [s](char *) { delete s; }); @@ -84,7 +84,7 @@ pegasus_extract_user_data(uint32_t version, std::string &&raw_value, ::dsn::blob } /// Extracts timetag from a v1 value. -inline uint64_t pegasus_extract_timetag(int version, absl::string_view value) +inline uint64_t pegasus_extract_timetag(int version, std::string_view value) { CHECK_EQ(version, 1); @@ -118,7 +118,7 @@ inline bool check_if_ts_expired(uint32_t epoch_now, uint32_t expire_ts) /// \return true if expired inline bool check_if_record_expired(uint32_t value_schema_version, uint32_t epoch_now, - absl::string_view raw_value) + std::string_view raw_value) { return check_if_ts_expired(epoch_now, pegasus_extract_expire_ts(value_schema_version, raw_value)); @@ -136,7 +136,7 @@ class pegasus_value_generator /// A higher level utility for generating value with given version. /// The value schema must be in v0 or v1. rocksdb::SliceParts generate_value(uint32_t value_schema_version, - absl::string_view user_data, + std::string_view user_data, uint32_t expire_ts, uint64_t timetag) { @@ -157,7 +157,7 @@ class pegasus_value_generator /// /// rocksdb value (ver 0) = [expire_ts(uint32_t)] [user_data(bytes)] /// \internal - rocksdb::SliceParts generate_value_v0(uint32_t expire_ts, absl::string_view user_data) + rocksdb::SliceParts generate_value_v0(uint32_t expire_ts, std::string_view user_data) { _write_buf.resize(sizeof(uint32_t)); _write_slices.clear(); @@ -210,7 +210,7 @@ class pegasus_value_generator /// /// \internal rocksdb::SliceParts - generate_value_v1(uint32_t expire_ts, uint64_t timetag, absl::string_view user_data) + generate_value_v1(uint32_t expire_ts, uint64_t timetag, std::string_view user_data) { _write_buf.resize(sizeof(uint32_t) + sizeof(uint64_t)); _write_slices.clear(); @@ -258,7 +258,7 @@ class value_schema public: virtual ~value_schema() = default; - virtual std::unique_ptr extract_field(absl::string_view value, + virtual std::unique_ptr extract_field(std::string_view value, value_field_type type) = 0; /// Extracts user value from the raw rocksdb value. /// In order to avoid data copy, the ownership of `raw_value` will be transferred diff --git a/src/base/test/value_manager_test.cpp b/src/base/test/value_manager_test.cpp index 35c1ce488c..f9e8a8dfd0 100644 --- a/src/base/test/value_manager_test.cpp +++ b/src/base/test/value_manager_test.cpp @@ -23,7 +23,7 @@ #include "base/value_schema_manager.h" #include "gtest/gtest.h" #include "pegasus_value_schema.h" -#include "absl/strings/string_view.h" +#include #include "value_field.h" using namespace pegasus; @@ -31,7 +31,7 @@ using namespace pegasus; extern std::string generate_value(value_schema *schema, uint32_t expire_ts, uint64_t time_tag, - absl::string_view user_data); + std::string_view user_data); TEST(value_schema_manager, get_latest_value_schema) { diff --git a/src/base/test/value_schema_test.cpp b/src/base/test/value_schema_test.cpp index 246eab5ef7..aa39bffccd 100644 --- a/src/base/test/value_schema_test.cpp +++ b/src/base/test/value_schema_test.cpp @@ -30,7 +30,7 @@ #include "base/value_schema_manager.h" #include "gtest/gtest.h" #include "utils/blob.h" -#include "absl/strings/string_view.h" +#include #include "value_field.h" using namespace pegasus; @@ -52,7 +52,7 @@ uint64_t extract_time_tag(value_schema *schema, const std::string &raw_value) std::string generate_value(value_schema *schema, uint32_t expire_ts, uint64_t time_tag, - absl::string_view user_data) + std::string_view user_data) { std::string write_buf; std::vector write_slices; diff --git a/src/base/value_field.h b/src/base/value_field.h index 11e99b8939..f5319a2261 100644 --- a/src/base/value_field.h +++ b/src/base/value_field.h @@ -56,9 +56,9 @@ struct time_tag_field : public value_field struct user_data_field : public value_field { - explicit user_data_field(absl::string_view data) : user_data(data) {} + explicit user_data_field(std::string_view data) : user_data(data) {} value_field_type type() { return value_field_type::USER_DATA; } - absl::string_view user_data; + std::string_view user_data; }; } // namespace pegasus diff --git a/src/base/value_schema_manager.cpp b/src/base/value_schema_manager.cpp index 2ce2f1db66..491230bdfa 100644 --- a/src/base/value_schema_manager.cpp +++ b/src/base/value_schema_manager.cpp @@ -19,7 +19,7 @@ #include "value_schema_manager.h" -#include +#include #include #include @@ -47,7 +47,7 @@ void value_schema_manager::register_schema(std::unique_ptr schema) } value_schema *value_schema_manager::get_value_schema(uint32_t meta_cf_data_version, - absl::string_view value) const + std::string_view value) const { dsn::data_input input(value); uint8_t first_byte = input.read_u8(); diff --git a/src/base/value_schema_manager.h b/src/base/value_schema_manager.h index 5afeef5740..86b834bfd6 100644 --- a/src/base/value_schema_manager.h +++ b/src/base/value_schema_manager.h @@ -25,7 +25,7 @@ #include "pegasus_value_schema.h" #include "utils/singleton.h" -#include "absl/strings/string_view.h" +#include namespace pegasus { @@ -35,7 +35,7 @@ class value_schema_manager : public dsn::utils::singleton void register_schema(std::unique_ptr schema); /// using the raw value in rocksdb and data version stored in meta column family to get data /// version - value_schema *get_value_schema(uint32_t meta_cf_data_version, absl::string_view value) const; + value_schema *get_value_schema(uint32_t meta_cf_data_version, std::string_view value) const; value_schema *get_value_schema(uint32_t version) const; value_schema *get_latest_value_schema() const; diff --git a/src/base/value_schema_v0.cpp b/src/base/value_schema_v0.cpp index b7cc30b5dc..1d9d0433bc 100644 --- a/src/base/value_schema_v0.cpp +++ b/src/base/value_schema_v0.cpp @@ -19,7 +19,7 @@ #include "value_schema_v0.h" -#include +#include #include #include #include @@ -32,7 +32,7 @@ #include "utils/ports.h" namespace pegasus { -std::unique_ptr value_schema_v0::extract_field(absl::string_view value, +std::unique_ptr value_schema_v0::extract_field(std::string_view value, value_field_type type) { std::unique_ptr field = nullptr; @@ -80,14 +80,14 @@ rocksdb::SliceParts value_schema_v0::generate_value(const value_params ¶ms) params.write_slices.clear(); params.write_slices.emplace_back(params.write_buf.data(), params.write_buf.size()); - absl::string_view user_data = data_field->user_data; + std::string_view user_data = data_field->user_data; if (user_data.length() > 0) { params.write_slices.emplace_back(user_data.data(), user_data.length()); } return {¶ms.write_slices[0], static_cast(params.write_slices.size())}; } -std::unique_ptr value_schema_v0::extract_timestamp(absl::string_view value) +std::unique_ptr value_schema_v0::extract_timestamp(std::string_view value) { uint32_t expire_ts = dsn::data_input(value).read_u32(); return std::make_unique(expire_ts); diff --git a/src/base/value_schema_v0.h b/src/base/value_schema_v0.h index 683afe8f92..750136916d 100644 --- a/src/base/value_schema_v0.h +++ b/src/base/value_schema_v0.h @@ -25,7 +25,7 @@ #include "pegasus_value_schema.h" #include "utils/blob.h" -#include "absl/strings/string_view.h" +#include #include "value_field.h" namespace pegasus { @@ -37,7 +37,7 @@ class value_schema_v0 : public value_schema public: value_schema_v0() = default; - std::unique_ptr extract_field(absl::string_view value, + std::unique_ptr extract_field(std::string_view value, value_field_type type) override; dsn::blob extract_user_data(std::string &&value) override; void update_field(std::string &value, std::unique_ptr field) override; @@ -45,7 +45,7 @@ class value_schema_v0 : public value_schema data_version version() const override { return data_version::VERSION_0; } private: - std::unique_ptr extract_timestamp(absl::string_view value); + std::unique_ptr extract_timestamp(std::string_view value); void update_expire_ts(std::string &value, std::unique_ptr field); }; } // namespace pegasus diff --git a/src/base/value_schema_v1.cpp b/src/base/value_schema_v1.cpp index 9e00369176..50ac2ba5bd 100644 --- a/src/base/value_schema_v1.cpp +++ b/src/base/value_schema_v1.cpp @@ -19,7 +19,7 @@ #include "value_schema_v1.h" -#include +#include #include #include #include @@ -32,7 +32,7 @@ #include "utils/ports.h" namespace pegasus { -std::unique_ptr value_schema_v1::extract_field(absl::string_view value, +std::unique_ptr value_schema_v1::extract_field(std::string_view value, value_field_type type) { std::unique_ptr field = nullptr; @@ -88,20 +88,20 @@ rocksdb::SliceParts value_schema_v1::generate_value(const value_params ¶ms) params.write_slices.clear(); params.write_slices.emplace_back(params.write_buf.data(), params.write_buf.size()); - absl::string_view user_data = data_field->user_data; + std::string_view user_data = data_field->user_data; if (user_data.length() > 0) { params.write_slices.emplace_back(user_data.data(), user_data.length()); } return {¶ms.write_slices[0], static_cast(params.write_slices.size())}; } -std::unique_ptr value_schema_v1::extract_timestamp(absl::string_view value) +std::unique_ptr value_schema_v1::extract_timestamp(std::string_view value) { uint32_t expire_ts = dsn::data_input(value).read_u32(); return std::make_unique(expire_ts); } -std::unique_ptr value_schema_v1::extract_time_tag(absl::string_view value) +std::unique_ptr value_schema_v1::extract_time_tag(std::string_view value) { dsn::data_input input(value); input.skip(sizeof(uint32_t)); diff --git a/src/base/value_schema_v1.h b/src/base/value_schema_v1.h index 7811b49f6b..88e8a03373 100644 --- a/src/base/value_schema_v1.h +++ b/src/base/value_schema_v1.h @@ -25,7 +25,7 @@ #include "pegasus_value_schema.h" #include "utils/blob.h" -#include "absl/strings/string_view.h" +#include #include "value_field.h" namespace pegasus { @@ -37,7 +37,7 @@ class value_schema_v1 : public value_schema public: value_schema_v1() = default; - std::unique_ptr extract_field(absl::string_view value, + std::unique_ptr extract_field(std::string_view value, value_field_type type) override; dsn::blob extract_user_data(std::string &&value) override; void update_field(std::string &value, std::unique_ptr field) override; @@ -45,8 +45,8 @@ class value_schema_v1 : public value_schema data_version version() const override { return data_version::VERSION_1; } private: - std::unique_ptr extract_timestamp(absl::string_view value); - std::unique_ptr extract_time_tag(absl::string_view value); + std::unique_ptr extract_timestamp(std::string_view value); + std::unique_ptr extract_time_tag(std::string_view value); void update_expire_ts(std::string &value, std::unique_ptr field); }; diff --git a/src/base/value_schema_v2.cpp b/src/base/value_schema_v2.cpp index 400cad75e7..82f4942a74 100644 --- a/src/base/value_schema_v2.cpp +++ b/src/base/value_schema_v2.cpp @@ -19,7 +19,7 @@ #include "value_schema_v2.h" -#include +#include #include #include #include @@ -33,7 +33,7 @@ namespace pegasus { -std::unique_ptr value_schema_v2::extract_field(absl::string_view value, +std::unique_ptr value_schema_v2::extract_field(std::string_view value, value_field_type type) { std::unique_ptr field = nullptr; @@ -91,21 +91,21 @@ rocksdb::SliceParts value_schema_v2::generate_value(const value_params ¶ms) params.write_slices.clear(); params.write_slices.emplace_back(params.write_buf.data(), params.write_buf.size()); - absl::string_view user_data = data_field->user_data; + std::string_view user_data = data_field->user_data; if (user_data.length() > 0) { params.write_slices.emplace_back(user_data.data(), user_data.length()); } return {¶ms.write_slices[0], static_cast(params.write_slices.size())}; } -std::unique_ptr value_schema_v2::extract_timestamp(absl::string_view value) +std::unique_ptr value_schema_v2::extract_timestamp(std::string_view value) { dsn::data_input input(value); input.skip(sizeof(uint8_t)); return std::make_unique(input.read_u32()); } -std::unique_ptr value_schema_v2::extract_time_tag(absl::string_view value) +std::unique_ptr value_schema_v2::extract_time_tag(std::string_view value) { dsn::data_input input(value); input.skip(sizeof(uint8_t)); diff --git a/src/base/value_schema_v2.h b/src/base/value_schema_v2.h index c3101baf3a..5415c5d648 100644 --- a/src/base/value_schema_v2.h +++ b/src/base/value_schema_v2.h @@ -25,7 +25,7 @@ #include "pegasus_value_schema.h" #include "utils/blob.h" -#include "absl/strings/string_view.h" +#include #include "value_field.h" namespace pegasus { @@ -38,7 +38,7 @@ class value_schema_v2 : public value_schema public: value_schema_v2() = default; - std::unique_ptr extract_field(absl::string_view value, + std::unique_ptr extract_field(std::string_view value, value_field_type type) override; dsn::blob extract_user_data(std::string &&value) override; void update_field(std::string &value, std::unique_ptr field) override; @@ -46,8 +46,8 @@ class value_schema_v2 : public value_schema data_version version() const override { return data_version::VERSION_2; } private: - std::unique_ptr extract_timestamp(absl::string_view value); - std::unique_ptr extract_time_tag(absl::string_view value); + std::unique_ptr extract_timestamp(std::string_view value); + std::unique_ptr extract_time_tag(std::string_view value); void update_expire_ts(std::string &value, std::unique_ptr field); }; } // namespace pegasus diff --git a/src/block_service/local/local_service.cpp b/src/block_service/local/local_service.cpp index f0dc35f141..0069a03773 100644 --- a/src/block_service/local/local_service.cpp +++ b/src/block_service/local/local_service.cpp @@ -21,7 +21,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "local_service.h" #include "nlohmann/json.hpp" #include "rocksdb/slice.h" @@ -279,7 +279,7 @@ dsn::task_ptr local_file_object::write(const write_request &req, write_future_ptr tsk(new write_future(code, cb, 0)); tsk->set_tracker(tracker); - FAIL_POINT_INJECT_F("mock_local_service_write_failed", [=](absl::string_view) { + FAIL_POINT_INJECT_F("mock_local_service_write_failed", [=](std::string_view) { auto write_failed = [=]() { write_response resp; resp.err = ERR_FS_INTERNAL; diff --git a/src/client/replication_ddl_client.cpp b/src/client/replication_ddl_client.cpp index 1844b87d94..27bfb2df0a 100644 --- a/src/client/replication_ddl_client.cpp +++ b/src/client/replication_ddl_client.cpp @@ -1457,7 +1457,7 @@ void replication_ddl_client::end_meta_request(const rpc_response_task_ptr &callb error_code err, dsn::message_ex *request, dsn::message_ex *response) mutable { FAIL_POINT_INJECT_NOT_RETURN_F( "ddl_client_request_meta", - [&err, this](absl::string_view str) { err = pop_mock_error(); }); + [&err, this](std::string_view str) { err = pop_mock_error(); }); end_meta_request(callback, attempt_count + 1, err, request, response); }); diff --git a/src/client/replication_ddl_client.h b/src/client/replication_ddl_client.h index a14b9a97d4..274605bb34 100644 --- a/src/client/replication_ddl_client.h +++ b/src/client/replication_ddl_client.h @@ -59,7 +59,7 @@ #include "utils/flags.h" #include "utils/fmt_logging.h" #include "utils/ports.h" -#include "absl/strings/string_view.h" +#include DSN_DECLARE_uint32(ddl_client_max_attempt_count); DSN_DECLARE_uint32(ddl_client_retry_interval_ms); @@ -301,7 +301,7 @@ class replication_ddl_client error_code err, dsn::message_ex *request, dsn::message_ex *response) mutable { FAIL_POINT_INJECT_NOT_RETURN_F( "ddl_client_request_meta", - [&err, this](absl::string_view str) { err = pop_mock_error(); }); + [&err, this](std::string_view str) { err = pop_mock_error(); }); end_meta_request(std::move(task), 1, err, request, response); }); @@ -342,7 +342,7 @@ class replication_ddl_client FAIL_POINT_INJECT_NOT_RETURN_F( "ddl_client_request_meta", - [&resp, this](absl::string_view str) { resp.err = pop_mock_error(); }); + [&resp, this](std::string_view str) { resp.err = pop_mock_error(); }); LOG_INFO("received response from meta server: rpc_code={}, err={}, attempt_count={}, " "max_attempt_count={}", diff --git a/src/client_lib/pegasus_client_impl.cpp b/src/client_lib/pegasus_client_impl.cpp index 2ad491c4f8..4ab09425bb 100644 --- a/src/client_lib/pegasus_client_impl.cpp +++ b/src/client_lib/pegasus_client_impl.cpp @@ -26,7 +26,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "common/common.h" #include "common/replication_other_types.h" #include "common/serialization_helper/dsn.layer2_types.h" diff --git a/src/common/fs_manager.cpp b/src/common/fs_manager.cpp index 4544c00985..1801d2c46b 100644 --- a/src/common/fs_manager.cpp +++ b/src/common/fs_manager.cpp @@ -31,7 +31,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "common/gpid.h" #include "common/replication_enums.h" #include "fmt/core.h" @@ -134,7 +134,7 @@ uint64_t dir_node::replicas_count(app_id id) const return iter->second.size(); } -std::string dir_node::replica_dir(absl::string_view app_type, const dsn::gpid &pid) const +std::string dir_node::replica_dir(std::string_view app_type, const dsn::gpid &pid) const { return utils::filesystem::path_combine(full_dir, fmt::format("{}.{}", pid, app_type)); } @@ -159,7 +159,7 @@ uint64_t dir_node::remove(const gpid &pid) void dir_node::update_disk_stat() { - FAIL_POINT_INJECT_F("update_disk_stat", [](absl::string_view) { return; }); + FAIL_POINT_INJECT_F("update_disk_stat", [](std::string_view) { return; }); dsn::utils::filesystem::disk_space_info dsi; if (!dsn::utils::filesystem::get_disk_space_info(full_dir, dsi)) { @@ -329,7 +329,7 @@ dir_node *fs_manager::find_best_dir_for_new_replica(const gpid &pid) const } void fs_manager::specify_dir_for_new_replica_for_test(dir_node *specified_dn, - absl::string_view app_type, + std::string_view app_type, const dsn::gpid &pid) const { bool dn_found = false; @@ -432,7 +432,7 @@ bool fs_manager::is_dir_node_exist(const std::string &data_dir, const std::strin return false; } -dir_node *fs_manager::find_replica_dir(absl::string_view app_type, gpid pid) +dir_node *fs_manager::find_replica_dir(std::string_view app_type, gpid pid) { std::string replica_dir; dir_node *replica_dn = nullptr; @@ -457,7 +457,7 @@ dir_node *fs_manager::find_replica_dir(absl::string_view app_type, gpid pid) return replica_dn; } -dir_node *fs_manager::create_replica_dir_if_necessary(absl::string_view app_type, gpid pid) +dir_node *fs_manager::create_replica_dir_if_necessary(std::string_view app_type, gpid pid) { // Try to find the replica directory. auto replica_dn = find_replica_dir(app_type, pid); @@ -489,7 +489,7 @@ dir_node *fs_manager::create_replica_dir_if_necessary(absl::string_view app_type return replica_dn; } -dir_node *fs_manager::create_child_replica_dir(absl::string_view app_type, +dir_node *fs_manager::create_child_replica_dir(std::string_view app_type, gpid child_pid, const std::string &parent_dir) { diff --git a/src/common/fs_manager.h b/src/common/fs_manager.h index 5820d26887..0ceecc3e43 100644 --- a/src/common/fs_manager.h +++ b/src/common/fs_manager.h @@ -31,7 +31,7 @@ #include "utils/autoref_ptr.h" #include "utils/error_code.h" #include "utils/flags.h" -#include "absl/strings/string_view.h" +#include #include "utils/metrics.h" #include "utils/ports.h" #include "utils/zlocks.h" @@ -105,7 +105,7 @@ struct dir_node uint64_t replicas_count() const; // Construct the replica dir for the given 'app_type' and 'pid'. // NOTE: Just construct the string, the directory will not be created. - std::string replica_dir(absl::string_view app_type, const dsn::gpid &pid) const; + std::string replica_dir(std::string_view app_type, const dsn::gpid &pid) const; bool has(const dsn::gpid &pid) const; uint64_t remove(const dsn::gpid &pid); void update_disk_stat(); @@ -133,18 +133,18 @@ class fs_manager // dir_nodes. // NOTE: only used in test. void specify_dir_for_new_replica_for_test(dir_node *specified_dn, - absl::string_view app_type, + std::string_view app_type, const dsn::gpid &pid) const; void add_replica(const dsn::gpid &pid, const std::string &pid_dir); // Find the replica instance directory. - dir_node *find_replica_dir(absl::string_view app_type, gpid pid); + dir_node *find_replica_dir(std::string_view app_type, gpid pid); // Similar to the above, but it will create a new directory if not found. - dir_node *create_replica_dir_if_necessary(absl::string_view app_type, gpid pid); + dir_node *create_replica_dir_if_necessary(std::string_view app_type, gpid pid); // Similar to the above, and will create a directory for the child on the same dir_node // of parent. // During partition split, we should guarantee child replica and parent replica share the // same data dir. - dir_node *create_child_replica_dir(absl::string_view app_type, + dir_node *create_child_replica_dir(std::string_view app_type, gpid child_pid, const std::string &parent_dir); void remove_replica(const dsn::gpid &pid); diff --git a/src/common/serialization_helper/thrift_helper.h b/src/common/serialization_helper/thrift_helper.h index c3e2bbfb0c..54737a30b6 100644 --- a/src/common/serialization_helper/thrift_helper.h +++ b/src/common/serialization_helper/thrift_helper.h @@ -37,7 +37,7 @@ #include #include -#include "absl/strings/string_view.h" +#include using namespace ::apache::thrift::transport; namespace dsn { @@ -498,7 +498,7 @@ inline uint32_t task_code::write(apache::thrift::protocol::TProtocol *oprot) con dynamic_cast(oprot); if (binary_proto != nullptr) { // the protocol is binary protocol - return binary_proto->writeString(absl::string_view(name)); + return binary_proto->writeString(std::string_view(name)); } else { // the protocol is json protocol uint32_t xfer = 0; @@ -566,7 +566,7 @@ inline uint32_t error_code::write(apache::thrift::protocol::TProtocol *oprot) co dynamic_cast(oprot); if (binary_proto != nullptr) { // the protocol is binary protocol - return binary_proto->writeString(absl::string_view(name)); + return binary_proto->writeString(std::string_view(name)); } else { // the protocol is json protocol uint32_t xfer = 0; diff --git a/src/failure_detector/failure_detector.cpp b/src/failure_detector/failure_detector.cpp index 62a0c7fc8a..b3d1378ebd 100644 --- a/src/failure_detector/failure_detector.cpp +++ b/src/failure_detector/failure_detector.cpp @@ -34,7 +34,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "failure_detector/fd.code.definition.h" #include "fd_types.h" #include "fmt/core.h" diff --git a/src/gutil/test/map_util_unittest.cpp b/src/gutil/test/map_util_unittest.cpp index 0c3e70011f..8f68bf50a2 100644 --- a/src/gutil/test/map_util_unittest.cpp +++ b/src/gutil/test/map_util_unittest.cpp @@ -17,7 +17,7 @@ #include "gutil/map_util.h" #include -#include +#include #include #include #include @@ -73,7 +73,7 @@ TEST(MapUtil, HeterogeneousLookup) // Verify that I can use a key type that's appropriate for heterogeneous // lookup, such as string_view -> string. - constexpr absl::string_view kLookupKey = "foo"; + constexpr std::string_view kLookupKey = "foo"; EXPECT_EQ(FindWithDefault(m, kLookupKey), ""); EXPECT_EQ(FindWithDefault(const_m, kLookupKey), ""); diff --git a/src/http/http_client.cpp b/src/http/http_client.cpp index 24dd575f5c..a369fe3e97 100644 --- a/src/http/http_client.cpp +++ b/src/http/http_client.cpp @@ -483,15 +483,15 @@ void http_client::free_header_list() _header_list = nullptr; } -void http_client::set_header_field(absl::string_view key, absl::string_view val) +void http_client::set_header_field(std::string_view key, std::string_view val) { _header_fields[std::string(key)] = std::string(val); _header_changed = true; } -void http_client::set_accept(absl::string_view val) { set_header_field("Accept", val); } +void http_client::set_accept(std::string_view val) { set_header_field("Accept", val); } -void http_client::set_content_type(absl::string_view val) { set_header_field("Content-Type", val); } +void http_client::set_content_type(std::string_view val) { set_header_field("Content-Type", val); } dsn::error_s http_client::process_header() { diff --git a/src/http/http_client.h b/src/http/http_client.h index 0d8b35e13c..9624f1e467 100644 --- a/src/http/http_client.h +++ b/src/http/http_client.h @@ -27,7 +27,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "http/http_method.h" #include "http/http_status_code.h" #include "utils/enum_helper.h" @@ -199,8 +199,8 @@ class http_client // Operations for the header fields. void clear_header_fields(); - void set_accept(absl::string_view val); - void set_content_type(absl::string_view val); + void set_accept(std::string_view val); + void set_content_type(std::string_view val); // Submit request to remote http service, with response processed by callback function. // @@ -235,7 +235,7 @@ class http_client dsn::error_s set_method(http_method method); void free_header_list(); - void set_header_field(absl::string_view key, absl::string_view val); + void set_header_field(std::string_view key, std::string_view val); dsn::error_s process_header(); // The size of a buffer that is used by libcurl to store human readable diff --git a/src/http/uri_decoder.cpp b/src/http/uri_decoder.cpp index d6a1a1de9f..63b784caec 100644 --- a/src/http/uri_decoder.cpp +++ b/src/http/uri_decoder.cpp @@ -40,7 +40,7 @@ error_with from_hex(const char c) } } -error_with decode_char(const absl::string_view &hex) +error_with decode_char(const std::string_view &hex) { CHECK_EQ(2, hex.size()); @@ -53,7 +53,7 @@ error_with decode_char(const absl::string_view &hex) return error_s::make(ERR_INVALID_PARAMETERS); } -error_with decode(const absl::string_view &encoded_uri) +error_with decode(const std::string_view &encoded_uri) { std::string decoded_uri; for (size_t i = 0; i < encoded_uri.size(); ++i) { @@ -64,7 +64,7 @@ error_with decode(const absl::string_view &encoded_uri) "Encountered partial escape sequence at end of string"); } - const absl::string_view encoded_char(encoded_uri.data() + i + 1, 2); + const std::string_view encoded_char(encoded_uri.data() + i + 1, 2); auto decoded_char = decode_char(encoded_char); if (!decoded_char.is_ok()) { return error_s::make( diff --git a/src/http/uri_decoder.h b/src/http/uri_decoder.h index b1d28a13ba..93266f4523 100644 --- a/src/http/uri_decoder.h +++ b/src/http/uri_decoder.h @@ -20,14 +20,14 @@ #include #include "utils/errors.h" -#include "absl/strings/string_view.h" +#include namespace dsn { namespace uri { /// \brief Decodes a sequence according to the percent decoding rules. /// \returns the decoded uri path -error_with decode(const absl::string_view &encoded_uri); +error_with decode(const std::string_view &encoded_uri); } // namespace uri } // namespace dsn diff --git a/src/meta/duplication/meta_duplication_service.cpp b/src/meta/duplication/meta_duplication_service.cpp index 4f0d7f816e..a2adf29469 100644 --- a/src/meta/duplication/meta_duplication_service.cpp +++ b/src/meta/duplication/meta_duplication_service.cpp @@ -21,7 +21,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "common//duplication_common.h" #include "common/common.h" #include "common/gpid.h" @@ -484,43 +484,43 @@ void meta_duplication_service::create_follower_app_for_duplication( dsn::message_ex *msg = dsn::message_ex::create_request(RPC_CM_CREATE_APP); dsn::marshall(msg, request); - rpc::call( - dsn::dns_resolver::instance().resolve_address(meta_servers), - msg, - _meta_svc->tracker(), - [=](error_code err, configuration_create_app_response &&resp) mutable { - FAIL_POINT_INJECT_NOT_RETURN_F("update_app_request_ok", - [&](absl::string_view s) -> void { err = ERR_OK; }); - error_code create_err = err == ERR_OK ? resp.err : err; - error_code update_err = ERR_NO_NEED_OPERATE; - - FAIL_POINT_INJECT_NOT_RETURN_F( - "persist_dup_status_failed", - [&](absl::string_view s) -> void { create_err = ERR_OK; }); - if (create_err == ERR_OK) { - update_err = dup->alter_status(duplication_status::DS_APP); - } + rpc::call(dsn::dns_resolver::instance().resolve_address(meta_servers), + msg, + _meta_svc->tracker(), + [=](error_code err, configuration_create_app_response &&resp) mutable { + FAIL_POINT_INJECT_NOT_RETURN_F("update_app_request_ok", + [&](std::string_view s) -> void { err = ERR_OK; }); + error_code create_err = err == ERR_OK ? resp.err : err; + error_code update_err = ERR_NO_NEED_OPERATE; - FAIL_POINT_INJECT_F("persist_dup_status_failed", - [&](absl::string_view s) -> void { return; }); - if (update_err == ERR_OK) { - blob value = dup->to_json_blob(); - // Note: this function is `async`, it may not be persisted completed - // after executing, now using `_is_altering` to judge whether `updating` or - // `completed`, if `_is_altering`, dup->alter_status() will return `ERR_BUSY` - _meta_svc->get_meta_storage()->set_data(std::string(dup->store_path), - std::move(value), - [=]() { dup->persist_status(); }); - } else { - LOG_ERROR("created follower app[{}.{}] to trigger duplicate checkpoint failed: " + FAIL_POINT_INJECT_NOT_RETURN_F( + "persist_dup_status_failed", + [&](std::string_view s) -> void { create_err = ERR_OK; }); + if (create_err == ERR_OK) { + update_err = dup->alter_status(duplication_status::DS_APP); + } + + FAIL_POINT_INJECT_F("persist_dup_status_failed", + [&](std::string_view s) -> void { return; }); + if (update_err == ERR_OK) { + blob value = dup->to_json_blob(); + // Note: this function is `async`, it may not be persisted completed + // after executing, now using `_is_altering` to judge whether `updating` or + // `completed`, if `_is_altering`, dup->alter_status() will return `ERR_BUSY` + _meta_svc->get_meta_storage()->set_data(std::string(dup->store_path), + std::move(value), + [=]() { dup->persist_status(); }); + } else { + LOG_ERROR( + "created follower app[{}.{}] to trigger duplicate checkpoint failed: " "duplication_status = {}, create_err = {}, update_err = {}", dup->remote_cluster_name, dup->app_name, duplication_status_to_string(dup->status()), create_err, update_err); - } - }); + } + }); } void meta_duplication_service::check_follower_app_if_create_completed( @@ -539,7 +539,7 @@ void meta_duplication_service::check_follower_app_if_create_completed( msg, _meta_svc->tracker(), [=](error_code err, query_cfg_response &&resp) mutable { - FAIL_POINT_INJECT_NOT_RETURN_F("create_app_ok", [&](absl::string_view s) -> void { + FAIL_POINT_INJECT_NOT_RETURN_F("create_app_ok", [&](std::string_view s) -> void { err = ERR_OK; int count = dup->partition_count; while (count-- > 0) { @@ -588,7 +588,7 @@ void meta_duplication_service::check_follower_app_if_create_completed( } FAIL_POINT_INJECT_F("persist_dup_status_failed", - [&](absl::string_view s) -> void { return; }); + [&](std::string_view s) -> void { return; }); if (update_err == ERR_OK) { blob value = dup->to_json_blob(); // Note: this function is `async`, it may not be persisted completed diff --git a/src/meta/load_balance_policy.cpp b/src/meta/load_balance_policy.cpp index 9295796746..1278b35f98 100644 --- a/src/meta/load_balance_policy.cpp +++ b/src/meta/load_balance_policy.cpp @@ -29,7 +29,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "dsn.layer2_types.h" #include "meta/meta_data.h" #include "meta_admin_types.h" @@ -146,8 +146,7 @@ generate_balancer_request(const app_mapper &apps, const host_port &from, const host_port &to) { - FAIL_POINT_INJECT_F("generate_balancer_request", - [](absl::string_view name) { return nullptr; }); + FAIL_POINT_INJECT_F("generate_balancer_request", [](std::string_view name) { return nullptr; }); configuration_balancer_request result; result.gpid = pc.pid; diff --git a/src/meta/meta_backup_service.cpp b/src/meta/meta_backup_service.cpp index 7264089e9e..2bfa64ede7 100644 --- a/src/meta/meta_backup_service.cpp +++ b/src/meta/meta_backup_service.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -#include #include #include #include @@ -23,6 +22,7 @@ #include #include #include +#include #include #include diff --git a/src/meta/meta_bulk_load_ingestion_context.cpp b/src/meta/meta_bulk_load_ingestion_context.cpp index 6921d4ea86..79d01a4f1e 100644 --- a/src/meta/meta_bulk_load_ingestion_context.cpp +++ b/src/meta/meta_bulk_load_ingestion_context.cpp @@ -26,7 +26,7 @@ #include "utils/fail_point.h" #include "utils/fmt_logging.h" #include "utils/string_conv.h" -#include "absl/strings/string_view.h" +#include DSN_DEFINE_uint32(meta_server, bulk_load_node_max_ingesting_count, @@ -73,7 +73,7 @@ uint32_t ingestion_context::node_context::get_max_disk_ingestion_count( const uint32_t max_node_ingestion_count) const { FAIL_POINT_INJECT_F("ingestion_node_context_disk_count", - [](absl::string_view count_str) -> uint32_t { + [](std::string_view count_str) -> uint32_t { uint32_t count = 0; buf2uint32(count_str, count); return count; @@ -123,7 +123,7 @@ void ingestion_context::node_context::decrease(const std::string &disk_tag) bool ingestion_context::try_partition_ingestion(const partition_configuration &pc, const config_context &cc) { - FAIL_POINT_INJECT_F("ingestion_try_partition_ingestion", [=](absl::string_view) -> bool { + FAIL_POINT_INJECT_F("ingestion_try_partition_ingestion", [=](std::string_view) -> bool { auto info = partition_node_info(); info.pid = pc.pid; _running_partitions[pc.pid] = info; @@ -158,7 +158,7 @@ void ingestion_context::add_partition(const partition_node_info &info) void ingestion_context::remove_partition(const gpid &pid) { FAIL_POINT_INJECT_F("ingestion_context_remove_partition", - [=](absl::string_view) { _running_partitions.erase(pid); }); + [=](std::string_view) { _running_partitions.erase(pid); }); if (_running_partitions.find(pid) == _running_partitions.end()) { return; diff --git a/src/meta/meta_bulk_load_service.cpp b/src/meta/meta_bulk_load_service.cpp index aa2688ce55..198fb003e7 100644 --- a/src/meta/meta_bulk_load_service.cpp +++ b/src/meta/meta_bulk_load_service.cpp @@ -26,7 +26,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "block_service/block_service.h" #include "block_service/block_service_manager.h" #include "common/replica_envs.h" @@ -101,7 +101,7 @@ void bulk_load_service::initialize_bulk_load_service() void bulk_load_service::on_start_bulk_load(start_bulk_load_rpc rpc) { FAIL_POINT_INJECT_F("meta_on_start_bulk_load", - [=](absl::string_view) { rpc.response().err = ERR_OK; }); + [=](std::string_view) { rpc.response().err = ERR_OK; }); const auto &request = rpc.request(); auto &response = rpc.response(); @@ -171,7 +171,7 @@ bulk_load_service::check_bulk_load_request_params(const start_bulk_load_request std::string &hint_msg) { FAIL_POINT_INJECT_F("meta_check_bulk_load_request_params", - [](absl::string_view) -> error_code { return ERR_OK; }); + [](std::string_view) -> error_code { return ERR_OK; }); if (!validate_ingest_behind(envs, request.ingest_behind)) { hint_msg = fmt::format("inconsistent ingestion behind option"); @@ -414,7 +414,7 @@ bool bulk_load_service::check_partition_status( // ThreadPool: THREAD_POOL_META_STATE void bulk_load_service::partition_bulk_load(const std::string &app_name, const gpid &pid) { - FAIL_POINT_INJECT_F("meta_bulk_load_partition_bulk_load", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("meta_bulk_load_partition_bulk_load", [](std::string_view) {}); partition_configuration pc; if (!check_partition_status(app_name, @@ -583,7 +583,7 @@ void bulk_load_service::on_partition_bulk_load_reply(error_code err, // ThreadPool: THREAD_POOL_META_STATE void bulk_load_service::try_resend_bulk_load_request(const std::string &app_name, const gpid &pid) { - FAIL_POINT_INJECT_F("meta_bulk_load_resend_request", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("meta_bulk_load_resend_request", [](std::string_view) {}); zauto_read_lock l(_lock); if (is_app_bulk_loading_unlocked(pid.get_app_id())) { tasking::enqueue(LPC_META_STATE_NORMAL, @@ -1091,7 +1091,7 @@ void bulk_load_service::update_app_status_on_remote_storage_unlocked( int32_t app_id, bulk_load_status::type new_status, error_code err, bool should_send_request) { FAIL_POINT_INJECT_F("meta_update_app_status_on_remote_storage_unlocked", - [](absl::string_view) {}); + [](std::string_view) {}); app_bulk_load_info ainfo = _app_bulk_load_info[app_id]; auto old_status = ainfo.status; @@ -1223,7 +1223,7 @@ bool bulk_load_service::check_ever_ingestion_succeed(const partition_configurati // ThreadPool: THREAD_POOL_META_STATE void bulk_load_service::partition_ingestion(const std::string &app_name, const gpid &pid) { - FAIL_POINT_INJECT_F("meta_bulk_load_partition_ingestion", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("meta_bulk_load_partition_ingestion", [](std::string_view) {}); auto app_status = get_app_bulk_load_status(pid.get_app_id()); if (app_status != bulk_load_status::BLS_INGESTING) { @@ -1659,7 +1659,7 @@ void bulk_load_service::on_clear_bulk_load(clear_bulk_load_rpc rpc) void bulk_load_service::do_clear_app_bulk_load_result(int32_t app_id, clear_bulk_load_rpc rpc) { FAIL_POINT_INJECT_F("meta_do_clear_app_bulk_load_result", - [rpc](absl::string_view) { rpc.response().err = ERR_OK; }); + [rpc](std::string_view) { rpc.response().err = ERR_OK; }); std::string bulk_load_path = get_app_bulk_load_path(app_id); _meta_svc->get_meta_storage()->delete_node_recursively( std::move(bulk_load_path), [this, app_id, bulk_load_path, rpc]() { @@ -1753,7 +1753,7 @@ void bulk_load_service::do_sync_partition(const gpid &pid, std::string &partitio // ThreadPool: THREAD_POOL_META_SERVER void bulk_load_service::try_to_continue_bulk_load() { - FAIL_POINT_INJECT_F("meta_try_to_continue_bulk_load", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("meta_try_to_continue_bulk_load", [](std::string_view) {}); zauto_read_lock l(_lock); for (const auto app_id : _bulk_load_app_id) { app_bulk_load_info ainfo = _app_bulk_load_info[app_id]; diff --git a/src/meta/meta_server_failure_detector.cpp b/src/meta/meta_server_failure_detector.cpp index 1d900fbcbe..9e198582cc 100644 --- a/src/meta/meta_server_failure_detector.cpp +++ b/src/meta/meta_server_failure_detector.cpp @@ -30,7 +30,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "fd_types.h" #include "meta/meta_options.h" #include "meta/meta_service.h" @@ -107,7 +107,7 @@ void meta_server_failure_detector::on_worker_connected(const host_port &node) bool meta_server_failure_detector::get_leader(host_port *leader) { - FAIL_POINT_INJECT_F("meta_server_failure_detector_get_leader", [leader](absl::string_view str) { + FAIL_POINT_INJECT_F("meta_server_failure_detector_get_leader", [leader](std::string_view str) { /// the format of str is : true#{ip}:{port} or false#{ip}:{port} auto pos = str.find("#"); // get leader host_port diff --git a/src/meta/meta_service.cpp b/src/meta/meta_service.cpp index 1de9069bb9..98f0339cee 100644 --- a/src/meta/meta_service.cpp +++ b/src/meta/meta_service.cpp @@ -24,7 +24,7 @@ * THE SOFTWARE. */ -#include +#include // IWYU pragma: no_include // IWYU pragma: no_include #include diff --git a/src/meta/table_metrics.cpp b/src/meta/table_metrics.cpp index 7b2a21af5c..2bacf28ac6 100644 --- a/src/meta/table_metrics.cpp +++ b/src/meta/table_metrics.cpp @@ -17,7 +17,7 @@ #include "table_metrics.h" -#include +#include // IWYU pragma: no_include #include #include diff --git a/src/nfs/nfs_client_impl.cpp b/src/nfs/nfs_client_impl.cpp index 867d1891a9..5b2bcd3657 100644 --- a/src/nfs/nfs_client_impl.cpp +++ b/src/nfs/nfs_client_impl.cpp @@ -30,7 +30,7 @@ // IWYU pragma: no_include #include -#include "absl/strings/string_view.h" +#include #include "fmt/core.h" #include "nfs/nfs_code_definition.h" #include "nfs/nfs_node.h" diff --git a/src/nfs/nfs_server_impl.cpp b/src/nfs/nfs_server_impl.cpp index 001090d095..d4f8568b81 100644 --- a/src/nfs/nfs_server_impl.cpp +++ b/src/nfs/nfs_server_impl.cpp @@ -32,7 +32,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "fmt/core.h" // IWYU pragma: keep #include "gutil/map_util.h" #include "nfs/nfs_code_definition.h" diff --git a/src/redis_protocol/proxy_lib/redis_parser.cpp b/src/redis_protocol/proxy_lib/redis_parser.cpp index 9aea607eac..99683d902c 100644 --- a/src/redis_protocol/proxy_lib/redis_parser.cpp +++ b/src/redis_protocol/proxy_lib/redis_parser.cpp @@ -32,7 +32,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "common/common.h" #include "common/replication_other_types.h" #include "pegasus/client.h" @@ -207,7 +207,7 @@ void redis_parser::eat_all(char *dest, size_t length) bool redis_parser::end_array_size() { int32_t count = 0; - if (dsn_unlikely(!dsn::buf2int32(absl::string_view(_current_size), count))) { + if (dsn_unlikely(!dsn::buf2int32(std::string_view(_current_size), count))) { LOG_ERROR_PREFIX("invalid size string \"{}\"", _current_size); return false; } @@ -242,7 +242,7 @@ void redis_parser::append_current_bulk_string() bool redis_parser::end_bulk_string_size() { int32_t length = 0; - if (dsn_unlikely(!dsn::buf2int32(absl::string_view(_current_size), length))) { + if (dsn_unlikely(!dsn::buf2int32(std::string_view(_current_size), length))) { LOG_ERROR_PREFIX("invalid size string \"{}\"", _current_size); return false; } diff --git a/src/replica/backup/replica_backup_manager.cpp b/src/replica/backup/replica_backup_manager.cpp index 42882a2597..e94cc3aea0 100644 --- a/src/replica/backup/replica_backup_manager.cpp +++ b/src/replica/backup/replica_backup_manager.cpp @@ -17,7 +17,7 @@ #include "replica_backup_manager.h" -#include +#include #include #include #include diff --git a/src/replica/bulk_load/replica_bulk_loader.cpp b/src/replica/bulk_load/replica_bulk_loader.cpp index e8eb03987e..b58afccc70 100644 --- a/src/replica/bulk_load/replica_bulk_loader.cpp +++ b/src/replica/bulk_load/replica_bulk_loader.cpp @@ -22,7 +22,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "block_service/block_service_manager.h" #include "common/bulk_load_common.h" #include "common/gpid.h" @@ -487,7 +487,7 @@ void replica_bulk_loader::download_files(const std::string &provider_name, const std::string &remote_dir, const std::string &local_dir) { - FAIL_POINT_INJECT_F("replica_bulk_loader_download_files", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_bulk_loader_download_files", [](std::string_view) {}); LOG_INFO_PREFIX("start to download files"); dist::block_service::block_filesystem *fs = diff --git a/src/replica/disk_cleaner.cpp b/src/replica/disk_cleaner.cpp index 805da23914..8a191d67fc 100644 --- a/src/replica/disk_cleaner.cpp +++ b/src/replica/disk_cleaner.cpp @@ -36,7 +36,7 @@ #include "utils/fmt_logging.h" #include "utils/macros.h" #include "utils/string_conv.h" -#include "absl/strings/string_view.h" +#include DSN_DEFINE_uint64(replication, gc_disk_error_replica_interval_seconds, @@ -148,7 +148,7 @@ bool parse_timestamp_us(const std::string &name, size_t suffix_size, uint64_t &t } const auto ok = - dsn::buf2uint64(absl::string_view(name.data() + begin_idx, length), timestamp_us); + dsn::buf2uint64(std::string_view(name.data() + begin_idx, length), timestamp_us); return ok ? timestamp_us > MIN_TIMESTAMP_US : false; } diff --git a/src/replica/duplication/duplication_pipeline.cpp b/src/replica/duplication/duplication_pipeline.cpp index 53e54ece33..81c1647632 100644 --- a/src/replica/duplication/duplication_pipeline.cpp +++ b/src/replica/duplication/duplication_pipeline.cpp @@ -17,7 +17,7 @@ #include "duplication_pipeline.h" -#include +#include #include #include #include @@ -47,7 +47,7 @@ namespace replication { // // /*static*/ std::function( - replica_base *, absl::string_view /*remote cluster*/, absl::string_view /*app*/)> + replica_base *, std::string_view /*remote cluster*/, std::string_view /*app*/)> mutation_duplicator::creator; // // diff --git a/src/replica/duplication/load_from_private_log.cpp b/src/replica/duplication/load_from_private_log.cpp index 6b0af82251..b0e501409f 100644 --- a/src/replica/duplication/load_from_private_log.cpp +++ b/src/replica/duplication/load_from_private_log.cpp @@ -20,7 +20,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "common/duplication_common.h" #include "duplication_types.h" #include "load_from_private_log.h" @@ -119,7 +119,7 @@ void load_from_private_log::run() repeat(1_s); FAIL_POINT_INJECT_NOT_RETURN_F( - "duplication_sync_complete", [&](absl::string_view s) -> void { + "duplication_sync_complete", [&](std::string_view s) -> void { if (_duplicator->progress().confirmed_decree == invalid_decree) { // set_confirmed_decree(9), the value must be equal (decree_start of // `test_start_duplication` in `load_from_private_log_test.cpp`) -1 diff --git a/src/replica/duplication/mutation_batch.cpp b/src/replica/duplication/mutation_batch.cpp index f50e7c113c..9a6493cb99 100644 --- a/src/replica/duplication/mutation_batch.cpp +++ b/src/replica/duplication/mutation_batch.cpp @@ -21,7 +21,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "common/replication.codes.h" #include "consensus_types.h" #include "metadata_types.h" diff --git a/src/replica/duplication/mutation_duplicator.h b/src/replica/duplication/mutation_duplicator.h index b1c29e00fa..cb6b3c65e2 100644 --- a/src/replica/duplication/mutation_duplicator.h +++ b/src/replica/duplication/mutation_duplicator.h @@ -68,7 +68,7 @@ class mutation_duplicator : public replica_base // Singleton creator of mutation_duplicator. static std::function( - replica_base *, absl::string_view /*remote cluster*/, absl::string_view /*app name*/)> + replica_base *, std::string_view /*remote cluster*/, std::string_view /*app name*/)> creator; explicit mutation_duplicator(replica_base *r) : replica_base(r) {} @@ -84,7 +84,7 @@ class mutation_duplicator : public replica_base }; inline std::unique_ptr new_mutation_duplicator( - replica_base *r, absl::string_view remote_cluster_address, absl::string_view app) + replica_base *r, std::string_view remote_cluster_address, std::string_view app) { return mutation_duplicator::creator(r, remote_cluster_address, app); } diff --git a/src/replica/duplication/replica_duplicator.cpp b/src/replica/duplication/replica_duplicator.cpp index 31d7e9d94b..6bb75b92cb 100644 --- a/src/replica/duplication/replica_duplicator.cpp +++ b/src/replica/duplication/replica_duplicator.cpp @@ -17,7 +17,7 @@ #include "replica_duplicator.h" -#include +#include #include #include #include diff --git a/src/replica/duplication/replica_duplicator_manager.cpp b/src/replica/duplication/replica_duplicator_manager.cpp index d7a6437c2e..2e1e61cc4d 100644 --- a/src/replica/duplication/replica_duplicator_manager.cpp +++ b/src/replica/duplication/replica_duplicator_manager.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include +#include #include #include #include diff --git a/src/replica/duplication/replica_follower.cpp b/src/replica/duplication/replica_follower.cpp index 4918f60e81..3373d9a86c 100644 --- a/src/replica/duplication/replica_follower.cpp +++ b/src/replica/duplication/replica_follower.cpp @@ -42,7 +42,7 @@ #include "utils/filesystem.h" #include "utils/fmt_logging.h" #include "utils/ports.h" -#include "absl/strings/string_view.h" +#include #include "utils/strings.h" namespace dsn { @@ -132,13 +132,13 @@ void replica_follower::async_duplicate_checkpoint_from_master_replica() msg, &_tracker, [&](error_code err, query_cfg_response &&resp) mutable { - FAIL_POINT_INJECT_F("duplicate_checkpoint_ok", [&](absl::string_view s) -> void { + FAIL_POINT_INJECT_F("duplicate_checkpoint_ok", [&](std::string_view s) -> void { _tracker.set_tasks_success(); return; }); FAIL_POINT_INJECT_F("duplicate_checkpoint_failed", - [&](absl::string_view s) -> void { return; }); + [&](std::string_view s) -> void { return; }); if (update_master_replica_config(err, std::move(resp)) == ERR_OK) { copy_master_replica_checkpoint(); } @@ -260,7 +260,7 @@ void replica_follower::nfs_copy_remote_files(const host_port &remote_node, &_tracker, [&, remote_dir](error_code err, size_t size) mutable { FAIL_POINT_INJECT_NOT_RETURN_F("nfs_copy_ok", - [&](absl::string_view s) -> void { err = ERR_OK; }); + [&](std::string_view s) -> void { err = ERR_OK; }); if (dsn_unlikely(err != ERR_OK)) { LOG_ERROR_PREFIX("nfs copy master[{}] checkpoint failed: checkpoint = {}, err = {}", diff --git a/src/replica/duplication/test/duplication_test_base.h b/src/replica/duplication/test/duplication_test_base.h index cd54fe9d8c..d4dacf9b9a 100644 --- a/src/replica/duplication/test/duplication_test_base.h +++ b/src/replica/duplication/test/duplication_test_base.h @@ -34,7 +34,7 @@ class duplication_test_base : public replica_test_base public: duplication_test_base() { - mutation_duplicator::creator = [](replica_base *r, absl::string_view, absl::string_view) { + mutation_duplicator::creator = [](replica_base *r, std::string_view, std::string_view) { return std::make_unique(r); }; stub->_duplication_sync_timer = std::make_unique(stub.get()); diff --git a/src/replica/log_file_stream.h b/src/replica/log_file_stream.h index 3b901aed73..2b2567229b 100644 --- a/src/replica/log_file_stream.h +++ b/src/replica/log_file_stream.h @@ -66,7 +66,7 @@ class log_file::file_streamer fill_buffers(); } - // TODO(wutao1): use absl::string_view instead of using blob. + // TODO(wutao1): use std::string_view instead of using blob. // WARNING: the resulted blob is not guaranteed to be reference counted. // possible error_code: // ERR_OK result would always size as expected diff --git a/src/replica/mutation_log_replay.cpp b/src/replica/mutation_log_replay.cpp index 126d19cdfc..ee891bcb73 100644 --- a/src/replica/mutation_log_replay.cpp +++ b/src/replica/mutation_log_replay.cpp @@ -38,7 +38,7 @@ #include "utils/errors.h" #include "utils/fail_point.h" #include "utils/fmt_logging.h" -#include "absl/strings/string_view.h" +#include namespace dsn { namespace replication { @@ -77,7 +77,7 @@ namespace replication { size_t start_offset, int64_t &end_offset) { - FAIL_POINT_INJECT_F("mutation_log_replay_block", [](absl::string_view) -> error_s { + FAIL_POINT_INJECT_F("mutation_log_replay_block", [](std::string_view) -> error_s { return error_s::make(ERR_INCOMPLETE_DATA, "mutation_log_replay_block"); }); diff --git a/src/replica/mutation_log_utils.cpp b/src/replica/mutation_log_utils.cpp index 9fed18ab8b..e69fd13909 100644 --- a/src/replica/mutation_log_utils.cpp +++ b/src/replica/mutation_log_utils.cpp @@ -24,7 +24,7 @@ * THE SOFTWARE. */ -#include +#include #include #include @@ -37,9 +37,9 @@ namespace dsn { namespace replication { namespace log_utils { -/*extern*/ error_s open_read(absl::string_view path, /*out*/ log_file_ptr &file) +/*extern*/ error_s open_read(std::string_view path, /*out*/ log_file_ptr &file) { - FAIL_POINT_INJECT_F("open_read", [](absl::string_view) -> error_s { + FAIL_POINT_INJECT_F("open_read", [](std::string_view) -> error_s { return error_s::make(ERR_FILE_OPERATION_FAILED, "open_read"); }); @@ -53,7 +53,7 @@ namespace log_utils { /*extern*/ error_s list_all_files(const std::string &dir, /*out*/ std::vector &files) { - FAIL_POINT_INJECT_F("list_all_files", [](absl::string_view) -> error_s { + FAIL_POINT_INJECT_F("list_all_files", [](std::string_view) -> error_s { return error_s::make(ERR_FILE_OPERATION_FAILED, "list_all_files"); }); diff --git a/src/replica/mutation_log_utils.h b/src/replica/mutation_log_utils.h index fafa4fbf56..56a5405439 100644 --- a/src/replica/mutation_log_utils.h +++ b/src/replica/mutation_log_utils.h @@ -34,13 +34,13 @@ #include "replica/mutation_log.h" #include "utils/autoref_ptr.h" #include "utils/errors.h" -#include "absl/strings/string_view.h" +#include namespace dsn { namespace replication { namespace log_utils { -extern error_s open_read(absl::string_view path, /*out*/ log_file_ptr &file); +extern error_s open_read(std::string_view path, /*out*/ log_file_ptr &file); extern error_s list_all_files(const std::string &dir, /*out*/ std::vector &files); diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp index 24d5cf77ef..f6d772169d 100644 --- a/src/replica/replica.cpp +++ b/src/replica/replica.cpp @@ -26,7 +26,7 @@ #include "replica.h" -#include +#include #include #include #include diff --git a/src/replica/replica_base.cpp b/src/replica/replica_base.cpp index ce4700abdd..f45d9b43f8 100644 --- a/src/replica/replica_base.cpp +++ b/src/replica/replica_base.cpp @@ -17,7 +17,7 @@ #include "replica_base.h" -#include +#include #include METRIC_DEFINE_entity(replica); @@ -41,7 +41,7 @@ metric_entity_ptr instantiate_replica_metric_entity(const gpid &id) } // anonymous namespace -replica_base::replica_base(gpid id, absl::string_view name, absl::string_view app_name) +replica_base::replica_base(gpid id, std::string_view name, std::string_view app_name) : _gpid(id), _name(name), _app_name(app_name), diff --git a/src/replica/replica_base.h b/src/replica/replica_base.h index 8909197a64..79583c13f1 100644 --- a/src/replica/replica_base.h +++ b/src/replica/replica_base.h @@ -29,7 +29,7 @@ #include #include "common/gpid.h" -#include "absl/strings/string_view.h" +#include #include "utils/fmt_logging.h" #include "utils/metrics.h" @@ -39,7 +39,7 @@ namespace replication { /// Base class for types that are one-instance-per-replica. struct replica_base { - replica_base(gpid id, absl::string_view name, absl::string_view app_name); + replica_base(gpid id, std::string_view name, std::string_view app_name); explicit replica_base(replica_base *rhs) : replica_base(rhs->get_gpid(), rhs->replica_name(), rhs->_app_name) diff --git a/src/replica/replica_check.cpp b/src/replica/replica_check.cpp index adb4771b52..32cf5f70fa 100644 --- a/src/replica/replica_check.cpp +++ b/src/replica/replica_check.cpp @@ -58,7 +58,7 @@ #include "utils/fail_point.h" #include "utils/flags.h" #include "utils/fmt_logging.h" -#include "absl/strings/string_view.h" +#include #include "utils/metrics.h" #include "utils/thread_access_checker.h" @@ -82,7 +82,7 @@ namespace replication { void replica::init_group_check() { - FAIL_POINT_INJECT_F("replica_init_group_check", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_init_group_check", [](std::string_view) {}); _checker.only_one_thread_access(); @@ -102,7 +102,7 @@ void replica::init_group_check() void replica::broadcast_group_check() { - FAIL_POINT_INJECT_F("replica_broadcast_group_check", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_broadcast_group_check", [](std::string_view) {}); CHECK_NOTNULL(_primary_states.group_check_task, ""); diff --git a/src/replica/replica_config.cpp b/src/replica/replica_config.cpp index 7abeaf4549..3bcd5faaff 100644 --- a/src/replica/replica_config.cpp +++ b/src/replica/replica_config.cpp @@ -40,7 +40,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "bulk_load/replica_bulk_loader.h" #include "common/gpid.h" #include "common/replica_envs.h" @@ -694,7 +694,7 @@ bool replica::is_same_ballot_status_change_allowed(partition_status::type olds, bool replica::update_local_configuration(const replica_configuration &config, bool same_ballot /* = false*/) { - FAIL_POINT_INJECT_F("replica_update_local_configuration", [=](absl::string_view) -> bool { + FAIL_POINT_INJECT_F("replica_update_local_configuration", [=](std::string_view) -> bool { auto old_status = status(); _config = config; LOG_INFO_PREFIX( diff --git a/src/replica/replica_disk_migrator.cpp b/src/replica/replica_disk_migrator.cpp index 189c81c42b..f0ff784381 100644 --- a/src/replica/replica_disk_migrator.cpp +++ b/src/replica/replica_disk_migrator.cpp @@ -20,7 +20,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "common/fs_manager.h" #include "common/gpid.h" #include "common/replication.codes.h" @@ -164,7 +164,7 @@ void replica_disk_migrator::migrate_replica(const replica_disk_migrate_request & // THREAD_POOL_REPLICATION_LONG bool replica_disk_migrator::init_target_dir(const replica_disk_migrate_request &req) { - FAIL_POINT_INJECT_F("init_target_dir", [this](absl::string_view) -> bool { + FAIL_POINT_INJECT_F("init_target_dir", [this](std::string_view) -> bool { reset_status(); return false; }); @@ -210,7 +210,7 @@ bool replica_disk_migrator::init_target_dir(const replica_disk_migrate_request & // THREAD_POOL_REPLICATION_LONG bool replica_disk_migrator::migrate_replica_checkpoint(const replica_disk_migrate_request &req) { - FAIL_POINT_INJECT_F("migrate_replica_checkpoint", [this](absl::string_view) -> bool { + FAIL_POINT_INJECT_F("migrate_replica_checkpoint", [this](std::string_view) -> bool { reset_status(); return false; }); @@ -246,7 +246,7 @@ bool replica_disk_migrator::migrate_replica_checkpoint(const replica_disk_migrat // THREAD_POOL_REPLICATION_LONG bool replica_disk_migrator::migrate_replica_app_info(const replica_disk_migrate_request &req) { - FAIL_POINT_INJECT_F("migrate_replica_app_info", [this](absl::string_view) -> bool { + FAIL_POINT_INJECT_F("migrate_replica_app_info", [this](std::string_view) -> bool { reset_status(); return false; }); diff --git a/src/replica/replica_stub.cpp b/src/replica/replica_stub.cpp index 32f5141e17..ebab553809 100644 --- a/src/replica/replica_stub.cpp +++ b/src/replica/replica_stub.cpp @@ -43,7 +43,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "backup/replica_backup_server.h" #include "bulk_load/replica_bulk_loader.h" #include "common/backup_common.h" @@ -1810,7 +1810,7 @@ void replica_stub::open_replica( dsn::utils::filesystem::rename_path(origin_tmp_dir, origin_dir); rep = load_replica(origin_dn, origin_dir.c_str()); - FAIL_POINT_INJECT_F("mock_replica_load", [&](absl::string_view) -> void {}); + FAIL_POINT_INJECT_F("mock_replica_load", [&](std::string_view) -> void {}); } } } @@ -1999,7 +1999,7 @@ bool replica_stub::validate_replica_dir(const std::string &dir, replica *replica_stub::load_replica(dir_node *dn, const char *dir) { FAIL_POINT_INJECT_F("mock_replica_load", - [&](absl::string_view) -> replica * { return nullptr; }); + [&](std::string_view) -> replica * { return nullptr; }); app_info ai; gpid pid; @@ -2671,7 +2671,7 @@ replica_ptr replica_stub::create_child_replica_if_not_found(gpid child_pid, const std::string &parent_dir) { FAIL_POINT_INJECT_F( - "replica_stub_create_child_replica_if_not_found", [=](absl::string_view) -> replica_ptr { + "replica_stub_create_child_replica_if_not_found", [=](std::string_view) -> replica_ptr { const auto dn = _fs_manager.create_child_replica_dir(app->app_type, child_pid, parent_dir); CHECK_NOTNULL(dn, ""); @@ -2716,8 +2716,7 @@ void replica_stub::split_replica_error_handler(gpid pid, local_execution handler dsn::error_code replica_stub::split_replica_exec(dsn::task_code code, gpid pid, local_execution handler) { - FAIL_POINT_INJECT_F("replica_stub_split_replica_exec", - [](absl::string_view) { return ERR_OK; }); + FAIL_POINT_INJECT_F("replica_stub_split_replica_exec", [](std::string_view) { return ERR_OK; }); replica_ptr replica = pid.get_app_id() == 0 ? nullptr : get_replica(pid); if (replica && handler) { tasking::enqueue( diff --git a/src/replica/replication_app_base.cpp b/src/replica/replication_app_base.cpp index 0455772599..e15df7829d 100644 --- a/src/replica/replication_app_base.cpp +++ b/src/replica/replication_app_base.cpp @@ -32,7 +32,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "common/bulk_load_common.h" #include "common/duplication_common.h" #include "common/replica_envs.h" @@ -281,7 +281,7 @@ int replication_app_base::on_batched_write_requests(int64_t decree, error_code replication_app_base::apply_mutation(const mutation *mu) { FAIL_POINT_INJECT_F("replication_app_base_apply_mutation", - [](absl::string_view) { return ERR_OK; }); + [](std::string_view) { return ERR_OK; }); CHECK_EQ_PREFIX(mu->data.header.decree, last_committed_decree() + 1); CHECK_EQ_PREFIX(mu->data.updates.size(), mu->client_requests.size()); diff --git a/src/replica/split/replica_split_manager.cpp b/src/replica/split/replica_split_manager.cpp index e59096e4d4..371d92f092 100644 --- a/src/replica/split/replica_split_manager.cpp +++ b/src/replica/split/replica_split_manager.cpp @@ -48,7 +48,7 @@ #include "utils/filesystem.h" #include "utils/flags.h" #include "utils/fmt_logging.h" -#include "absl/strings/string_view.h" +#include #include "utils/thread_access_checker.h" METRIC_DEFINE_counter(replica, @@ -172,7 +172,7 @@ void replica_split_manager::child_init_replica(gpid parent_gpid, const host_port &primary_host_port, ballot init_ballot) // on child partition { - FAIL_POINT_INJECT_F("replica_child_init_replica", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_child_init_replica", [](std::string_view) {}); if (status() != partition_status::PS_INACTIVE) { LOG_WARNING_PREFIX("wrong status({})", enum_to_string(status())); @@ -218,7 +218,7 @@ void replica_split_manager::child_init_replica(gpid parent_gpid, // ThreadPool: THREAD_POOL_REPLICATION void replica_split_manager::child_check_split_context() // on child partition { - FAIL_POINT_INJECT_F("replica_child_check_split_context", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_child_check_split_context", [](std::string_view) {}); if (status() != partition_status::PS_PARTITION_SPLIT) { LOG_ERROR_PREFIX("wrong status({})", enum_to_string(status())); @@ -246,7 +246,7 @@ void replica_split_manager::child_check_split_context() // on child partition // ThreadPool: THREAD_POOL_REPLICATION bool replica_split_manager::parent_check_states() // on parent partition { - FAIL_POINT_INJECT_F("replica_parent_check_states", [](absl::string_view) { return true; }); + FAIL_POINT_INJECT_F("replica_parent_check_states", [](std::string_view) { return true; }); if (_split_status != split_status::SPLITTING || _child_init_ballot != get_ballot() || _child_gpid.get_app_id() == 0 || @@ -387,7 +387,7 @@ void replica_split_manager::child_learn_states(learn_state lstate, uint64_t total_file_size, decree last_committed_decree) // on child partition { - FAIL_POINT_INJECT_F("replica_child_learn_states", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_child_learn_states", [](std::string_view) {}); if (status() != partition_status::PS_PARTITION_SPLIT) { LOG_ERROR_PREFIX("wrong status({})", enum_to_string(status())); @@ -455,7 +455,7 @@ replica_split_manager::child_apply_private_logs(std::vector plog_fi uint64_t total_file_size, decree last_committed_decree) // on child partition { - FAIL_POINT_INJECT_F("replica_child_apply_private_logs", [](absl::string_view arg) { + FAIL_POINT_INJECT_F("replica_child_apply_private_logs", [](std::string_view arg) { return error_code::try_get(arg.data(), ERR_OK); }); @@ -547,7 +547,7 @@ replica_split_manager::child_apply_private_logs(std::vector plog_fi // ThreadPool: THREAD_POOL_REPLICATION void replica_split_manager::child_catch_up_states() // on child partition { - FAIL_POINT_INJECT_F("replica_child_catch_up_states", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_child_catch_up_states", [](std::string_view) {}); if (status() != partition_status::PS_PARTITION_SPLIT) { LOG_ERROR_PREFIX("wrong status, status is {}", enum_to_string(status())); @@ -612,7 +612,7 @@ void replica_split_manager::child_catch_up_states() // on child partition // ThreadPool: THREAD_POOL_REPLICATION void replica_split_manager::child_notify_catch_up() // on child partition { - FAIL_POINT_INJECT_F("replica_child_notify_catch_up", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_child_notify_catch_up", [](std::string_view) {}); std::unique_ptr request = std::make_unique(); request->parent_gpid = _replica->_split_states.parent_gpid; @@ -737,7 +737,7 @@ void replica_split_manager::parent_handle_child_catch_up( // ThreadPool: THREAD_POOL_REPLICATION void replica_split_manager::parent_check_sync_point_commit(decree sync_point) // on primary parent { - FAIL_POINT_INJECT_F("replica_parent_check_sync_point_commit", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_parent_check_sync_point_commit", [](std::string_view) {}); if (status() != partition_status::PS_PRIMARY) { LOG_ERROR_PREFIX("wrong status({})", enum_to_string(status())); parent_handle_split_error("check_sync_point_commit failed, primary changed", false); @@ -803,7 +803,7 @@ void replica_split_manager::parent_send_update_partition_count_request( int32_t new_partition_count, std::shared_ptr> ¬_replied_addresses) // on primary parent { - FAIL_POINT_INJECT_F("replica_parent_update_partition_count_request", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_parent_update_partition_count_request", [](std::string_view) {}); CHECK_EQ_PREFIX(status(), partition_status::PS_PRIMARY); @@ -964,7 +964,7 @@ void replica_split_manager::on_update_child_group_partition_count_reply( // ThreadPool: THREAD_POOL_REPLICATION void replica_split_manager::register_child_on_meta(ballot b) // on primary parent { - FAIL_POINT_INJECT_F("replica_register_child_on_meta", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_register_child_on_meta", [](std::string_view) {}); if (status() != partition_status::PS_PRIMARY || _split_status != split_status::SPLITTING) { LOG_ERROR_PREFIX( @@ -1013,7 +1013,7 @@ void replica_split_manager::register_child_on_meta(ballot b) // on primary paren void replica_split_manager::parent_send_register_request( const register_child_request &request) // on primary parent { - FAIL_POINT_INJECT_F("replica_parent_send_register_request", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_parent_send_register_request", [](std::string_view) {}); CHECK_EQ_PREFIX(status(), partition_status::PS_INACTIVE); LOG_INFO_PREFIX( @@ -1043,7 +1043,7 @@ void replica_split_manager::on_register_child_on_meta_reply( const register_child_request &request, const register_child_response &response) // on primary parent { - FAIL_POINT_INJECT_F("replica_on_register_child_on_meta_reply", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_on_register_child_on_meta_reply", [](std::string_view) {}); _replica->_checker.only_one_thread_access(); @@ -1517,7 +1517,7 @@ void replica_split_manager::primary_parent_handle_stop_split( void replica_split_manager::parent_send_notify_stop_request( split_status::type meta_split_status) // on primary parent { - FAIL_POINT_INJECT_F("replica_parent_send_notify_stop_request", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("replica_parent_send_notify_stop_request", [](std::string_view) {}); auto meta_address = dsn::dns_resolver::instance().resolve_address(_stub->_failure_detector->get_servers()); std::unique_ptr req = std::make_unique(); diff --git a/src/runtime/profiler.cpp b/src/runtime/profiler.cpp index 7852f1fcb1..2228cc35b3 100644 --- a/src/runtime/profiler.cpp +++ b/src/runtime/profiler.cpp @@ -57,7 +57,7 @@ START<== queue(server) == ENQUEUE <===== net(reply) ======= REPLY <============= #include #include -#include "absl/strings/string_view.h" +#include #include "aio/aio_task.h" #include "fmt/core.h" #include "profiler_header.h" diff --git a/src/runtime/rpc/dns_resolver.cpp b/src/runtime/rpc/dns_resolver.cpp index 2eab9d8327..bfd7f5206d 100644 --- a/src/runtime/rpc/dns_resolver.cpp +++ b/src/runtime/rpc/dns_resolver.cpp @@ -22,7 +22,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "fmt/core.h" #include "fmt/format.h" #include "runtime/rpc/dns_resolver.h" diff --git a/src/runtime/rpc/network.cpp b/src/runtime/rpc/network.cpp index 2c506fd720..c572b19a8d 100644 --- a/src/runtime/rpc/network.cpp +++ b/src/runtime/rpc/network.cpp @@ -32,7 +32,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "message_parser_manager.h" #include "runtime/api_task.h" #include "runtime/rpc/rpc_address.h" diff --git a/src/runtime/rpc/rpc_address.cpp b/src/runtime/rpc/rpc_address.cpp index f8d02d7c66..064dd458d5 100644 --- a/src/runtime/rpc/rpc_address.cpp +++ b/src/runtime/rpc/rpc_address.cpp @@ -35,7 +35,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "runtime/rpc/group_address.h" #include "utils/error_code.h" #include "utils/fixed_size_buffer_pool.h" @@ -143,7 +143,7 @@ bool rpc_address::is_site_local_address(uint32_t ip_net) /*static*/ bool rpc_address::is_docker_netcard(const char *netcard_interface, uint32_t ip_net) { - if (absl::string_view(netcard_interface).find("docker") != absl::string_view::npos) { + if (std::string_view(netcard_interface).find("docker") != std::string_view::npos) { return true; } uint32_t iphost = ntohl(ip_net); diff --git a/src/runtime/rpc/thrift_message_parser.cpp b/src/runtime/rpc/thrift_message_parser.cpp index 08bbfcb872..d547075025 100644 --- a/src/runtime/rpc/thrift_message_parser.cpp +++ b/src/runtime/rpc/thrift_message_parser.cpp @@ -46,7 +46,7 @@ #include "utils/endians.h" #include "utils/fmt_logging.h" #include "utils/fmt_utils.h" -#include "absl/strings/string_view.h" +#include #include "utils/strings.h" namespace dsn { @@ -340,7 +340,7 @@ void thrift_message_parser::prepare_on_send(message_ex *msg) // first total length, but we don't know the length, so firstly we put a placeholder header_proto.writeI32(0); // then the error_message - header_proto.writeString(absl::string_view(header->server.error_name)); + header_proto.writeString(std::string_view(header->server.error_name)); // then the thrift message begin header_proto.writeMessageBegin( header->rpc_name, ::apache::thrift::protocol::T_REPLY, header->id); diff --git a/src/runtime/task/task_queue.cpp b/src/runtime/task/task_queue.cpp index 8b0de42b34..b22e01bdb8 100644 --- a/src/runtime/task/task_queue.cpp +++ b/src/runtime/task/task_queue.cpp @@ -26,7 +26,7 @@ #include "task_queue.h" -#include "absl/strings/string_view.h" +#include #include "fmt/core.h" #include "runtime/rpc/network.h" #include "runtime/rpc/rpc_engine.h" diff --git a/src/security/sasl_client_wrapper.cpp b/src/security/sasl_client_wrapper.cpp index 50296826da..acdce301e0 100644 --- a/src/security/sasl_client_wrapper.cpp +++ b/src/security/sasl_client_wrapper.cpp @@ -23,7 +23,7 @@ #include "utils/error_code.h" #include "utils/fail_point.h" #include "utils/flags.h" -#include "absl/strings/string_view.h" +#include DSN_DECLARE_string(service_fqdn); DSN_DECLARE_string(service_name); @@ -33,7 +33,7 @@ namespace security { error_s sasl_client_wrapper::init() { - FAIL_POINT_INJECT_F("sasl_client_wrapper_init", [](absl::string_view str) { + FAIL_POINT_INJECT_F("sasl_client_wrapper_init", [](std::string_view str) { error_code err = error_code::try_get(str.data(), ERR_UNKNOWN); return error_s::make(err); }); @@ -45,7 +45,7 @@ error_s sasl_client_wrapper::init() error_s sasl_client_wrapper::start(const std::string &mechanism, const blob &input, blob &output) { - FAIL_POINT_INJECT_F("sasl_client_wrapper_start", [](absl::string_view str) { + FAIL_POINT_INJECT_F("sasl_client_wrapper_start", [](std::string_view str) { error_code err = error_code::try_get(str.data(), ERR_UNKNOWN); return error_s::make(err); }); @@ -62,7 +62,7 @@ error_s sasl_client_wrapper::start(const std::string &mechanism, const blob &inp error_s sasl_client_wrapper::step(const blob &input, blob &output) { - FAIL_POINT_INJECT_F("sasl_client_wrapper_step", [](absl::string_view str) { + FAIL_POINT_INJECT_F("sasl_client_wrapper_step", [](std::string_view str) { error_code err = error_code::try_get(str.data(), ERR_UNKNOWN); return error_s::make(err); }); diff --git a/src/security/sasl_server_wrapper.cpp b/src/security/sasl_server_wrapper.cpp index 4cda84241b..67498926ff 100644 --- a/src/security/sasl_server_wrapper.cpp +++ b/src/security/sasl_server_wrapper.cpp @@ -23,7 +23,7 @@ #include "utils/error_code.h" #include "utils/fail_point.h" #include "utils/flags.h" -#include "absl/strings/string_view.h" +#include DSN_DECLARE_string(service_fqdn); DSN_DECLARE_string(service_name); @@ -33,7 +33,7 @@ namespace security { error_s sasl_server_wrapper::init() { - FAIL_POINT_INJECT_F("sasl_server_wrapper_init", [](absl::string_view str) { + FAIL_POINT_INJECT_F("sasl_server_wrapper_init", [](std::string_view str) { error_code err = error_code::try_get(str.data(), ERR_UNKNOWN); return error_s::make(err); }); @@ -45,7 +45,7 @@ error_s sasl_server_wrapper::init() error_s sasl_server_wrapper::start(const std::string &mechanism, const blob &input, blob &output) { - FAIL_POINT_INJECT_F("sasl_server_wrapper_start", [](absl::string_view str) { + FAIL_POINT_INJECT_F("sasl_server_wrapper_start", [](std::string_view str) { error_code err = error_code::try_get(str.data(), ERR_UNKNOWN); return error_s::make(err); }); @@ -61,7 +61,7 @@ error_s sasl_server_wrapper::start(const std::string &mechanism, const blob &inp error_s sasl_server_wrapper::step(const blob &input, blob &output) { - FAIL_POINT_INJECT_F("sasl_server_wrapper_step", [](absl::string_view str) { + FAIL_POINT_INJECT_F("sasl_server_wrapper_step", [](std::string_view str) { error_code err = error_code::try_get(str.data(), ERR_UNKNOWN); return error_s::make(err); }); diff --git a/src/security/sasl_wrapper.cpp b/src/security/sasl_wrapper.cpp index 14c28f7b95..7d7d222c51 100644 --- a/src/security/sasl_wrapper.cpp +++ b/src/security/sasl_wrapper.cpp @@ -23,7 +23,7 @@ #include "sasl_server_wrapper.h" #include "utils/error_code.h" #include "utils/fail_point.h" -#include "absl/strings/string_view.h" +#include namespace dsn { namespace security { @@ -44,7 +44,7 @@ sasl_wrapper::~sasl_wrapper() error_s sasl_wrapper::retrieve_username(std::string &output) { - FAIL_POINT_INJECT_F("sasl_wrapper_retrieve_username", [](absl::string_view str) { + FAIL_POINT_INJECT_F("sasl_wrapper_retrieve_username", [](std::string_view str) { error_code err = error_code::try_get(str.data(), ERR_UNKNOWN); return error_s::make(err); }); diff --git a/src/server/capacity_unit_calculator.cpp b/src/server/capacity_unit_calculator.cpp index cfbc086f41..8a4745f4ba 100644 --- a/src/server/capacity_unit_calculator.cpp +++ b/src/server/capacity_unit_calculator.cpp @@ -19,7 +19,7 @@ #include "capacity_unit_calculator.h" -#include +#include #include #include #include diff --git a/src/server/compaction_filter_rule.cpp b/src/server/compaction_filter_rule.cpp index cf369da96b..d04b67e70b 100644 --- a/src/server/compaction_filter_rule.cpp +++ b/src/server/compaction_filter_rule.cpp @@ -22,14 +22,14 @@ #include "base/pegasus_utils.h" #include "base/pegasus_value_schema.h" #include "utils/fmt_logging.h" -#include "absl/strings/string_view.h" +#include #include "utils/strings.h" namespace pegasus { namespace server { -bool string_pattern_match(absl::string_view value, +bool string_pattern_match(std::string_view value, string_match_type type, - absl::string_view filter_pattern) + std::string_view filter_pattern) { if (filter_pattern.empty()) return false; @@ -38,7 +38,7 @@ bool string_pattern_match(absl::string_view value, switch (type) { case string_match_type::SMT_MATCH_ANYWHERE: - return value.find(filter_pattern) != absl::string_view::npos; + return value.find(filter_pattern) != std::string_view::npos; case string_match_type::SMT_MATCH_PREFIX: return dsn::utils::mequals(value.data(), filter_pattern.data(), filter_pattern.length()); case string_match_type::SMT_MATCH_POSTFIX: @@ -53,27 +53,27 @@ bool string_pattern_match(absl::string_view value, hashkey_pattern_rule::hashkey_pattern_rule(uint32_t data_version) {} -bool hashkey_pattern_rule::match(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value) const +bool hashkey_pattern_rule::match(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value) const { return string_pattern_match(hash_key, match_type, pattern); } sortkey_pattern_rule::sortkey_pattern_rule(uint32_t data_version) {} -bool sortkey_pattern_rule::match(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value) const +bool sortkey_pattern_rule::match(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value) const { return string_pattern_match(sort_key, match_type, pattern); } ttl_range_rule::ttl_range_rule(uint32_t data_version) : data_version(data_version) {} -bool ttl_range_rule::match(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value) const +bool ttl_range_rule::match(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value) const { uint32_t expire_ts = pegasus_extract_expire_ts(data_version, existing_value); // if start_ttl and stop_ttl = 0, it means we want to delete keys which have no ttl diff --git a/src/server/compaction_filter_rule.h b/src/server/compaction_filter_rule.h index be211ba6b0..e471a759fd 100644 --- a/src/server/compaction_filter_rule.h +++ b/src/server/compaction_filter_rule.h @@ -30,7 +30,7 @@ #include "utils/enum_helper.h" #include "utils/factory_store.h" #include "utils/fmt_utils.h" -#include "absl/strings/string_view.h" +#include namespace pegasus { namespace server { @@ -76,9 +76,9 @@ class compaction_filter_rule // TODO(zhaoliwei): we can use `value_filed` to replace existing_value in the later, // after the refactor of value schema - virtual bool match(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value) const = 0; + virtual bool match(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value) const = 0; }; enum string_match_type @@ -102,9 +102,9 @@ class hashkey_pattern_rule : public compaction_filter_rule public: hashkey_pattern_rule(uint32_t data_version = VERSION_MAX); - bool match(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value) const; + bool match(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value) const; DEFINE_JSON_SERIALIZATION(pattern, match_type) private: @@ -124,9 +124,9 @@ class sortkey_pattern_rule : public compaction_filter_rule public: sortkey_pattern_rule(uint32_t data_version = VERSION_MAX); - bool match(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value) const; + bool match(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value) const; DEFINE_JSON_SERIALIZATION(pattern, match_type) private: @@ -144,9 +144,9 @@ class ttl_range_rule : public compaction_filter_rule public: explicit ttl_range_rule(uint32_t data_version); - bool match(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value) const; + bool match(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value) const; DEFINE_JSON_SERIALIZATION(start_ttl, stop_ttl) private: diff --git a/src/server/compaction_operation.cpp b/src/server/compaction_operation.cpp index 5c139073c1..a7d21bd326 100644 --- a/src/server/compaction_operation.cpp +++ b/src/server/compaction_operation.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include +#include #include #include "base/pegasus_utils.h" @@ -30,9 +30,9 @@ namespace pegasus { namespace server { compaction_operation::~compaction_operation() = default; -bool compaction_operation::all_rules_match(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value) const +bool compaction_operation::all_rules_match(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value) const { if (rules.empty()) { return false; @@ -55,9 +55,9 @@ delete_key::delete_key(filter_rules &&rules, uint32_t data_version) delete_key::delete_key(uint32_t data_version) : compaction_operation(data_version) {} -bool delete_key::filter(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value, +bool delete_key::filter(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value, std::string *new_value, bool *value_changed) const { @@ -74,9 +74,9 @@ update_ttl::update_ttl(filter_rules &&rules, uint32_t data_version) update_ttl::update_ttl(uint32_t data_version) : compaction_operation(data_version) {} -bool update_ttl::filter(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value, +bool update_ttl::filter(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value, std::string *new_value, bool *value_changed) const { diff --git a/src/server/compaction_operation.h b/src/server/compaction_operation.h index d7c9d811eb..1101be1fa6 100644 --- a/src/server/compaction_operation.h +++ b/src/server/compaction_operation.h @@ -21,13 +21,12 @@ #include #include -#include #include #include +#include #include #include -#include "absl/strings/string_view.h" #include "common/json_helper.h" #include "compaction_filter_rule.h" #include "utils/blob.h" @@ -75,18 +74,18 @@ class compaction_operation explicit compaction_operation(uint32_t data_version) : data_version(data_version) {} virtual ~compaction_operation() = 0; - bool all_rules_match(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value) const; + bool all_rules_match(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value) const; void set_rules(filter_rules &&rules); /** * @return false indicates that this key-value should be removed * If you want to modify the existing_value, you can pass it back through new_value and * value_changed needs to be set to true in this case. */ - virtual bool filter(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value, + virtual bool filter(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value, std::string *new_value, bool *value_changed) const = 0; @@ -106,9 +105,9 @@ class delete_key : public compaction_operation delete_key(filter_rules &&rules, uint32_t data_version); explicit delete_key(uint32_t data_version); - bool filter(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value, + bool filter(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value, std::string *new_value, bool *value_changed) const; @@ -154,9 +153,9 @@ class update_ttl : public compaction_operation update_ttl(filter_rules &&rules, uint32_t data_version); explicit update_ttl(uint32_t data_version); - bool filter(absl::string_view hash_key, - absl::string_view sort_key, - absl::string_view existing_value, + bool filter(std::string_view hash_key, + std::string_view sort_key, + std::string_view existing_value, std::string *new_value, bool *value_changed) const; DEFINE_JSON_SERIALIZATION(type, value) diff --git a/src/server/hotkey_collector.cpp b/src/server/hotkey_collector.cpp index a05c3012d8..ca67a74d03 100644 --- a/src/server/hotkey_collector.cpp +++ b/src/server/hotkey_collector.cpp @@ -17,7 +17,7 @@ #include "hotkey_collector.h" -#include +#include #include // IWYU pragma: no_include #include @@ -129,7 +129,7 @@ find_outlier_index(const std::vector &captured_keys, int threshold, in // TODO: (Tangyanzhao) replace it to xxhash -/*extern*/ int get_bucket_id(absl::string_view data, int bucket_num) +/*extern*/ int get_bucket_id(std::string_view data, int bucket_num) { return static_cast(boost::hash_range(data.begin(), data.end()) % bucket_num); } @@ -398,7 +398,7 @@ struct blob_hash { std::size_t operator()(const dsn::blob &str) const { - absl::string_view cp = str.to_string_view(); + std::string_view cp = str.to_string_view(); return boost::hash_range(cp.begin(), cp.end()); } }; @@ -429,8 +429,8 @@ void hotkey_fine_data_collector::analyse_data(detect_hotkey_result &result) // the weight of all the collected hash keys std::vector weights; weights.reserve(hash_keys_weight.size()); - absl::string_view weight_max_key; // the hashkey with the max weight - uint64_t weight_max = 0; // the max weight by far + std::string_view weight_max_key; // the hashkey with the max weight + uint64_t weight_max = 0; // the max weight by far for (const auto &iter : hash_keys_weight) { weights.push_back(iter.second); if (iter.second > weight_max) { diff --git a/src/server/hotkey_collector.h b/src/server/hotkey_collector.h index 826bf157b8..b8b74ccf51 100644 --- a/src/server/hotkey_collector.h +++ b/src/server/hotkey_collector.h @@ -29,7 +29,7 @@ #include "replica/replica_base.h" #include "replica_admin_types.h" #include "utils/blob.h" -#include "absl/strings/string_view.h" +#include namespace pegasus { namespace server { @@ -50,7 +50,7 @@ struct detect_hotkey_result } }; -extern int get_bucket_id(absl::string_view data, int bucket_num); +extern int get_bucket_id(std::string_view data, int bucket_num); extern bool find_outlier_index(const std::vector &captured_keys, int threshold, int &hot_index); diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index b2b71e6987..9b99abfabb 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -22,7 +22,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "client/replication_ddl_client.h" #include "common/gpid.h" #include "common/serialization_helper/dsn.layer2_types.h" @@ -212,7 +212,7 @@ void hotspot_partition_calculator::send_detect_hotkey_request( const dsn::replication::hotkey_type::type hotkey_type, const dsn::replication::detect_action::type action) { - FAIL_POINT_INJECT_F("send_detect_hotkey_request", [](absl::string_view) {}); + FAIL_POINT_INJECT_F("send_detect_hotkey_request", [](std::string_view) {}); int app_id = -1; int partition_count = -1; diff --git a/src/server/key_ttl_compaction_filter.h b/src/server/key_ttl_compaction_filter.h index 8df418de9d..6f4f07f05c 100644 --- a/src/server/key_ttl_compaction_filter.h +++ b/src/server/key_ttl_compaction_filter.h @@ -79,7 +79,7 @@ class KeyWithTTLCompactionFilter : public rocksdb::CompactionFilter } if (!_user_specified_operations.empty()) { - absl::string_view value_view = utils::to_string_view(existing_value); + std::string_view value_view = utils::to_string_view(existing_value); if (*value_changed) { value_view = *new_value; } @@ -92,7 +92,7 @@ class KeyWithTTLCompactionFilter : public rocksdb::CompactionFilter } bool user_specified_operation_filter(const rocksdb::Slice &key, - absl::string_view existing_value, + std::string_view existing_value, std::string *new_value, bool *value_changed) const { diff --git a/src/server/pegasus_event_listener.cpp b/src/server/pegasus_event_listener.cpp index 1c4ebbfd66..ba6b622c91 100644 --- a/src/server/pegasus_event_listener.cpp +++ b/src/server/pegasus_event_listener.cpp @@ -19,7 +19,7 @@ #include "pegasus_event_listener.h" -#include +#include #include #include #include diff --git a/src/server/pegasus_manual_compact_service.cpp b/src/server/pegasus_manual_compact_service.cpp index f4d143690b..e55077e7a4 100644 --- a/src/server/pegasus_manual_compact_service.cpp +++ b/src/server/pegasus_manual_compact_service.cpp @@ -19,7 +19,7 @@ #include "pegasus_manual_compact_service.h" -#include +#include #include #include #include diff --git a/src/server/pegasus_mutation_duplicator.cpp b/src/server/pegasus_mutation_duplicator.cpp index 6b29e4e3ec..8b82557bb4 100644 --- a/src/server/pegasus_mutation_duplicator.cpp +++ b/src/server/pegasus_mutation_duplicator.cpp @@ -19,7 +19,7 @@ #include "pegasus_mutation_duplicator.h" -#include +#include #include #include #include @@ -75,9 +75,9 @@ DSN_TAG_VARIABLE(dup_max_allowed_write_size, FT_MUTABLE); /// static definition of mutation_duplicator::creator. /*static*/ std::function( - replica_base *, absl::string_view, absl::string_view)> + replica_base *, std::string_view, std::string_view)> mutation_duplicator::creator = - [](replica_base *r, absl::string_view remote, absl::string_view app) { + [](replica_base *r, std::string_view remote, std::string_view app) { return std::make_unique(r, remote, app); }; @@ -116,8 +116,8 @@ using namespace dsn::literals::chrono_literals; } pegasus_mutation_duplicator::pegasus_mutation_duplicator(dsn::replication::replica_base *r, - absl::string_view remote_cluster, - absl::string_view app) + std::string_view remote_cluster, + std::string_view app) : mutation_duplicator(r), _remote_cluster(remote_cluster), METRIC_VAR_INIT_replica(dup_shipped_successful_requests), diff --git a/src/server/pegasus_mutation_duplicator.h b/src/server/pegasus_mutation_duplicator.h index 2d81a6a37c..0ac19c68df 100644 --- a/src/server/pegasus_mutation_duplicator.h +++ b/src/server/pegasus_mutation_duplicator.h @@ -31,7 +31,7 @@ #include "runtime/task/task_code.h" #include "runtime/task/task_tracker.h" #include "utils/chrono_literals.h" -#include "absl/strings/string_view.h" +#include #include "utils/metrics.h" #include "utils/zlocks.h" @@ -61,8 +61,8 @@ class pegasus_mutation_duplicator : public dsn::replication::mutation_duplicator public: pegasus_mutation_duplicator(dsn::replication::replica_base *r, - absl::string_view remote_cluster, - absl::string_view app); + std::string_view remote_cluster, + std::string_view app); void duplicate(mutation_tuple_set muts, callback cb) override; diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index e3a75c2fef..ffb4b8c1c9 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -44,7 +44,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "base/idl_utils.h" // IWYU pragma: keep #include "base/meta_store.h" #include "base/pegasus_key_schema.h" @@ -2354,7 +2354,7 @@ bool pegasus_server_impl::validate_filter(::dsn::apps::filter_type::type filter_ return false; if (filter_type == ::dsn::apps::filter_type::FT_MATCH_ANYWHERE) { return value.to_string_view().find(filter_pattern.to_string_view()) != - absl::string_view::npos; + std::string_view::npos; } else if (filter_type == ::dsn::apps::filter_type::FT_MATCH_PREFIX) { return dsn::utils::mequals( value.data(), filter_pattern.data(), filter_pattern.length()); diff --git a/src/server/pegasus_server_impl_init.cpp b/src/server/pegasus_server_impl_init.cpp index d7611e2660..d679d6ce3d 100644 --- a/src/server/pegasus_server_impl_init.cpp +++ b/src/server/pegasus_server_impl_init.cpp @@ -17,7 +17,6 @@ * under the License. */ -#include #include #include #include @@ -27,12 +26,12 @@ #include #include #include -#include #include #include #include #include #include +#include #include #include #include diff --git a/src/server/pegasus_server_write.cpp b/src/server/pegasus_server_write.cpp index 1bb6c968ad..33437ca692 100644 --- a/src/server/pegasus_server_write.cpp +++ b/src/server/pegasus_server_write.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include +#include #include #include #include diff --git a/src/server/pegasus_write_service.cpp b/src/server/pegasus_write_service.cpp index d688ab6e45..2fb5733265 100644 --- a/src/server/pegasus_write_service.cpp +++ b/src/server/pegasus_write_service.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include +#include #include #include #include diff --git a/src/server/pegasus_write_service_impl.h b/src/server/pegasus_write_service_impl.h index 2169b273e2..35a9c6399d 100644 --- a/src/server/pegasus_write_service_impl.h +++ b/src/server/pegasus_write_service_impl.h @@ -90,7 +90,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base int empty_put(int64_t decree) { int err = - _rocksdb_wrapper->write_batch_put(decree, absl::string_view(), absl::string_view(), 0); + _rocksdb_wrapper->write_batch_put(decree, std::string_view(), std::string_view(), 0); auto cleanup = dsn::defer([this]() { _rocksdb_wrapper->clear_up_write_batch(); }); if (err != rocksdb::Status::kOk) { return err; @@ -178,7 +178,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base resp.decree = decree; resp.server = _primary_host_port; - absl::string_view raw_key = update.key.to_string_view(); + std::string_view raw_key = update.key.to_string_view(); int64_t new_value = 0; uint32_t new_expire_ts = 0; db_get_context get_ctx; @@ -275,7 +275,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base pegasus_generate_key(check_key, update.hash_key, update.check_sort_key); db_get_context get_context; - absl::string_view check_raw_key = check_key.to_string_view(); + std::string_view check_raw_key = check_key.to_string_view(); int err = _rocksdb_wrapper->get(check_raw_key, &get_context); if (err != rocksdb::Status::kOk) { // read check value failed @@ -327,7 +327,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base } else { // check not passed, write empty record to update rocksdb's last flushed decree resp.error = _rocksdb_wrapper->write_batch_put( - decree, absl::string_view(), absl::string_view(), 0); + decree, std::string_view(), std::string_view(), 0); } auto cleanup = dsn::defer([this]() { _rocksdb_wrapper->clear_up_write_batch(); }); @@ -395,7 +395,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base pegasus_generate_key(check_key, update.hash_key, update.check_sort_key); db_get_context get_context; - absl::string_view check_raw_key = check_key.to_string_view(); + std::string_view check_raw_key = check_key.to_string_view(); int err = _rocksdb_wrapper->get(check_raw_key, &get_context); if (err != rocksdb::Status::kOk) { // read check value failed @@ -453,7 +453,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base } else { // check not passed, write empty record to update rocksdb's last flushed decree resp.error = _rocksdb_wrapper->write_batch_put( - decree, absl::string_view(), absl::string_view(), 0); + decree, std::string_view(), std::string_view(), 0); } auto cleanup = dsn::defer([this]() { _rocksdb_wrapper->clear_up_write_batch(); }); @@ -562,7 +562,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base _rocksdb_wrapper->clear_up_write_batch(); } - static dsn::blob composite_raw_key(absl::string_view hash_key, absl::string_view sort_key) + static dsn::blob composite_raw_key(std::string_view hash_key, std::string_view sort_key) { dsn::blob raw_key; pegasus_generate_key(raw_key, hash_key, sort_key); @@ -609,7 +609,7 @@ class pegasus_write_service::impl : public dsn::replication::replica_base return false; if (check_type == ::dsn::apps::cas_check_type::CT_VALUE_MATCH_ANYWHERE) { return value.to_string_view().find(check_operand.to_string_view()) != - absl::string_view::npos; + std::string_view::npos; } else if (check_type == ::dsn::apps::cas_check_type::CT_VALUE_MATCH_PREFIX) { return dsn::utils::mequals( value.data(), check_operand.data(), check_operand.length()); diff --git a/src/server/rocksdb_wrapper.cpp b/src/server/rocksdb_wrapper.cpp index 59203836be..078b2b9d8a 100644 --- a/src/server/rocksdb_wrapper.cpp +++ b/src/server/rocksdb_wrapper.cpp @@ -19,7 +19,7 @@ #include "rocksdb_wrapper.h" -#include +#include #include #include #include @@ -74,9 +74,9 @@ rocksdb_wrapper::rocksdb_wrapper(pegasus_server_impl *server) _wt_opts->disableWAL = true; } -int rocksdb_wrapper::get(absl::string_view raw_key, /*out*/ db_get_context *ctx) +int rocksdb_wrapper::get(std::string_view raw_key, /*out*/ db_get_context *ctx) { - FAIL_POINT_INJECT_F("db_get", [](absl::string_view) -> int { return FAIL_DB_GET; }); + FAIL_POINT_INJECT_F("db_get", [](std::string_view) -> int { return FAIL_DB_GET; }); rocksdb::Status s = _db->Get(_rd_opts, utils::to_rocksdb_slice(raw_key), &(ctx->raw_value)); if (dsn_likely(s.ok())) { @@ -105,20 +105,20 @@ int rocksdb_wrapper::get(absl::string_view raw_key, /*out*/ db_get_context *ctx) } int rocksdb_wrapper::write_batch_put(int64_t decree, - absl::string_view raw_key, - absl::string_view value, + std::string_view raw_key, + std::string_view value, uint32_t expire_sec) { return write_batch_put_ctx(db_write_context::empty(decree), raw_key, value, expire_sec); } int rocksdb_wrapper::write_batch_put_ctx(const db_write_context &ctx, - absl::string_view raw_key, - absl::string_view value, + std::string_view raw_key, + std::string_view value, uint32_t expire_sec) { FAIL_POINT_INJECT_F("db_write_batch_put", - [](absl::string_view) -> int { return FAIL_DB_WRITE_BATCH_PUT; }); + [](std::string_view) -> int { return FAIL_DB_WRITE_BATCH_PUT; }); uint64_t new_timetag = ctx.remote_timetag; if (!ctx.is_duplicated_write()) { // local write @@ -143,7 +143,7 @@ int rocksdb_wrapper::write_batch_put_ctx(const db_write_context &ctx, if (local_timetag >= new_timetag) { // ignore this stale update with lower timetag, // and write an empty record instead - raw_key = value = absl::string_view(); + raw_key = value = std::string_view(); } } } @@ -175,7 +175,7 @@ int rocksdb_wrapper::write(int64_t decree) return FLAGS_inject_write_error_for_test; } - FAIL_POINT_INJECT_F("db_write", [](absl::string_view) -> int { return FAIL_DB_WRITE; }); + FAIL_POINT_INJECT_F("db_write", [](std::string_view) -> int { return FAIL_DB_WRITE; }); rocksdb::Status status = _write_batch->Put(_meta_cf, meta_store::LAST_FLUSHED_DECREE, std::to_string(decree)); @@ -194,10 +194,10 @@ int rocksdb_wrapper::write(int64_t decree) return status.code(); } -int rocksdb_wrapper::write_batch_delete(int64_t decree, absl::string_view raw_key) +int rocksdb_wrapper::write_batch_delete(int64_t decree, std::string_view raw_key) { FAIL_POINT_INJECT_F("db_write_batch_delete", - [](absl::string_view) -> int { return FAIL_DB_WRITE_BATCH_DELETE; }); + [](std::string_view) -> int { return FAIL_DB_WRITE_BATCH_DELETE; }); rocksdb::Status s = _write_batch->Delete(utils::to_rocksdb_slice(raw_key)); if (dsn_unlikely(!s.ok())) { diff --git a/src/server/rocksdb_wrapper.h b/src/server/rocksdb_wrapper.h index 00a682a540..c73f5cb918 100644 --- a/src/server/rocksdb_wrapper.h +++ b/src/server/rocksdb_wrapper.h @@ -29,7 +29,7 @@ #include "pegasus_value_schema.h" #include "replica/replica_base.h" -#include "absl/strings/string_view.h" +#include #include "utils/metrics.h" namespace rocksdb { @@ -54,18 +54,18 @@ class rocksdb_wrapper : public dsn::replication::replica_base /// is returned. /// \result ctx.expired=true if record expired. Still rocksdb::Status::kOk is returned. /// \result ctx.found=false if record is not found. Still rocksdb::Status::kOk is returned. - int get(absl::string_view raw_key, /*out*/ db_get_context *ctx); + int get(std::string_view raw_key, /*out*/ db_get_context *ctx); int write_batch_put(int64_t decree, - absl::string_view raw_key, - absl::string_view value, + std::string_view raw_key, + std::string_view value, uint32_t expire_sec); int write_batch_put_ctx(const db_write_context &ctx, - absl::string_view raw_key, - absl::string_view value, + std::string_view raw_key, + std::string_view value, uint32_t expire_sec); int write(int64_t decree); - int write_batch_delete(int64_t decree, absl::string_view raw_key); + int write_batch_delete(int64_t decree, std::string_view raw_key); void clear_up_write_batch(); int ingest_files(int64_t decree, const std::vector &sst_file_list, diff --git a/src/server/test/hotkey_collector_test.cpp b/src/server/test/hotkey_collector_test.cpp index c8eb49cadb..56864962af 100644 --- a/src/server/test/hotkey_collector_test.cpp +++ b/src/server/test/hotkey_collector_test.cpp @@ -17,7 +17,7 @@ #include "server/hotkey_collector.h" -#include +#include #include #include #include @@ -70,7 +70,7 @@ TEST(hotkey_collector_public_func_test, get_bucket_id_test) { int bucket_id = -1; for (int i = 0; i < 1000000; i++) { - bucket_id = get_bucket_id(absl::string_view(generate_hash_key_by_random(false)), + bucket_id = get_bucket_id(std::string_view(generate_hash_key_by_random(false)), FLAGS_hotkey_buckets_num); ASSERT_GE(bucket_id, 0); ASSERT_LT(bucket_id, FLAGS_hotkey_buckets_num); diff --git a/src/server/test/pegasus_mutation_duplicator_test.cpp b/src/server/test/pegasus_mutation_duplicator_test.cpp index e93e3bfc3e..f4e6d871db 100644 --- a/src/server/test/pegasus_mutation_duplicator_test.cpp +++ b/src/server/test/pegasus_mutation_duplicator_test.cpp @@ -19,12 +19,11 @@ #include "server/pegasus_mutation_duplicator.h" -#include #include #include #include -#include #include +#include #include #include #include diff --git a/src/server/test/pegasus_write_service_impl_test.cpp b/src/server/test/pegasus_write_service_impl_test.cpp index a076699ef1..0ac33faa40 100644 --- a/src/server/test/pegasus_write_service_impl_test.cpp +++ b/src/server/test/pegasus_write_service_impl_test.cpp @@ -34,7 +34,7 @@ #include "server/rocksdb_wrapper.h" #include "utils/blob.h" #include "utils/fail_point.h" -#include "absl/strings/string_view.h" +#include namespace pegasus { namespace server { @@ -55,7 +55,7 @@ class pegasus_write_service_impl_test : public pegasus_server_test_base _rocksdb_wrapper = _write_impl->_rocksdb_wrapper.get(); } - int db_get(absl::string_view raw_key, db_get_context *get_ctx) + int db_get(std::string_view raw_key, db_get_context *get_ctx) { return _rocksdb_wrapper->get(raw_key, get_ctx); } @@ -79,7 +79,7 @@ class incr_test : public pegasus_write_service_impl_test { pegasus_write_service_impl_test::SetUp(); pegasus::pegasus_generate_key( - req.key, absl::string_view("hash_key"), absl::string_view("sort_key")); + req.key, std::string_view("hash_key"), std::string_view("sort_key")); } dsn::apps::incr_request req; diff --git a/src/server/test/rocksdb_wrapper_test.cpp b/src/server/test/rocksdb_wrapper_test.cpp index e7f4b0006c..7831afc713 100644 --- a/src/server/test/rocksdb_wrapper_test.cpp +++ b/src/server/test/rocksdb_wrapper_test.cpp @@ -39,7 +39,7 @@ #include "utils/blob.h" #include "utils/error_code.h" #include "utils/fmt_logging.h" -#include "absl/strings/string_view.h" +#include namespace pegasus { namespace server { @@ -60,12 +60,12 @@ class rocksdb_wrapper_test : public pegasus_server_test_base _rocksdb_wrapper = _server_write->_write_svc->_impl->_rocksdb_wrapper.get(); pegasus::pegasus_generate_key( - _raw_key, absl::string_view("hash_key"), absl::string_view("sort_key")); + _raw_key, std::string_view("hash_key"), std::string_view("sort_key")); } void single_set(db_write_context write_ctx, dsn::blob raw_key, - absl::string_view user_value, + std::string_view user_value, int32_t expire_ts_seconds) { ASSERT_EQ(_rocksdb_wrapper->write_batch_put_ctx( @@ -93,7 +93,7 @@ class rocksdb_wrapper_test : public pegasus_server_test_base SetUp(); } - uint64_t read_timestamp_from(absl::string_view raw_value) + uint64_t read_timestamp_from(std::string_view raw_value) { uint64_t local_timetag = pegasus_extract_timetag(_rocksdb_wrapper->_pegasus_data_version, raw_value); diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index 0a5440b400..912fae9172 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -56,7 +56,7 @@ #include "runtime/task/async_calls.h" #include "tools/mutation_log_tool.h" #include "utils/fmt_utils.h" -#include "absl/strings/string_view.h" +#include #include "utils/errors.h" #include "utils/metrics.h" #include "utils/ports.h" @@ -286,7 +286,7 @@ inline bool validate_filter(pegasus::pegasus_client::filter_type filter_type, if (value.length() < filter_pattern.length()) return false; if (filter_type == pegasus::pegasus_client::FT_MATCH_ANYWHERE) { - return absl::string_view(value).find(filter_pattern) != absl::string_view::npos; + return std::string_view(value).find(filter_pattern) != std::string_view::npos; } else if (filter_type == pegasus::pegasus_client::FT_MATCH_PREFIX) { return dsn::utils::mequals( value.data(), filter_pattern.data(), filter_pattern.length()); diff --git a/src/shell/commands.h b/src/shell/commands.h index 3ea3c132c0..2e6044b064 100644 --- a/src/shell/commands.h +++ b/src/shell/commands.h @@ -29,7 +29,7 @@ #include "utils/filesystem.h" #include "utils/output_utils.h" #include "utils/string_conv.h" -#include "absl/strings/string_view.h" +#include #include "client/replication_ddl_client.h" #include "tools/mutation_log_tool.h" diff --git a/src/utils/blob.h b/src/utils/blob.h index fea1a7a0e0..3cdbce320a 100644 --- a/src/utils/blob.h +++ b/src/utils/blob.h @@ -29,7 +29,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include #include @@ -59,7 +59,7 @@ class blob { } - /// NOTE: Use absl::string_view whenever possible. + /// NOTE: Use std::string_view whenever possible. /// blob is designed for shared buffer, never use it as constant view. /// Maybe we could deprecate this function in the future. blob(const char *buffer, int offset, unsigned int length) @@ -100,7 +100,7 @@ class blob _length = length; } - /// Deprecated. Use absl::string_view whenever possible. + /// Deprecated. Use std::string_view whenever possible. void assign(const char *buffer, int offset, unsigned int length) { _holder = nullptr; @@ -165,7 +165,7 @@ class blob return os << bb.to_string(); } - absl::string_view to_string_view() const { return absl::string_view(_data, _length); } + std::string_view to_string_view() const { return std::string_view(_data, _length); } uint32_t read(::apache::thrift::protocol::TProtocol *iprot); uint32_t write(::apache::thrift::protocol::TProtocol *oprot) const; diff --git a/src/utils/builtin_metrics.cpp b/src/utils/builtin_metrics.cpp index 3ffb8f1474..6c8b8ee5a8 100644 --- a/src/utils/builtin_metrics.cpp +++ b/src/utils/builtin_metrics.cpp @@ -17,7 +17,7 @@ #include "utils/builtin_metrics.h" -#include +#include #include #include diff --git a/src/utils/endians.h b/src/utils/endians.h index f41a4b7759..1d0739b4ea 100644 --- a/src/utils/endians.h +++ b/src/utils/endians.h @@ -23,7 +23,7 @@ #include "api_utilities.h" #include "fmt_logging.h" #include "ports.h" -#include "absl/strings/string_view.h" +#include namespace dsn { @@ -108,7 +108,7 @@ class data_output class data_input { public: - explicit data_input(absl::string_view s) : _p(s.data()), _size(s.size()) {} + explicit data_input(std::string_view s) : _p(s.data()), _size(s.size()) {} uint8_t read_u8() { return read_unsigned(); } @@ -118,7 +118,7 @@ class data_input uint64_t read_u64() { return read_unsigned(); } - absl::string_view read_str() { return {_p, _size}; } + std::string_view read_str() { return {_p, _size}; } void skip(size_t sz) { diff --git a/src/utils/errors.h b/src/utils/errors.h index 8d5806efa5..734ee9375b 100644 --- a/src/utils/errors.h +++ b/src/utils/errors.h @@ -33,7 +33,7 @@ #include "utils/fmt_logging.h" #include "utils/fmt_utils.h" #include "utils/ports.h" -#include "absl/strings/string_view.h" +#include namespace dsn { @@ -72,7 +72,7 @@ class error_s error_s(error_s &&rhs) noexcept = default; error_s &operator=(error_s &&) noexcept = default; - static error_s make(error_code code, absl::string_view reason) { return error_s(code, reason); } + static error_s make(error_code code, std::string_view reason) { return error_s(code, reason); } static error_s make(error_code code) { @@ -145,14 +145,14 @@ class error_s } private: - error_s(error_code code, absl::string_view msg) noexcept : _info(new error_info(code, msg)) {} + error_s(error_code code, std::string_view msg) noexcept : _info(new error_info(code, msg)) {} struct error_info { error_code code; std::string msg; // TODO(wutao1): use raw char* to improve performance? - error_info(error_code c, absl::string_view s) : code(c), msg(s) {} + error_info(error_code c, std::string_view s) : code(c), msg(s) {} }; void copy(const error_s &rhs) diff --git a/src/utils/fail_point.cpp b/src/utils/fail_point.cpp index 45c809c020..49790832c8 100644 --- a/src/utils/fail_point.cpp +++ b/src/utils/fail_point.cpp @@ -34,7 +34,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "fail_point_impl.h" #include "utils/fail_point.h" #include "utils/fmt_logging.h" @@ -45,7 +45,7 @@ namespace fail { static fail_point_registry REGISTRY; -/*extern*/ const std::string *eval(absl::string_view name) +/*extern*/ const std::string *eval(std::string_view name) { fail_point *p = REGISTRY.try_get(name); if (!p) { @@ -71,7 +71,7 @@ inline const char *task_type_to_string(fail_point::task_type t) } } -/*extern*/ void cfg(absl::string_view name, absl::string_view action) +/*extern*/ void cfg(std::string_view name, std::string_view action) { fail_point &p = REGISTRY.create_if_not_exists(name); p.set_action(action); @@ -93,14 +93,14 @@ inline const char *task_type_to_string(fail_point::task_type t) _S_FAIL_POINT_ENABLED = false; } -void fail_point::set_action(absl::string_view action) +void fail_point::set_action(std::string_view action) { if (!parse_from_string(action)) { LOG_FATAL("unrecognized command: {}", action); } } -bool fail_point::parse_from_string(absl::string_view action) +bool fail_point::parse_from_string(std::string_view action) { _max_cnt = -1; _freq = 100; diff --git a/src/utils/fail_point.h b/src/utils/fail_point.h index 663dcbd5a3..2c0e4111bf 100644 --- a/src/utils/fail_point.h +++ b/src/utils/fail_point.h @@ -36,7 +36,7 @@ #include #include "utils/ports.h" -#include "absl/strings/string_view.h" +#include // The only entry to define a fail point with `return` function: lambda function must be // return non-void type. When a fail point is defined, it's referenced via the name. @@ -75,14 +75,14 @@ namespace dsn { namespace fail { -extern const std::string *eval(absl::string_view name); +extern const std::string *eval(std::string_view name); /// Set new actions to a fail point at runtime. /// The format of an action is `[p%][cnt*]task[(arg)]`. `p%` is the expected probability that /// the action is triggered, and `cnt*` is the max times the action can be triggered. /// For example, `20%3*print(still alive!)` means the fail point has 20% chance to print a /// message "still alive!". And the message will be printed at most 3 times. -extern void cfg(absl::string_view name, absl::string_view action); +extern void cfg(std::string_view name, std::string_view action); extern void setup(); diff --git a/src/utils/fail_point_impl.h b/src/utils/fail_point_impl.h index 75ad2ef399..449d8fd534 100644 --- a/src/utils/fail_point_impl.h +++ b/src/utils/fail_point_impl.h @@ -61,11 +61,11 @@ struct fail_point Void, }; - void set_action(absl::string_view action); + void set_action(std::string_view action); const std::string *eval(); - explicit fail_point(absl::string_view name) : _name(name) {} + explicit fail_point(std::string_view name) : _name(name) {} /// for test only fail_point(task_type t, std::string arg, int freq, int max_cnt) @@ -76,7 +76,7 @@ struct fail_point /// for test only fail_point() = default; - bool parse_from_string(absl::string_view action); + bool parse_from_string(std::string_view action); friend inline bool operator==(const fail_point &p1, const fail_point &p2) { @@ -103,7 +103,7 @@ USER_DEFINED_ENUM_FORMATTER(fail_point::task_type) struct fail_point_registry { - fail_point &create_if_not_exists(absl::string_view name) + fail_point &create_if_not_exists(std::string_view name) { std::lock_guard guard(_mu); @@ -111,7 +111,7 @@ struct fail_point_registry return it->second; } - fail_point *try_get(absl::string_view name) + fail_point *try_get(std::string_view name) { std::lock_guard guard(_mu); diff --git a/src/utils/filesystem.cpp b/src/utils/filesystem.cpp index 1a7b3dd214..75ad72f5a1 100644 --- a/src/utils/filesystem.cpp +++ b/src/utils/filesystem.cpp @@ -43,7 +43,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "errors.h" #include "utils/defer.h" #include "utils/env.h" @@ -648,9 +648,9 @@ error_code get_process_image_path(int pid, std::string &path) bool get_disk_space_info(const std::string &path, disk_space_info &info) { - FAIL_POINT_INJECT_F("filesystem_get_disk_space_info", [&info](absl::string_view str) { + FAIL_POINT_INJECT_F("filesystem_get_disk_space_info", [&info](std::string_view str) { info.capacity = 100 * 1024 * 1024; - if (str.find("insufficient") != absl::string_view::npos) { + if (str.find("insufficient") != std::string_view::npos) { info.available = 512 * 1024; } else { info.available = 50 * 1024 * 1024; @@ -850,11 +850,11 @@ bool verify_file_size(const std::string &fname, FileDataType type, const int64_t bool create_directory(const std::string &path, std::string &absolute_path, std::string &err_msg) { - FAIL_POINT_INJECT_F("filesystem_create_directory", [path](absl::string_view str) { + FAIL_POINT_INJECT_F("filesystem_create_directory", [path](std::string_view str) { // when str contains 'false', and path contains broken_disk_dir, mock create fail(return // false) std::string broken_disk_dir = "disk1"; - return str.find("false") == absl::string_view::npos || + return str.find("false") == std::string_view::npos || path.find(broken_disk_dir) == std::string::npos; }); @@ -871,11 +871,11 @@ bool create_directory(const std::string &path, std::string &absolute_path, std:: bool check_dir_rw(const std::string &path, std::string &err_msg) { - FAIL_POINT_INJECT_F("filesystem_check_dir_rw", [path](absl::string_view str) { + FAIL_POINT_INJECT_F("filesystem_check_dir_rw", [path](std::string_view str) { // when str contains 'false', and path contains broken_disk_dir, mock check fail(return // false) std::string broken_disk_dir = "disk1"; - return str.find("false") == absl::string_view::npos || + return str.find("false") == std::string_view::npos || path.find(broken_disk_dir) == std::string::npos; }); diff --git a/src/utils/latency_tracer.cpp b/src/utils/latency_tracer.cpp index e8408dd938..f5c97f511b 100644 --- a/src/utils/latency_tracer.cpp +++ b/src/utils/latency_tracer.cpp @@ -17,7 +17,7 @@ #include "utils/latency_tracer.h" -#include +#include #include #include #include diff --git a/src/utils/metrics.cpp b/src/utils/metrics.cpp index 1677584bed..89e3ff1f7f 100644 --- a/src/utils/metrics.cpp +++ b/src/utils/metrics.cpp @@ -17,7 +17,7 @@ #include "utils/metrics.h" -#include +#include #include #include #include diff --git a/src/utils/metrics.h b/src/utils/metrics.h index 199d11d483..9d9b0671d3 100644 --- a/src/utils/metrics.h +++ b/src/utils/metrics.h @@ -40,7 +40,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "common/json_helper.h" #include "http/http_server.h" #include "utils/alloc.h" @@ -853,22 +853,22 @@ class metric_prototype public: struct ctor_args { - const absl::string_view entity_type; + const std::string_view entity_type; const metric_type type; - const absl::string_view name; + const std::string_view name; const metric_unit unit; - const absl::string_view desc; + const std::string_view desc; }; - absl::string_view entity_type() const { return _args.entity_type; } + std::string_view entity_type() const { return _args.entity_type; } metric_type type() const { return _args.type; } - absl::string_view name() const { return _args.name; } + std::string_view name() const { return _args.name; } metric_unit unit() const { return _args.unit; } - absl::string_view description() const { return _args.desc; } + std::string_view description() const { return _args.desc; } protected: explicit metric_prototype(const ctor_args &args); diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index f67132408e..881b97ecd8 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -41,7 +41,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "runtime/api_layer1.h" #include "utils/command_manager.h" #include "utils/errors.h" @@ -165,7 +165,7 @@ inline void process_fatal_log(log_level_t log_level) } bool coredump = true; - FAIL_POINT_INJECT_NOT_RETURN_F("coredump_for_fatal_log", [&coredump](absl::string_view str) { + FAIL_POINT_INJECT_NOT_RETURN_F("coredump_for_fatal_log", [&coredump](std::string_view str) { CHECK(buf2bool(str, coredump), "invalid coredump toggle for fatal log, should be true or false: {}", str); diff --git a/src/utils/string_conv.h b/src/utils/string_conv.h index a92b1e6bb3..16b0793b28 100644 --- a/src/utils/string_conv.h +++ b/src/utils/string_conv.h @@ -27,14 +27,14 @@ #include #include -#include "absl/strings/string_view.h" +#include namespace dsn { namespace internal { template -bool buf2signed(absl::string_view buf, T &result) +bool buf2signed(std::string_view buf, T &result) { static_assert(std::is_signed::value, "buf2signed works only with signed integer"); @@ -65,7 +65,7 @@ bool buf2signed(absl::string_view buf, T &result) } template -bool buf2unsigned(absl::string_view buf, T &result) +bool buf2unsigned(std::string_view buf, T &result) { static_assert(std::is_unsigned::value, "buf2unsigned works only with unsigned integer"); @@ -104,32 +104,32 @@ bool buf2unsigned(absl::string_view buf, T &result) /// buf2*: `result` will keep unmodified if false is returned. -inline bool buf2int32(absl::string_view buf, int32_t &result) +inline bool buf2int32(std::string_view buf, int32_t &result) { return internal::buf2signed(buf, result); } -inline bool buf2int64(absl::string_view buf, int64_t &result) +inline bool buf2int64(std::string_view buf, int64_t &result) { return internal::buf2signed(buf, result); } -inline bool buf2uint32(absl::string_view buf, uint32_t &result) +inline bool buf2uint32(std::string_view buf, uint32_t &result) { return internal::buf2unsigned(buf, result); } -inline bool buf2uint64(absl::string_view buf, uint64_t &result) +inline bool buf2uint64(std::string_view buf, uint64_t &result) { return internal::buf2unsigned(buf, result); } -inline bool buf2uint16(absl::string_view buf, uint16_t &result) +inline bool buf2uint16(std::string_view buf, uint16_t &result) { return internal::buf2unsigned(buf, result); } -inline bool buf2bool(absl::string_view buf, bool &result, bool ignore_case = true) +inline bool buf2bool(std::string_view buf, bool &result, bool ignore_case = true) { std::string data(buf.data(), buf.length()); if (ignore_case) { @@ -146,7 +146,7 @@ inline bool buf2bool(absl::string_view buf, bool &result, bool ignore_case = tru return false; } -inline bool buf2double(absl::string_view buf, double &result) +inline bool buf2double(std::string_view buf, double &result) { if (buf.empty()) { return false; @@ -175,7 +175,7 @@ inline bool buf2double(absl::string_view buf, double &result) } #define DEF_BUF2NUMERIC_FUNC(type, postfix) \ - inline bool buf2numeric(absl::string_view buf, type &result) \ + inline bool buf2numeric(std::string_view buf, type &result) \ { \ return buf2##postfix(buf, result); \ } diff --git a/src/utils/test/fail_point_test.cpp b/src/utils/test/fail_point_test.cpp index 62b1c0c507..e04d39f54c 100644 --- a/src/utils/test/fail_point_test.cpp +++ b/src/utils/test/fail_point_test.cpp @@ -33,7 +33,7 @@ #include "gtest/gtest.h" #include "utils/fail_point.h" #include "utils/fail_point_impl.h" -#include "absl/strings/string_view.h" +#include namespace dsn { namespace fail { @@ -119,12 +119,12 @@ TEST(fail_point, parse) int test_func() { - FAIL_POINT_INJECT_F("test_1", [](absl::string_view str) -> int { + FAIL_POINT_INJECT_F("test_1", [](std::string_view str) -> int { EXPECT_EQ(str, "1"); return 1; }); - FAIL_POINT_INJECT_F("test_2", [](absl::string_view str) -> int { + FAIL_POINT_INJECT_F("test_2", [](std::string_view str) -> int { EXPECT_EQ(str, "2"); return 2; }); @@ -148,7 +148,7 @@ TEST(fail_point, macro_use) void test_func_return_void(int &a) { - FAIL_POINT_INJECT_F("test_1", [](absl::string_view str) {}); + FAIL_POINT_INJECT_F("test_1", [](std::string_view str) {}); a++; } TEST(fail_point, return_void) diff --git a/src/utils/test/fmt_logging_test.cpp b/src/utils/test/fmt_logging_test.cpp index 20017ddecd..c19126c7c5 100644 --- a/src/utils/test/fmt_logging_test.cpp +++ b/src/utils/test/fmt_logging_test.cpp @@ -27,7 +27,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "common/gpid.h" #include "common/replication.codes.h" #include "gtest/gtest.h" @@ -46,8 +46,8 @@ TEST(fmt_logging, basic) ASSERT_EQ(fmt::format("{}", error_s::make(ERR_TIMEOUT, "yes")), "ERR_TIMEOUT: yes"); ASSERT_EQ(fmt::format("{}", ERR_OK), "ERR_OK"); ASSERT_EQ(fmt::format("{}", LPC_REPLICATION_LOW), "LPC_REPLICATION_LOW"); - ASSERT_EQ(absl::string_view("yes"), "yes"); - ASSERT_EQ(fmt::format("{}", absl::string_view("yes\0yes")), "yes\0yes"); + ASSERT_EQ(std::string_view("yes"), "yes"); + ASSERT_EQ(fmt::format("{}", std::string_view("yes\0yes")), "yes\0yes"); ASSERT_DEATH(CHECK(false, "CHECK false in test"), "CHECK false in test"); } diff --git a/src/utils/test/string_conv_test.cpp b/src/utils/test/string_conv_test.cpp index c9fd5fc2d7..5e564b0d2b 100644 --- a/src/utils/test/string_conv_test.cpp +++ b/src/utils/test/string_conv_test.cpp @@ -27,7 +27,7 @@ #include "utils/string_conv.h" #include "gtest/gtest.h" -#include "absl/strings/string_view.h" +#include TEST(string_conv, buf2bool) { @@ -50,13 +50,13 @@ TEST(string_conv, buf2bool) ASSERT_FALSE(dsn::buf2bool("TrUe", result, false)); std::string str("true\0false", 10); - ASSERT_FALSE(dsn::buf2bool(absl::string_view(str.data(), 3), result)); - ASSERT_TRUE(dsn::buf2bool(absl::string_view(str.data(), 4), result)); + ASSERT_FALSE(dsn::buf2bool(std::string_view(str.data(), 3), result)); + ASSERT_TRUE(dsn::buf2bool(std::string_view(str.data(), 4), result)); ASSERT_EQ(result, true); - ASSERT_FALSE(dsn::buf2bool(absl::string_view(str.data(), 5), result)); - ASSERT_FALSE(dsn::buf2bool(absl::string_view(str.data(), 6), result)); - ASSERT_FALSE(dsn::buf2bool(absl::string_view(str.data() + 5, 4), result)); - ASSERT_TRUE(dsn::buf2bool(absl::string_view(str.data() + 5, 5), result)); + ASSERT_FALSE(dsn::buf2bool(std::string_view(str.data(), 5), result)); + ASSERT_FALSE(dsn::buf2bool(std::string_view(str.data(), 6), result)); + ASSERT_FALSE(dsn::buf2bool(std::string_view(str.data() + 5, 4), result)); + ASSERT_TRUE(dsn::buf2bool(std::string_view(str.data() + 5, 5), result)); ASSERT_EQ(result, false); } @@ -92,12 +92,12 @@ TEST(string_conv, buf2int32) // "\045" is "%", so the string length=5, otherwise(2th argument > 5) it will be reported // "global-buffer-overflow" error under AddressSanitizer check std::string str("123\0456", 5); - ASSERT_TRUE(dsn::buf2int32(absl::string_view(str.data(), 2), result)); + ASSERT_TRUE(dsn::buf2int32(std::string_view(str.data(), 2), result)); ASSERT_EQ(result, 12); - ASSERT_TRUE(dsn::buf2int32(absl::string_view(str.data(), 3), result)); + ASSERT_TRUE(dsn::buf2int32(std::string_view(str.data(), 3), result)); ASSERT_EQ(result, 123); - ASSERT_FALSE(dsn::buf2int32(absl::string_view(str.data(), 4), result)); - ASSERT_FALSE(dsn::buf2int32(absl::string_view(str.data(), 5), result)); + ASSERT_FALSE(dsn::buf2int32(std::string_view(str.data(), 4), result)); + ASSERT_FALSE(dsn::buf2int32(std::string_view(str.data(), 5), result)); } TEST(string_conv, buf2int64) @@ -139,12 +139,12 @@ TEST(string_conv, buf2int64) // "\045" is "%", so the string length=5, otherwise(2th argument > 5) it will be reported // "global-buffer-overflow" error under AddressSanitizer check std::string str("123\0456", 5); - ASSERT_TRUE(dsn::buf2int64(absl::string_view(str.data(), 2), result)); + ASSERT_TRUE(dsn::buf2int64(std::string_view(str.data(), 2), result)); ASSERT_EQ(result, 12); - ASSERT_TRUE(dsn::buf2int64(absl::string_view(str.data(), 3), result)); + ASSERT_TRUE(dsn::buf2int64(std::string_view(str.data(), 3), result)); ASSERT_EQ(result, 123); - ASSERT_FALSE(dsn::buf2int64(absl::string_view(str.data(), 4), result)); - ASSERT_FALSE(dsn::buf2int64(absl::string_view(str.data(), 5), result)); + ASSERT_FALSE(dsn::buf2int64(std::string_view(str.data(), 4), result)); + ASSERT_FALSE(dsn::buf2int64(std::string_view(str.data(), 5), result)); } TEST(string_conv, buf2uint64) @@ -183,12 +183,12 @@ TEST(string_conv, buf2uint64) // "\045" is "%", so the string length=5, otherwise(2th argument > 5) it will be reported // "global-buffer-overflow" error under AddressSanitizer check std::string str("123\0456", 5); - ASSERT_TRUE(dsn::buf2uint64(absl::string_view(str.data(), 2), result)); + ASSERT_TRUE(dsn::buf2uint64(std::string_view(str.data(), 2), result)); ASSERT_EQ(result, 12); - ASSERT_TRUE(dsn::buf2uint64(absl::string_view(str.data(), 3), result)); + ASSERT_TRUE(dsn::buf2uint64(std::string_view(str.data(), 3), result)); ASSERT_EQ(result, 123); - ASSERT_FALSE(dsn::buf2uint64(absl::string_view(str.data(), 4), result)); - ASSERT_FALSE(dsn::buf2uint64(absl::string_view(str.data(), 5), result)); + ASSERT_FALSE(dsn::buf2uint64(std::string_view(str.data(), 4), result)); + ASSERT_FALSE(dsn::buf2uint64(std::string_view(str.data(), 5), result)); } TEST(string_conv, buf2uint32) @@ -229,12 +229,12 @@ TEST(string_conv, buf2uint32) // "\045" is "%", so the string length=5, otherwise(2th argument > 5) it will be reported // "global-buffer-overflow" error under AddressSanitizer check std::string str("123\0456", 5); - ASSERT_TRUE(dsn::buf2uint32(absl::string_view(str.data(), 2), result)); + ASSERT_TRUE(dsn::buf2uint32(std::string_view(str.data(), 2), result)); ASSERT_EQ(result, 12); - ASSERT_TRUE(dsn::buf2uint32(absl::string_view(str.data(), 3), result)); + ASSERT_TRUE(dsn::buf2uint32(std::string_view(str.data(), 3), result)); ASSERT_EQ(result, 123); - ASSERT_FALSE(dsn::buf2uint32(absl::string_view(str.data(), 4), result)); - ASSERT_FALSE(dsn::buf2uint32(absl::string_view(str.data(), 5), result)); + ASSERT_FALSE(dsn::buf2uint32(std::string_view(str.data(), 4), result)); + ASSERT_FALSE(dsn::buf2uint32(std::string_view(str.data(), 5), result)); } TEST(string_conv, int64_partial) diff --git a/src/utils/time_utils.h b/src/utils/time_utils.h index 546bea0ef2..d36aa98640 100644 --- a/src/utils/time_utils.h +++ b/src/utils/time_utils.h @@ -33,7 +33,7 @@ #include #include -#include "absl/strings/string_view.h" +#include #include "runtime/api_layer1.h" #include "utils/fmt_logging.h" #include "utils/ports.h" @@ -115,7 +115,7 @@ inline int64_t get_unix_sec_today_midnight() // `hh:mm` (range in [00:00, 23:59]) to seconds since 00:00:00 // eg. `01:00` => `3600` // Return: -1 when invalid -inline int hh_mm_to_seconds(absl::string_view hhmm) +inline int hh_mm_to_seconds(std::string_view hhmm) { int hour = 0, min = 0, sec = -1; if (::sscanf(hhmm.data(), "%d:%d", &hour, &min) == 2 && (0 <= hour && hour <= 23) && @@ -128,7 +128,7 @@ inline int hh_mm_to_seconds(absl::string_view hhmm) // local time `hh:mm` to unix timestamp. // eg. `18:10` => `1525947000` when called on May 10, 2018, CST // Return: -1 when invalid -inline int64_t hh_mm_today_to_unix_sec(absl::string_view hhmm_of_day) +inline int64_t hh_mm_today_to_unix_sec(std::string_view hhmm_of_day) { int sec_of_day = hh_mm_to_seconds(hhmm_of_day); if (sec_of_day == -1) { From 3d39af1a4b0000bb9924b7ee206e35b2a6ad9164 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Mon, 12 Aug 2024 21:28:02 +0800 Subject: [PATCH 10/15] chore(CI): Introduce clang tidy linter (#2094) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > clang-tidy is a clang-based C++ “linter” tool. Its purpose is to provide an extensible framework for diagnosing and fixing typical programming errors, like style violations, interface misuse, or bugs that can be deduced via static analysis. clang-tidy is modular and provides a convenient interface for writing new checks. This patch introduces clang-tidy to lint the Pegasus code, there are many issues are needed to be fixed, it would be a very large patch if fix them all in one patch. So only changed files will be checked in the newly added GitHub action. To lint the code localy before submitting the code, please follow the steps: 1. Install `clang-tidy`, now we are using clang-tidy-14. `sudo apt-get install clang-tidy -y` 2. Setup the compilation database the clang-tidy needed. `./run.sh build --test --compiler clang-14,clang++-14 -t debug --skip_thirdparty -c --cmake_only` 3. Lint the code. (The example checked the code modified from the local `origin/master` branch). `./scripts/clang_tidy.py --rev-range $(git log origin/master -n1 --format=format:"%H")` --- .clang-tidy | 29 +++++++ .github/workflows/lint_and_test_cpp.yaml | 52 +++++++++++-- run.sh | 11 +++ scripts/clang_tidy.py | 97 ++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 .clang-tidy create mode 100755 scripts/clang_tidy.py diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000000..a2a2c95c1b --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# https://releases.llvm.org/14.0.0/tools/clang/tools/extra/docs/clang-tidy/index.html + +CheckOptions: [] +Checks: 'abseil-*,boost-*,bugprone-*,cert-*,clang-analyzer-*,concurrency-*,cppcoreguidelines-*,darwin-*,fuchsia-*,google-*,hicpp-*,linuxkernel-*,llvm-*,misc-*,modernize-*,performance-*,portability-*,readability-*' +ExtraArgs: +ExtraArgsBefore: [] +FormatStyle: none +HeaderFilterRegex: '' +InheritParentConfig: true +UseColor: true +User: 'clang-tidy' +WarningsAsErrors: '' diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 799263566a..5e4fb7d468 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -50,16 +50,40 @@ env: jobs: cpp_clang_format_linter: - name: Lint + name: Format runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: clang-format run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . + cpp_clang_tidy_linter: + name: Tidy + runs-on: ubuntu-22.04 + container: + image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }} + steps: + - name: Install Softwares + run: | + apt-get update + apt-get install clang-tidy -y + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Rebuild thirdparty if needed + uses: "./.github/actions/rebuild_thirdparty_if_needed" + - name: clang-tidy + run: | + git config --global --add safe.directory $(pwd) + ./run.sh build --test --compiler clang-14,clang++-14 -t debug --skip_thirdparty -c --cmake_only + ./scripts/clang_tidy.py --rev-range $(git log origin/${{ github.base_ref }} -n1 --format=format:"%H") + shell: bash + iwyu: name: IWYU - needs: cpp_clang_format_linter + needs: + - cpp_clang_format_linter + - cpp_clang_tidy_linter runs-on: ubuntu-latest env: USE_JEMALLOC: OFF @@ -87,7 +111,9 @@ jobs: build_Release: name: Build Release - needs: cpp_clang_format_linter + needs: + - cpp_clang_format_linter + - cpp_clang_tidy_linter runs-on: ubuntu-latest env: USE_JEMALLOC: OFF @@ -170,7 +196,9 @@ jobs: build_ASAN: name: Build ASAN - needs: cpp_clang_format_linter + needs: + - cpp_clang_format_linter + - cpp_clang_tidy_linter runs-on: ubuntu-latest env: USE_JEMALLOC: OFF @@ -257,7 +285,9 @@ jobs: # before we find any way to reduce the time cost. # build_UBSAN: # name: Build UBSAN -# needs: cpp_clang_format_linter +# needs: +# - cpp_clang_format_linter +# - cpp_clang_tidy_linter # runs-on: ubuntu-latest # env: # USE_JEMALLOC: OFF @@ -337,7 +367,9 @@ jobs: build_with_jemalloc: name: Build with jemalloc - needs: cpp_clang_format_linter + needs: + - cpp_clang_format_linter + - cpp_clang_tidy_linter runs-on: ubuntu-latest env: USE_JEMALLOC: ON @@ -378,7 +410,9 @@ jobs: build_release_on_macos: name: Build Release on macOS - needs: cpp_clang_format_linter + needs: + - cpp_clang_format_linter + - cpp_clang_tidy_linter runs-on: macos-12 steps: - name: Install Softwares @@ -412,7 +446,9 @@ jobs: build_debug_on_centos7: name: Build Debug on CentOS 7 - needs: cpp_clang_format_linter + needs: + - cpp_clang_format_linter + - cpp_clang_tidy_linter runs-on: ubuntu-latest env: USE_JEMALLOC: OFF diff --git a/run.sh b/run.sh index dfe4f7ea98..8c6048b019 100755 --- a/run.sh +++ b/run.sh @@ -114,6 +114,7 @@ function usage_build() echo " --enable_rocksdb_portable build a portable rocksdb binary" echo " --test whether to build test binaries" echo " --iwyu specify the binary path of 'include-what-you-use' when build with IWYU" + echo " --cmake_only whether to run cmake only, default no" } function exit_if_fail() { @@ -131,6 +132,7 @@ function run_build() C_COMPILER="gcc" CXX_COMPILER="g++" BUILD_TYPE="release" + # TODO(yingchun): some boolean variables are using YES/NO, some are using ON/OFF, should be unified. CLEAR=NO CLEAR_THIRDPARTY=NO JOB_NUM=8 @@ -145,6 +147,7 @@ function run_build() BUILD_TEST=OFF IWYU="" BUILD_MODULES="" + CMAKE_ONLY=NO while [[ $# > 0 ]]; do key="$1" case $key in @@ -220,6 +223,9 @@ function run_build() IWYU="$2" shift ;; + --cmake_only) + CMAKE_ONLY=YES + ;; *) echo "ERROR: unknown option \"$key\"" echo @@ -353,6 +359,11 @@ function run_build() rm -f ${BUILD_LATEST_DIR} ln -s ${BUILD_DIR} ${BUILD_LATEST_DIR} + if [ "$CMAKE_ONLY" == "YES" ]; then + echo "CMake only, exit" + return + fi + echo "[$(date)] Building Pegasus ..." pushd $BUILD_DIR if [ ! -z "${IWYU}" ]; then diff --git a/scripts/clang_tidy.py b/scripts/clang_tidy.py new file mode 100755 index 0000000000..2a24eb8129 --- /dev/null +++ b/scripts/clang_tidy.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Most of the code are inspired by https://github.com/apache/kudu/blob/856fa3404b00ee02bd3bc1d77d414ede2b2cd02e/build-support/clang_tidy_gerrit.py + +import argparse +import collections +import json +import multiprocessing +from multiprocessing.pool import ThreadPool +import os +import re +import subprocess +import sys +import tempfile + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + +BUILD_PATH = os.path.join(ROOT, "build", "latest") + +def run_tidy(sha="HEAD", is_rev_range=False): + diff_cmdline = ["git", "diff" if is_rev_range else "show", sha] + + # Figure out which paths changed in the given diff. + changed_paths = subprocess.check_output(diff_cmdline + ["--name-only", "--pretty=format:"]).splitlines() + changed_paths = [p for p in changed_paths if p] + + # Produce a separate diff for each file and run clang-tidy-diff on it + # in parallel. + # + # Note: this will incorporate any configuration from .clang-tidy. + def tidy_on_path(path): + patch_file = tempfile.NamedTemporaryFile() + cmd = diff_cmdline + [ + "--src-prefix=%s/" % ROOT, + "--dst-prefix=%s/" % ROOT, + "--", + path] + subprocess.check_call(cmd, stdout=patch_file, cwd=ROOT) + # TODO(yingchun): some checks could be disabled before we fix them. + # "-checks=-llvm-include-order,-modernize-concat-nested-namespaces,-cppcoreguidelines-pro-type-union-access,-cppcoreguidelines-macro-usage,-cppcoreguidelines-special-member-functions,-hicpp-special-member-functions,-modernize-use-trailing-return-type,-bugprone-easily-swappable-parameters,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-llvm-header-guard,-cppcoreguidelines-pro-bounds-pointer-arithmetic", + cmdline = ["clang-tidy-diff", + "-clang-tidy-binary", + "clang-tidy", + "-p0", + "-path", BUILD_PATH, + "-extra-arg=-language=c++", + "-extra-arg=-std=c++17", + "-extra-arg=-Ithirdparty/output/include"] + return subprocess.check_output( + cmdline, + stdin=open(patch_file.name), + cwd=ROOT).decode() + pool = ThreadPool(multiprocessing.cpu_count()) + try: + return "".join(pool.imap(tidy_on_path, changed_paths)) + except KeyboardInterrupt as ki: + sys.exit(1) + finally: + pool.terminate() + pool.join() + + +if __name__ == "__main__": + # Basic setup and argument parsing. + parser = argparse.ArgumentParser(description="Run clang-tidy on a patch") + parser.add_argument("--rev-range", action="store_true", + default=False, + help="Whether the revision specifies the 'rev..' range") + parser.add_argument('rev', help="The git revision (or range of revisions) to process") + args = parser.parse_args() + + # Run clang-tidy and parse the output. + clang_output = run_tidy(args.rev, args.rev_range) + parsed = re.match(r'.+(warning|error): .+', clang_output, re.MULTILINE | re.DOTALL) + print(clang_output, file=sys.stderr) + if not parsed: + print("No warnings", file=sys.stderr) + sys.exit(0) + sys.exit(1) + From 01b8754509796d80fa4e3849b4463ae726ae53b9 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 14 Aug 2024 17:13:35 +0800 Subject: [PATCH 11/15] refactor(scripts): Separate admin and build scripts to independent directories (#2096) Separate the `scripts` directory to `admin_tools` and `build_tools` directories. --- .../rebuild_thirdparty_if_needed/action.yaml | 4 +-- .github/workflows/lint_and_test_cpp.yaml | 8 ++--- .github/workflows/module_labeler_conf.yml | 3 +- .github/workflows/regular-build.yml | 2 +- .licenserc.yaml | 6 ++-- LICENSE | 2 +- {scripts => admin_tools}/config_hdfs.sh | 0 {scripts => admin_tools}/downgrade_node.sh | 0 {scripts => admin_tools}/learn_stat.py | 0 {scripts => admin_tools}/migrate_node.sh | 0 {scripts => admin_tools}/minos_common.sh | 0 .../pegasus_add_node_list.sh | 4 +-- {scripts => admin_tools}/pegasus_bench_run.sh | 0 .../pegasus_check_arguments.sh | 2 +- {scripts => admin_tools}/pegasus_kill_test.sh | 0 .../pegasus_manual_compact.sh | 0 .../pegasus_migrate_zookeeper.sh | 2 +- .../pegasus_offline_node.sh | 2 +- .../pegasus_offline_node_list.sh | 4 +-- .../pegasus_rebalance_cluster.sh | 0 .../pegasus_replica_thread.sh | 0 .../pegasus_restart_node.sh | 2 +- .../pegasus_rolling_update.sh | 4 +-- .../pegasus_set_usage_scenario.sh | 0 .../pegasus_stat_available.sh | 0 .../pegasus_update_ingest_behind.sh | 2 +- {scripts => admin_tools}/sendmail.sh | 0 {scripts => build_tools}/bump_version.sh | 0 {scripts => build_tools}/check_license.py | 0 {scripts => build_tools}/clang_tidy.py | 3 +- {scripts => build_tools}/clear_zk.sh | 0 .../collector_table_counter_gen.py | 2 +- {scripts => build_tools}/compile_thrift.py | 0 {scripts => build_tools}/download_hadoop.sh | 0 {scripts => build_tools}/download_package.sh | 0 {scripts => build_tools}/download_zk.sh | 0 {scripts => build_tools}/format_files.sh | 0 {scripts => build_tools}/pack_client.sh | 0 {scripts => build_tools}/pack_common.sh | 0 {scripts => build_tools}/pack_server.sh | 4 +-- {scripts => build_tools}/pack_tools.sh | 11 +++++-- {scripts => build_tools}/recompile_thrift.sh | 0 {scripts => build_tools}/redis_proto_check.py | 0 {scripts => build_tools}/run-clang-format.py | 0 {scripts => build_tools}/start_zk.sh | 0 {scripts => build_tools}/stop_zk.sh | 0 docker/thirdparties-bin/Dockerfile | 4 +-- java-client/README.md | 2 +- run.sh | 30 +++++++++---------- src/replica/replica.h | 4 +-- 50 files changed, 57 insertions(+), 50 deletions(-) rename {scripts => admin_tools}/config_hdfs.sh (100%) rename {scripts => admin_tools}/downgrade_node.sh (100%) rename {scripts => admin_tools}/learn_stat.py (100%) rename {scripts => admin_tools}/migrate_node.sh (100%) rename {scripts => admin_tools}/minos_common.sh (100%) rename {scripts => admin_tools}/pegasus_add_node_list.sh (92%) rename {scripts => admin_tools}/pegasus_bench_run.sh (100%) rename {scripts => admin_tools}/pegasus_check_arguments.sh (98%) rename {scripts => admin_tools}/pegasus_kill_test.sh (100%) rename {scripts => admin_tools}/pegasus_manual_compact.sh (100%) rename {scripts => admin_tools}/pegasus_migrate_zookeeper.sh (99%) rename {scripts => admin_tools}/pegasus_offline_node.sh (99%) rename {scripts => admin_tools}/pegasus_offline_node_list.sh (96%) rename {scripts => admin_tools}/pegasus_rebalance_cluster.sh (100%) rename {scripts => admin_tools}/pegasus_replica_thread.sh (100%) rename {scripts => admin_tools}/pegasus_restart_node.sh (99%) rename {scripts => admin_tools}/pegasus_rolling_update.sh (98%) rename {scripts => admin_tools}/pegasus_set_usage_scenario.sh (100%) rename {scripts => admin_tools}/pegasus_stat_available.sh (100%) rename {scripts => admin_tools}/pegasus_update_ingest_behind.sh (99%) rename {scripts => admin_tools}/sendmail.sh (100%) rename {scripts => build_tools}/bump_version.sh (100%) rename {scripts => build_tools}/check_license.py (100%) rename {scripts => build_tools}/clang_tidy.py (88%) rename {scripts => build_tools}/clear_zk.sh (100%) rename {scripts => build_tools}/collector_table_counter_gen.py (98%) rename {scripts => build_tools}/compile_thrift.py (100%) rename {scripts => build_tools}/download_hadoop.sh (100%) rename {scripts => build_tools}/download_package.sh (100%) rename {scripts => build_tools}/download_zk.sh (100%) rename {scripts => build_tools}/format_files.sh (100%) rename {scripts => build_tools}/pack_client.sh (100%) rename {scripts => build_tools}/pack_common.sh (100%) rename {scripts => build_tools}/pack_server.sh (98%) rename {scripts => build_tools}/pack_tools.sh (94%) rename {scripts => build_tools}/recompile_thrift.sh (100%) rename {scripts => build_tools}/redis_proto_check.py (100%) rename {scripts => build_tools}/run-clang-format.py (100%) rename {scripts => build_tools}/start_zk.sh (100%) rename {scripts => build_tools}/stop_zk.sh (100%) diff --git a/.github/actions/rebuild_thirdparty_if_needed/action.yaml b/.github/actions/rebuild_thirdparty_if_needed/action.yaml index 6e64f7db46..838ac3f615 100644 --- a/.github/actions/rebuild_thirdparty_if_needed/action.yaml +++ b/.github/actions/rebuild_thirdparty_if_needed/action.yaml @@ -42,8 +42,8 @@ runs: cmake --build build/ -j $(nproc) rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]* find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - ../scripts/download_hadoop.sh hadoop-bin - ../scripts/download_zk.sh zookeeper-bin + ../build_tools/download_hadoop.sh hadoop-bin + ../build_tools/download_zk.sh zookeeper-bin rm -rf hadoop-bin/share/doc rm -rf zookeeper-bin/docs shell: bash diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 5e4fb7d468..1ae74b268f 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -30,13 +30,13 @@ on: - .github/actions - .github/workflows/lint_and_test_cpp.yaml - .github/workflows/thirdparty-regular-push.yml + - build_tools/pack_server.sh + - build_tools/pack_tools.sh - docker/thirdparties-bin/** - docker/thirdparties-src/** - CMakeLists.txt - cmake_modules/** - run.sh - - scripts/pack_server.sh - - scripts/pack_tools.sh - src/** - thirdparty/** @@ -55,7 +55,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: clang-format - run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . + run: ./build_tools/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . cpp_clang_tidy_linter: name: Tidy @@ -76,7 +76,7 @@ jobs: run: | git config --global --add safe.directory $(pwd) ./run.sh build --test --compiler clang-14,clang++-14 -t debug --skip_thirdparty -c --cmake_only - ./scripts/clang_tidy.py --rev-range $(git log origin/${{ github.base_ref }} -n1 --format=format:"%H") + ./build_tools/clang_tidy.py --rev-range $(git log origin/${{ github.base_ref }} -n1 --format=format:"%H") shell: bash iwyu: diff --git a/.github/workflows/module_labeler_conf.yml b/.github/workflows/module_labeler_conf.yml index ad23e4bf08..b002b2e3cd 100644 --- a/.github/workflows/module_labeler_conf.yml +++ b/.github/workflows/module_labeler_conf.yml @@ -42,7 +42,8 @@ thrift: docs: - '**/*.md' scripts: - - 'scripts/**/*' + - 'admin_tools/**/*' + - 'build_tools/**/*' - '**/*.sh' build: - 'cmake_modules/**/*' diff --git a/.github/workflows/regular-build.yml b/.github/workflows/regular-build.yml index 04468e4d39..01c17f4fc7 100644 --- a/.github/workflows/regular-build.yml +++ b/.github/workflows/regular-build.yml @@ -41,7 +41,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: clang-format - run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . + run: ./build_tools/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . build_cpp: name: Build Cpp diff --git a/.licenserc.yaml b/.licenserc.yaml index b8e45b5ec4..71376cd101 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -110,8 +110,10 @@ header: - 'cmake_modules/FindRT.cmake' - 'cmake_modules/FindDL.cmake' # Copyright (c) 2017 Guillaume Papin - - 'scripts/run-clang-format.py' + - 'build_tools/run-clang-format.py' # The MIT License (MIT), Copyright (c) 2015 Microsoft Corporation + - 'admin_tools/learn_stat.py' + - 'build_tools/compile_thrift.py' - 'cmake_modules/BaseFunctions.cmake' - 'docs/rdsn-README.md' - 'idl/command.thrift' @@ -120,8 +122,6 @@ header: - 'idl/metadata.thrift' - 'idl/meta_admin.thrift' - 'idl/replica_admin.thrift' - - 'scripts/compile_thrift.py' - - 'scripts/learn_stat.py' - 'src/runtime/api_layer1.h' - 'src/runtime/api_task.h' - 'src/utils/api_utilities.h' diff --git a/LICENSE b/LICENSE index a8e975f469..eb22d54de3 100644 --- a/LICENSE +++ b/LICENSE @@ -514,7 +514,7 @@ RESULTING FROM THE USE OF THIS SOFTWARE. -------------------------------------------------------------------------------- -scripts/run-clang-format.py - MIT License +build_tools/run-clang-format.py - MIT License MIT License diff --git a/scripts/config_hdfs.sh b/admin_tools/config_hdfs.sh similarity index 100% rename from scripts/config_hdfs.sh rename to admin_tools/config_hdfs.sh diff --git a/scripts/downgrade_node.sh b/admin_tools/downgrade_node.sh similarity index 100% rename from scripts/downgrade_node.sh rename to admin_tools/downgrade_node.sh diff --git a/scripts/learn_stat.py b/admin_tools/learn_stat.py similarity index 100% rename from scripts/learn_stat.py rename to admin_tools/learn_stat.py diff --git a/scripts/migrate_node.sh b/admin_tools/migrate_node.sh similarity index 100% rename from scripts/migrate_node.sh rename to admin_tools/migrate_node.sh diff --git a/scripts/minos_common.sh b/admin_tools/minos_common.sh similarity index 100% rename from scripts/minos_common.sh rename to admin_tools/minos_common.sh diff --git a/scripts/pegasus_add_node_list.sh b/admin_tools/pegasus_add_node_list.sh similarity index 92% rename from scripts/pegasus_add_node_list.sh rename to admin_tools/pegasus_add_node_list.sh index ed40a8b6d2..1665d5a6df 100755 --- a/scripts/pegasus_add_node_list.sh +++ b/admin_tools/pegasus_add_node_list.sh @@ -52,7 +52,7 @@ shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir echo "Check the argument..." -source ./scripts/pegasus_check_arguments.sh add_node_list $cluster $meta_list $replica_task_id_list +source ./admin_tools/pegasus_check_arguments.sh add_node_list $cluster $meta_list $replica_task_id_list if [ $? -ne 0 ]; then echo "ERROR: the argument check failed" @@ -78,7 +78,7 @@ do echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" done -./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list true $nfs_rate_megabytes_per_disk +./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list true $nfs_rate_megabytes_per_disk echo "Finish time: `date`" add_node_finish_time=$((`date +%s`)) diff --git a/scripts/pegasus_bench_run.sh b/admin_tools/pegasus_bench_run.sh similarity index 100% rename from scripts/pegasus_bench_run.sh rename to admin_tools/pegasus_bench_run.sh diff --git a/scripts/pegasus_check_arguments.sh b/admin_tools/pegasus_check_arguments.sh similarity index 98% rename from scripts/pegasus_check_arguments.sh rename to admin_tools/pegasus_check_arguments.sh index 4b0231d87c..9574051c58 100755 --- a/scripts/pegasus_check_arguments.sh +++ b/admin_tools/pegasus_check_arguments.sh @@ -41,7 +41,7 @@ if [ "$check_type" != "add_node_list" -a "$check_type" != "offline_node_list" ]; exit 1 fi -source ./scripts/minos_common.sh +source ./admin_tools/minos_common.sh find_cluster $cluster if [ $? -ne 0 ]; then echo "ERROR: cluster \"$cluster\" not found" diff --git a/scripts/pegasus_kill_test.sh b/admin_tools/pegasus_kill_test.sh similarity index 100% rename from scripts/pegasus_kill_test.sh rename to admin_tools/pegasus_kill_test.sh diff --git a/scripts/pegasus_manual_compact.sh b/admin_tools/pegasus_manual_compact.sh similarity index 100% rename from scripts/pegasus_manual_compact.sh rename to admin_tools/pegasus_manual_compact.sh diff --git a/scripts/pegasus_migrate_zookeeper.sh b/admin_tools/pegasus_migrate_zookeeper.sh similarity index 99% rename from scripts/pegasus_migrate_zookeeper.sh rename to admin_tools/pegasus_migrate_zookeeper.sh index 52e7c0952f..d4b78f5e24 100755 --- a/scripts/pegasus_migrate_zookeeper.sh +++ b/admin_tools/pegasus_migrate_zookeeper.sh @@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0" )" && pwd )" shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir -source ./scripts/minos_common.sh +source ./admin_tools/minos_common.sh find_cluster $cluster if [ $? -ne 0 ]; then echo "ERROR: cluster \"$cluster\" not found" diff --git a/scripts/pegasus_offline_node.sh b/admin_tools/pegasus_offline_node.sh similarity index 99% rename from scripts/pegasus_offline_node.sh rename to admin_tools/pegasus_offline_node.sh index d16f368f1a..fee4107af4 100755 --- a/scripts/pegasus_offline_node.sh +++ b/admin_tools/pegasus_offline_node.sh @@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0" )" && pwd )" shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir -source ./scripts/minos_common.sh +source ./admin_tools/minos_common.sh find_cluster $cluster if [ $? -ne 0 ]; then echo "ERROR: cluster \"$cluster\" not found" diff --git a/scripts/pegasus_offline_node_list.sh b/admin_tools/pegasus_offline_node_list.sh similarity index 96% rename from scripts/pegasus_offline_node_list.sh rename to admin_tools/pegasus_offline_node_list.sh index 7a2483f491..4da739c698 100755 --- a/scripts/pegasus_offline_node_list.sh +++ b/admin_tools/pegasus_offline_node_list.sh @@ -52,7 +52,7 @@ shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir echo "Check the argument..." -source ./scripts/pegasus_check_arguments.sh offline_node_list $cluster $meta_list $replica_task_id_list +source ./admin_tools/pegasus_check_arguments.sh offline_node_list $cluster $meta_list $replica_task_id_list if [ $? -ne 0 ]; then echo "ERROR: the argument check failed" @@ -93,7 +93,7 @@ echo for id in $id_list do echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - ./scripts/pegasus_offline_node.sh $cluster $meta_list $id + ./admin_tools/pegasus_offline_node.sh $cluster $meta_list $id if [ $? -ne 0 ]; then echo "ERROR: offline replica task $id failed" exit 1 diff --git a/scripts/pegasus_rebalance_cluster.sh b/admin_tools/pegasus_rebalance_cluster.sh similarity index 100% rename from scripts/pegasus_rebalance_cluster.sh rename to admin_tools/pegasus_rebalance_cluster.sh diff --git a/scripts/pegasus_replica_thread.sh b/admin_tools/pegasus_replica_thread.sh similarity index 100% rename from scripts/pegasus_replica_thread.sh rename to admin_tools/pegasus_replica_thread.sh diff --git a/scripts/pegasus_restart_node.sh b/admin_tools/pegasus_restart_node.sh similarity index 99% rename from scripts/pegasus_restart_node.sh rename to admin_tools/pegasus_restart_node.sh index b65e6ec18a..27d3588c1c 100755 --- a/scripts/pegasus_restart_node.sh +++ b/admin_tools/pegasus_restart_node.sh @@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0" )" && pwd )" shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir -source ./scripts/minos_common.sh +source ./admin_tools/minos_common.sh find_cluster $cluster if [ $? -ne 0 ]; then echo "ERROR: cluster \"$cluster\" not found" diff --git a/scripts/pegasus_rolling_update.sh b/admin_tools/pegasus_rolling_update.sh similarity index 98% rename from scripts/pegasus_rolling_update.sh rename to admin_tools/pegasus_rolling_update.sh index 9e207fc303..c805426db4 100755 --- a/scripts/pegasus_rolling_update.sh +++ b/admin_tools/pegasus_rolling_update.sh @@ -72,7 +72,7 @@ pwd="$( cd "$( dirname "$0" )" && pwd )" shell_dir="$( cd $pwd/.. && pwd )" cd $shell_dir -source ./scripts/minos_common.sh +source ./admin_tools/minos_common.sh find_cluster $cluster if [ $? -ne 0 ]; then echo "ERROR: cluster \"$cluster\" not found" @@ -338,7 +338,7 @@ fi if [ "$rebalance_cluster_after_rolling" == "true" ]; then echo "Start to rebalance cluster..." - ./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary + ./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary fi echo "Finish time: `date`" diff --git a/scripts/pegasus_set_usage_scenario.sh b/admin_tools/pegasus_set_usage_scenario.sh similarity index 100% rename from scripts/pegasus_set_usage_scenario.sh rename to admin_tools/pegasus_set_usage_scenario.sh diff --git a/scripts/pegasus_stat_available.sh b/admin_tools/pegasus_stat_available.sh similarity index 100% rename from scripts/pegasus_stat_available.sh rename to admin_tools/pegasus_stat_available.sh diff --git a/scripts/pegasus_update_ingest_behind.sh b/admin_tools/pegasus_update_ingest_behind.sh similarity index 99% rename from scripts/pegasus_update_ingest_behind.sh rename to admin_tools/pegasus_update_ingest_behind.sh index 8932f667e8..1000a058a5 100755 --- a/scripts/pegasus_update_ingest_behind.sh +++ b/admin_tools/pegasus_update_ingest_behind.sh @@ -327,7 +327,7 @@ echo if [ "$rebalance_cluster_after_rolling" == "true" ]; then echo "Start to rebalance cluster..." - ./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary + ./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary fi echo "Finish time: `date`" diff --git a/scripts/sendmail.sh b/admin_tools/sendmail.sh similarity index 100% rename from scripts/sendmail.sh rename to admin_tools/sendmail.sh diff --git a/scripts/bump_version.sh b/build_tools/bump_version.sh similarity index 100% rename from scripts/bump_version.sh rename to build_tools/bump_version.sh diff --git a/scripts/check_license.py b/build_tools/check_license.py similarity index 100% rename from scripts/check_license.py rename to build_tools/check_license.py diff --git a/scripts/clang_tidy.py b/build_tools/clang_tidy.py similarity index 88% rename from scripts/clang_tidy.py rename to build_tools/clang_tidy.py index 2a24eb8129..4071497659 100755 --- a/scripts/clang_tidy.py +++ b/build_tools/clang_tidy.py @@ -54,12 +54,13 @@ def tidy_on_path(path): path] subprocess.check_call(cmd, stdout=patch_file, cwd=ROOT) # TODO(yingchun): some checks could be disabled before we fix them. - # "-checks=-llvm-include-order,-modernize-concat-nested-namespaces,-cppcoreguidelines-pro-type-union-access,-cppcoreguidelines-macro-usage,-cppcoreguidelines-special-member-functions,-hicpp-special-member-functions,-modernize-use-trailing-return-type,-bugprone-easily-swappable-parameters,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-llvm-header-guard,-cppcoreguidelines-pro-bounds-pointer-arithmetic", + # "-checks=-llvm-include-order,-modernize-concat-nested-namespaces,-cppcoreguidelines-macro-usage,-cppcoreguidelines-special-member-functions,-hicpp-special-member-functions,-bugprone-easily-swappable-parameters,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-llvm-header-guard,-cppcoreguidelines-pro-bounds-pointer-arithmetic", cmdline = ["clang-tidy-diff", "-clang-tidy-binary", "clang-tidy", "-p0", "-path", BUILD_PATH, + "-checks=-cppcoreguidelines-pro-type-union-access,-modernize-use-trailing-return-type", "-extra-arg=-language=c++", "-extra-arg=-std=c++17", "-extra-arg=-Ithirdparty/output/include"] diff --git a/scripts/clear_zk.sh b/build_tools/clear_zk.sh similarity index 100% rename from scripts/clear_zk.sh rename to build_tools/clear_zk.sh diff --git a/scripts/collector_table_counter_gen.py b/build_tools/collector_table_counter_gen.py similarity index 98% rename from scripts/collector_table_counter_gen.py rename to build_tools/collector_table_counter_gen.py index 03a3f9f297..0cb96b66c1 100755 --- a/scripts/collector_table_counter_gen.py +++ b/build_tools/collector_table_counter_gen.py @@ -118,7 +118,7 @@ def generate_code_in_command_helper_header(replica_counter): # python3 ./collector_table_counter_gen.py counter1,counter2 -# please use `./scripts/format_files.sh` to format after generate code +# please use `./build_tools/format_files.sh` to format after generate code if __name__ == '__main__': if len(sys.argv) != 2: print("python3 ./collector_table_counter_gen.py {counter1,counter2..}") diff --git a/scripts/compile_thrift.py b/build_tools/compile_thrift.py similarity index 100% rename from scripts/compile_thrift.py rename to build_tools/compile_thrift.py diff --git a/scripts/download_hadoop.sh b/build_tools/download_hadoop.sh similarity index 100% rename from scripts/download_hadoop.sh rename to build_tools/download_hadoop.sh diff --git a/scripts/download_package.sh b/build_tools/download_package.sh similarity index 100% rename from scripts/download_package.sh rename to build_tools/download_package.sh diff --git a/scripts/download_zk.sh b/build_tools/download_zk.sh similarity index 100% rename from scripts/download_zk.sh rename to build_tools/download_zk.sh diff --git a/scripts/format_files.sh b/build_tools/format_files.sh similarity index 100% rename from scripts/format_files.sh rename to build_tools/format_files.sh diff --git a/scripts/pack_client.sh b/build_tools/pack_client.sh similarity index 100% rename from scripts/pack_client.sh rename to build_tools/pack_client.sh diff --git a/scripts/pack_common.sh b/build_tools/pack_common.sh similarity index 100% rename from scripts/pack_common.sh rename to build_tools/pack_common.sh diff --git a/scripts/pack_server.sh b/build_tools/pack_server.sh similarity index 98% rename from scripts/pack_server.sh rename to build_tools/pack_server.sh index 5f1172e918..1c27ac32fb 100755 --- a/scripts/pack_server.sh +++ b/build_tools/pack_server.sh @@ -130,10 +130,10 @@ fi copy_file ${THIRDPARTY_ROOT}/output/lib/libboost*.so.1.69.0 ${pack}/bin copy_file ${THIRDPARTY_ROOT}/output/lib/libhdfs* ${pack}/bin copy_file ${THIRDPARTY_ROOT}/output/lib/librocksdb.so.8 ${pack}/bin -copy_file ./scripts/sendmail.sh ${pack}/bin +copy_file ./admin_tools/config_hdfs.sh ${pack}/bin +copy_file ./admin_tools/sendmail.sh ${pack}/bin copy_file ./src/server/config.ini ${pack}/bin copy_file ./src/server/config.min.ini ${pack}/bin -copy_file ./scripts/config_hdfs.sh ${pack}/bin copy_file "$(get_stdcpp_lib $custom_gcc $separate_servers)" "${pack}/bin" diff --git a/scripts/pack_tools.sh b/build_tools/pack_tools.sh similarity index 94% rename from scripts/pack_tools.sh rename to build_tools/pack_tools.sh index de1d6cdf2c..24d78f2efe 100755 --- a/scripts/pack_tools.sh +++ b/build_tools/pack_tools.sh @@ -155,9 +155,14 @@ pack_tools_lib ssl $separate_servers chmod -x ${pack}/lib/* -mkdir -p ${pack}/scripts -copy_file ./scripts/* ${pack}/scripts/ -chmod +x ${pack}/scripts/*.sh +mkdir -p ${pack}/admin_tools +copy_file ./admin_tools/* ${pack}/admin_tools/ +chmod +x ${pack}/admin_tools/*.sh + +mkdir -p ${pack}/build_tools +copy_file ./build_tools/download_*.sh ${pack}/build_tools/ +copy_file ./build_tools/*_zk.sh ${pack}/build_tools/ +chmod +x ${pack}/build_tools/*.sh mkdir -p ${pack}/src/server copy_file ./src/server/*.ini ${pack}/src/server/ diff --git a/scripts/recompile_thrift.sh b/build_tools/recompile_thrift.sh similarity index 100% rename from scripts/recompile_thrift.sh rename to build_tools/recompile_thrift.sh diff --git a/scripts/redis_proto_check.py b/build_tools/redis_proto_check.py similarity index 100% rename from scripts/redis_proto_check.py rename to build_tools/redis_proto_check.py diff --git a/scripts/run-clang-format.py b/build_tools/run-clang-format.py similarity index 100% rename from scripts/run-clang-format.py rename to build_tools/run-clang-format.py diff --git a/scripts/start_zk.sh b/build_tools/start_zk.sh similarity index 100% rename from scripts/start_zk.sh rename to build_tools/start_zk.sh diff --git a/scripts/stop_zk.sh b/build_tools/stop_zk.sh similarity index 100% rename from scripts/stop_zk.sh rename to build_tools/stop_zk.sh diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile index 85137791bc..f080da3f03 100644 --- a/docker/thirdparties-bin/Dockerfile +++ b/docker/thirdparties-bin/Dockerfile @@ -35,8 +35,8 @@ RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} \ && unzip /root/thirdparties-src.zip -d . \ && cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=${ROCKSDB_PORTABLE} -DUSE_JEMALLOC=${USE_JEMALLOC} -B build/ . \ && cmake --build build/ -j $(($(nproc)/2+1)) \ - && ../scripts/download_hadoop.sh ${HADOOP_BIN_PATH} \ - && ../scripts/download_zk.sh ${ZOOKEEPER_BIN_PATH} \ + && ../build_tools/download_hadoop.sh ${HADOOP_BIN_PATH} \ + && ../build_tools/download_zk.sh ${ZOOKEEPER_BIN_PATH} \ && zip -r ~/thirdparties-bin.zip output/ build/Source/rocksdb/cmake build/Source/http-parser build/Source/hadoop build/Download/zookeeper ${HADOOP_BIN_PATH} ${ZOOKEEPER_BIN_PATH} \ && cd ~ \ && rm -rf incubator-pegasus; diff --git a/java-client/README.md b/java-client/README.md index 9cfdb9a703..c03a93541c 100644 --- a/java-client/README.md +++ b/java-client/README.md @@ -60,7 +60,7 @@ mvn clean package -Dtest=TestPing ### Install ``` -cd scripts && bash recompile_thrift.sh && cd - +cd scripts && bash download_thrift.sh && cd - mvn clean install -DskipTests ``` diff --git a/run.sh b/run.sh index 8c6048b019..f01b9c022f 100755 --- a/run.sh +++ b/run.sh @@ -324,8 +324,8 @@ function run_build() if [ ! -f "${ROOT}/src/common/serialization_helper/dsn.layer2_types.h" ]; then echo "Gen thrift" # TODO(yingchun): should be optimized - python3 $ROOT/scripts/compile_thrift.py - sh ${ROOT}/scripts/recompile_thrift.sh + python3 $ROOT/build_tools/compile_thrift.py + sh ${ROOT}/build_tools/recompile_thrift.sh fi if [ ! -d "$BUILD_DIR" ]; then @@ -656,7 +656,7 @@ function run_start_zk() fi fi - INSTALL_DIR="$INSTALL_DIR" PORT="$PORT" $ROOT/scripts/start_zk.sh + INSTALL_DIR="$INSTALL_DIR" PORT="$PORT" $ROOT/build_tools/start_zk.sh } ##################### @@ -693,7 +693,7 @@ function run_stop_zk() esac shift done - INSTALL_DIR="$INSTALL_DIR" $ROOT/scripts/stop_zk.sh + INSTALL_DIR="$INSTALL_DIR" $ROOT/build_tools/stop_zk.sh } ##################### @@ -730,7 +730,7 @@ function run_clear_zk() esac shift done - INSTALL_DIR="$INSTALL_DIR" $ROOT/scripts/clear_zk.sh + INSTALL_DIR="$INSTALL_DIR" $ROOT/build_tools/clear_zk.sh } ##################### @@ -853,7 +853,7 @@ function run_start_onebox() exit 1 fi - source "${ROOT}"/scripts/config_hdfs.sh + source "${ROOT}"/admin_tools/config_hdfs.sh if [ $USE_PRODUCT_CONFIG == "true" ]; then [ -z "${CONFIG_FILE}" ] && CONFIG_FILE=${ROOT}/src/server/config.ini [ ! -f "${CONFIG_FILE}" ] && { echo "${CONFIG_FILE} is not exist"; exit 1; } @@ -1097,7 +1097,7 @@ function run_start_onebox_instance() esac shift done - source "${ROOT}"/scripts/config_hdfs.sh + source "${ROOT}"/admin_tools/config_hdfs.sh if [ $META_ID = "0" -a $REPLICA_ID = "0" -a $COLLECTOR_ID = "0" ]; then echo "ERROR: no meta_id or replica_id or collector set" exit 1 @@ -1887,9 +1887,9 @@ function run_migrate_node() cd ${ROOT} echo "------------------------------" if [ "$CLUSTER" != "" ]; then - ./scripts/migrate_node.sh $CLUSTER $NODE "$APP" $TYPE + ./admin_tools/migrate_node.sh $CLUSTER $NODE "$APP" $TYPE else - ./scripts/migrate_node.sh $CONFIG $NODE "$APP" $TYPE -f + ./admin_tools/migrate_node.sh $CONFIG $NODE "$APP" $TYPE -f fi echo "------------------------------" echo @@ -1995,9 +1995,9 @@ function run_downgrade_node() cd ${ROOT} echo "------------------------------" if [ "$CLUSTER" != "" ]; then - ./scripts/downgrade_node.sh $CLUSTER $NODE "$APP" $TYPE + ./admin_tools/downgrade_node.sh $CLUSTER $NODE "$APP" $TYPE else - ./scripts/downgrade_node.sh $CONFIG $NODE "$APP" $TYPE -f + ./admin_tools/downgrade_node.sh $CONFIG $NODE "$APP" $TYPE -f fi echo "------------------------------" echo @@ -2105,19 +2105,19 @@ case $cmd in ;; pack_server) shift - PEGASUS_ROOT=$ROOT ./scripts/pack_server.sh $* + PEGASUS_ROOT=$ROOT ./build_tools/pack_server.sh $* ;; pack_client) shift - PEGASUS_ROOT=$ROOT ./scripts/pack_client.sh $* + PEGASUS_ROOT=$ROOT ./build_tools/pack_client.sh $* ;; pack_tools) shift - PEGASUS_ROOT=$ROOT ./scripts/pack_tools.sh $* + PEGASUS_ROOT=$ROOT ./build_tools/pack_tools.sh $* ;; bump_version) shift - ./scripts/bump_version.sh $* + ./build_tools/bump_version.sh $* ;; *) echo "ERROR: unknown command $cmd" diff --git a/src/replica/replica.h b/src/replica/replica.h index 4bb2ca5892..eb177f5454 100644 --- a/src/replica/replica.h +++ b/src/replica/replica.h @@ -532,8 +532,8 @@ class replica : public serverlet, public ref_counter, public replica_ba void update_restore_progress(uint64_t f_size); // Used for remote command - // TODO: remove this interface and only expose the http interface - // now this remote commend will be used by `scripts/pegasus_manual_compact.sh` + // TODO(clang-tidy): remove this interface and only expose the http interface + // now this remote commend will be used by `admin_tools/pegasus_manual_compact.sh` std::string query_manual_compact_state() const; manual_compaction_status::type get_manual_compact_status() const; From 9efe8d6b5463c05f1dba5617f656ac645d59ab4f Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Thu, 15 Aug 2024 11:01:15 +0800 Subject: [PATCH 12/15] fix(duplication): fix unexpected ERR_NOT_ENOUGH_MEMBER while checking if the remote table with only one replica is ready for duplication (#2093) https://github.com/apache/incubator-pegasus/issues/2092 While the remote table created with only one replica for the full duplication (copying checkpoints), the duplication would get stuck in `DS_APP` status forever due to `ERR_NOT_ENOUGH_MEMBER`. The reason is that the check would fail once there is no secondary replica. However, for single-replica tables, there is certainly not any secondary replica. --- src/meta/duplication/duplication_info.cpp | 1 + .../duplication/meta_duplication_service.cpp | 267 +++++++++++------- .../test/meta_duplication_service_test.cpp | 91 +++++- src/meta/test/meta_test_base.cpp | 6 +- src/meta/test/meta_test_base.h | 9 +- 5 files changed, 256 insertions(+), 118 deletions(-) diff --git a/src/meta/duplication/duplication_info.cpp b/src/meta/duplication/duplication_info.cpp index 4036cf2936..7f7bb62b69 100644 --- a/src/meta/duplication/duplication_info.cpp +++ b/src/meta/duplication/duplication_info.cpp @@ -173,6 +173,7 @@ void duplication_info::persist_status() _is_altering = false; _status = _next_status; + // Now we don't know what exactly is the next status, thus set DS_INIT temporarily. _next_status = duplication_status::DS_INIT; _fail_mode = _next_fail_mode; } diff --git a/src/meta/duplication/meta_duplication_service.cpp b/src/meta/duplication/meta_duplication_service.cpp index a2adf29469..76472a04d8 100644 --- a/src/meta/duplication/meta_duplication_service.cpp +++ b/src/meta/duplication/meta_duplication_service.cpp @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include #include +#include #include #include +#include #include +#include #include -#include #include "common//duplication_common.h" #include "common/common.h" #include "common/gpid.h" @@ -54,6 +57,7 @@ #include "utils/fmt_logging.h" #include "utils/ports.h" #include "utils/string_conv.h" +#include "utils/strings.h" #include "utils/zlocks.h" DSN_DECLARE_bool(dup_ignore_other_cluster_ids); @@ -336,10 +340,12 @@ void meta_duplication_service::do_add_duplication(std::shared_ptr &ap std::queue nodes({get_duplication_path(*app), std::to_string(dup->id)}); _meta_svc->get_meta_storage()->create_node_recursively( std::move(nodes), std::move(value), [app, this, dup, rpc, resp_err]() mutable { - LOG_INFO("[{}] add duplication successfully [app_name: {}, remote_cluster_name: {}]", + LOG_INFO("[{}] add duplication successfully [app_name: {}, remote_cluster_name: {}, " + "remote_app_name: {}]", dup->log_prefix(), app->app_name, - dup->remote_cluster_name); + dup->remote_cluster_name, + dup->remote_app_name); // The duplication starts only after it's been persisted. dup->persist_status(); @@ -484,45 +490,103 @@ void meta_duplication_service::create_follower_app_for_duplication( dsn::message_ex *msg = dsn::message_ex::create_request(RPC_CM_CREATE_APP); dsn::marshall(msg, request); - rpc::call(dsn::dns_resolver::instance().resolve_address(meta_servers), - msg, - _meta_svc->tracker(), - [=](error_code err, configuration_create_app_response &&resp) mutable { - FAIL_POINT_INJECT_NOT_RETURN_F("update_app_request_ok", - [&](std::string_view s) -> void { err = ERR_OK; }); - error_code create_err = err == ERR_OK ? resp.err : err; - error_code update_err = ERR_NO_NEED_OPERATE; - - FAIL_POINT_INJECT_NOT_RETURN_F( - "persist_dup_status_failed", - [&](std::string_view s) -> void { create_err = ERR_OK; }); - if (create_err == ERR_OK) { - update_err = dup->alter_status(duplication_status::DS_APP); - } - - FAIL_POINT_INJECT_F("persist_dup_status_failed", - [&](std::string_view s) -> void { return; }); - if (update_err == ERR_OK) { - blob value = dup->to_json_blob(); - // Note: this function is `async`, it may not be persisted completed - // after executing, now using `_is_altering` to judge whether `updating` or - // `completed`, if `_is_altering`, dup->alter_status() will return `ERR_BUSY` - _meta_svc->get_meta_storage()->set_data(std::string(dup->store_path), - std::move(value), - [=]() { dup->persist_status(); }); - } else { - LOG_ERROR( - "created follower app[{}.{}] to trigger duplicate checkpoint failed: " + rpc::call( + dsn::dns_resolver::instance().resolve_address(meta_servers), + msg, + _meta_svc->tracker(), + [dup, this](error_code err, configuration_create_app_response &&resp) mutable { + FAIL_POINT_INJECT_NOT_RETURN_F("update_app_request_ok", + [&err](std::string_view) -> void { err = ERR_OK; }); + + error_code create_err = err == ERR_OK ? resp.err : err; + FAIL_POINT_INJECT_NOT_RETURN_F( + "persist_dup_status_failed", + [&create_err](std::string_view) -> void { create_err = ERR_OK; }); + + error_code update_err = ERR_NO_NEED_OPERATE; + if (create_err == ERR_OK) { + update_err = dup->alter_status(duplication_status::DS_APP); + } + + FAIL_POINT_INJECT_F("persist_dup_status_failed", + [](std::string_view) -> void { return; }); + + if (update_err == ERR_OK) { + blob value = dup->to_json_blob(); + // Note: this function is `async`, it may not be persisted completed + // after executing, now using `_is_altering` to judge whether `updating` or + // `completed`, if `_is_altering`, dup->alter_status() will return `ERR_BUSY` + _meta_svc->get_meta_storage()->set_data(std::string(dup->store_path), + std::move(value), + [dup]() { dup->persist_status(); }); + } else { + LOG_ERROR("create follower app[{}.{}] to trigger duplicate checkpoint failed: " "duplication_status = {}, create_err = {}, update_err = {}", dup->remote_cluster_name, - dup->app_name, + dup->remote_app_name, duplication_status_to_string(dup->status()), create_err, update_err); - } - }); + } + }); +} + +namespace { + +// The format of `replica_state_str` is ",,": +// +// : bool, true means if the address of primary replica is valid, +// otherwise false. +// : uint32_t, the number of secondaries whose address are valid. +// : uint32_t, the number of secondaries whose address are invalid. +void mock_create_app(std::string_view replica_state_str, + const std::shared_ptr &dup, + dsn::query_cfg_response &resp, + dsn::error_code &err) +{ + std::vector strs; + utils::split_args(replica_state_str.data(), strs, ','); + CHECK_EQ(strs.size(), 3); + + bool has_primary = 0; + CHECK_TRUE(buf2bool(strs[0], has_primary)); + + uint32_t valid_secondaries = 0; + CHECK_TRUE(buf2uint32(strs[1], valid_secondaries)); + + uint32_t invalid_secondaries = 0; + CHECK_TRUE(buf2uint32(strs[2], invalid_secondaries)); + + std::vector nodes; + if (has_primary) { + nodes.emplace_back("localhost", 34801); + } else { + nodes.emplace_back(); + } + for (uint32_t i = 0; i < valid_secondaries; ++i) { + nodes.emplace_back("localhost", static_cast(34802 + i)); + } + for (uint32_t i = 0; i < invalid_secondaries; ++i) { + nodes.emplace_back(); + } + + for (int32_t i = 0; i < dup->partition_count; ++i) { + partition_configuration pc; + pc.max_replica_count = dup->remote_replica_count; + + SET_IP_AND_HOST_PORT_BY_DNS(pc, primary, nodes[0]); + for (size_t j = 1; j < nodes.size(); ++j) { + ADD_IP_AND_HOST_PORT_BY_DNS(pc, secondaries, nodes[j]); + } + + resp.partitions.push_back(std::move(pc)); + } + + err = ERR_OK; } +} // anonymous namespace + void meta_duplication_service::check_follower_app_if_create_completed( const std::shared_ptr &dup) { @@ -535,79 +599,78 @@ void meta_duplication_service::check_follower_app_if_create_completed( dsn::message_ex *msg = dsn::message_ex::create_request(RPC_CM_QUERY_PARTITION_CONFIG_BY_INDEX); dsn::marshall(msg, meta_config_request); - rpc::call(dsn::dns_resolver::instance().resolve_address(meta_servers), - msg, - _meta_svc->tracker(), - [=](error_code err, query_cfg_response &&resp) mutable { - FAIL_POINT_INJECT_NOT_RETURN_F("create_app_ok", [&](std::string_view s) -> void { - err = ERR_OK; - int count = dup->partition_count; - while (count-- > 0) { - const host_port primary("localhost", 34801); - const host_port secondary1("localhost", 34802); - const host_port secondary2("localhost", 34803); - - partition_configuration pc; - SET_IP_AND_HOST_PORT_BY_DNS(pc, primary, primary); - SET_IPS_AND_HOST_PORTS_BY_DNS(pc, secondaries, secondary1, secondary2); - resp.partitions.emplace_back(pc); - } - }); - - // - ERR_INCONSISTENT_STATE: partition count of response isn't equal with local - // - ERR_INACTIVE_STATE: the follower table hasn't been healthy - error_code query_err = err == ERR_OK ? resp.err : err; - if (query_err == ERR_OK) { - if (resp.partitions.size() != dup->partition_count) { - query_err = ERR_INCONSISTENT_STATE; - } else { - for (const auto &pc : resp.partitions) { - if (!pc.hp_primary) { - query_err = ERR_INACTIVE_STATE; - break; - } - - if (pc.hp_secondaries.empty()) { - query_err = ERR_NOT_ENOUGH_MEMBER; - break; - } - - for (const auto &secondary : pc.hp_secondaries) { - if (!secondary) { - query_err = ERR_INACTIVE_STATE; - break; - } - } - } - } - } - - error_code update_err = ERR_NO_NEED_OPERATE; - if (query_err == ERR_OK) { - update_err = dup->alter_status(duplication_status::DS_LOG); - } - - FAIL_POINT_INJECT_F("persist_dup_status_failed", - [&](std::string_view s) -> void { return; }); - if (update_err == ERR_OK) { - blob value = dup->to_json_blob(); - // Note: this function is `async`, it may not be persisted completed - // after executing, now using `_is_altering` to judge whether `updating` or - // `completed`, if `_is_altering`, dup->alter_status() will return `ERR_BUSY` - _meta_svc->get_meta_storage()->set_data(std::string(dup->store_path), - std::move(value), - [dup]() { dup->persist_status(); }); - } else { - LOG_ERROR( - "query follower app[{}.{}] replica configuration completed, result: " + rpc::call( + dsn::dns_resolver::instance().resolve_address(meta_servers), + msg, + _meta_svc->tracker(), + [dup, this](error_code err, query_cfg_response &&resp) mutable { + FAIL_POINT_INJECT_NOT_RETURN_F( + "create_app_ok", + std::bind( + mock_create_app, std::placeholders::_1, dup, std::ref(resp), std::ref(err))); + + // - ERR_INCONSISTENT_STATE: partition count of response isn't equal with local + // - ERR_INACTIVE_STATE: the follower table hasn't been healthy + error_code query_err = err == ERR_OK ? resp.err : err; + if (query_err == ERR_OK) { + if (resp.partitions.size() != dup->partition_count) { + query_err = ERR_INCONSISTENT_STATE; + } else { + for (const auto &pc : resp.partitions) { + if (!pc.hp_primary) { + // Fail once the primary replica is unavailable. + query_err = ERR_INACTIVE_STATE; + break; + } + + // Once replica count is more than 1, at least one secondary replica + // is required. + if (1 + pc.hp_secondaries.size() < pc.max_replica_count && + pc.hp_secondaries.empty()) { + query_err = ERR_NOT_ENOUGH_MEMBER; + break; + } + + for (const auto &secondary : pc.hp_secondaries) { + if (!secondary) { + // Fail once any secondary replica is unavailable. + query_err = ERR_INACTIVE_STATE; + break; + } + } + if (query_err != ERR_OK) { + break; + } + } + } + } + + error_code update_err = ERR_NO_NEED_OPERATE; + if (query_err == ERR_OK) { + update_err = dup->alter_status(duplication_status::DS_LOG); + } + + FAIL_POINT_INJECT_F("persist_dup_status_failed", + [](std::string_view) -> void { return; }); + + if (update_err == ERR_OK) { + blob value = dup->to_json_blob(); + // Note: this function is `async`, it may not be persisted completed + // after executing, now using `_is_altering` to judge whether `updating` or + // `completed`, if `_is_altering`, dup->alter_status() will return `ERR_BUSY` + _meta_svc->get_meta_storage()->set_data(std::string(dup->store_path), + std::move(value), + [dup]() { dup->persist_status(); }); + } else { + LOG_ERROR("query follower app[{}.{}] replica configuration completed, result: " "duplication_status = {}, query_err = {}, update_err = {}", dup->remote_cluster_name, dup->remote_app_name, duplication_status_to_string(dup->status()), query_err, update_err); - } - }); + } + }); } void meta_duplication_service::do_update_partition_confirmed( diff --git a/src/meta/test/meta_duplication_service_test.cpp b/src/meta/test/meta_duplication_service_test.cpp index 54a58af87d..8fbcddf09f 100644 --- a/src/meta/test/meta_duplication_service_test.cpp +++ b/src/meta/test/meta_duplication_service_test.cpp @@ -119,6 +119,12 @@ class meta_duplication_service_test : public meta_test_base return create_dup(app_name, remote_cluster, app_name, remote_replica_count); } + duplication_add_response create_dup(const std::string &app_name, + const int32_t remote_replica_count) + { + return create_dup(app_name, kTestRemoteClusterName, remote_replica_count); + } + duplication_add_response create_dup(const std::string &app_name) { return create_dup(app_name, kTestRemoteClusterName, kTestRemoteReplicaCount); @@ -963,39 +969,99 @@ TEST_F(meta_duplication_service_test, check_follower_app_if_create_completed) { struct test_case { + int32_t remote_replica_count; std::vector fail_cfg_name; std::vector fail_cfg_action; bool is_altering; duplication_status::type cur_status; duplication_status::type next_status; - } test_cases[] = {{{"create_app_ok"}, - {"void()"}, + } test_cases[] = {// 3 remote replicas with both primary and secondaries valid. + {3, + {"create_app_ok"}, + {"void(true,2,0)"}, false, duplication_status::DS_LOG, duplication_status::DS_INIT}, - // the case just `palace holder`, actually - // `check_follower_app_if_create_completed` is failed by default in unit test - {{"create_app_failed"}, + // 3 remote replicas with primary invalid and all secondaries valid. + {3, + {"create_app_ok"}, + {"void(false,2,0)"}, + false, + duplication_status::DS_APP, + duplication_status::DS_INIT}, + // 3 remote replicas with primary valid and only one secondary present + // and valid. + {3, + {"create_app_ok"}, + {"void(true,1,0)"}, + false, + duplication_status::DS_LOG, + duplication_status::DS_INIT}, + // 3 remote replicas with primary valid and one secondary invalid. + {3, + {"create_app_ok"}, + {"void(true,1,1)"}, + false, + duplication_status::DS_APP, + duplication_status::DS_INIT}, + // 3 remote replicas with primary valid and only one secondary present + // and invalid. + {3, + {"create_app_ok"}, + {"void(true,0,1)"}, + false, + duplication_status::DS_APP, + duplication_status::DS_INIT}, + // 3 remote replicas with primary valid and both secondaries absent. + {3, + {"create_app_ok"}, + {"void(true,0,0)"}, + false, + duplication_status::DS_APP, + duplication_status::DS_INIT}, + // 1 remote replicas with primary valid. + {1, + {"create_app_ok"}, + {"void(true,0,0)"}, + false, + duplication_status::DS_LOG, + duplication_status::DS_INIT}, + // 1 remote replicas with primary invalid. + {1, + {"create_app_ok"}, + {"void(false,0,0)"}, + false, + duplication_status::DS_APP, + duplication_status::DS_INIT}, + // The case is just a "palace holder", actually + // `check_follower_app_if_create_completed` would fail by default + // in unit test. + {3, + {"create_app_failed"}, {"off()"}, false, duplication_status::DS_APP, duplication_status::DS_INIT}, - {{"create_app_ok", "persist_dup_status_failed"}, - {"void()", "return()"}, + {3, + {"create_app_ok", "persist_dup_status_failed"}, + {"void(true,2,0)", "return()"}, true, duplication_status::DS_APP, duplication_status::DS_LOG}}; + size_t i = 0; for (const auto &test : test_cases) { - const auto test_app = fmt::format("{}{}", test.fail_cfg_name[0], test.fail_cfg_name.size()); - create_app(test_app); - auto app = find_app(test_app); + const auto &app_name = fmt::format("check_follower_app_if_create_completed_test_{}", i++); + create_app(app_name); - auto dup_add_resp = create_dup(test_app); + auto app = find_app(app_name); + auto dup_add_resp = create_dup(app_name, test.remote_replica_count); auto dup = app->duplications[dup_add_resp.dupid]; + // 'check_follower_app_if_create_completed' must execute under duplication_status::DS_APP, - // so force update it + // so force update it. force_update_dup_status(dup, duplication_status::DS_APP); + fail::setup(); for (int i = 0; i < test.fail_cfg_name.size(); i++) { fail::cfg(test.fail_cfg_name[i], test.fail_cfg_action[i]); @@ -1003,6 +1069,7 @@ TEST_F(meta_duplication_service_test, check_follower_app_if_create_completed) check_follower_app_if_create_completed(dup); wait_all(); fail::teardown(); + ASSERT_EQ(dup->is_altering(), test.is_altering); ASSERT_EQ(next_status(dup), test.next_status); ASSERT_EQ(dup->status(), test.cur_status); diff --git a/src/meta/test/meta_test_base.cpp b/src/meta/test/meta_test_base.cpp index 965939d566..49cf729d12 100644 --- a/src/meta/test/meta_test_base.cpp +++ b/src/meta/test/meta_test_base.cpp @@ -182,14 +182,16 @@ std::vector meta_test_base::ensure_enough_alive_nodes(int min_node_co return nodes; } -void meta_test_base::create_app(const std::string &name, uint32_t partition_count) +void meta_test_base::create_app(const std::string &name, + int32_t partition_count, + int32_t replica_count) { configuration_create_app_request req; configuration_create_app_response resp; req.app_name = name; req.options.app_type = "simple_kv"; req.options.partition_count = partition_count; - req.options.replica_count = 3; + req.options.replica_count = replica_count; req.options.success_if_exist = false; req.options.is_stateful = true; req.options.envs["value_version"] = "1"; diff --git a/src/meta/test/meta_test_base.h b/src/meta/test/meta_test_base.h index 2f27e883a4..25f3eed42f 100644 --- a/src/meta/test/meta_test_base.h +++ b/src/meta/test/meta_test_base.h @@ -57,8 +57,13 @@ class meta_test_base : public testing::Test std::vector ensure_enough_alive_nodes(int min_node_count); - // create an app for test with specified name and specified partition count - void create_app(const std::string &name, uint32_t partition_count); + // Create an app for test with specified name, partition count and replica count. + void create_app(const std::string &name, int32_t partition_count, int32_t replica_count); + + void create_app(const std::string &name, int32_t partition_count) + { + create_app(name, partition_count, 3); + } void create_app(const std::string &name) { create_app(name, 8); } From e68c2b9b95082d265f99f2dd2ff04148513d847e Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Thu, 15 Aug 2024 11:02:23 +0800 Subject: [PATCH 13/15] fix(duplication): fix the dangling pointer after `blob` object is moved (#2088) https://github.com/apache/incubator-pegasus/issues/2089 Refactor `blob` object to avoid the dangling pointer leading to heap-use-after-free error which happened in #2089 after `blob` object is moved. Also refactor and add some tests for `blob` and `mutation_batch`. --- .clang-tidy | 2 +- build_tools/clang_tidy.py | 2 +- src/replica/duplication/mutation_batch.cpp | 18 +- .../duplication/test/duplication_test_base.h | 7 +- .../test/load_from_private_log_test.cpp | 13 +- .../duplication/test/mutation_batch_test.cpp | 160 +++++++++++------- src/replica/mutation_log_replay.cpp | 70 +++++--- src/replica/test/log_block_test.cpp | 6 +- src/replica/test/log_file_test.cpp | 3 +- src/replica/test/mutation_log_test.cpp | 10 +- src/replica/test/replica_test_base.h | 8 +- src/runtime/rpc/network.sim.cpp | 2 +- src/utils/binary_reader.cpp | 4 +- src/utils/binary_reader.h | 8 +- src/utils/blob.h | 120 +++++++++---- src/utils/test/blob_test.cpp | 126 ++++++++++++++ 16 files changed, 397 insertions(+), 162 deletions(-) create mode 100644 src/utils/test/blob_test.cpp diff --git a/.clang-tidy b/.clang-tidy index a2a2c95c1b..2e072d7d84 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -18,7 +18,7 @@ # https://releases.llvm.org/14.0.0/tools/clang/tools/extra/docs/clang-tidy/index.html CheckOptions: [] -Checks: 'abseil-*,boost-*,bugprone-*,cert-*,clang-analyzer-*,concurrency-*,cppcoreguidelines-*,darwin-*,fuchsia-*,google-*,hicpp-*,linuxkernel-*,llvm-*,misc-*,modernize-*,performance-*,portability-*,readability-*' +Checks: 'abseil-*,boost-*,bugprone-*,cert-*,clang-analyzer-*,concurrency-*,cppcoreguidelines-*,darwin-*,fuchsia-*,google-*,hicpp-*,linuxkernel-*,llvm-*,misc-*,modernize-*,performance-*,portability-*,readability-*,-cppcoreguidelines-pro-type-union-access,-llvm-include-order,-modernize-use-trailing-return-type,-cppcoreguidelines-avoid-non-const-global-variables,-fuchsia-statically-constructed-objects,-fuchsia-overloaded-operator,-bugprone-easily-swappable-parameters,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-hicpp-no-array-decay,-hicpp-named-parameter,-readability-named-parameter,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-readability-function-cognitive-complexity,-cert-err58-cpp,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-cppcoreguidelines-owning-memory,-cppcoreguidelines-pro-type-const-cast,-readability-identifier-length,-fuchsia-default-arguments-calls,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers' ExtraArgs: ExtraArgsBefore: [] FormatStyle: none diff --git a/build_tools/clang_tidy.py b/build_tools/clang_tidy.py index 4071497659..d4c8964a0c 100755 --- a/build_tools/clang_tidy.py +++ b/build_tools/clang_tidy.py @@ -60,7 +60,7 @@ def tidy_on_path(path): "clang-tidy", "-p0", "-path", BUILD_PATH, - "-checks=-cppcoreguidelines-pro-type-union-access,-modernize-use-trailing-return-type", + "-checks=-cppcoreguidelines-pro-type-union-access,-llvm-include-order,-modernize-use-trailing-return-type,-cppcoreguidelines-avoid-non-const-global-variables,-fuchsia-statically-constructed-objects,-fuchsia-overloaded-operator,-bugprone-easily-swappable-parameters,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-hicpp-no-array-decay,-hicpp-named-parameter,-readability-named-parameter,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-readability-function-cognitive-complexity,-cert-err58-cpp,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-cppcoreguidelines-owning-memory,-cppcoreguidelines-pro-type-const-cast,-readability-identifier-length,-fuchsia-default-arguments-calls,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers", "-extra-arg=-language=c++", "-extra-arg=-std=c++17", "-extra-arg=-Ithirdparty/output/include"] diff --git a/src/replica/duplication/mutation_batch.cpp b/src/replica/duplication/mutation_batch.cpp index 9a6493cb99..efcd1eeac3 100644 --- a/src/replica/duplication/mutation_batch.cpp +++ b/src/replica/duplication/mutation_batch.cpp @@ -17,23 +17,24 @@ #include #include +#include #include #include #include -#include #include "common/replication.codes.h" #include "consensus_types.h" #include "metadata_types.h" #include "mutation_batch.h" -#include "replica_duplicator.h" #include "replica/replica.h" +#include "replica_duplicator.h" #include "runtime/task/task_code.h" #include "runtime/task/task_spec.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" #include "utils/error_code.h" #include "utils/fmt_logging.h" +#include "utils/ports.h" METRIC_DEFINE_gauge_int64(replica, dup_recent_lost_mutations, @@ -196,10 +197,19 @@ void mutation_batch::add_mutation_if_valid(mutation_ptr &mu, decree start_decree } blob bb; - if (update.data.buffer() != nullptr) { + if (update.data.buffer()) { + // ATTENTION: instead of copy, move could optimize the performance. However, this + // would nullify the elements of mu->data.updates. bb = std::move(update.data); } else { - bb = blob::create_from_bytes(update.data.data(), update.data.length()); + // TODO(wangdan): if update.data.buffer() is nullptr, the blob object must have + // been used as `string_view`. + // + // Once `string_view` function is removed from blob, consider dropping following + // statements. + if (dsn_likely(update.data.data() != nullptr && !update.data.empty())) { + bb = blob::create_from_bytes(update.data.data(), update.data.length()); + } } _total_bytes += bb.length(); diff --git a/src/replica/duplication/test/duplication_test_base.h b/src/replica/duplication/test/duplication_test_base.h index d4dacf9b9a..49101341ed 100644 --- a/src/replica/duplication/test/duplication_test_base.h +++ b/src/replica/duplication/test/duplication_test_base.h @@ -75,16 +75,15 @@ class duplication_test_base : public replica_test_base return log_file_map; } - mutation_ptr create_test_mutation(int64_t decree, - int64_t last_committed_decree, - const std::string &data) override + mutation_ptr + create_test_mutation(int64_t decree, int64_t last_committed_decree, const char *data) override { auto mut = replica_test_base::create_test_mutation(decree, last_committed_decree, data); mut->data.updates[0].code = RPC_DUPLICATION_IDEMPOTENT_WRITE; // must be idempotent write return mut; } - mutation_ptr create_test_mutation(int64_t decree, const std::string &data) override + mutation_ptr create_test_mutation(int64_t decree, const char *data) override { return duplication_test_base::create_test_mutation(decree, decree - 1, data); } diff --git a/src/replica/duplication/test/load_from_private_log_test.cpp b/src/replica/duplication/test/load_from_private_log_test.cpp index 0d7141745d..f2eaf253bf 100644 --- a/src/replica/duplication/test/load_from_private_log_test.cpp +++ b/src/replica/duplication/test/load_from_private_log_test.cpp @@ -92,8 +92,7 @@ class load_from_private_log_test : public duplication_test_base // each round mlog will replay the former logs, and create new file mutation_log_ptr mlog = create_private_log(); for (int i = 1; i <= 10; i++) { - std::string msg = "hello!"; - mutation_ptr mu = create_test_mutation(10 * f + i, msg); + auto mu = create_test_mutation(10 * f + i, "hello!"); mlog->append(mu, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr, 0); } mlog->tracker()->wait_outstanding_tasks(); @@ -149,9 +148,8 @@ class load_from_private_log_test : public duplication_test_base auto reserved_plog_force_flush = FLAGS_plog_force_flush; FLAGS_plog_force_flush = true; for (int i = decree_start; i <= num_entries + decree_start; i++) { - std::string msg = "hello!"; // decree - last_commit_decree = 1 by default - mutation_ptr mu = create_test_mutation(i, msg); + auto mu = create_test_mutation(i, "hello!"); // mock the last_commit_decree of first mu equal with `last_commit_decree_start` if (i == decree_start) { mu->data.header.last_committed_decree = last_commit_decree_start; @@ -160,7 +158,7 @@ class load_from_private_log_test : public duplication_test_base } // commit the last entry - mutation_ptr mu = create_test_mutation(decree_start + num_entries + 1, "hello!"); + auto mu = create_test_mutation(decree_start + num_entries + 1, "hello!"); mlog->append(mu, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr, 0); FLAGS_plog_force_flush = reserved_plog_force_flush; @@ -362,13 +360,12 @@ TEST_P(load_from_private_log_test, ignore_useless) int num_entries = 100; for (int i = 1; i <= num_entries; i++) { - std::string msg = "hello!"; - mutation_ptr mu = create_test_mutation(i, msg); + auto mu = create_test_mutation(i, "hello!"); mlog->append(mu, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr, 0); } // commit the last entry - mutation_ptr mu = create_test_mutation(1 + num_entries, "hello!"); + auto mu = create_test_mutation(1 + num_entries, "hello!"); mlog->append(mu, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr, 0); mlog->close(); diff --git a/src/replica/duplication/test/mutation_batch_test.cpp b/src/replica/duplication/test/mutation_batch_test.cpp index dd5c277387..c865de20b0 100644 --- a/src/replica/duplication/test/mutation_batch_test.cpp +++ b/src/replica/duplication/test/mutation_batch_test.cpp @@ -31,138 +31,178 @@ #include "gtest/gtest.h" #include "replica/duplication/mutation_batch.h" #include "replica/duplication/mutation_duplicator.h" +#include "replica/duplication/replica_duplicator.h" #include "replica/mutation.h" #include "replica/prepare_list.h" #include "runtime/task/task_code.h" #include "utils/autoref_ptr.h" +#include "utils/blob.h" -namespace dsn { -namespace replication { +namespace dsn::replication { class mutation_batch_test : public duplication_test_base { public: - void reset_buffer(const decree last_commit, - const decree start, - const decree end, - mutation_batch &batcher) + mutation_batch_test() : _duplicator(create_test_duplicator(0)), _batcher(_duplicator.get()) {} + + void reset_buffer(const decree last_commit, const decree start, const decree end) { - batcher._mutation_buffer->reset(last_commit); - batcher._mutation_buffer->_start_decree = start; - batcher._mutation_buffer->_end_decree = end; + _batcher._mutation_buffer->reset(last_commit); + _batcher._mutation_buffer->_start_decree = start; + _batcher._mutation_buffer->_end_decree = end; } - void commit_buffer(const decree current_decree, mutation_batch &batcher) + void commit_buffer(const decree current_decree) { - batcher._mutation_buffer->commit(current_decree, COMMIT_TO_DECREE_HARD); + _batcher._mutation_buffer->commit(current_decree, COMMIT_TO_DECREE_HARD); } - void check_mutation_contents(const std::set &expected_mutations, - mutation_batch &batcher) + void check_mutation_contents(const std::vector &expected_mutations) { - const auto all_mutations = batcher.move_all_mutations(); + const auto all_mutations = _batcher.move_all_mutations(); - std::set actual_mutations; + std::vector actual_mutations; std::transform(all_mutations.begin(), all_mutations.end(), - std::inserter(actual_mutations, actual_mutations.end()), + std::back_inserter(actual_mutations), [](const mutation_tuple &tuple) { return std::get<2>(tuple).to_string(); }); ASSERT_EQ(expected_mutations, actual_mutations); } + + std::unique_ptr _duplicator; + mutation_batch _batcher; }; INSTANTIATE_TEST_SUITE_P(, mutation_batch_test, ::testing::Values(false, true)); TEST_P(mutation_batch_test, prepare_mutation) { - auto duplicator = create_test_duplicator(0); - mutation_batch batcher(duplicator.get()); - auto mu1 = create_test_mutation(1, 0, "first mutation"); set_last_applied_decree(1); - ASSERT_TRUE(batcher.add(mu1)); - ASSERT_EQ(1, batcher.last_decree()); + ASSERT_TRUE(_batcher.add(mu1)); + ASSERT_EQ(1, _batcher.last_decree()); auto mu2 = create_test_mutation(2, 1, "abcde"); set_last_applied_decree(2); - ASSERT_TRUE(batcher.add(mu2)); - ASSERT_EQ(2, batcher.last_decree()); + ASSERT_TRUE(_batcher.add(mu2)); + ASSERT_EQ(2, _batcher.last_decree()); auto mu3 = create_test_mutation(3, 2, "hello world"); - ASSERT_TRUE(batcher.add(mu3)); + ASSERT_TRUE(_batcher.add(mu3)); // The last decree has not been updated. - ASSERT_EQ(2, batcher.last_decree()); + ASSERT_EQ(2, _batcher.last_decree()); auto mu4 = create_test_mutation(4, 2, "foo bar"); - ASSERT_TRUE(batcher.add(mu4)); - ASSERT_EQ(2, batcher.last_decree()); + ASSERT_TRUE(_batcher.add(mu4)); + ASSERT_EQ(2, _batcher.last_decree()); // The committed mutation would be ignored. auto mu2_another = create_test_mutation(2, 1, "another second mutation"); - ASSERT_TRUE(batcher.add(mu2_another)); - ASSERT_EQ(2, batcher.last_decree()); + ASSERT_TRUE(_batcher.add(mu2_another)); + ASSERT_EQ(2, _batcher.last_decree()); // The mutation with duplicate decree would be ignored. auto mu3_another = create_test_mutation(3, 2, "123 xyz"); - ASSERT_TRUE(batcher.add(mu3_another)); - ASSERT_EQ(2, batcher.last_decree()); + ASSERT_TRUE(_batcher.add(mu3_another)); + ASSERT_EQ(2, _batcher.last_decree()); auto mu5 = create_test_mutation(5, 2, "5th mutation"); set_last_applied_decree(5); - ASSERT_TRUE(batcher.add(mu5)); - ASSERT_EQ(5, batcher.last_decree()); + ASSERT_TRUE(_batcher.add(mu5)); + ASSERT_EQ(5, _batcher.last_decree()); + + check_mutation_contents({"first mutation", "abcde", "hello world", "foo bar", "5th mutation"}); +} + +TEST_P(mutation_batch_test, add_null_mutation) +{ + auto mu = create_test_mutation(1, nullptr); + _batcher.add_mutation_if_valid(mu, 0); + + check_mutation_contents({""}); +} + +TEST_P(mutation_batch_test, add_empty_mutation) +{ + auto mu = create_test_mutation(1, ""); + _batcher.add_mutation_if_valid(mu, 0); + + check_mutation_contents({""}); +} + +// TODO(wangdan): once `string_view` function is removed from blob, drop this test. +TEST_P(mutation_batch_test, add_string_view_mutation) +{ + auto mu = create_test_mutation(1, nullptr); + const std::string data("hello"); + mu->data.updates.back().data = blob(data.data(), 0, data.size()); + _batcher.add_mutation_if_valid(mu, 0); - check_mutation_contents({"first mutation", "abcde", "hello world", "foo bar", "5th mutation"}, - batcher); + check_mutation_contents({"hello"}); } -TEST_P(mutation_batch_test, add_mutation_if_valid) +TEST_P(mutation_batch_test, add_a_valid_mutation) { - auto duplicator = create_test_duplicator(0); - mutation_batch batcher(duplicator.get()); + auto mu = create_test_mutation(1, "hello"); + _batcher.add_mutation_if_valid(mu, 0); + + check_mutation_contents({"hello"}); +} +TEST_P(mutation_batch_test, add_multiple_valid_mutations) +{ + // The mutation could not be reused, since in mutation_batch::add_mutation_if_valid + // the elements of mutation::data::updates would be moved and nullified. auto mu1 = create_test_mutation(1, "hello"); - batcher.add_mutation_if_valid(mu1, 0); - check_mutation_contents({"hello"}, batcher); + _batcher.add_mutation_if_valid(mu1, 0); auto mu2 = create_test_mutation(2, "world"); - batcher.add_mutation_if_valid(mu2, 0); - check_mutation_contents({"world"}, batcher); + _batcher.add_mutation_if_valid(mu2, 2); + + auto mu3 = create_test_mutation(3, "hi"); + _batcher.add_mutation_if_valid(mu3, 2); + + check_mutation_contents({"hello", "world", "hi"}); +} + +TEST_P(mutation_batch_test, add_invalid_mutation) +{ + auto mu2 = create_test_mutation(2, "world"); + _batcher.add_mutation_if_valid(mu2, 2); // mu1 would be ignored, since its decree is less than the start decree. - batcher.add_mutation_if_valid(mu1, 2); - batcher.add_mutation_if_valid(mu2, 2); + auto mu1 = create_test_mutation(1, "hello"); + _batcher.add_mutation_if_valid(mu1, 2); + + auto mu3 = create_test_mutation(3, "hi"); + _batcher.add_mutation_if_valid(mu3, 2); - auto mu3 = create_test_mutation(1, "hi"); - batcher.add_mutation_if_valid(mu3, 1); - check_mutation_contents({"hi", "world"}, batcher); + auto mu4 = create_test_mutation(1, "ok"); + _batcher.add_mutation_if_valid(mu4, 1); + + // "ok" would be the first, since its timestamp (i.e. decree in create_test_mutation) + // is the smallest. + check_mutation_contents({"ok", "world", "hi"}); } TEST_P(mutation_batch_test, ignore_non_idempotent_write) { - auto duplicator = create_test_duplicator(0); - mutation_batch batcher(duplicator.get()); - auto mu = create_test_mutation(1, "hello"); mu->data.updates[0].code = RPC_DUPLICATION_NON_IDEMPOTENT_WRITE; - batcher.add_mutation_if_valid(mu, 0); - check_mutation_contents({}, batcher); + _batcher.add_mutation_if_valid(mu, 0); + check_mutation_contents({}); } TEST_P(mutation_batch_test, mutation_buffer_commit) { - auto duplicator = create_test_duplicator(0); - mutation_batch batcher(duplicator.get()); - // Mock mutation_buffer[last=10, start=15, end=20], last + 1(next commit decree) is out of // [start~end], then last would become min_decree() - 1, see mutation_buffer::commit() for // details. - reset_buffer(10, 15, 20, batcher); - commit_buffer(15, batcher); - ASSERT_EQ(14, batcher.last_decree()); + reset_buffer(10, 15, 20); + commit_buffer(15); + ASSERT_EQ(14, _batcher.last_decree()); } -} // namespace replication -} // namespace dsn +} // namespace dsn::replication diff --git a/src/replica/mutation_log_replay.cpp b/src/replica/mutation_log_replay.cpp index ee891bcb73..0e7956c8d7 100644 --- a/src/replica/mutation_log_replay.cpp +++ b/src/replica/mutation_log_replay.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -38,10 +39,9 @@ #include "utils/errors.h" #include "utils/fail_point.h" #include "utils/fmt_logging.h" -#include +#include "utils/ports.h" -namespace dsn { -namespace replication { +namespace dsn::replication { /*static*/ error_code mutation_log::replay(log_file_ptr log, replay_callback callback, @@ -72,40 +72,55 @@ namespace replication { return err.code(); } -/*static*/ error_s mutation_log::replay_block(log_file_ptr &log, - replay_callback &callback, - size_t start_offset, - int64_t &end_offset) -{ - FAIL_POINT_INJECT_F("mutation_log_replay_block", [](std::string_view) -> error_s { - return error_s::make(ERR_INCOMPLETE_DATA, "mutation_log_replay_block"); - }); - - blob bb; - std::unique_ptr reader; +namespace { - log->reset_stream(start_offset); // start reading from given offset - int64_t global_start_offset = start_offset + log->start_offset(); - end_offset = global_start_offset; // reset end_offset to the start. - - // reads the entire block into memory - error_code err = log->read_next_log_block(bb); - if (err != ERR_OK) { - return error_s::make(err, "failed to read log block"); +dsn::error_s read_block(dsn::replication::log_file_ptr &log, + size_t start_offset, + int64_t &end_offset, + std::unique_ptr &reader) +{ + log->reset_stream(start_offset); // Start reading from given offset. + int64_t global_start_offset = static_cast(start_offset) + log->start_offset(); + end_offset = global_start_offset; // Reset end_offset to the start. + + { + // Read the entire block into memory. + blob bb; + const auto err = log->read_next_log_block(bb); + if (dsn_unlikely(err != dsn::ERR_OK)) { + return FMT_ERR(err, "failed to read log block"); + } + reader = std::make_unique(std::move(bb)); } - reader = std::make_unique(bb); - end_offset += sizeof(log_block_header); + end_offset += sizeof(dsn::replication::log_block_header); // The first block is log_file_header. if (global_start_offset == log->start_offset()) { end_offset += log->read_file_header(*reader); if (!log->is_right_header()) { - return error_s::make(ERR_INVALID_DATA, "failed to read log file header"); + return FMT_ERR(dsn::ERR_INVALID_DATA, "failed to read log file header"); } - // continue to parsing the data block + // Continue to parsing the data block. } + return dsn::error_s::ok(); +} + +} // anonymous namespace + +/*static*/ error_s mutation_log::replay_block(log_file_ptr &log, + replay_callback &callback, + size_t start_offset, + int64_t &end_offset) +{ + FAIL_POINT_INJECT_F("mutation_log_replay_block", [](std::string_view) -> error_s { + return error_s::make(ERR_INCOMPLETE_DATA, "mutation_log_replay_block"); + }); + + std::unique_ptr reader; + RETURN_NOT_OK(read_block(log, start_offset, end_offset, reader)); + while (!reader->is_eof()) { auto old_size = reader->get_remaining_size(); mutation_ptr mu = mutation::read_from(*reader, nullptr); @@ -218,5 +233,4 @@ namespace replication { return err; } -} // namespace replication -} // namespace dsn +} // namespace dsn::replication diff --git a/src/replica/test/log_block_test.cpp b/src/replica/test/log_block_test.cpp index e667a59cef..7499e13c87 100644 --- a/src/replica/test/log_block_test.cpp +++ b/src/replica/test/log_block_test.cpp @@ -112,7 +112,8 @@ TEST_P(log_appender_test, log_block_full) { log_appender appender(10); for (int i = 0; i < 1024; i++) { // more than DEFAULT_MAX_BLOCK_BYTES - appender.append_mutation(create_test_mutation(1 + i, std::string(1024, 'a')), nullptr); + appender.append_mutation(create_test_mutation(1 + i, std::string(1024, 'a').c_str()), + nullptr); } ASSERT_EQ(appender.mutations().size(), 1024); // two log_block_header blobs @@ -136,7 +137,8 @@ TEST_P(log_appender_test, read_log_block) { log_appender appender(10); for (int i = 0; i < 1024; i++) { // more than DEFAULT_MAX_BLOCK_BYTES - appender.append_mutation(create_test_mutation(1 + i, std::string(1024, 'a')), nullptr); + appender.append_mutation(create_test_mutation(1 + i, std::string(1024, 'a').c_str()), + nullptr); } ASSERT_EQ(appender.all_blocks().size(), 2); diff --git a/src/replica/test/log_file_test.cpp b/src/replica/test/log_file_test.cpp index 3ac902451a..c4abb02b20 100644 --- a/src/replica/test/log_file_test.cpp +++ b/src/replica/test/log_file_test.cpp @@ -81,7 +81,8 @@ TEST_P(log_file_test, commit_log_blocks) size_t written_sz = appender->size(); appender = std::make_shared(_start_offset + written_sz); for (int i = 0; i < 1024; i++) { // more than DEFAULT_MAX_BLOCK_BYTES - appender->append_mutation(create_test_mutation(1 + i, std::string(1024, 'a')), nullptr); + appender->append_mutation(create_test_mutation(1 + i, std::string(1024, 'a').c_str()), + nullptr); } ASSERT_GT(appender->all_blocks().size(), 1); tsk = _logf->commit_log_blocks( diff --git a/src/replica/test/mutation_log_test.cpp b/src/replica/test/mutation_log_test.cpp index 0009e800e7..6d92e042bf 100644 --- a/src/replica/test/mutation_log_test.cpp +++ b/src/replica/test/mutation_log_test.cpp @@ -291,9 +291,8 @@ class mutation_log_test : public replica_test_base void TearDown() override { utils::filesystem::remove_path(_log_dir); } - mutation_ptr create_test_mutation(int64_t decree, - int64_t last_committed_decree, - const std::string &data) override + mutation_ptr + create_test_mutation(int64_t decree, int64_t last_committed_decree, const char *data) override { mutation_ptr mu(new mutation()); mu->data.header.ballot = 1; @@ -315,7 +314,7 @@ class mutation_log_test : public replica_test_base return mu; } - mutation_ptr create_test_mutation(int64_t decree, const std::string &data) override + mutation_ptr create_test_mutation(int64_t decree, const char *data) override { return mutation_log_test::create_test_mutation(decree, decree - 1, data); } @@ -333,8 +332,7 @@ class mutation_log_test : public replica_test_base // each round mlog will replay the former logs, and create new file mutation_log_ptr mlog = create_private_log(); for (int i = 1; i <= 10; i++) { - std::string msg = "hello!"; - mutation_ptr mu = create_test_mutation(10 * f + i, msg); + auto mu = create_test_mutation(10 * f + i, "hello!"); mlog->append(mu, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr, 0); } mlog->tracker()->wait_outstanding_tasks(); diff --git a/src/replica/test/replica_test_base.h b/src/replica/test/replica_test_base.h index 0fe479d8a7..374e87e82e 100644 --- a/src/replica/test/replica_test_base.h +++ b/src/replica/test/replica_test_base.h @@ -62,7 +62,7 @@ class replica_test_base : public replica_stub_test_base } virtual mutation_ptr - create_test_mutation(int64_t decree, int64_t last_committed_decree, const std::string &data) + create_test_mutation(int64_t decree, int64_t last_committed_decree, const char *data) { mutation_ptr mu(new mutation()); mu->data.header.ballot = 1; @@ -75,7 +75,9 @@ class replica_test_base : public replica_stub_test_base mu->data.updates.emplace_back(mutation_update()); mu->data.updates.back().code = RPC_COLD_BACKUP; // whatever code it is, but never be WRITE_EMPTY - mu->data.updates.back().data = blob::create_from_bytes(std::string(data)); + if (data != nullptr) { + mu->data.updates.back().data = blob::create_from_bytes(data); + } mu->client_requests.push_back(nullptr); // replica_duplicator always loads from hard disk, @@ -85,7 +87,7 @@ class replica_test_base : public replica_stub_test_base return mu; } - virtual mutation_ptr create_test_mutation(int64_t decree, const std::string &data) + virtual mutation_ptr create_test_mutation(int64_t decree, const char *data) { return replica_test_base::create_test_mutation(decree, decree - 1, data); } diff --git a/src/runtime/rpc/network.sim.cpp b/src/runtime/rpc/network.sim.cpp index 595f86cb1d..ecde4fb171 100644 --- a/src/runtime/rpc/network.sim.cpp +++ b/src/runtime/rpc/network.sim.cpp @@ -83,7 +83,7 @@ static message_ex *virtual_send_message(message_ex *msg) tmp += buf.length(); } - blob bb(buffer, 0, msg->header->body_length + sizeof(message_header)); + blob bb(buffer, msg->header->body_length + sizeof(message_header)); message_ex *recv_msg = message_ex::create_receive_message(bb); recv_msg->to_address = msg->to_address; recv_msg->to_host_port = msg->to_host_port; diff --git a/src/utils/binary_reader.cpp b/src/utils/binary_reader.cpp index 227a98ed31..64e1e3a788 100644 --- a/src/utils/binary_reader.cpp +++ b/src/utils/binary_reader.cpp @@ -102,10 +102,10 @@ int binary_reader::inner_read(blob &blob, int len) blob = _blob.range(static_cast(_ptr - _blob.data()), len); // optimization: zero-copy - if (!blob.buffer_ptr()) { + if (!blob.buffer()) { std::shared_ptr buffer(::dsn::utils::make_shared_array(len)); memcpy(buffer.get(), blob.data(), blob.length()); - blob = ::dsn::blob(buffer, 0, blob.length()); + blob = ::dsn::blob(buffer, blob.length()); } _ptr += len; diff --git a/src/utils/binary_reader.h b/src/utils/binary_reader.h index 65d0749fb2..415da55935 100644 --- a/src/utils/binary_reader.h +++ b/src/utils/binary_reader.h @@ -39,13 +39,13 @@ class binary_reader { public: // given bb on ctor - binary_reader(const blob &blob); - binary_reader(blob &&blob); + explicit binary_reader(const blob &blob); + explicit binary_reader(blob &&blob); // or delayed init - binary_reader() {} + binary_reader() = default; - virtual ~binary_reader() {} + virtual ~binary_reader() = default; void init(const blob &bb); void init(blob &&bb); diff --git a/src/utils/blob.h b/src/utils/blob.h index 3cdbce320a..62bc0a8a51 100644 --- a/src/utils/blob.h +++ b/src/utils/blob.h @@ -33,6 +33,7 @@ #include #include +#include "utils/fmt_logging.h" #include "utils/fmt_utils.h" #include "utils.h" @@ -51,14 +52,6 @@ class blob { } - blob(std::shared_ptr buffer, int offset, unsigned int length) - : _holder(std::move(buffer)), - _buffer(_holder.get()), - _data(_holder.get() + offset), - _length(length) - { - } - /// NOTE: Use std::string_view whenever possible. /// blob is designed for shared buffer, never use it as constant view. /// Maybe we could deprecate this function in the future. @@ -67,21 +60,68 @@ class blob { } + blob(const blob &rhs) noexcept = default; + + blob &operator=(const blob &rhs) noexcept + { + if (this == &rhs) { + return *this; + } + + _holder = rhs._holder; + _buffer = rhs._buffer; + _data = rhs._data; + _length = rhs._length; + + return *this; + } + + blob(blob &&rhs) noexcept + : _holder(std::move(rhs._holder)), + _buffer(rhs._buffer), + _data(rhs._data), + _length(rhs._length) + { + rhs._buffer = nullptr; + rhs._data = nullptr; + rhs._length = 0; + } + + blob &operator=(blob &&rhs) noexcept + { + if (this == &rhs) { + return *this; + } + + _holder = std::move(rhs._holder); + _buffer = rhs._buffer; + _data = rhs._data; + _length = rhs._length; + + rhs._buffer = nullptr; + rhs._data = nullptr; + rhs._length = 0; + + return *this; + } + /// Create shared buffer from allocated raw bytes. /// NOTE: this operation is not efficient since it involves a memory copy. - static blob create_from_bytes(const char *s, size_t len) + [[nodiscard]] static blob create_from_bytes(const char *s, size_t len) { + CHECK_NOTNULL(s, "null source pointer would lead to undefined behaviour"); + std::shared_ptr s_arr(new char[len], std::default_delete()); memcpy(s_arr.get(), s, len); - return blob(std::move(s_arr), 0, static_cast(len)); + return {std::move(s_arr), static_cast(len)}; } /// Create shared buffer without copying data. - static blob create_from_bytes(std::string &&bytes) + [[nodiscard]] static blob create_from_bytes(std::string &&bytes) { - auto s = new std::string(std::move(bytes)); + auto *s = new std::string(std::move(bytes)); std::shared_ptr buf(const_cast(s->data()), [s](char *) { delete s; }); - return blob(std::move(buf), 0, static_cast(s->length())); + return {std::move(buf), static_cast(s->length())}; } void assign(const std::shared_ptr &buffer, int offset, unsigned int length) @@ -95,8 +135,8 @@ class blob void assign(std::shared_ptr &&buffer, int offset, unsigned int length) { _holder = std::move(buffer); - _buffer = (_holder.get()); - _data = (_holder.get() + offset); + _buffer = _holder.get(); + _data = _holder.get() + offset; _length = length; } @@ -109,21 +149,22 @@ class blob _length = length; } - const char *data() const noexcept { return _data; } + [[nodiscard]] const char *data() const noexcept { return _data; } - unsigned int length() const noexcept { return _length; } - unsigned int size() const noexcept { return _length; } - bool empty() const noexcept { return _length == 0; } + [[nodiscard]] unsigned int length() const noexcept { return _length; } + [[nodiscard]] unsigned int size() const noexcept { return _length; } + [[nodiscard]] bool empty() const noexcept { return _length == 0; } - std::shared_ptr buffer() const { return _holder; } + [[nodiscard]] std::shared_ptr buffer() const { return _holder; } - const char *buffer_ptr() const { return _holder.get(); } + [[nodiscard]] const char *buffer_ptr() const { return _holder.get(); } - // offset can be negative for buffer dereference - blob range(int offset) const + // `offset` can be negative for buffer dereference. + [[nodiscard]] blob range(int offset) const { - // offset cannot exceed the current length value - assert(offset <= static_cast(_length)); + DCHECK_LE_MSG(offset, + static_cast(_length), + "the required offset cannot exceed the current length"); blob temp = *this; temp._data += offset; @@ -131,33 +172,38 @@ class blob return temp; } - blob range(int offset, unsigned int len) const + [[nodiscard]] blob range(int offset, unsigned int len) const { - // offset cannot exceed the current length value - assert(offset <= static_cast(_length)); + DCHECK_LE_MSG(offset, + static_cast(_length), + "the required offset cannot exceed the current length"); blob temp = *this; temp._data += offset; temp._length -= offset; - // buffer length must exceed the required length - assert(temp._length >= len); + DCHECK_LE_MSG( + len, temp._length, "the required length cannot exceed remaining buffer length"); + temp._length = len; return temp; } - bool operator==(const blob &r) const + // Could NOT be declared with "= delete", since many thrift-generated classes would + // access this in their own `operator==`. + bool operator==(const blob &) const { - // not implemented - assert(false); + CHECK(false, "not implemented"); return false; } - std::string to_string() const + [[nodiscard]] std::string to_string() const { - if (_length == 0) + if (_length == 0) { return {}; - return std::string(_data, _length); + } + + return {_data, _length}; } friend std::ostream &operator<<(std::ostream &os, const blob &bb) @@ -165,7 +211,7 @@ class blob return os << bb.to_string(); } - std::string_view to_string_view() const { return std::string_view(_data, _length); } + [[nodiscard]] std::string_view to_string_view() const { return {_data, _length}; } uint32_t read(::apache::thrift::protocol::TProtocol *iprot); uint32_t write(::apache::thrift::protocol::TProtocol *oprot) const; diff --git a/src/utils/test/blob_test.cpp b/src/utils/test/blob_test.cpp new file mode 100644 index 0000000000..8ae2c8b7f6 --- /dev/null +++ b/src/utils/test/blob_test.cpp @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +#include "gtest/gtest.h" +#include "utils/blob.h" + +namespace dsn { + +struct blob_base_case +{ + std::string expected_str; +}; + +class BlobBaseTest : public testing::TestWithParam +{ +public: + void SetUp() override + { + const auto &test_case = GetParam(); + _expected_str = test_case.expected_str; + } + + void check_blob_value(const blob &obj) const + { + EXPECT_EQ(_expected_str, obj.to_string()); + + EXPECT_EQ(_expected_str.size(), obj.length()); + EXPECT_EQ(_expected_str.size(), obj.size()); + + if (_expected_str.empty()) { + EXPECT_TRUE(obj.empty()); + } else { + EXPECT_FALSE(obj.empty()); + } + } + +protected: + std::string _expected_str; +}; + +const std::vector blob_base_tests = { + // Test empty case. + {""}, + // Test non-empty case. + {"hello"}, +}; + +class BlobCreateTest : public BlobBaseTest +{ +}; + +TEST_P(BlobCreateTest, CreateFromCString) +{ + const auto &obj = blob::create_from_bytes(_expected_str.data(), _expected_str.size()); + check_blob_value(obj); +} + +TEST_P(BlobCreateTest, CreateFromString) +{ + const auto &obj = blob::create_from_bytes(std::string(_expected_str)); + check_blob_value(obj); +} + +INSTANTIATE_TEST_SUITE_P(BlobTest, BlobCreateTest, testing::ValuesIn(blob_base_tests)); + +class BlobInitTest : public BlobBaseTest +{ +public: + blob create() { return blob::create_from_bytes(std::string(_expected_str)); } +}; + +TEST_P(BlobInitTest, CopyConstructor) +{ + const auto &obj = create(); + + blob copy(obj); + check_blob_value(copy); +} + +TEST_P(BlobInitTest, CopyAssignment) +{ + const auto &obj = create(); + + blob copy; + copy = obj; + check_blob_value(copy); +} + +TEST_P(BlobInitTest, MoveConstructor) +{ + auto obj = create(); + + blob move(std::move(obj)); + check_blob_value(move); +} + +TEST_P(BlobInitTest, MoveAssignment) +{ + auto obj = create(); + + blob move; + move = std::move(obj); + check_blob_value(move); +} + +INSTANTIATE_TEST_SUITE_P(BlobTest, BlobInitTest, testing::ValuesIn(blob_base_tests)); + +} // namespace dsn From 2f843db2a5dde325822bf423db7f848ec186a3d6 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Mon, 19 Aug 2024 19:54:20 +0800 Subject: [PATCH 14/15] fix(duplication): `min_checkpoint_decree` was reset to the invalid decree after replica server was restarted (#2097) https://github.com/apache/incubator-pegasus/issues/2098 After the replica server was restarted, `min_checkpoint_decree` would be reset to the invalid decree(-1), which is wrong for an existing duplication that is still in the status of `DS_PREPARE`, meaning the replica would continue to generate checkpoints for the full duplication. The duplication in the status of `DS_PREPARE` would call `replica_duplicator::prepare_dup()` which makes an assertion that `min_checkpoint_decree` must be greater than 0. Therefore, the replica server would always exit abnormally due to the failed assertion. --- .../duplication/replica_duplicator.cpp | 106 +++++++++++------- src/replica/duplication/replica_duplicator.h | 4 +- .../test/dup_replica_http_service_test.cpp | 3 +- .../test/duplication_sync_timer_test.cpp | 36 +++--- .../duplication/test/mutation_batch_test.cpp | 82 +++++++------- .../test/replica_duplicator_manager_test.cpp | 64 ++++++----- .../test/replica_duplicator_test.cpp | 31 ++++- 7 files changed, 194 insertions(+), 132 deletions(-) diff --git a/src/replica/duplication/replica_duplicator.cpp b/src/replica/duplication/replica_duplicator.cpp index 6bb75b92cb..d19033d4ef 100644 --- a/src/replica/duplication/replica_duplicator.cpp +++ b/src/replica/duplication/replica_duplicator.cpp @@ -59,42 +59,66 @@ replica_duplicator::replica_duplicator(const duplication_entry &ent, replica *r) _stub(r->get_replica_stub()), METRIC_VAR_INIT_replica(dup_confirmed_mutations) { + // Ensure that the checkpoint decree is at least 1. Otherwise, the checkpoint could not be + // created in time for empty replica; in consequence, the remote cluster would inevitably + // fail to pull the checkpoint files. + // + // The max decree in rocksdb memtable (the last applied decree) is considered as the min + // decree that should be covered by the checkpoint, which means currently all of the data + // in current rocksdb should be included into the created checkpoint. + // + // `_min_checkpoint_decree` is not persisted into zk. Once replica server was restarted, + // it would be reset to the decree that is applied most recently. + const auto last_applied_decree = _replica->last_applied_decree(); + _min_checkpoint_decree = std::max(last_applied_decree, static_cast(1)); + LOG_INFO_PREFIX("initialize checkpoint decree: min_checkpoint_decree={}, " + "last_committed_decree={}, last_applied_decree={}, " + "last_flushed_decree={}, last_durable_decree={}, " + "plog_max_decree_on_disk={}, plog_max_commit_on_disk={}", + _min_checkpoint_decree, + _replica->last_committed_decree(), + last_applied_decree, + _replica->last_flushed_decree(), + _replica->last_durable_decree(), + _replica->private_log()->max_decree_on_disk(), + _replica->private_log()->max_commit_on_disk()); + _status = ent.status; - auto it = ent.progress.find(get_gpid().get_partition_index()); + const auto it = ent.progress.find(get_gpid().get_partition_index()); + CHECK_PREFIX_MSG(it != ent.progress.end(), + "partition({}) not found in duplication progress: " + "app_name={}, dup_id={}, remote_cluster_name={}, remote_app_name={}", + get_gpid(), + r->get_app_info()->app_name, + id(), + _remote_cluster_name, + _remote_app_name); + + // Initial progress would be `invalid_decree` which was synced from meta server + // immediately after the duplication was created. + // See `init_progress()` in `meta_duplication_service::new_dup_from_init()`. + // + // _progress.last_decree would be used to update the state in meta server. + // See `replica_duplicator_manager::get_duplication_confirms_to_update()`. if (it->second == invalid_decree) { - // Ensure that the checkpoint decree is at least 1. Otherwise, the checkpoint could not be - // created in time for empty replica; in consequence, the remote cluster would inevitably - // fail to pull the checkpoint files. - // - // The max decree in rocksdb memtable (the last applied decree) is considered as the min - // decree that should be covered by the checkpoint, which means currently all of the data - // in current rocksdb should be included into the created checkpoint. - // - // TODO(jiashuo1): _min_checkpoint_decree hasn't be ready to persist zk, so if master - // restart, the value will be reset to 0. - const auto last_applied_decree = _replica->last_applied_decree(); - _min_checkpoint_decree = std::max(last_applied_decree, static_cast(1)); - _progress.last_decree = last_applied_decree; - LOG_INFO_PREFIX("initialize checkpoint decree: min_checkpoint_decree={}, " - "last_committed_decree={}, last_applied_decree={}, " - "last_flushed_decree={}, last_durable_decree={}, " - "plog_max_decree_on_disk={}, plog_max_commit_on_disk={}", - _min_checkpoint_decree, - _replica->last_committed_decree(), - last_applied_decree, - _replica->last_flushed_decree(), - _replica->last_durable_decree(), - _replica->private_log()->max_decree_on_disk(), - _replica->private_log()->max_commit_on_disk()); - + _progress.last_decree = _min_checkpoint_decree; } else { _progress.last_decree = _progress.confirmed_decree = it->second; } - LOG_INFO_PREFIX("initialize replica_duplicator[{}] [dupid:{}, meta_confirmed_decree:{}]", - duplication_status_to_string(_status), + + LOG_INFO_PREFIX("initialize replica_duplicator: app_name={}, dup_id={}, " + "remote_cluster_name={}, remote_app_name={}, status={}, " + "replica_confirmed_decree={}, meta_persisted_decree={}/{}", + r->get_app_info()->app_name, id(), - it->second); + _remote_cluster_name, + _remote_app_name, + duplication_status_to_string(_status), + _progress.last_decree, + it->second, + _progress.confirmed_decree); + thread_pool(LPC_REPLICATION_LOW).task_tracker(tracker()).thread_hash(get_gpid().thread_hash()); if (_status == duplication_status::DS_PREPARE) { @@ -123,7 +147,8 @@ void replica_duplicator::prepare_dup() void replica_duplicator::start_dup_log() { - LOG_INFO_PREFIX("starting duplication {} [last_decree: {}, confirmed_decree: {}]", + LOG_INFO_PREFIX("starting duplication: {}, replica_confirmed_decree={}, " + "meta_persisted_decree={}", to_string(), _progress.last_decree, _progress.confirmed_decree); @@ -261,17 +286,16 @@ error_s replica_duplicator::update_progress(const duplication_progress &p) void replica_duplicator::verify_start_decree(decree start_decree) { - decree confirmed_decree = progress().confirmed_decree; - decree last_decree = progress().last_decree; - decree max_gced_decree = get_max_gced_decree(); - CHECK_LT_MSG(max_gced_decree, - start_decree, - "the logs haven't yet duplicated were accidentally truncated " - "[max_gced_decree: {}, start_decree: {}, confirmed_decree: {}, last_decree: {}]", - max_gced_decree, - start_decree, - confirmed_decree, - last_decree); + const auto max_gced_decree = get_max_gced_decree(); + CHECK_LT_PREFIX_MSG( + max_gced_decree, + start_decree, + "the logs haven't yet duplicated were accidentally truncated [max_gced_decree: {}, " + "start_decree: {}, replica_confirmed_decree: {}, meta_persisted_decree: {}]", + max_gced_decree, + start_decree, + progress().last_decree, + progress().confirmed_decree); } decree replica_duplicator::get_max_gced_decree() const diff --git a/src/replica/duplication/replica_duplicator.h b/src/replica/duplication/replica_duplicator.h index 9a8deed0d9..1516ec520c 100644 --- a/src/replica/duplication/replica_duplicator.h +++ b/src/replica/duplication/replica_duplicator.h @@ -157,8 +157,8 @@ class replica_duplicator : public replica_base, public pipeline::base { zauto_read_lock l(_lock); - JSON_ENCODE_OBJ(writer, confirmed_decree, _progress.last_decree); - JSON_ENCODE_OBJ(writer, persisted_decree, _progress.confirmed_decree); + JSON_ENCODE_OBJ(writer, replica_confirmed_decree, _progress.last_decree); + JSON_ENCODE_OBJ(writer, meta_persisted_decree, _progress.confirmed_decree); } writer.EndObject(); diff --git a/src/replica/duplication/test/dup_replica_http_service_test.cpp b/src/replica/duplication/test/dup_replica_http_service_test.cpp index 43a0a35153..5fbffbd49b 100644 --- a/src/replica/duplication/test/dup_replica_http_service_test.cpp +++ b/src/replica/duplication/test/dup_replica_http_service_test.cpp @@ -42,7 +42,8 @@ INSTANTIATE_TEST_SUITE_P(, dup_replica_http_service_test, ::testing::Values(fals TEST_P(dup_replica_http_service_test, query_duplication_handler) { - auto pri = stub->add_primary_replica(1, 1); + auto *pri = stub->add_primary_replica(1, 1); + pri->init_private_log(pri->dir()); // primary confirmed_decree duplication_entry ent; diff --git a/src/replica/duplication/test/duplication_sync_timer_test.cpp b/src/replica/duplication/test/duplication_sync_timer_test.cpp index 0f7855ed72..2714af6bd5 100644 --- a/src/replica/duplication/test/duplication_sync_timer_test.cpp +++ b/src/replica/duplication/test/duplication_sync_timer_test.cpp @@ -52,7 +52,8 @@ class duplication_sync_timer_test : public duplication_test_base static const std::string kTestRemoteAppName = "temp"; // replica: {app_id:2, partition_id:1, duplications:{}} - stub->add_primary_replica(2, 1); + auto *rep = stub->add_primary_replica(2, 1); + rep->init_private_log(rep->dir()); ASSERT_NE(stub->find_replica(2, 1), nullptr); // appid:2 -> dupid:1 @@ -82,15 +83,16 @@ class duplication_sync_timer_test : public duplication_test_base { int total_app_num = 4; for (int appid = 1; appid <= total_app_num; appid++) { - auto r = stub->add_non_primary_replica(appid, 1); + auto *rep = stub->add_non_primary_replica(appid, 1); + rep->init_private_log(rep->dir()); // trigger duplication sync on partition 1 duplication_entry ent; ent.dupid = 1; - ent.progress[r->get_gpid().get_partition_index()] = 1000; + ent.progress[rep->get_gpid().get_partition_index()] = 1000; ent.status = duplication_status::DS_PAUSE; - auto dup = std::make_unique(ent, r); - add_dup(r, std::move(dup)); + auto dup = std::make_unique(ent, rep); + add_dup(rep, std::move(dup)); } RPC_MOCKING(duplication_sync_rpc) @@ -164,7 +166,8 @@ class duplication_sync_timer_test : public duplication_test_base std::map> dup_map; for (int32_t appid = 1; appid <= 10; appid++) { for (int partition_id = 0; partition_id < 3; partition_id++) { - stub->add_primary_replica(appid, partition_id); + auto *rep = stub->add_primary_replica(appid, partition_id); + rep->init_private_log(rep->dir()); } } @@ -254,19 +257,20 @@ class duplication_sync_timer_test : public duplication_test_base void test_update_confirmed_points() { for (int32_t appid = 1; appid <= 10; appid++) { - stub->add_primary_replica(appid, 1); + auto *rep = stub->add_primary_replica(appid, 1); + rep->init_private_log(rep->dir()); } for (int appid = 1; appid <= 3; appid++) { - auto r = stub->find_replica(appid, 1); + auto *rep = stub->find_replica(appid, 1); duplication_entry ent; ent.dupid = 1; ent.status = duplication_status::DS_PAUSE; - ent.progress[r->get_gpid().get_partition_index()] = 0; - auto dup = std::make_unique(ent, r); + ent.progress[rep->get_gpid().get_partition_index()] = 0; + auto dup = std::make_unique(ent, rep); dup->update_progress(dup->progress().set_last_decree(3).set_confirmed_decree(1)); - add_dup(r, std::move(dup)); + add_dup(rep, std::move(dup)); } duplication_entry ent; @@ -280,8 +284,8 @@ class duplication_sync_timer_test : public duplication_test_base dup_sync->on_duplication_sync_reply(ERR_OK, resp); for (int appid = 1; appid <= 3; appid++) { - auto r = stub->find_replica(appid, 1); - auto dup = find_dup(r, 1); + auto *rep = stub->find_replica(appid, 1); + auto *dup = find_dup(rep, 1); ASSERT_EQ(3, dup->progress().confirmed_decree); } @@ -294,7 +298,8 @@ class duplication_sync_timer_test : public duplication_test_base // 10 primaries int appid = 1; for (int partition_id = 0; partition_id < 10; partition_id++) { - stub->add_primary_replica(appid, partition_id); + auto *r = stub->add_primary_replica(appid, partition_id); + r->init_private_log(r->dir()); } duplication_entry ent; @@ -353,7 +358,8 @@ class duplication_sync_timer_test : public duplication_test_base // there must be some internal problems. void test_receive_illegal_duplication_status() { - stub->add_primary_replica(1, 0); + auto *rep = stub->add_primary_replica(1, 0); + rep->init_private_log(rep->dir()); duplication_entry ent; ent.dupid = 2; diff --git a/src/replica/duplication/test/mutation_batch_test.cpp b/src/replica/duplication/test/mutation_batch_test.cpp index c865de20b0..862e6276b3 100644 --- a/src/replica/duplication/test/mutation_batch_test.cpp +++ b/src/replica/duplication/test/mutation_batch_test.cpp @@ -34,6 +34,7 @@ #include "replica/duplication/replica_duplicator.h" #include "replica/mutation.h" #include "replica/prepare_list.h" +#include "replica/test/mock_utils.h" #include "runtime/task/task_code.h" #include "utils/autoref_ptr.h" #include "utils/blob.h" @@ -42,24 +43,29 @@ namespace dsn::replication { class mutation_batch_test : public duplication_test_base { -public: - mutation_batch_test() : _duplicator(create_test_duplicator(0)), _batcher(_duplicator.get()) {} +protected: + mutation_batch_test() + { + _replica->init_private_log(_replica->dir()); + _duplicator = create_test_duplicator(0); + _batcher = std::make_unique(_duplicator.get()); + } - void reset_buffer(const decree last_commit, const decree start, const decree end) + void reset_buffer(const decree last_commit, const decree start, const decree end) const { - _batcher._mutation_buffer->reset(last_commit); - _batcher._mutation_buffer->_start_decree = start; - _batcher._mutation_buffer->_end_decree = end; + _batcher->_mutation_buffer->reset(last_commit); + _batcher->_mutation_buffer->_start_decree = start; + _batcher->_mutation_buffer->_end_decree = end; } - void commit_buffer(const decree current_decree) + void commit_buffer(const decree current_decree) const { - _batcher._mutation_buffer->commit(current_decree, COMMIT_TO_DECREE_HARD); + _batcher->_mutation_buffer->commit(current_decree, COMMIT_TO_DECREE_HARD); } - void check_mutation_contents(const std::vector &expected_mutations) + void check_mutation_contents(const std::vector &expected_mutations) const { - const auto all_mutations = _batcher.move_all_mutations(); + const auto all_mutations = _batcher->move_all_mutations(); std::vector actual_mutations; std::transform(all_mutations.begin(), @@ -71,7 +77,7 @@ class mutation_batch_test : public duplication_test_base } std::unique_ptr _duplicator; - mutation_batch _batcher; + std::unique_ptr _batcher; }; INSTANTIATE_TEST_SUITE_P(, mutation_batch_test, ::testing::Values(false, true)); @@ -80,38 +86,38 @@ TEST_P(mutation_batch_test, prepare_mutation) { auto mu1 = create_test_mutation(1, 0, "first mutation"); set_last_applied_decree(1); - ASSERT_TRUE(_batcher.add(mu1)); - ASSERT_EQ(1, _batcher.last_decree()); + ASSERT_TRUE(_batcher->add(mu1)); + ASSERT_EQ(1, _batcher->last_decree()); auto mu2 = create_test_mutation(2, 1, "abcde"); set_last_applied_decree(2); - ASSERT_TRUE(_batcher.add(mu2)); - ASSERT_EQ(2, _batcher.last_decree()); + ASSERT_TRUE(_batcher->add(mu2)); + ASSERT_EQ(2, _batcher->last_decree()); auto mu3 = create_test_mutation(3, 2, "hello world"); - ASSERT_TRUE(_batcher.add(mu3)); + ASSERT_TRUE(_batcher->add(mu3)); // The last decree has not been updated. - ASSERT_EQ(2, _batcher.last_decree()); + ASSERT_EQ(2, _batcher->last_decree()); auto mu4 = create_test_mutation(4, 2, "foo bar"); - ASSERT_TRUE(_batcher.add(mu4)); - ASSERT_EQ(2, _batcher.last_decree()); + ASSERT_TRUE(_batcher->add(mu4)); + ASSERT_EQ(2, _batcher->last_decree()); // The committed mutation would be ignored. auto mu2_another = create_test_mutation(2, 1, "another second mutation"); - ASSERT_TRUE(_batcher.add(mu2_another)); - ASSERT_EQ(2, _batcher.last_decree()); + ASSERT_TRUE(_batcher->add(mu2_another)); + ASSERT_EQ(2, _batcher->last_decree()); // The mutation with duplicate decree would be ignored. auto mu3_another = create_test_mutation(3, 2, "123 xyz"); - ASSERT_TRUE(_batcher.add(mu3_another)); - ASSERT_EQ(2, _batcher.last_decree()); + ASSERT_TRUE(_batcher->add(mu3_another)); + ASSERT_EQ(2, _batcher->last_decree()); auto mu5 = create_test_mutation(5, 2, "5th mutation"); set_last_applied_decree(5); - ASSERT_TRUE(_batcher.add(mu5)); - ASSERT_EQ(5, _batcher.last_decree()); + ASSERT_TRUE(_batcher->add(mu5)); + ASSERT_EQ(5, _batcher->last_decree()); check_mutation_contents({"first mutation", "abcde", "hello world", "foo bar", "5th mutation"}); } @@ -119,7 +125,7 @@ TEST_P(mutation_batch_test, prepare_mutation) TEST_P(mutation_batch_test, add_null_mutation) { auto mu = create_test_mutation(1, nullptr); - _batcher.add_mutation_if_valid(mu, 0); + _batcher->add_mutation_if_valid(mu, 0); check_mutation_contents({""}); } @@ -127,7 +133,7 @@ TEST_P(mutation_batch_test, add_null_mutation) TEST_P(mutation_batch_test, add_empty_mutation) { auto mu = create_test_mutation(1, ""); - _batcher.add_mutation_if_valid(mu, 0); + _batcher->add_mutation_if_valid(mu, 0); check_mutation_contents({""}); } @@ -138,7 +144,7 @@ TEST_P(mutation_batch_test, add_string_view_mutation) auto mu = create_test_mutation(1, nullptr); const std::string data("hello"); mu->data.updates.back().data = blob(data.data(), 0, data.size()); - _batcher.add_mutation_if_valid(mu, 0); + _batcher->add_mutation_if_valid(mu, 0); check_mutation_contents({"hello"}); } @@ -146,7 +152,7 @@ TEST_P(mutation_batch_test, add_string_view_mutation) TEST_P(mutation_batch_test, add_a_valid_mutation) { auto mu = create_test_mutation(1, "hello"); - _batcher.add_mutation_if_valid(mu, 0); + _batcher->add_mutation_if_valid(mu, 0); check_mutation_contents({"hello"}); } @@ -156,13 +162,13 @@ TEST_P(mutation_batch_test, add_multiple_valid_mutations) // The mutation could not be reused, since in mutation_batch::add_mutation_if_valid // the elements of mutation::data::updates would be moved and nullified. auto mu1 = create_test_mutation(1, "hello"); - _batcher.add_mutation_if_valid(mu1, 0); + _batcher->add_mutation_if_valid(mu1, 0); auto mu2 = create_test_mutation(2, "world"); - _batcher.add_mutation_if_valid(mu2, 2); + _batcher->add_mutation_if_valid(mu2, 2); auto mu3 = create_test_mutation(3, "hi"); - _batcher.add_mutation_if_valid(mu3, 2); + _batcher->add_mutation_if_valid(mu3, 2); check_mutation_contents({"hello", "world", "hi"}); } @@ -170,17 +176,17 @@ TEST_P(mutation_batch_test, add_multiple_valid_mutations) TEST_P(mutation_batch_test, add_invalid_mutation) { auto mu2 = create_test_mutation(2, "world"); - _batcher.add_mutation_if_valid(mu2, 2); + _batcher->add_mutation_if_valid(mu2, 2); // mu1 would be ignored, since its decree is less than the start decree. auto mu1 = create_test_mutation(1, "hello"); - _batcher.add_mutation_if_valid(mu1, 2); + _batcher->add_mutation_if_valid(mu1, 2); auto mu3 = create_test_mutation(3, "hi"); - _batcher.add_mutation_if_valid(mu3, 2); + _batcher->add_mutation_if_valid(mu3, 2); auto mu4 = create_test_mutation(1, "ok"); - _batcher.add_mutation_if_valid(mu4, 1); + _batcher->add_mutation_if_valid(mu4, 1); // "ok" would be the first, since its timestamp (i.e. decree in create_test_mutation) // is the smallest. @@ -191,7 +197,7 @@ TEST_P(mutation_batch_test, ignore_non_idempotent_write) { auto mu = create_test_mutation(1, "hello"); mu->data.updates[0].code = RPC_DUPLICATION_NON_IDEMPOTENT_WRITE; - _batcher.add_mutation_if_valid(mu, 0); + _batcher->add_mutation_if_valid(mu, 0); check_mutation_contents({}); } @@ -202,7 +208,7 @@ TEST_P(mutation_batch_test, mutation_buffer_commit) // details. reset_buffer(10, 15, 20); commit_buffer(15); - ASSERT_EQ(14, _batcher.last_decree()); + ASSERT_EQ(14, _batcher->last_decree()); } } // namespace dsn::replication diff --git a/src/replica/duplication/test/replica_duplicator_manager_test.cpp b/src/replica/duplication/test/replica_duplicator_manager_test.cpp index 8123cd623d..1f69997576 100644 --- a/src/replica/duplication/test/replica_duplicator_manager_test.cpp +++ b/src/replica/duplication/test/replica_duplicator_manager_test.cpp @@ -44,14 +44,15 @@ class replica_duplicator_manager_test : public duplication_test_base void test_remove_non_existed_duplications() { - auto r = stub->add_primary_replica(2, 1); - auto &d = r->get_replica_duplicator_manager(); + auto *rep = stub->add_primary_replica(2, 1); + rep->init_private_log(rep->dir()); + auto &d = rep->get_replica_duplicator_manager(); duplication_entry ent; ent.dupid = 1; ent.status = duplication_status::DS_PAUSE; ent.remote = "dsn://slave-cluster"; - ent.progress[r->get_gpid().get_partition_index()] = 0; + ent.progress[rep->get_gpid().get_partition_index()] = 0; d.sync_duplication(ent); ASSERT_EQ(d._duplications.size(), 1); @@ -66,20 +67,21 @@ class replica_duplicator_manager_test : public duplication_test_base void test_set_confirmed_decree_non_primary() { - auto r = stub->add_primary_replica(2, 1); - auto &d = r->get_replica_duplicator_manager(); + auto *rep = stub->add_primary_replica(2, 1); + rep->init_private_log(rep->dir()); + auto &d = rep->get_replica_duplicator_manager(); duplication_entry ent; ent.dupid = 1; ent.status = duplication_status::DS_PAUSE; ent.remote = "dsn://slave-cluster"; - ent.progress[r->get_gpid().get_partition_index()] = 100; + ent.progress[rep->get_gpid().get_partition_index()] = 100; d.sync_duplication(ent); ASSERT_EQ(d._duplications.size(), 1); ASSERT_EQ(d._primary_confirmed_decree, invalid_decree); // replica failover - r->as_secondary(); + rep->as_secondary(); d.update_confirmed_decree_if_secondary(99); ASSERT_EQ(d._duplications.size(), 0); @@ -103,7 +105,8 @@ class replica_duplicator_manager_test : public duplication_test_base void test_get_duplication_confirms() { - auto r = stub->add_primary_replica(2, 1); + auto *rep = stub->add_primary_replica(2, 1); + rep->init_private_log(rep->dir()); int total_dup_num = 10; int update_dup_num = 4; // the number of dups that will be updated @@ -112,25 +115,25 @@ class replica_duplicator_manager_test : public duplication_test_base duplication_entry ent; ent.dupid = id; ent.status = duplication_status::DS_PAUSE; - ent.progress[r->get_gpid().get_partition_index()] = 0; + ent.progress[rep->get_gpid().get_partition_index()] = 0; - auto dup = std::make_unique(ent, r); + auto dup = std::make_unique(ent, rep); dup->update_progress(dup->progress().set_last_decree(2).set_confirmed_decree(1)); - add_dup(r, std::move(dup)); + add_dup(rep, std::move(dup)); } for (dupid_t id = update_dup_num + 1; id <= total_dup_num; id++) { duplication_entry ent; ent.dupid = id; ent.status = duplication_status::DS_PAUSE; - ent.progress[r->get_gpid().get_partition_index()] = 0; + ent.progress[rep->get_gpid().get_partition_index()] = 0; - auto dup = std::make_unique(ent, r); + auto dup = std::make_unique(ent, rep); dup->update_progress(dup->progress().set_last_decree(1).set_confirmed_decree(1)); - add_dup(r, std::move(dup)); + add_dup(rep, std::move(dup)); } - auto result = r->get_replica_duplicator_manager().get_duplication_confirms_to_update(); + auto result = rep->get_replica_duplicator_manager().get_duplication_confirms_to_update(); ASSERT_EQ(result.size(), update_dup_num); } @@ -142,24 +145,25 @@ class replica_duplicator_manager_test : public duplication_test_base int64_t min_confirmed_decree; }; - auto r = stub->add_non_primary_replica(2, 1); - auto assert_test = [r, this](test_case tt) { + auto *rep = stub->add_non_primary_replica(2, 1); + rep->init_private_log(rep->dir()); + auto assert_test = [rep, this](test_case tt) { for (int id = 1; id <= tt.confirmed_decree.size(); id++) { duplication_entry ent; ent.dupid = id; ent.status = duplication_status::DS_PAUSE; - ent.progress[r->get_gpid().get_partition_index()] = 0; + ent.progress[rep->get_gpid().get_partition_index()] = 0; - auto dup = std::make_unique(ent, r); + auto dup = std::make_unique(ent, rep); dup->update_progress(dup->progress() .set_last_decree(tt.confirmed_decree[id - 1]) .set_confirmed_decree(tt.confirmed_decree[id - 1])); - add_dup(r, std::move(dup)); + add_dup(rep, std::move(dup)); } - ASSERT_EQ(r->get_replica_duplicator_manager().min_confirmed_decree(), + ASSERT_EQ(rep->get_replica_duplicator_manager().min_confirmed_decree(), tt.min_confirmed_decree); - r->get_replica_duplicator_manager()._duplications.clear(); + rep->get_replica_duplicator_manager()._duplications.clear(); }; { @@ -169,7 +173,7 @@ class replica_duplicator_manager_test : public duplication_test_base } { // primary - r->as_primary(); + rep->as_primary(); test_case tt{{1, 2, 3}, 1}; assert_test(tt); @@ -203,17 +207,19 @@ TEST_P(replica_duplicator_manager_test, min_confirmed_decree) { test_min_confirm TEST_P(replica_duplicator_manager_test, update_checkpoint_prepared) { - auto r = stub->add_primary_replica(2, 1); + auto *rep = stub->add_primary_replica(2, 1); + rep->init_private_log(rep->dir()); + duplication_entry ent; ent.dupid = 1; ent.status = duplication_status::DS_PAUSE; - ent.progress[r->get_gpid().get_partition_index()] = 0; + ent.progress[rep->get_gpid().get_partition_index()] = 0; - auto dup = std::make_unique(ent, r); - r->update_last_durable_decree(100); + auto dup = std::make_unique(ent, rep); + rep->update_last_durable_decree(100); dup->update_progress(dup->progress().set_last_decree(2).set_confirmed_decree(1)); - add_dup(r, std::move(dup)); - auto updates = r->get_replica_duplicator_manager().get_duplication_confirms_to_update(); + add_dup(rep, std::move(dup)); + auto updates = rep->get_replica_duplicator_manager().get_duplication_confirms_to_update(); for (const auto &update : updates) { ASSERT_TRUE(update.checkpoint_prepared); } diff --git a/src/replica/duplication/test/replica_duplicator_test.cpp b/src/replica/duplication/test/replica_duplicator_test.cpp index 78e1aabfb2..2942d89150 100644 --- a/src/replica/duplication/test/replica_duplicator_test.cpp +++ b/src/replica/duplication/test/replica_duplicator_test.cpp @@ -69,12 +69,13 @@ class replica_duplicator_test : public duplication_test_base return dup->_min_checkpoint_decree; } - void test_new_duplicator(const std::string &remote_app_name, bool specify_remote_app_name) + void test_new_duplicator(const std::string &remote_app_name, + bool specify_remote_app_name, + int64_t confirmed_decree) { const dupid_t dupid = 1; const std::string remote = "remote_address"; const duplication_status::type status = duplication_status::DS_PAUSE; - const int64_t confirmed_decree = 100; duplication_entry dup_ent; dup_ent.dupid = dupid; @@ -90,8 +91,13 @@ class replica_duplicator_test : public duplication_test_base ASSERT_EQ(remote, duplicator->remote_cluster_name()); ASSERT_EQ(remote_app_name, duplicator->remote_app_name()); ASSERT_EQ(status, duplicator->_status); + ASSERT_EQ(1, duplicator->_min_checkpoint_decree); ASSERT_EQ(confirmed_decree, duplicator->progress().confirmed_decree); - ASSERT_EQ(confirmed_decree, duplicator->progress().last_decree); + if (confirmed_decree == invalid_decree) { + ASSERT_EQ(1, duplicator->progress().last_decree); + } else { + ASSERT_EQ(confirmed_decree, duplicator->progress().last_decree); + } auto &expected_env = *duplicator; ASSERT_EQ(duplicator->tracker(), expected_env.__conf.tracker); @@ -144,12 +150,25 @@ INSTANTIATE_TEST_SUITE_P(, replica_duplicator_test, ::testing::Values(false, tru TEST_P(replica_duplicator_test, new_duplicator_without_remote_app_name) { - test_new_duplicator("temp", false); + test_new_duplicator("temp", false, 100); } TEST_P(replica_duplicator_test, new_duplicator_with_remote_app_name) { - test_new_duplicator("another_test_app", true); + test_new_duplicator("another_test_app", true, 100); +} + +// Initial confirmed decree immediately after the duplication was created is `invalid_decree` +// which was synced from meta server. +TEST_P(replica_duplicator_test, new_duplicator_with_initial_confirmed_decree) +{ + test_new_duplicator("test_initial_confirmed_decree", true, invalid_decree); +} + +// The duplication progressed and confirmed decree became valid. +TEST_P(replica_duplicator_test, new_duplicator_with_non_initial_confirmed_decree) +{ + test_new_duplicator("test_non_initial_confirmed_decree", true, 1); } TEST_P(replica_duplicator_test, pause_start_duplication) { test_pause_start_duplication(); } @@ -160,7 +179,7 @@ TEST_P(replica_duplicator_test, duplication_progress) // Start duplication from empty replica. ASSERT_EQ(1, min_checkpoint_decree(duplicator)); - ASSERT_EQ(0, duplicator->progress().last_decree); + ASSERT_EQ(1, duplicator->progress().last_decree); ASSERT_EQ(invalid_decree, duplicator->progress().confirmed_decree); // Update the max decree that has been duplicated to the remote cluster. From a8d99fd531d5cd596d461d319c6cf67f64ed20dd Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Mon, 19 Aug 2024 19:55:40 +0800 Subject: [PATCH 15/15] feat(duplication): add a new option to configure the duration of the delay until the next execution if there is no mutation to be loaded (#2099) A new option is added for duplication to configure the duration of the delay until the next execution if there is no mutation to be loaded: ```diff [replication] + dup_no_mutation_load_delay_ms = 100 ``` --- .../duplication/duplication_pipeline.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/replica/duplication/duplication_pipeline.cpp b/src/replica/duplication/duplication_pipeline.cpp index 81c1647632..739b1cc8f6 100644 --- a/src/replica/duplication/duplication_pipeline.cpp +++ b/src/replica/duplication/duplication_pipeline.cpp @@ -17,22 +17,30 @@ #include "duplication_pipeline.h" -#include -#include #include +#include +#include #include #include +#include #include #include "load_from_private_log.h" #include "replica/duplication/replica_duplicator.h" #include "replica/mutation_log.h" #include "replica/replica.h" -#include "runtime/rpc/rpc_holder.h" #include "utils/autoref_ptr.h" #include "utils/errors.h" +#include "utils/flags.h" #include "utils/fmt_logging.h" +DSN_DEFINE_uint64( + replication, + dup_no_mutation_load_delay_ms, + 100, + "The duration of the delay until the next execution if there is no mutation to be loaded."); +DSN_TAG_VARIABLE(dup_no_mutation_load_delay_ms, FT_MUTABLE); + METRIC_DEFINE_counter(replica, dup_shipped_bytes, dsn::metric_unit::kBytes, @@ -63,8 +71,8 @@ void load_mutation::run() const auto max_plog_committed_decree = std::min(_replica->private_log()->max_decree_on_disk(), _replica->last_applied_decree()); if (_start_decree > max_plog_committed_decree) { - // wait 100ms for next try if no mutation was added. - repeat(100_ms); + // Wait for a while if no mutation was added. + repeat(std::chrono::milliseconds(FLAGS_dup_no_mutation_load_delay_ms)); return; }