Merge branch 'apache:master' into bulkload_coredump_fix

.clang-tidy

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# https://releases.llvm.org/14.0.0/tools/clang/tools/extra/docs/clang-tidy/index.html
+
+CheckOptions: []
+Checks: 'abseil-*,boost-*,bugprone-*,cert-*,clang-analyzer-*,concurrency-*,cppcoreguidelines-*,darwin-*,fuchsia-*,google-*,hicpp-*,linuxkernel-*,llvm-*,misc-*,modernize-*,performance-*,portability-*,readability-*,-cppcoreguidelines-pro-type-union-access,-llvm-include-order,-modernize-use-trailing-return-type,-cppcoreguidelines-avoid-non-const-global-variables,-fuchsia-statically-constructed-objects,-fuchsia-overloaded-operator,-bugprone-easily-swappable-parameters,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-hicpp-no-array-decay,-hicpp-named-parameter,-readability-named-parameter,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-readability-function-cognitive-complexity,-cert-err58-cpp,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-cppcoreguidelines-owning-memory,-cppcoreguidelines-pro-type-const-cast,-readability-identifier-length,-fuchsia-default-arguments-calls,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers'
+ExtraArgs:
+ExtraArgsBefore: []
+FormatStyle: none
+HeaderFilterRegex: ''
+InheritParentConfig: true
+UseColor: true
+User: 'clang-tidy'
+WarningsAsErrors: ''

.github/actions/rebuild_thirdparty_if_needed/action.yaml

@@ -42,8 +42,8 @@ runs:
         cmake --build build/ -j $(nproc)
         rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
         find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-        ../scripts/download_hadoop.sh hadoop-bin
-        ../scripts/download_zk.sh zookeeper-bin
+        ../build_tools/download_hadoop.sh hadoop-bin
+        ../build_tools/download_zk.sh zookeeper-bin
         rm -rf hadoop-bin/share/doc
         rm -rf zookeeper-bin/docs
       shell: bash

.github/workflows/build-push-env-docker.yml

@@ -52,19 +52,20 @@ jobs:
       - name: Checkout
         # The glibc version on ubuntu1804 and centos7 is lower than the actions/checkout@v4 required, so
         # we need to force to use actions/checkout@v3.
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build and push
-        uses: docker/build-push-action@v2.10.0
+        uses: docker/build-push-action@v6
         with:
+          platforms: linux/amd64,linux/arm64
           context: .
           file: ./docker/pegasus-build-env/${{ matrix.dockertag }}/Dockerfile
           push: true

.github/workflows/lint_and_test_cpp.yaml

@@ -30,13 +30,13 @@ on:
       - .github/actions
       - .github/workflows/lint_and_test_cpp.yaml
       - .github/workflows/thirdparty-regular-push.yml
+      - build_tools/pack_server.sh
+      - build_tools/pack_tools.sh
       - docker/thirdparties-bin/**
       - docker/thirdparties-src/**
       - CMakeLists.txt
       - cmake_modules/**
       - run.sh
-      - scripts/pack_server.sh
-      - scripts/pack_tools.sh
       - src/**
       - thirdparty/**
 
@@ -50,16 +50,40 @@ env:
 
 jobs:
   cpp_clang_format_linter:
-    name: Lint
+    name: Format
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - name: clang-format
-        run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
+        run: ./build_tools/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
+
+  cpp_clang_tidy_linter:
+    name: Tidy
+    runs-on: ubuntu-22.04
+    container:
+      image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }}
+    steps:
+      - name: Install Softwares
+        run: |
+          apt-get update
+          apt-get install clang-tidy -y
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Rebuild thirdparty if needed
+        uses: "./.github/actions/rebuild_thirdparty_if_needed"
+      - name: clang-tidy
+        run: |
+          git config --global --add safe.directory $(pwd)
+          ./run.sh build --test --compiler clang-14,clang++-14 -t debug --skip_thirdparty -c --cmake_only
+          ./build_tools/clang_tidy.py --rev-range $(git log origin/${{ github.base_ref }} -n1 --format=format:"%H")
+        shell: bash
 
   iwyu:
     name: IWYU
-    needs: cpp_clang_format_linter
+    needs:
+      - cpp_clang_format_linter
+      - cpp_clang_tidy_linter
     runs-on: ubuntu-latest
     env:
       USE_JEMALLOC: OFF
@@ -87,7 +111,9 @@ jobs:
 
   build_Release:
     name: Build Release
-    needs: cpp_clang_format_linter
+    needs:
+      - cpp_clang_format_linter
+      - cpp_clang_tidy_linter
     runs-on: ubuntu-latest
     env:
       USE_JEMALLOC: OFF
@@ -170,7 +196,9 @@ jobs:
 
   build_ASAN:
     name: Build ASAN
-    needs: cpp_clang_format_linter
+    needs:
+      - cpp_clang_format_linter
+      - cpp_clang_tidy_linter
     runs-on: ubuntu-latest
     env:
       USE_JEMALLOC: OFF
@@ -257,7 +285,9 @@ jobs:
 #                 before we find any way to reduce the time cost.
 #  build_UBSAN:
 #    name: Build UBSAN
-#    needs: cpp_clang_format_linter
+#    needs:
+#      - cpp_clang_format_linter
+#      - cpp_clang_tidy_linter
 #    runs-on: ubuntu-latest
 #    env:
 #      USE_JEMALLOC: OFF
@@ -337,7 +367,9 @@ jobs:
 
   build_with_jemalloc:
     name: Build with jemalloc
-    needs: cpp_clang_format_linter
+    needs:
+      - cpp_clang_format_linter
+      - cpp_clang_tidy_linter
     runs-on: ubuntu-latest
     env:
       USE_JEMALLOC: ON
@@ -378,7 +410,9 @@ jobs:
 
   build_release_on_macos:
     name: Build Release on macOS
-    needs: cpp_clang_format_linter
+    needs:
+      - cpp_clang_format_linter
+      - cpp_clang_tidy_linter
     runs-on: macos-12
     steps:
       - name: Install Softwares
@@ -412,7 +446,9 @@ jobs:
 
   build_debug_on_centos7:
     name: Build Debug on CentOS 7
-    needs: cpp_clang_format_linter
+    needs:
+      - cpp_clang_format_linter
+      - cpp_clang_tidy_linter
     runs-on: ubuntu-latest
     env:
       USE_JEMALLOC: OFF

.github/workflows/module_labeler_conf.yml

@@ -42,7 +42,8 @@ thrift:
 docs:
   - '**/*.md'
 scripts:
-  - 'scripts/**/*'
+  - 'admin_tools/**/*'
+  - 'build_tools/**/*'
   - '**/*.sh'
 build:
   - 'cmake_modules/**/*'

.github/workflows/regular-build.yml

@@ -41,7 +41,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: clang-format
-        run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
+        run: ./build_tools/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .
 
   build_cpp:
     name: Build Cpp

.licenserc.yaml

@@ -110,8 +110,10 @@ header:
     - 'cmake_modules/FindRT.cmake'
     - 'cmake_modules/FindDL.cmake'
     # Copyright (c) 2017 Guillaume Papin
-    - 'scripts/run-clang-format.py'
+    - 'build_tools/run-clang-format.py'
     # The MIT License (MIT), Copyright (c) 2015 Microsoft Corporation
+    - 'admin_tools/learn_stat.py'
+    - 'build_tools/compile_thrift.py'
     - 'cmake_modules/BaseFunctions.cmake'
     - 'docs/rdsn-README.md'
     - 'idl/command.thrift'
@@ -120,8 +122,6 @@ header:
     - 'idl/metadata.thrift'
     - 'idl/meta_admin.thrift'
     - 'idl/replica_admin.thrift'
-    - 'scripts/compile_thrift.py'
-    - 'scripts/learn_stat.py'
     - 'src/runtime/api_layer1.h'
     - 'src/runtime/api_task.h'
     - 'src/utils/api_utilities.h'
@@ -667,5 +667,13 @@ header:
     - 'src/zookeeper/zookeeper_session.h'
     - 'src/zookeeper/zookeeper_session_mgr.cpp'
     - 'src/zookeeper/zookeeper_session_mgr.h'
+    # Apache License, Version 2.0, Copyright 2018 Google LLC
+    - 'src/gutil/test/map_traits_test.cpp'
+    - 'src/gutil/test/map_util_test.h'
+    - 'src/gutil/test/map_util_unittest.cpp'
+    - 'src/gutil/test/no_destructor_test.cpp'
+    - 'src/gutil/map_traits.h'
+    - 'src/gutil/map_util.h'
+    - 'src/gutil/no_destructor.h'
 
   comment: on-failure

LICENSE

@@ -514,7 +514,7 @@ RESULTING FROM THE USE OF THIS SOFTWARE.
 
 --------------------------------------------------------------------------------
 
-scripts/run-clang-format.py - MIT License
+build_tools/run-clang-format.py - MIT License
 
 MIT License
 
@@ -570,3 +570,26 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+--------------------------------------------------------------------------------
+
+src/gutil/test/map_traits_test.cpp
+src/gutil/test/map_util_test.h
+src/gutil/test/map_util_unittest.cpp
+src/gutil/test/no_destructor_test.cpp
+src/gutil/map_traits.h
+src/gutil/map_util.h
+src/gutil/no_destructor.h
+
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

scripts/pegasus_add_node_list.sh → admin_tools/pegasus_add_node_list.sh

@@ -52,7 +52,7 @@ shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
 echo "Check the argument..."
-source ./scripts/pegasus_check_arguments.sh add_node_list $cluster $meta_list $replica_task_id_list
+source ./admin_tools/pegasus_check_arguments.sh add_node_list $cluster $meta_list $replica_task_id_list
 
 if [ $? -ne 0 ]; then
     echo "ERROR: the argument check failed"
@@ -78,7 +78,7 @@ do
   echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 done
 
-./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list true $nfs_rate_megabytes_per_disk
+./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list true $nfs_rate_megabytes_per_disk
 
 echo "Finish time: `date`"
 add_node_finish_time=$((`date +%s`))

scripts/pegasus_check_arguments.sh → admin_tools/pegasus_check_arguments.sh

@@ -41,7 +41,7 @@ if [ "$check_type" != "add_node_list" -a "$check_type" != "offline_node_list" ];
   exit 1
 fi
 
-source ./scripts/minos_common.sh
+source ./admin_tools/minos_common.sh
 find_cluster $cluster
 if [ $? -ne 0 ]; then
   echo "ERROR: cluster \"$cluster\" not found"

scripts/pegasus_migrate_zookeeper.sh → admin_tools/pegasus_migrate_zookeeper.sh

@@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0"  )" && pwd )"
 shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
-source ./scripts/minos_common.sh
+source ./admin_tools/minos_common.sh
 find_cluster $cluster
 if [ $? -ne 0 ]; then
   echo "ERROR: cluster \"$cluster\" not found"

scripts/pegasus_offline_node.sh → admin_tools/pegasus_offline_node.sh

@@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0"  )" && pwd )"
 shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
-source ./scripts/minos_common.sh
+source ./admin_tools/minos_common.sh
 find_cluster $cluster
 if [ $? -ne 0 ]; then
   echo "ERROR: cluster \"$cluster\" not found"

scripts/pegasus_offline_node_list.sh → admin_tools/pegasus_offline_node_list.sh

@@ -52,7 +52,7 @@ shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
 echo "Check the argument..."
-source ./scripts/pegasus_check_arguments.sh offline_node_list $cluster $meta_list $replica_task_id_list
+source ./admin_tools/pegasus_check_arguments.sh offline_node_list $cluster $meta_list $replica_task_id_list
 
 if [ $? -ne 0 ]; then
     echo "ERROR: the argument check failed"
@@ -93,7 +93,7 @@ echo
 for id in $id_list
 do
   echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-  ./scripts/pegasus_offline_node.sh $cluster $meta_list $id
+  ./admin_tools/pegasus_offline_node.sh $cluster $meta_list $id
   if [ $? -ne 0 ]; then
     echo "ERROR: offline replica task $id failed"
     exit 1

scripts/pegasus_restart_node.sh → admin_tools/pegasus_restart_node.sh

@@ -39,7 +39,7 @@ pwd="$( cd "$( dirname "$0"  )" && pwd )"
 shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
-source ./scripts/minos_common.sh
+source ./admin_tools/minos_common.sh
 find_cluster $cluster
 if [ $? -ne 0 ]; then
   echo "ERROR: cluster \"$cluster\" not found"

scripts/pegasus_rolling_update.sh → admin_tools/pegasus_rolling_update.sh

@@ -72,7 +72,7 @@ pwd="$( cd "$( dirname "$0"  )" && pwd )"
 shell_dir="$( cd $pwd/.. && pwd )"
 cd $shell_dir
 
-source ./scripts/minos_common.sh
+source ./admin_tools/minos_common.sh
 find_cluster $cluster
 if [ $? -ne 0 ]; then
   echo "ERROR: cluster \"$cluster\" not found"
@@ -338,7 +338,7 @@ fi
 
 if [ "$rebalance_cluster_after_rolling" == "true" ]; then
   echo "Start to rebalance cluster..."
-  ./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary
+  ./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary
 fi
 
 echo "Finish time: `date`"

scripts/pegasus_update_ingest_behind.sh → admin_tools/pegasus_update_ingest_behind.sh

@@ -327,7 +327,7 @@ echo
 
 if [ "$rebalance_cluster_after_rolling" == "true" ]; then
   echo "Start to rebalance cluster..."
-  ./scripts/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary
+  ./admin_tools/pegasus_rebalance_cluster.sh $cluster $meta_list $rebalance_only_move_primary
 fi
 
 echo "Finish time: `date`"