[Enhancement] support lake_clear_corrupted_cache option for BE

Signed-off-by: starrocks-xupeng <[email protected]>
starrocks-xupeng · Dec 23, 2024 · cf587a0 · cf587a0
1 parent fe991f1
commit cf587a0
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 1 deletion.
diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -1062,6 +1062,7 @@ CONF_mInt32(lake_pk_index_sst_max_compaction_versions, "100");
 // When the ratio of cumulative level to base level is greater than this config, use base merge.
 CONF_mDouble(lake_pk_index_cumulative_base_compaction_ratio, "0.1");
 CONF_Int32(lake_pk_index_block_cache_limit_percent, "10");
+CONF_mBool(lake_clear_corrupted_cache, "true");
 
 CONF_mBool(dependency_librdkafka_debug_enable, "false");
 

diff --git a/be/src/storage/lake/tablet_manager.cpp b/be/src/storage/lake/tablet_manager.cpp
@@ -22,6 +22,7 @@
 
 #include "agent/master_info.h"
 #include "common/compiler_util.h"
+#include "common/config.h"
 #include "fmt/format.h"
 #include "fs/fs.h"
 #include "fs/fs_util.h"
@@ -243,7 +244,30 @@ StatusOr<TabletMetadataPtr> TabletManager::load_tablet_metadata(std::shared_ptr<
     auto t0 = butil::gettimeofday_us();
     auto metadata = std::make_shared<TabletMetadataPB>();
     ProtobufFile file(metadata_location, std::move(fs));
-    RETURN_IF_ERROR(file.load(metadata.get(), fill_cache));
+    auto s = file.load(metadata.get(), fill_cache);
+    if (!s.ok()) {
+        if (s.is_corruption() && config::lake_clear_corrupted_cache) {
+            auto tmp_fs = FileSystem::CreateSharedFromString(metadata_location);
+            if (!tmp_fs.ok()) {
+                LOG(WARNING) << "fail to get file system to clear corrupted cache for " << metadata_location
+                             << ", error: " << tmp_fs.status();
+                return s;
+            }
+            auto drop_status = (*tmp_fs)->drop_local_cache(metadata_location);
+            if (drop_status.ok()) {
+                // reset metadata
+                metadata = std::make_shared<TabletMetadataPB>();
+                // read again
+                RETURN_IF_ERROR(file.load(metadata.get(), fill_cache));
+            } else {
+                LOG(WARNING) << "clear corrupted cache for " << metadata_location << " failed, "
+                             << "error: " << drop_status;
+                return s; // return error so load tablet meta can be retried
+            }
+        } else {
+            return s;
+        }
+    }
     g_get_tablet_metadata_latency << (butil::gettimeofday_us() - t0);
     return std::move(metadata);
 }