diff --git a/be/src/common/config.h b/be/src/common/config.h index 92722c6cf2c51f..dae92c4cab9e79 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1062,6 +1062,7 @@ CONF_mInt32(lake_pk_index_sst_max_compaction_versions, "100"); // When the ratio of cumulative level to base level is greater than this config, use base merge. CONF_mDouble(lake_pk_index_cumulative_base_compaction_ratio, "0.1"); CONF_Int32(lake_pk_index_block_cache_limit_percent, "10"); +CONF_mBool(lake_clear_corrupted_cache, "true"); CONF_mBool(dependency_librdkafka_debug_enable, "false"); diff --git a/be/src/storage/lake/tablet_manager.cpp b/be/src/storage/lake/tablet_manager.cpp index c1e01cdba0c517..828361ca36ab86 100644 --- a/be/src/storage/lake/tablet_manager.cpp +++ b/be/src/storage/lake/tablet_manager.cpp @@ -22,6 +22,7 @@ #include "agent/master_info.h" #include "common/compiler_util.h" +#include "common/config.h" #include "fmt/format.h" #include "fs/fs.h" #include "fs/fs_util.h" @@ -243,7 +244,30 @@ StatusOr TabletManager::load_tablet_metadata(std::shared_ptr< auto t0 = butil::gettimeofday_us(); auto metadata = std::make_shared(); ProtobufFile file(metadata_location, std::move(fs)); - RETURN_IF_ERROR(file.load(metadata.get(), fill_cache)); + auto s = file.load(metadata.get(), fill_cache); + if (!s.ok()) { + if (s.is_corruption() && config::lake_clear_corrupted_cache) { + auto tmp_fs = FileSystem::CreateSharedFromString(metadata_location); + if (!tmp_fs.ok()) { + LOG(WARNING) << "fail to get file system to clear corrupted cache for " << metadata_location + << ", error: " << tmp_fs.status(); + return s; + } + auto drop_status = (*tmp_fs)->drop_local_cache(metadata_location); + if (drop_status.ok()) { + // reset metadata + metadata = std::make_shared(); + // read again + RETURN_IF_ERROR(file.load(metadata.get(), fill_cache)); + } else { + LOG(WARNING) << "clear corrupted cache for " << metadata_location << " failed, " + << "error: " << drop_status; + return s; // return error so load tablet meta can be retried + } + } else { + return s; + } + } g_get_tablet_metadata_latency << (butil::gettimeofday_us() - t0); return std::move(metadata); }