diff --git a/.licenserc.yaml b/.licenserc.yaml index 3bb82270bc..185ee7757b 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -65,7 +65,6 @@ header: - 'src/replica/duplication/test/log.1.0.handle_real_private_log2' - 'src/replica/duplication/test/log.1.0.all_loaded_are_write_empties' # Used for patches for thirdparties. - - 'thirdparty/fix_fds_for_macos.patch' - 'thirdparty/fix_jemalloc_for_m1_on_macos.patch' - 'thirdparty/fix_libevent_for_macos.patch' - 'thirdparty/fix_prometheus-cpp_limits.patch' diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index b55bd05768..1882353c83 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -222,9 +222,6 @@ function(dsn_setup_compiler_flags) # use frame pointers to allow simple stack frame walking for backtraces. # This has a small perf hit but worth it for the ability to profile in production add_compile_options( -fno-omit-frame-pointer) - # -Wno-implicit-float-conversion - # Poco/Dynamic/VarHolder.h uses 'unsigned long' to 'float' conversion - add_compile_options(-Wno-implicit-float-conversion) find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh index 200fbbacb5..d39f9ac0f0 100755 --- a/scripts/pack_server.sh +++ b/scripts/pack_server.sh @@ -114,7 +114,6 @@ copy_file ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server ${pack}/b copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_meta_server.so ${pack}/bin copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_replica_server.so ${pack}/bin copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_utils.so ${pack}/bin -copy_file ./thirdparty/output/lib/libPoco*.so.* ${pack}/bin if [ "$use_jemalloc" == "on" ]; then copy_file ./thirdparty/output/lib/libjemalloc.so.2 ${pack}/bin diff --git a/scripts/pack_tools.sh b/scripts/pack_tools.sh index 793322435b..c3cc9e5d2e 100755 --- a/scripts/pack_tools.sh +++ b/scripts/pack_tools.sh @@ -123,7 +123,6 @@ cp -v -r ${BUILD_LATEST_DIR}/output/bin/pegasus_pressureclient ${pack}/bin/ mkdir -p ${pack}/lib copy_file ${BUILD_LATEST_DIR}/output/lib/*.so* ${pack}/lib/ -copy_file ./thirdparty/output/lib/libPoco*.so.* ${pack}/lib/ if [ "$use_jemalloc" == "on" ]; then copy_file ./thirdparty/output/lib/libjemalloc.so.2 ${pack}/lib/ diff --git a/src/block_service/CMakeLists.txt b/src/block_service/CMakeLists.txt index 1bcbba9ac6..0ce357ce1c 100644 --- a/src/block_service/CMakeLists.txt +++ b/src/block_service/CMakeLists.txt @@ -37,7 +37,6 @@ set(MY_BOOST_LIBS Boost::system Boost::filesystem) dsn_add_static_library() -add_subdirectory(fds) add_subdirectory(hdfs) add_subdirectory(local) add_subdirectory(test) diff --git a/src/block_service/block_service_manager.cpp b/src/block_service/block_service_manager.cpp index 07e1f8c866..a6c48509d2 100644 --- a/src/block_service/block_service_manager.cpp +++ b/src/block_service/block_service_manager.cpp @@ -22,7 +22,6 @@ #include #include "block_service/block_service.h" -#include "block_service/fds/fds_service.h" #include "block_service/hdfs/hdfs_service.h" #include "block_service/local/local_service.h" #include "runtime/task/task_code.h" @@ -39,10 +38,6 @@ namespace block_service { block_service_registry::block_service_registry() { - CHECK(utils::factory_store::register_factory( - "fds_service", block_filesystem::create, PROVIDER_TYPE_MAIN), - "register fds_service failed"); - CHECK(utils::factory_store::register_factory( "hdfs_service", block_filesystem::create, PROVIDER_TYPE_MAIN), "register hdfs_service failed"); diff --git a/src/block_service/fds/CMakeLists.txt b/src/block_service/fds/CMakeLists.txt deleted file mode 100644 index 6058605d1a..0000000000 --- a/src/block_service/fds/CMakeLists.txt +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set(MY_PROJ_NAME dsn.block_service.fds) - -#Source files under CURRENT project directory will be automatically included. -#You can manually set MY_PROJ_SRC to include source files under other directories. -set(MY_PROJ_SRC "") - -#Search mode for source files under CURRENT project directory ? -#"GLOB_RECURSE" for recursive search -#"GLOB" for non - recursive search -set(MY_SRC_SEARCH_MODE "GLOB") - -set(MY_PROJ_LIBS "") - -#Extra files that will be installed -set(MY_BINPLACES "") - -dsn_add_static_library() -target_include_directories(dsn.block_service.fds PRIVATE ${THIRDPARTY_INSTALL_DIR}/include/fds) diff --git a/src/block_service/fds/fds_service.cpp b/src/block_service/fds/fds_service.cpp deleted file mode 100644 index 8be5709ab5..0000000000 --- a/src/block_service/fds/fds_service.cpp +++ /dev/null @@ -1,720 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "block_service/fds/fds_service.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Poco/Exception.h" -#include "runtime/task/async_calls.h" -#include "utils/TokenBucket.h" -#include "utils/autoref_ptr.h" -#include "utils/blob.h" -#include "utils/env.h" -#include "utils/error_code.h" -#include "utils/filesystem.h" -#include "utils/flags.h" -#include "utils/fmt_logging.h" -#include "utils/safe_strerror_posix.h" -#include "utils/string_conv.h" - -namespace dsn { -namespace dist { -namespace block_service { - -DSN_DEFINE_uint32(replication, fds_write_limit_rate, 100, "write rate limit of fds(MB/s)"); -DSN_TAG_VARIABLE(fds_write_limit_rate, FT_MUTABLE); - -DSN_DEFINE_uint32(replication, fds_write_burst_size, 500, "write burst size of fds(MB)"); -DSN_TAG_VARIABLE(fds_write_burst_size, FT_MUTABLE); - -DSN_DEFINE_uint32(replication, fds_read_limit_rate, 100, "read rate limit of fds(MB/s)"); -DSN_TAG_VARIABLE(fds_read_limit_rate, FT_MUTABLE); - -DSN_DEFINE_uint32(replication, fds_read_batch_size, 100, "read batch size of fds(MB)"); -DSN_TAG_VARIABLE(fds_read_batch_size, FT_MUTABLE); - -class utils -{ -public: - /* - * read data from inputstream 'istr' then write to outputstream 'ostr' piece by piece until - * reach EOF, the size of each piece is specified by 'piece_size' - */ - static size_t copy_stream(std::istream &istr, std::ostream &ostr, size_t piece_size); - /* - * normalize a absolute path to a valid fds object path: - * 1. the output_path shouldn't start with / - * 2. the path shoudn't contain "." or ".." or "//" - * 3. if the path is a dir, the path should ends with "/" - * 4. if the path is root("/"), then return an empty string - */ - static std::string path_to_fds(const std::string &input, bool is_dir); - /* - * normalize a fds object path to a absolute path: - * 1. the output_path starts with / - * 2. all the postfix / are removed if the path marks a dir - */ - static std::string path_from_fds(const std::string &input, bool is_dir); -}; - -/*static*/ -size_t utils::copy_stream(std::istream &is, std::ostream &os, size_t piece_size) -{ - std::unique_ptr buffer(new char[piece_size]); - size_t length = 0; - is.read(buffer.get(), piece_size); - size_t got_length = is.gcount(); - while (got_length > 0) { - length += got_length; - os.write(buffer.get(), got_length); - if (is && os) { - is.read(buffer.get(), piece_size); - got_length = is.gcount(); - } else - got_length = 0; - } - return length; -} - -/*static*/ -std::string utils::path_to_fds(const std::string &input, bool is_dir) -{ - // TODO: handle the "." and ".." and "//" - if (input.size() < 1 || input == "/") - return std::string(); - std::string result; - if (input.front() == '/') - result = input.substr(1); - else - result = input; - - if (is_dir) - result.push_back('/'); - return result; -} - -/*static*/ -std::string utils::path_from_fds(const std::string &input, bool /*is_dir*/) -{ - std::string result = input; - if (!input.empty() && input.back() == '/') - result.pop_back(); - return result; -} - -DEFINE_TASK_CODE(LPC_FDS_CALL, TASK_PRIORITY_COMMON, THREAD_POOL_BLOCK_SERVICE) - -const std::string fds_service::FILE_LENGTH_CUSTOM_KEY = "x-xiaomi-meta-content-length"; -const std::string fds_service::FILE_MD5_KEY = "content-md5"; - -fds_service::fds_service() -{ - _write_token_bucket.reset(new folly::DynamicTokenBucket()); - _read_token_bucket.reset(new folly::DynamicTokenBucket()); -} - -fds_service::~fds_service() {} - -/** - * @brief fds_service::initialize - * @param args: {httpServer, accessKey, secretKey, bucket} - * @return - */ -error_code fds_service::initialize(const std::vector &args) -{ - galaxy::fds::FDSClientConfiguration config; - config.enableHttps(true); - config.setEndpoint(args[0]); - const std::string &access_key = args[1]; - const std::string &secret_key = args[2]; - - _client.reset(new galaxy::fds::GalaxyFDSClient(access_key, secret_key, config)); - _bucket_name = args[3]; - return dsn::ERR_OK; -} - -#define FDS_EXCEPTION_HANDLE(ERR_REFERENCE, OPERATION, INPUT_PARAMETER) \ - catch (const Poco::TimeoutException &ex) \ - { \ - LOG_ERROR("fds {} timeout: parameter({}), code({}), msg({})", \ - OPERATION, \ - INPUT_PARAMETER, \ - ex.code(), \ - ex.message()); \ - ERR_REFERENCE = ERR_TIMEOUT; \ - } \ - catch (const Poco::Exception &ex) \ - { \ - LOG_ERROR("fds {} get poco exception: parameter({}), code({}), msg({}), what({})", \ - OPERATION, \ - INPUT_PARAMETER, \ - ex.code(), \ - ex.message(), \ - ex.what()); \ - ERR_REFERENCE = ERR_FS_INTERNAL; \ - } \ - catch (...) \ - { \ - LOG_ERROR("fds {} get unknown exception: parameter({})", OPERATION, INPUT_PARAMETER); \ - ERR_REFERENCE = ERR_FS_INTERNAL; \ - } - -dsn::task_ptr fds_service::list_dir(const ls_request &req, - dsn::task_code code, - const ls_callback &callback, - dsn::task_tracker *tracker = nullptr) -{ - ls_future_ptr t(new ls_future(code, callback, 0)); - t->set_tracker(tracker); - - auto list_dir_in_background = [this, req, t]() { - ls_response resp; - std::string fds_path = utils::path_to_fds(req.dir_name, true); - try { - std::shared_ptr result = - _client->listObjects(_bucket_name, fds_path); - - while (true) { - const std::vector &objs = result->objectSummaries(); - const std::vector &common_prefix = result->commonPrefixes(); - resp.err = dsn::ERR_OK; - - // fds listing's objects are with full-path, we must extract the postfix to emulate - // the filesystem structure - for (const galaxy::fds::FDSObjectSummary &obj : objs) { - CHECK(fds_path.empty() || boost::starts_with(obj.objectName(), fds_path), - "invalid path({}) in parent({})", - obj.objectName(), - fds_path); - resp.entries->push_back( - {utils::path_from_fds(obj.objectName().substr(fds_path.size()), false), - false}); - } - for (const std::string &s : common_prefix) { - CHECK(fds_path.empty() || boost::starts_with(s, fds_path), - "invalid path({}) in parent({})", - s, - fds_path); - resp.entries->push_back( - {utils::path_from_fds(s.substr(fds_path.size()), true), true}); - } - - // list result may be paged - if (result->truncated()) { - auto res_temp = _client->listNextBatchOfObjects(*result); - result.swap(res_temp); - } else { - break; - } - } - } catch (const galaxy::fds::GalaxyFDSClientException &ex) { - LOG_ERROR("fds listObjects failed: parameter({}), code({}), msg({})", - req.dir_name, - ex.code(), - ex.what()); - resp.err = ERR_FS_INTERNAL; - } - FDS_EXCEPTION_HANDLE(resp.err, "listObject", req.dir_name.c_str()) - - if (resp.err == dsn::ERR_OK && resp.entries->empty()) { - try { - if (_client->doesObjectExist(_bucket_name, - utils::path_to_fds(req.dir_name, false))) { - LOG_ERROR("fds list_dir failed: path not dir, parameter({})", req.dir_name); - resp.err = ERR_INVALID_PARAMETERS; - } else { - LOG_ERROR("fds list_dir failed: path not found, parameter({})", req.dir_name); - resp.err = ERR_OBJECT_NOT_FOUND; - } - } catch (const galaxy::fds::GalaxyFDSClientException &ex) { - LOG_ERROR("fds doesObjectExist failed: parameter({}), code({}), msg({})", - req.dir_name, - ex.code(), - ex.what()); - resp.err = ERR_FS_INTERNAL; - } - FDS_EXCEPTION_HANDLE(resp.err, "doesObjectExist", req.dir_name.c_str()) - } - - t->enqueue_with(resp); - }; - - dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, list_dir_in_background); - return t; -} - -dsn::task_ptr fds_service::create_file(const create_file_request &req, - dsn::task_code code, - const create_file_callback &cb, - dsn::task_tracker *tracker = nullptr) -{ - create_file_future_ptr t(new create_file_future(code, cb, 0)); - t->set_tracker(tracker); - if (req.ignore_metadata) { - create_file_response resp; - resp.err = dsn::ERR_OK; - resp.file_handle = - new fds_file_object(this, req.file_name, utils::path_to_fds(req.file_name, false)); - t->enqueue_with(resp); - return t; - } - - auto create_file_in_background = [this, req, t]() { - create_file_response resp; - resp.err = ERR_IO_PENDING; - std::string fds_path = utils::path_to_fds(req.file_name, false); - - dsn::ref_ptr f = new fds_file_object(this, req.file_name, fds_path); - resp.err = f->get_file_meta(); - if (resp.err == ERR_OK || resp.err == ERR_OBJECT_NOT_FOUND) { - resp.err = ERR_OK; - resp.file_handle = f; - } - - t->enqueue_with(resp); - }; - - dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, create_file_in_background); - return t; -} - -dsn::task_ptr fds_service::remove_path(const remove_path_request &req, - dsn::task_code code, - const remove_path_callback &cb, - dsn::task_tracker *tracker) -{ - remove_path_future_ptr callback(new remove_path_future(code, cb, 0)); - callback->set_tracker(tracker); - auto remove_path_background = [this, req, callback]() { - remove_path_response resp; - resp.err = ERR_OK; - std::string fds_path = utils::path_to_fds(req.path, true); - bool should_remove_path = false; - - try { - std::shared_ptr result = - _client->listObjects(_bucket_name, fds_path); - while (result->objectSummaries().size() <= 0 && result->commonPrefixes().size() <= 0 && - result->truncated()) { - result = _client->listNextBatchOfObjects(*result); - } - const std::vector &objs = result->objectSummaries(); - const std::vector &common_prefix = result->commonPrefixes(); - - if (!objs.empty() || !common_prefix.empty()) { - // path is non-empty directory - if (req.recursive) { - should_remove_path = true; - } else { - LOG_ERROR("fds remove_path failed: dir not empty, parameter({})", req.path); - resp.err = ERR_DIR_NOT_EMPTY; - } - } else { - if (_client->doesObjectExist(_bucket_name, utils::path_to_fds(req.path, false))) { - should_remove_path = true; - } else { - LOG_ERROR("fds remove_path failed: path not found, parameter({})", req.path); - resp.err = ERR_OBJECT_NOT_FOUND; - } - } - } catch (const galaxy::fds::GalaxyFDSClientException &ex) { - LOG_ERROR("fds remove_path failed: parameter({}), code({}), msg({})", - req.path, - ex.code(), - ex.what()); - resp.err = ERR_FS_INTERNAL; - } - FDS_EXCEPTION_HANDLE(resp.err, "remove_path", req.path.c_str()); - - if (resp.err == ERR_OK && should_remove_path) { - fds_path = utils::path_to_fds(req.path, false); - try { - auto deleting = _client->deleteObjects(_bucket_name, fds_path, false); - if (deleting->countFailedObjects() <= 0) { - resp.err = ERR_OK; - } else { - LOG_ERROR("fds remove_path failed: countFailedObjects = {}, parameter({})", - deleting->countFailedObjects(), - req.path); - resp.err = ERR_FS_INTERNAL; - } - } catch (const galaxy::fds::GalaxyFDSClientException &ex) { - LOG_ERROR("fds remove_path failed: parameter({}), code({}), msg({})", - req.path, - ex.code(), - ex.what()); - resp.err = ERR_FS_INTERNAL; - } - FDS_EXCEPTION_HANDLE(resp.err, "remove_path", req.path.c_str()); - } - - callback->enqueue_with(resp); - return; - }; - - dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, remove_path_background); - return callback; -} - -fds_file_object::fds_file_object(fds_service *s, - const std::string &name, - const std::string &fds_path) - : block_file(name), - _service(s), - _fds_path(fds_path), - _md5sum(""), - _size(0), - _has_meta_synced(false) -{ -} - -fds_file_object::~fds_file_object() {} - -error_code fds_file_object::get_file_meta() -{ - error_code err = ERR_OK; - galaxy::fds::GalaxyFDSClient *c = _service->get_client(); - try { - auto meta = c->getObjectMetadata(_service->get_bucket_name(), _fds_path)->metadata(); - - // get file length - auto iter = meta.find(fds_service::FILE_LENGTH_CUSTOM_KEY); - CHECK(iter != meta.end(), - "can't find {} in object({})'s metadata", - fds_service::FILE_LENGTH_CUSTOM_KEY, - _fds_path); - bool valid = dsn::buf2uint64(iter->second, _size); - CHECK(valid, "error to get file size"); - - // get md5 key - iter = meta.find(fds_service::FILE_MD5_KEY); - CHECK(iter != meta.end(), - "can't find {} in object({})'s metadata", - fds_service::FILE_MD5_KEY, - _fds_path); - _md5sum = iter->second; - - _has_meta_synced = true; - } catch (const galaxy::fds::GalaxyFDSClientException &ex) { - if (ex.code() == Poco::Net::HTTPResponse::HTTP_NOT_FOUND) { - err = ERR_OBJECT_NOT_FOUND; - } else { - LOG_ERROR("fds getObjectMetadata failed: parameter({}), code({}), msg({})", - _name.c_str(), - ex.code(), - ex.what()); - err = ERR_FS_INTERNAL; - } - } - FDS_EXCEPTION_HANDLE(err, "getObjectMetadata", _fds_path.c_str()); - return err; -} - -error_code fds_file_object::get_content_in_batches(uint64_t start, - int64_t length, - /*out*/ std::ostream &os, - /*out*/ uint64_t &transfered_bytes) -{ - error_code err = ERR_OK; - transfered_bytes = 0; - - // get file meta if it is not synced - if (!_has_meta_synced) { - err = get_file_meta(); - if (ERR_OK != err) { - return err; - } - } - - // if length = -1, it means we should transfer the whole file - uint64_t to_transfer_bytes = (length == -1 ? _size : length); - - uint64_t pos = start; - uint64_t once_transfered_bytes = 0; - while (pos < start + to_transfer_bytes) { - const uint64_t BATCH_SIZE = FLAGS_fds_read_batch_size << 20; - uint64_t batch_size = std::min(BATCH_SIZE, start + to_transfer_bytes - pos); - - // burst size should not be less than consume size - const uint64_t rate = FLAGS_fds_read_limit_rate << 20; - _service->_read_token_bucket->consumeWithBorrowAndWait( - batch_size, rate, std::max(2 * rate, batch_size)); - - err = get_content(pos, batch_size, os, once_transfered_bytes); - transfered_bytes += once_transfered_bytes; - if (err != ERR_OK || once_transfered_bytes < batch_size) { - return err; - } - pos += batch_size; - } - - return ERR_OK; -} - -error_code fds_file_object::get_content(uint64_t pos, - uint64_t length, - /*out*/ std::ostream &os, - /*out*/ uint64_t &transfered_bytes) -{ - error_code err = ERR_OK; - transfered_bytes = 0; - while (true) { - // if we have download enough or we have reach the end - if (transfered_bytes >= length || transfered_bytes + pos >= _size) { - return ERR_OK; - } - - try { - galaxy::fds::GalaxyFDSClient *c = _service->get_client(); - std::shared_ptr obj; - obj = c->getObject(_service->get_bucket_name(), - _fds_path, - pos + transfered_bytes, - length - transfered_bytes); - LOG_DEBUG("get object from fds succeed, remote_file({})", _fds_path); - std::istream &is = obj->objectContent(); - transfered_bytes += utils::copy_stream(is, os, PIECE_SIZE); - err = ERR_OK; - } catch (const galaxy::fds::GalaxyFDSClientException &ex) { - LOG_ERROR("fds getObject error: remote_file({}), code({}), msg({})", - file_name(), - ex.code(), - ex.what()); - if (ex.code() == Poco::Net::HTTPResponse::HTTP_NOT_FOUND) { - _has_meta_synced = true; - _md5sum = ""; - _size = 0; - err = ERR_OBJECT_NOT_FOUND; - } else { - err = ERR_FS_INTERNAL; - } - } - FDS_EXCEPTION_HANDLE(err, "getObject", file_name().c_str()) - - if (err != ERR_OK) { - return err; - } - } -} - -error_code fds_file_object::put_content(/*in-out*/ std::istream &is, - int64_t to_transfer_bytes, - uint64_t &transfered_bytes) -{ - error_code err = ERR_OK; - transfered_bytes = 0; - galaxy::fds::GalaxyFDSClient *c = _service->get_client(); - - // get tokens from token bucket - if (!_service->_write_token_bucket->consumeWithBorrowAndWait(to_transfer_bytes, - FLAGS_fds_write_limit_rate << 20, - FLAGS_fds_write_burst_size - << 20)) { - LOG_INFO("the transfer count({}B) is greater than burst size({}MB), so it is rejected by " - "token bucket", - to_transfer_bytes, - FLAGS_fds_write_burst_size); - return ERR_BUSY; - } - - try { - c->putObject(_service->get_bucket_name(), _fds_path, is, galaxy::fds::FDSObjectMetadata()); - } catch (const galaxy::fds::GalaxyFDSClientException &ex) { - LOG_ERROR("fds putObject error: remote_file({}), code({}), msg({})", - file_name(), - ex.code(), - ex.what()); - err = ERR_FS_INTERNAL; - } - FDS_EXCEPTION_HANDLE(err, "putObject", file_name().c_str()) - - if (err != ERR_OK) { - return err; - } - - LOG_INFO("start to synchronize meta data after successfully wrote data to fds"); - err = get_file_meta(); - if (err == ERR_OK) { - transfered_bytes = _size; - } - return err; -} - -dsn::task_ptr fds_file_object::write(const write_request &req, - dsn::task_code code, - const write_callback &cb, - dsn::task_tracker *tracker = nullptr) -{ - write_future_ptr t(new write_future(code, cb, 0)); - t->set_tracker(tracker); - - add_ref(); - auto write_in_background = [this, req, t]() { - write_response resp; - std::istringstream is; - is.str(std::string(req.buffer.data(), req.buffer.length())); - resp.err = put_content(is, req.buffer.length(), resp.written_size); - - t->enqueue_with(resp); - release_ref(); - }; - - dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, write_in_background); - return t; -} - -// TODO: handle the localfile path -dsn::task_ptr fds_file_object::upload(const upload_request &req, - dsn::task_code code, - const upload_callback &cb, - dsn::task_tracker *tracker = nullptr) -{ - upload_future_ptr t(new upload_future(code, cb, 0)); - t->set_tracker(tracker); - - add_ref(); - auto upload_background = [this, req, t]() { - const std::string &local_file = req.input_local_name; - // get file size - int64_t file_sz = 0; - dsn::utils::filesystem::file_size( - local_file, dsn::utils::FileDataType::kSensitive, file_sz); - - upload_response resp; - // TODO: we can cache the whole file in buffer, then upload the buffer rather than the - // ifstream, because if ifstream read file beyond 60s, fds-server will reset the session, - // then upload will fail with error broken-pipe - std::ifstream is(local_file, std::ios::binary | std::ios::in); - - if (!is.is_open()) { - LOG_ERROR("fds upload failed: open local file({}) failed when upload to({}), error({})", - local_file, - file_name(), - ::dsn::utils::safe_strerror(errno)); - resp.err = dsn::ERR_FILE_OPERATION_FAILED; - } else { - resp.err = put_content(is, file_sz, resp.uploaded_size); - is.close(); - } - - t->enqueue_with(resp); - release_ref(); - }; - - dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, upload_background); - return t; -} - -dsn::task_ptr fds_file_object::read(const read_request &req, - dsn::task_code code, - const read_callback &cb, - dsn::task_tracker *tracker = nullptr) -{ - read_future_ptr t(new read_future(code, cb, 0)); - t->set_tracker(tracker); - - add_ref(); - auto read_in_background = [this, req, t]() { - read_response resp; - std::ostringstream os; - uint64_t transferd_size; - resp.err = get_content_in_batches(req.remote_pos, req.remote_length, os, transferd_size); - if (os.tellp() > 0) { - std::string *output = new std::string(); - *output = os.str(); - std::shared_ptr ptr((char *)output->c_str(), [output](char *) { delete output; }); - resp.buffer.assign(std::move(ptr), 0, output->length()); - } - t->enqueue_with(resp); - release_ref(); - }; - - dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, read_in_background); - return t; -} - -// TODO: handle the localfile path -dsn::task_ptr fds_file_object::download(const download_request &req, - dsn::task_code code, - const download_callback &cb, - dsn::task_tracker *tracker = nullptr) -{ - download_future_ptr t(new download_future(code, cb, 0)); - t->set_tracker(tracker); - download_response resp; - - // TODO(yingchun): use rocksdb API to implement this. - std::shared_ptr handle(new std::ofstream( - req.output_local_name, std::ios::binary | std::ios::out | std::ios::trunc)); - if (!handle->is_open()) { - LOG_ERROR("fds download failed: fail to open localfile({}) when download({}), error({})", - req.output_local_name, - _fds_path, - ::dsn::utils::safe_strerror(errno)); - resp.err = ERR_FILE_OPERATION_FAILED; - resp.downloaded_size = 0; - t->enqueue_with(resp); - return t; - } - - add_ref(); - auto download_background = [this, req, handle, t]() { - download_response resp; - uint64_t transfered_size; - resp.err = - get_content_in_batches(req.remote_pos, req.remote_length, *handle, transfered_size); - resp.downloaded_size = 0; - if (resp.err == ERR_OK && handle->tellp() != -1) { - resp.downloaded_size = handle->tellp(); - } - handle->close(); - if (resp.err != ERR_OK && dsn::utils::filesystem::file_exists(req.output_local_name)) { - LOG_ERROR("fail to download file {} from fds, remove localfile {}", - _fds_path, - req.output_local_name); - dsn::utils::filesystem::remove_path(req.output_local_name); - } else if ((resp.err = dsn::utils::filesystem::md5sum(req.output_local_name, - resp.file_md5)) != ERR_OK) { - LOG_ERROR("download failed when calculate the md5sum of local file {}", - req.output_local_name); - } - t->enqueue_with(resp); - release_ref(); - }; - - dsn::tasking::enqueue(LPC_FDS_CALL, nullptr, download_background); - return t; -} -} // namespace block_service -} // namespace dist -} // namespace dsn diff --git a/src/block_service/fds/fds_service.h b/src/block_service/fds/fds_service.h deleted file mode 100644 index 916cd57efc..0000000000 --- a/src/block_service/fds/fds_service.h +++ /dev/null @@ -1,156 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef FDS_SERVICE_H -#define FDS_SERVICE_H - -#include -#include -#include -#include -#include -#include -#include - -#include "block_service/block_service.h" -#include "runtime/task/task.h" -#include "runtime/task/task_code.h" -#include "utils/error_code.h" -#include "utils/TokenBucket.h" - -namespace dsn { -class task_tracker; -} // namespace dsn - -namespace folly { -using DynamicTokenBucket = folly::BasicDynamicTokenBucket; -} // namespace folly - -namespace galaxy { -namespace fds { -class GalaxyFDSClient; -} -} // namespace galaxy - -namespace dsn { -namespace dist { -namespace block_service { - -class fds_service : public block_filesystem -{ -public: - static const std::string FILE_MD5_KEY; - static const std::string FILE_LENGTH_KEY; - static const std::string FILE_LENGTH_CUSTOM_KEY; - -public: - fds_service(); - galaxy::fds::GalaxyFDSClient *get_client() { return _client.get(); } - const std::string &get_bucket_name() { return _bucket_name; } - - virtual ~fds_service() override; - virtual error_code initialize(const std::vector &args) override; - virtual dsn::task_ptr list_dir(const ls_request &req, - dsn::task_code code, - const ls_callback &callback, - dsn::task_tracker *tracker) override; - - virtual dsn::task_ptr create_file(const create_file_request &req, - dsn::task_code code, - const create_file_callback &cb, - dsn::task_tracker *tracker) override; - - // - // Attention： - // -- remove the path directly on fds, will not enter trash - // -- when req.path is a directory, this operation may consume much time if there are many - // files under this directory - // - virtual dsn::task_ptr remove_path(const remove_path_request &req, - dsn::task_code code, - const remove_path_callback &cb, - dsn::task_tracker *tracker) override; - -private: - std::shared_ptr _client; - std::string _bucket_name; - std::unique_ptr _write_token_bucket; - std::unique_ptr _read_token_bucket; - - friend class fds_file_object; -}; - -class fds_file_object : public block_file -{ -public: - fds_file_object(fds_service *s, const std::string &name, const std::string &fds_path); - - virtual ~fds_file_object(); - virtual uint64_t get_size() override { return _size; } - virtual const std::string &get_md5sum() override { return _md5sum; } - - virtual dsn::task_ptr write(const write_request &req, - dsn::task_code code, - const write_callback &cb, - dsn::task_tracker *tracker) override; - - virtual dsn::task_ptr read(const read_request &req, - dsn::task_code code, - const read_callback &cb, - dsn::task_tracker *tracker) override; - - virtual dsn::task_ptr upload(const upload_request &req, - dsn::task_code code, - const upload_callback &cb, - dsn::task_tracker *tracker) override; - - virtual dsn::task_ptr download(const download_request &req, - dsn::task_code code, - const download_callback &cb, - dsn::task_tracker *tracker) override; - - // Possible errors: - // - ERR_FS_INTERNAL - // - ERR_OBJECT_NOT_FOUND - // - ERR_TIMEOUT - error_code get_file_meta(); - -private: - error_code get_content_in_batches(uint64_t start, - int64_t length, - /*out*/ std::ostream &os, - /*out*/ uint64_t &transfered_bytes); - error_code get_content(uint64_t pos, - uint64_t length, - /*out*/ std::ostream &os, - /*out*/ uint64_t &transfered_bytes); - error_code put_content(/*in-out*/ std::istream &is, - /*int*/ int64_t to_transfer_bytes, - /*out*/ uint64_t &transfered_bytes); - - fds_service *_service; - std::string _fds_path; - std::string _md5sum; - uint64_t _size; - bool _has_meta_synced; - - static const size_t PIECE_SIZE = 16384; // 16k -}; -} // namespace block_service -} // namespace dist -} // namespace dsn -#endif // FDS_SERVICE_H diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt index 78202667dd..6fbfa9ca11 100644 --- a/src/block_service/test/CMakeLists.txt +++ b/src/block_service/test/CMakeLists.txt @@ -25,15 +25,9 @@ set(MY_PROJ_LIBS dsn_replication_common dsn.block_service dsn.block_service.local - dsn.block_service.fds dsn.block_service.hdfs dsn_runtime dsn_utils - galaxy-fds-sdk-cpp - PocoNet - PocoFoundation - PocoNetSSL - PocoJSON gtest gtest_main hdfs diff --git a/src/block_service/test/config-test.ini b/src/block_service/test/config-test.ini index 2acb86e2bb..19a33003c3 100644 --- a/src/block_service/test/config-test.ini +++ b/src/block_service/test/config-test.ini @@ -44,11 +44,6 @@ fast_flush = true short_header = false stderr_start_level = LOG_LEVEL_WARNING -[fds_concurrent_test] -total_files = 64 -min_size = 100 -max_size = 150 - [threadpool.THREAD_POOL_BLOCK_SERVICE] worker_count = 8 diff --git a/src/block_service/test/fds_service_test.cpp b/src/block_service/test/fds_service_test.cpp deleted file mode 100644 index 3c8d15b2b4..0000000000 --- a/src/block_service/test/fds_service_test.cpp +++ /dev/null @@ -1,819 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "block_service/fds/fds_service.h" - -#include -#include -#include -#include -#include -#include -#include -#include // IWYU pragma: keep -#include -#include - -#include "block_service/block_service.h" -#include "gtest/gtest.h" -#include "utils/autoref_ptr.h" -#include "utils/blob.h" -#include "utils/enum_helper.h" -#include "utils/filesystem.h" -#include "utils/flags.h" -#include "utils/fmt_logging.h" -#include "utils/process_utils.h" -#include "utils/rand.h" -#include "utils/safe_strerror_posix.h" -#include "utils/strings.h" -#include "utils/threadpool_code.h" -#include "utils/utils.h" - -using namespace dsn; -using namespace dsn::dist::block_service; - -DSN_DEFINE_uint64(fds_concurrent_test, min_size, 64, ""); -DSN_DEFINE_uint64(fds_concurrent_test, max_size, 64, ""); - -static std::string example_server_address = ""; -// please modify the the paras below to enable fds_service_test, default fds_service_test will be -// skipped and return true -static std::string server_address = ""; -static std::string access_key = ""; -static std::string access_secret = ""; -static std::string bucket_name = ""; - -static void file_eq_compare(const std::string &fname1, const std::string &fname2) -{ - static const int length = 4096; - std::shared_ptr buffer(dsn::utils::make_shared_array(length * 2)); - char *buf1 = buffer.get(), *buf2 = buffer.get() + length; - - std::ifstream ifile1(fname1.c_str(), std::ios::in | std::ios::binary); - std::ifstream ifile2(fname2.c_str(), std::ios::in | std::ios::binary); - - auto file_length = [](std::ifstream &is) { - is.seekg(0, is.end); - int result = is.tellg(); - is.seekg(0, is.beg); - return result; - }; - - int l = file_length(ifile1); - ASSERT_EQ(l, file_length(ifile2)); - - for (int i = 0; i < l; i += length) { - int up_to_bytes = length < (l - i) ? length : (l - i); - ifile1.read(buf1, up_to_bytes); - ifile2.read(buf2, up_to_bytes); - ASSERT_TRUE(dsn::utils::mequals(buf1, buf2, up_to_bytes)); - } -} - -class FDSClientTest : public testing::Test -{ -protected: - virtual void SetUp() override; - virtual void TearDown() override; - - struct file_desc - { - std::string filename; - std::string md5; - size_t length; - }; - - file_desc f1; - file_desc f2; - std::string local_file_for_download; -}; - -void FDSClientTest::SetUp() -{ - f1.filename = "test_file"; - f2.filename = "test_2"; - local_file_for_download = "local_download"; - - // generate a test file - { - int lines = rand::next_u32(1000, 2000); - FILE *fp = fopen(f1.filename.c_str(), "wb"); - for (int i = 0; i < lines; ++i) { - fprintf(fp, "%04d_this_is_a_simple_test_file\n", i); - } - fclose(fp); - - std::stringstream ss; - CHECK_EQ(utils::pipe_execute((std::string("md5sum ") + f1.filename).c_str(), ss), 0); - ss >> f1.md5; - // well, the string of each line in _test_file is 32 - f1.length = 32 * lines; - } - - // generate another test file - { - int lines = rand::next_u32(10, 20); - FILE *fp = fopen(f2.filename.c_str(), "wb"); - for (int i = 0; i < lines; ++i) { - fprintf(fp, "%04d_this_is_a_simple_test_file\n", i); - } - fclose(fp); - - std::stringstream ss; - CHECK_EQ(utils::pipe_execute((std::string("md5sum ") + f2.filename).c_str(), ss), 0); - ss >> f2.md5; - // well, the string of each line in _test_file is 32 - f2.length = 32 * lines; - } -} - -void FDSClientTest::TearDown() {} - -DEFINE_TASK_CODE(lpc_btest, TASK_PRIORITY_HIGH, dsn::THREAD_POOL_DEFAULT) - -// TODO(yingchun): add encryption test when FDSClient supports encryption. -TEST_F(FDSClientTest, test_basic_operation) -{ - const char *files[] = {"/fdstest/fdstest1/test1/test1", - "/fdstest/fdstest1/test1/test2", - "/fdstest/fdstest1/test2/test1", - "/fdstest/fdstest1/test2/test2", - "/fdstest/fdstest2/test2", - "/fdstest/fdstest3", - "/fdstest/fds_rootfile", - nullptr}; - // ensure prefix_path is the prefix of some file in files - std::string prefix_path = std::string("/fdstest/fdstest1/test1"); - int total_files; - - std::shared_ptr s = std::make_shared(); - // server, access-key, access-secret, bucket_name - std::vector args = {server_address, access_key, access_secret, bucket_name}; - - if (server_address == example_server_address) { - // user don't specify the server-address, we just return true - return; - } - - s->initialize(args); - - create_file_response cf_resp; - ls_response l_resp; - upload_response u_resp; - download_response d_resp; - read_response r_resp; - write_response w_resp; - remove_path_response rem_resp; - - auto entry_cmp = [](const ls_entry &entry1, const ls_entry &entry2) { - return entry1.entry_name < entry2.entry_name; - }; - auto entry_vec_eq = [](const std::vector &entry_vec1, - const std::vector &entry_vec2) { - ASSERT_EQ(entry_vec1.size(), entry_vec2.size()); - for (unsigned int i = 0; i < entry_vec1.size(); ++i) { - ASSERT_EQ(entry_vec1[i].entry_name, entry_vec2[i].entry_name); - ASSERT_EQ(entry_vec1[i].is_directory, entry_vec2[i].is_directory) - << "on index " << i << ", name " << entry_vec1[i].entry_name; - } - }; - - // first clean all - { - std::cout << "clean all old files" << std::endl; - for (int i = 0; files[i]; ++i) { - std::cout << "delete file " << files[i] << std::endl; - s->remove_path(remove_path_request{std::string(files[i]), true}, - lpc_btest, - [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, - nullptr) - ->wait(); - ASSERT_TRUE(dsn::ERR_OK == rem_resp.err || dsn::ERR_OBJECT_NOT_FOUND == rem_resp.err); - } - } - - // first upload all these files - { - std::cout << "Test upload files" << std::endl; - for (total_files = 0; files[total_files]; ++total_files) { - std::cout << "create and upload: " << files[total_files] << std::endl; - s->create_file(create_file_request{std::string(files[total_files]), true}, - lpc_btest, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(cf_resp.err, dsn::ERR_OK); - - cf_resp.file_handle - ->upload(upload_request{FDSClientTest::f1.filename}, - lpc_btest, - [&u_resp](const upload_response &r) { u_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, u_resp.err); - ASSERT_EQ(FDSClientTest::f1.length, cf_resp.file_handle->get_size()); - ASSERT_EQ(FDSClientTest::f1.md5, cf_resp.file_handle->get_md5sum()); - } - - // create a non-exist file for read - { - std::cout << "create a non-exist file for read: fds_hellworld" << std::endl; - s->create_file(create_file_request{std::string("fds_helloworld"), false}, - lpc_btest, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_TRUE(cf_resp.file_handle->get_md5sum().empty()); - } - - // create an exist-file for write - { - std::cout << "create an exist file for write: " << files[total_files - 1] << std::endl; - s->create_file(create_file_request{std::string(files[total_files - 1]), false}, - lpc_btest, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_EQ(FDSClientTest::f1.md5, cf_resp.file_handle->get_md5sum()); - - cf_resp.file_handle - ->upload(upload_request{FDSClientTest::f2.filename}, - lpc_btest, - [&u_resp](const upload_response &r) { u_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, u_resp.err); - ASSERT_EQ(FDSClientTest::f2.length, cf_resp.file_handle->get_size()); - ASSERT_EQ(FDSClientTest::f2.md5, cf_resp.file_handle->get_md5sum()); - - // upload an non-exist local file - cf_resp.file_handle - ->upload(upload_request{"joke_file"}, - lpc_btest, - [&u_resp](const upload_response &r) { u_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_FILE_OPERATION_FAILED, u_resp.err); - - // upload an local file which we don't have read-permission - cf_resp.file_handle - ->upload(upload_request{"/root/.profile"}, - lpc_btest, - [&u_resp](const upload_response &r) { u_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_FILE_OPERATION_FAILED, u_resp.err); - } - } - - // then test list files - { - std::cout << "test ls files" << std::endl; - - // list the root - std::cout << "list the test root" << std::endl; - std::vector root = { - {"fdstest1", true}, {"fdstest2", true}, {"fdstest3", false}, {"fds_rootfile", false}}; - std::sort(root.begin(), root.end(), entry_cmp); - - s->list_dir(ls_request{"/fdstest"}, - lpc_btest, - [&l_resp](const ls_response &resp) { l_resp = resp; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, l_resp.err); - std::sort(l_resp.entries->begin(), l_resp.entries->end(), entry_cmp); - entry_vec_eq(root, *l_resp.entries); - - // list the fdstest1 - std::cout << "list the fdstest1" << std::endl; - std::vector fdstest1 = {{"test1", true}, {"test2", true}}; - std::sort(fdstest1.begin(), fdstest1.end(), entry_cmp); - - s->list_dir(ls_request{"/fdstest1"}, - lpc_btest, - [&l_resp](const ls_response &resp) { l_resp = resp; }, - nullptr) - ->wait(); - std::sort(l_resp.entries->begin(), l_resp.entries->end(), entry_cmp); - entry_vec_eq(fdstest1, *l_resp.entries); - - // list the fdstest1/test2 - std::cout << "list the fdstest1/test2" << std::endl; - std::vector fdstest1_slash_test2 = {{"test1", false}, {"test2", false}}; - std::sort(fdstest1_slash_test2.begin(), fdstest1_slash_test2.end(), entry_cmp); - - s->list_dir(ls_request{"/fdstest1/test2"}, - lpc_btest, - [&l_resp](const ls_response &resp) { l_resp = resp; }, - nullptr) - ->wait(); - std::sort(l_resp.entries->begin(), l_resp.entries->end(), entry_cmp); - entry_vec_eq(fdstest1_slash_test2, *l_resp.entries); - - // list a non-exist dir - std::cout << "list a non-exist dir /fds_hehe" << std::endl; - s->list_dir(ls_request{"/fds_hehe"}, - lpc_btest, - [&l_resp](const ls_response &resp) { l_resp = resp; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OBJECT_NOT_FOUND, l_resp.err); - - // list a regular file - std::cout << "list a regular file /fds_rootfile" << std::endl; - s->list_dir(ls_request{"/fds_rootfile"}, - lpc_btest, - [&l_resp](const ls_response &resp) { l_resp = resp; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_INVALID_PARAMETERS, l_resp.err); - } - - // then test download files - { - std::cout << "test download file, don't ignore metadata" << std::endl; - for (int i = 0; i < total_files - 1; ++i) { - std::cout << "test download " << files[i] << std::endl; - s->create_file(create_file_request{files[i], false}, - lpc_btest, - [&cf_resp](const create_file_response &resp) { cf_resp = resp; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_NE(nullptr, cf_resp.file_handle.get()); - ASSERT_EQ(f1.length, cf_resp.file_handle->get_size()); - ASSERT_EQ(f1.md5, cf_resp.file_handle->get_md5sum()); - - cf_resp.file_handle - ->download(download_request{local_file_for_download, 0, -1}, - lpc_btest, - [&d_resp](const download_response &resp) { d_resp = resp; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, d_resp.err); - ASSERT_EQ(cf_resp.file_handle->get_size(), d_resp.downloaded_size); - ASSERT_EQ(cf_resp.file_handle->get_md5sum(), d_resp.file_md5); - file_eq_compare(f1.filename, local_file_for_download); - } - - std::cout << "test download file, ignore metadata: " << files[total_files - 1] << std::endl; - s->create_file(create_file_request{files[total_files - 1], true}, - lpc_btest, - [&cf_resp](const create_file_response &resp) { cf_resp = resp; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_NE(nullptr, cf_resp.file_handle.get()); - ASSERT_TRUE(cf_resp.file_handle->get_md5sum().empty()); - - cf_resp.file_handle - ->download(download_request{local_file_for_download, 0, -1}, - lpc_btest, - [&d_resp](const download_response &resp) { d_resp = resp; }, - nullptr) - ->wait(); - ASSERT_EQ(f2.length, cf_resp.file_handle->get_size()); - ASSERT_EQ(f2.md5, cf_resp.file_handle->get_md5sum()); - file_eq_compare(f2.filename, local_file_for_download); - - std::cout << "test partitial download " << std::endl; - cf_resp.file_handle - ->download(download_request{local_file_for_download, 32, 32}, - lpc_btest, - [&d_resp](const download_response &resp) { d_resp = resp; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, d_resp.err); - ASSERT_EQ(32, d_resp.downloaded_size); - { - std::shared_ptr f(fopen("tmp_generate", "wb"), [](FILE *p) { fclose(p); }); - fprintf(f.get(), "%04d_this_is_a_simple_test_file\n", 1); - } - file_eq_compare(std::string("tmp_generate"), local_file_for_download); - } - - // try to read a non-exist file - { - std::cout << "test try to read non-exist file" << std::endl; - s->create_file(create_file_request{"non_exist_file", true}, - lpc_btest, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_NE(nullptr, cf_resp.file_handle.get()); - ASSERT_TRUE(cf_resp.file_handle->get_md5sum().empty()); - - cf_resp.file_handle - ->read(read_request{0, -1}, - lpc_btest, - [&r_resp](const read_response &r) { r_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OBJECT_NOT_FOUND, r_resp.err); - - // now file handle has been synced from remote - cf_resp.file_handle - ->download(download_request{"local_file", 0, -1}, - lpc_btest, - [&d_resp](const download_response &r) { d_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OBJECT_NOT_FOUND, d_resp.err); - // so we expect the file doesn't create - ASSERT_FALSE(dsn::utils::filesystem::file_exists("local_file")); - } - - // try to download to a path where we can't create the file - { - std::cout << "test try to download to a path where we can't create the file" << std::endl; - s->create_file(create_file_request{files[0], false}, - lpc_btest, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_NE(nullptr, cf_resp.file_handle.get()); - ASSERT_EQ(cf_resp.file_handle->get_size(), f1.length); - ASSERT_EQ(cf_resp.file_handle->get_md5sum(), f1.md5); - - cf_resp.file_handle - ->download(download_request{"test_dir/test_file.txt", 0, -1}, - lpc_btest, - [&d_resp](const download_response &r) { d_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_FILE_OPERATION_FAILED, d_resp.err); - ASSERT_EQ(0, d_resp.downloaded_size); - - cf_resp.file_handle - ->download(download_request{"/root/.profile", 0, -1}, - lpc_btest, - [&d_resp](const download_response &r) { d_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_FILE_OPERATION_FAILED, d_resp.err); - ASSERT_EQ(0, d_resp.downloaded_size); - } - - // try to do write/read - { - std::cout << "test read write operation" << std::endl; - s->create_file(create_file_request{files[0], false}, - lpc_btest, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_NE(nullptr, cf_resp.file_handle.get()); - ASSERT_EQ(cf_resp.file_handle->get_size(), f1.length); - ASSERT_EQ(cf_resp.file_handle->get_md5sum(), f1.md5); - - const char *test_buffer = "1234567890qwertyuiopasdfghjklzxcvbnm"; - int length = strlen(test_buffer); - dsn::blob bb(test_buffer, 0, length); - - cf_resp.file_handle - ->write(write_request{bb}, - lpc_btest, - [&w_resp](const write_response &w) { w_resp = w; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, w_resp.err); - ASSERT_EQ(length, w_resp.written_size); - ASSERT_EQ(length, cf_resp.file_handle->get_size()); - ASSERT_NE(f1.md5, cf_resp.file_handle->get_md5sum()); - - std::cout << "test read just written value" << std::endl; - cf_resp.file_handle - ->read(read_request{0, -1}, - lpc_btest, - [&r_resp](const read_response &r) { r_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, r_resp.err); - ASSERT_EQ(length, r_resp.buffer.length()); - ASSERT_TRUE(dsn::utils::mequals(r_resp.buffer.data(), test_buffer, length)); - - // partitial read - cf_resp.file_handle - ->read(read_request{5, 10}, - lpc_btest, - [&r_resp](const read_response &r) { r_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, r_resp.err); - ASSERT_EQ(10, r_resp.buffer.length()); - ASSERT_TRUE(dsn::utils::mequals(r_resp.buffer.data(), test_buffer + 5, 10)); - } - - // then test remove path - { - // test remove_path - { - std::cout << "Test remove non-empty path with recusive = false" << std::endl; - s->remove_path(remove_path_request{std::string(prefix_path), false}, - lpc_btest, - [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, - nullptr) - ->wait(); - ASSERT_EQ(rem_resp.err, ERR_DIR_NOT_EMPTY); - - std::cout << "Test remove non-empty path with recusive = true" << std::endl; - s->remove_path(remove_path_request{std::string(prefix_path), true}, - lpc_btest, - [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, - nullptr) - ->wait(); - ASSERT_EQ(rem_resp.err, ERR_OK); - - std::cout << "Test remove a common path" << std::endl; - for (total_files = 0; files[total_files]; total_files++) { - std::string filename = files[total_files]; - // file under prefix_path already removed - if (filename.find(prefix_path) == std::string::npos) { - // remove a single file with recusive = true/false - bool recursive = ((total_files % 2) == 0); - s->remove_path( - remove_path_request{std::string(files[total_files]), recursive}, - lpc_btest, - [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, - nullptr) - ->wait(); - ASSERT_EQ(rem_resp.err, ERR_OK); - } - s->create_file(create_file_request{files[total_files], false}, - lpc_btest, - [&cf_resp](const create_file_response &resp) { cf_resp = resp; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_TRUE(cf_resp.file_handle->get_md5sum().empty()); - } - } - - // test list_dir that total file/dir count greater than 1000 - { - int total_file_cnt = 30; - std::cout << "upload " << total_file_cnt << " to fds server" << std::endl; - std::string prefix = "/fdstest_prefix"; - std::string file_name_prefix = "file_"; - std::vector t_entries; - - // first clean in case last test failed - std::cout << "first clean the dir in case old test failed" << std::endl; - s->remove_path(remove_path_request{std::string(prefix), true}, - lpc_btest, - [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, - nullptr) - ->wait(); - - // generate total_file_cnt files - std::cout << "then start to upload files" << std::endl; - for (int i = 1; i <= total_file_cnt; i++) { - std::string filename = file_name_prefix + std::to_string(i); - t_entries.emplace_back(ls_entry{std::string(filename), false}); - - s->create_file(create_file_request{std::string(prefix + "/" + filename), true}, - lpc_btest, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(cf_resp.err, dsn::ERR_OK); - - cf_resp.file_handle - ->upload(upload_request{FDSClientTest::f1.filename}, - lpc_btest, - [&u_resp](const upload_response &r) { u_resp = r; }, - nullptr) - ->wait(); - - ASSERT_EQ(dsn::ERR_OK, u_resp.err); - ASSERT_EQ(FDSClientTest::f1.length, cf_resp.file_handle->get_size()); - ASSERT_EQ(FDSClientTest::f1.md5, cf_resp.file_handle->get_md5sum()); - } - l_resp.entries->clear(); - s->list_dir(ls_request{prefix}, - lpc_btest, - [&l_resp](const ls_response &resp) { l_resp = resp; }, - nullptr) - ->wait(); - ASSERT_EQ(l_resp.err, ERR_OK); - ASSERT_EQ(l_resp.entries->size(), total_file_cnt); - std::sort(l_resp.entries->begin(), l_resp.entries->end(), entry_cmp); - std::sort(t_entries.begin(), t_entries.end(), entry_cmp); - entry_vec_eq(t_entries, *l_resp.entries); - - // then remove all the file, using remove_path - std::cout << "then remove all the files, using remove path" << std::endl; - s->remove_path(remove_path_request{std::string(prefix), true}, - lpc_btest, - [&rem_resp](const remove_path_response &resp) { rem_resp = resp; }, - nullptr) - ->wait(); - ASSERT_EQ(rem_resp.err, ERR_OK); - } - } -} - -static void -generate_file(const char *filename, unsigned long long file_size, char *block, unsigned block_size) -{ - int fd = ::open(filename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); - ASSERT_GT(fd, 0) << utils::safe_strerror(errno); - for (unsigned long long i = 0; i < file_size;) { - int batch_size = (file_size - i); - if (batch_size > block_size) - batch_size = block_size; - i += batch_size; - - for (int j = 0; j < batch_size; ++j) { - block[j] = (char)rand::next_u32(0, 255); - } - ASSERT_EQ(batch_size, ::write(fd, block, batch_size)) - << "write file " << filename << " failed, err = " << utils::safe_strerror(errno); - } - ASSERT_EQ(0, ::close(fd)) << "close file " << filename - << " failed, err = " << utils::safe_strerror(errno); -} - -TEST_F(FDSClientTest, test_concurrent_upload_download) -{ - char block[1024]; - const char *str = "test_str"; - for (int i = 0; i < 128; ++i) { - memcpy(block + i * 8, str, 8); - } - - std::shared_ptr _service = std::make_shared(); - std::vector init_str = {server_address, access_key, access_secret, bucket_name}; - - if (server_address == example_server_address) { - // user don't specify the server-address, we just return true - return; - } - - _service->initialize(init_str); - - DSN_DEFINE_int32(fds_concurrent_test, total_files, 64, ""); - int total_files = FLAGS_total_files; - - std::vector filenames; - filenames.reserve(total_files); - std::vector filesize; - filesize.reserve(total_files); - std::vector md5; - md5.reserve(total_files); - - for (int i = 0; i < total_files; ++i) { - char index[64]; - snprintf(index, 64, "%04d", i); - unsigned long random_size = rand::next_u64(FLAGS_min_size, FLAGS_max_size); - std::string filename = "randomfile" + std::string(index); - filenames.push_back(filename); - filesize.push_back(random_size); - generate_file(filename.c_str(), random_size, block, 1024); - - std::string md5result; - dsn::utils::filesystem::md5sum(filename, md5result); - md5.push_back(md5result); - } - - printf("start delete phase\n"); - { - for (unsigned int i = 0; i < total_files; ++i) { - _service - ->remove_path(remove_path_request{filenames[i], true}, - lpc_btest, - [i, &filenames](const remove_path_response &resp) { - printf("file %s delete finished, err(%s)\n", - filenames[i].c_str(), - resp.err.to_string()); - }, - nullptr) - ->wait(); - } - } - - printf("start upload phase\n"); - { - std::vector block_files; - for (unsigned int i = 0; i < total_files; ++i) { - create_file_response cf_resp; - _service - ->create_file(create_file_request{filenames[i], true}, - lpc_btest, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_NE(nullptr, cf_resp.file_handle.get()); - block_files.push_back(cf_resp.file_handle); - } - - std::vector callbacks; - for (unsigned int i = 0; i < total_files; ++i) { - block_file_ptr p = block_files[i]; - dsn::task_ptr t = - p->upload(upload_request{filenames[i]}, - lpc_btest, - [p, &filenames, &filesize, &md5, i](const upload_response &ur) { - printf("file %s upload finished\n", filenames[i].c_str()); - ASSERT_EQ(dsn::ERR_OK, ur.err); - ASSERT_EQ(filesize[i], ur.uploaded_size); - ASSERT_EQ(filesize[i], p->get_size()); - ASSERT_EQ(md5[i], p->get_md5sum()); - }); - callbacks.push_back(t); - } - - for (unsigned int i = 0; i < total_files; ++i) { - callbacks[i]->wait(); - } - } - - printf("start download phase\n"); - { - std::vector block_files; - for (unsigned int i = 0; i < total_files; ++i) { - create_file_response cf_resp; - _service - ->create_file(create_file_request{filenames[i], true}, - lpc_btest, - [&cf_resp](const create_file_response &r) { cf_resp = r; }, - nullptr) - ->wait(); - ASSERT_EQ(dsn::ERR_OK, cf_resp.err); - ASSERT_NE(nullptr, cf_resp.file_handle.get()); - block_files.push_back(cf_resp.file_handle); - } - - std::vector callbacks; - for (unsigned int i = 0; i < total_files; ++i) { - block_file_ptr p = block_files[i]; - dsn::task_ptr t = - p->download(download_request{filenames[i] + ".b", 0, -1}, - lpc_btest, - [&filenames, &filesize, &md5, i, p](const download_response &dr) { - printf("file %s download finished\n", filenames[i].c_str()); - ASSERT_EQ(dsn::ERR_OK, dr.err); - ASSERT_EQ(filesize[i], dr.downloaded_size); - ASSERT_EQ(filesize[i], p->get_size()); - ASSERT_EQ(md5[i], p->get_md5sum()); - }); - callbacks.push_back(t); - } - - for (unsigned int i = 0; i < total_files; ++i) { - callbacks[i]->wait(); - } - } - - printf("start cleanup phase\n"); - { - for (unsigned int i = 0; i < total_files; ++i) { - _service - ->remove_path(remove_path_request{filenames[i], true}, - lpc_btest, - [i, &filenames](const remove_path_response &resp) { - printf("file %s delete finished, err(%s)\n", - filenames[i].c_str(), - resp.err.to_string()); - }, - nullptr) - ->wait(); - } - } -} diff --git a/src/failure_detector/CMakeLists.txt b/src/failure_detector/CMakeLists.txt index cbeb139ccb..702f02fd98 100644 --- a/src/failure_detector/CMakeLists.txt +++ b/src/failure_detector/CMakeLists.txt @@ -38,9 +38,7 @@ seT(MY_PROJ_SRC ${FD_THRIFT_SRCS}) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS - PocoFoundation - PocoJSON) +set(MY_PROJ_LIBS "") # Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/failure_detector/test/config-test.ini b/src/failure_detector/test/config-test.ini index d1edd63850..e29d6777ec 100644 --- a/src/failure_detector/test/config-test.ini +++ b/src/failure_detector/test/config-test.ini @@ -124,9 +124,3 @@ partitioned = true hosts_list = localhost:22181 timeout_ms = 30000 logfile = zoolog.log - -[fds_concurrent_test] -total_files = 64 -min_size = 100 -max_size = 150 - diff --git a/src/failure_detector/test/config-whitelist-test-failed.ini b/src/failure_detector/test/config-whitelist-test-failed.ini index 8332ceac01..8b60870c5d 100644 --- a/src/failure_detector/test/config-whitelist-test-failed.ini +++ b/src/failure_detector/test/config-whitelist-test-failed.ini @@ -125,9 +125,3 @@ partitioned = true hosts_list = localhost:22181 timeout_ms = 30000 logfile = zoolog.log - -[fds_concurrent_test] -total_files = 64 -min_size = 100 -max_size = 150 - diff --git a/src/failure_detector/test/config-whitelist-test.ini b/src/failure_detector/test/config-whitelist-test.ini index 4ee04d7af1..728507e9c5 100644 --- a/src/failure_detector/test/config-whitelist-test.ini +++ b/src/failure_detector/test/config-whitelist-test.ini @@ -125,9 +125,3 @@ partitioned = true hosts_list = localhost:22181 timeout_ms = 30000 logfile = zoolog.log - -[fds_concurrent_test] -total_files = 64 -min_size = 100 -max_size = 150 - diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 3e4d4d4134..477671acb3 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -37,7 +37,6 @@ set(MY_PROJ_LIBS dsn_replication_common dsn.block_service dsn.block_service.local - dsn.block_service.fds dsn.block_service.hdfs dsn.failure_detector dsn.replication.zookeeper_provider @@ -46,11 +45,6 @@ set(MY_PROJ_LIBS dsn_runtime dsn_aio zookeeper - galaxy-fds-sdk-cpp - PocoNet - PocoFoundation - PocoNetSSL - PocoJSON hashtable hdfs rocksdb) diff --git a/src/meta/test/CMakeLists.txt b/src/meta/test/CMakeLists.txt index 60f02c691c..49c9e4994d 100644 --- a/src/meta/test/CMakeLists.txt +++ b/src/meta/test/CMakeLists.txt @@ -42,7 +42,6 @@ set(MY_PROJ_LIBS dsn_replication_common dsn.block_service dsn.block_service.local - dsn.block_service.fds dsn.block_service.hdfs dsn.failure_detector dsn_dist_cmd @@ -52,11 +51,6 @@ set(MY_PROJ_LIBS dsn_aio zookeeper hashtable - galaxy-fds-sdk-cpp - PocoNet - PocoFoundation - PocoNetSSL - PocoJSON gtest hdfs) diff --git a/src/meta/test/config-test.ini b/src/meta/test/config-test.ini index d0a8656567..ca5fdf2c75 100644 --- a/src/meta/test/config-test.ini +++ b/src/meta/test/config-test.ini @@ -129,10 +129,6 @@ args = ./block_service type = local_service args = -[block_service.fds_service] -type = fds_service -args = - [duplication-group] master-cluster = 1 slave-cluster = 2 diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt index 005967def1..fcf7891017 100644 --- a/src/replica/CMakeLists.txt +++ b/src/replica/CMakeLists.txt @@ -62,7 +62,6 @@ set(MY_PROJ_LIBS dsn.failure_detector dsn.block_service dsn.block_service.local - dsn.block_service.fds dsn.block_service.hdfs dsn_nfs dsn_dist_cmd @@ -70,11 +69,6 @@ set(MY_PROJ_LIBS dsn_runtime dsn_aio dsn_meta_server - galaxy-fds-sdk-cpp - PocoNet - PocoFoundation - PocoNetSSL - PocoJSON rocksdb) set(MY_BOOST_LIBS Boost::filesystem) diff --git a/src/replica/backup/test/CMakeLists.txt b/src/replica/backup/test/CMakeLists.txt index 063246df02..47158ca93c 100644 --- a/src/replica/backup/test/CMakeLists.txt +++ b/src/replica/backup/test/CMakeLists.txt @@ -26,7 +26,6 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_replication_common dsn.block_service dsn.block_service.local - dsn.block_service.fds dsn_utils hashtable gtest diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt index cd309ba2e8..83af542d10 100644 --- a/src/replica/test/CMakeLists.txt +++ b/src/replica/test/CMakeLists.txt @@ -39,7 +39,6 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_replication_common dsn.block_service dsn.block_service.local - dsn.block_service.fds dsn.block_service.hdfs dsn.failure_detector dsn_http diff --git a/src/replica/test/replica_http_service_test.cpp b/src/replica/test/replica_http_service_test.cpp index d4b49667fa..d6a80544c9 100644 --- a/src/replica/test/replica_http_service_test.cpp +++ b/src/replica/test/replica_http_service_test.cpp @@ -112,7 +112,7 @@ TEST_P(replica_http_service_test, update_config_handler) // Update config failed and value not changed. NO_FATALS(test_update_config( - {{"config_sync_interval_ms", "10"}, {"fds_write_limit_rate", "50"}}, + {{"config_sync_interval_ms", "10"}, {"hdfs_write_limit_rate_mb_per_sec", "50"}}, R"({"update_status":"ERR_INVALID_PARAMETERS: there should be exactly one config to be updated once"})" "\n")); NO_FATALS(test_check_config("config_sync_interval_ms", "30000")); diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index a61b75d36e..ff48c181e3 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -32,7 +32,6 @@ set(MY_PROJ_LIBS dsn_replication_common dsn_client dsn.block_service.local - dsn.block_service.fds dsn.block_service dsn.failure_detector dsn.replication.zookeeper_provider @@ -42,14 +41,6 @@ set(MY_PROJ_LIBS pegasus_base pegasus_client_static event - galaxy-fds-sdk-cpp - PocoNet - PocoFoundation - PocoNetSSL - PocoJSON - PocoCrypto - PocoUtil - PocoXML hashtable ) diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt index 4c3ea8a838..e9b9200f3e 100644 --- a/src/server/test/CMakeLists.txt +++ b/src/server/test/CMakeLists.txt @@ -41,7 +41,6 @@ set(MY_PROJ_LIBS dsn_replication_common dsn_client dsn.block_service.local - dsn.block_service.fds dsn.block_service dsn.failure_detector dsn.replication.zookeeper_provider @@ -50,11 +49,6 @@ set(MY_PROJ_LIBS RocksDB::rocksdb pegasus_client_static event - galaxy-fds-sdk-cpp - PocoNet - PocoFoundation - PocoNetSSL - PocoJSON pegasus_base gtest gmock diff --git a/src/server/test/config.ini b/src/server/test/config.ini index 1ec547264a..fe4bdb9078 100644 --- a/src/server/test/config.ini +++ b/src/server/test/config.ini @@ -42,10 +42,6 @@ logging_factory_name = dsn::tools::simple_logger ;logging_factory_name = dsn::tools::screen_logger logging_flush_on_exit = true -[block_service.fds_service] -type = fds_service -args = - [block_service.local_service] type = local_service args = diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index f35ead8e85..909fda9ab6 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -36,19 +36,10 @@ set(MY_PROJ_LIBS dsn_client dsn_utils dsn.block_service.local - dsn.block_service.fds dsn.block_service.hdfs dsn.block_service dsn.failure_detector pegasus_client_static - galaxy-fds-sdk-cpp - PocoNet - PocoFoundation - PocoNetSSL - PocoJSON - PocoCrypto - PocoUtil - PocoXML pegasus_geo_lib RocksDB::rocksdb absl::flat_hash_set diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index bbe99ece78..e95e24f83e 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -220,54 +220,6 @@ ExternalProject_Add(libevent DOWNLOAD_NO_PROGRESS true ) -ExternalProject_Add(poco - URL ${OSS_URL_PREFIX}/poco-1.11.1-release.tar.gz - https://github.com/pocoproject/poco/archive/refs/tags/poco-1.11.1-release.tar.gz - URL_MD5 a96210b60a675c4a8183ad1f2099d549 - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} - -DENABLE_MONGODB=OFF - -DENABLE_PDF=OFF - -DENABLE_DATA=OFF - -DENABLE_DATA_SQLITE=OFF - -DENABLE_DATA_MYSQL=OFF - -DENABLE_DATA_ODBC=OFF - -DENABLE_SEVENZIP=OFF - -DENABLE_ZIP=OFF - -DENABLE_APACHECONNECTOR=OFF - -DENABLE_CPPPARSER=OFF - -DENABLE_POCODOC=OFF - -DENABLE_PAGECOMPILER=OFF - -DENABLE_PAGECOMPILER_FILE2PAGE=OFF - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - DEPENDS boost - DOWNLOAD_EXTRACT_TIMESTAMP true - DOWNLOAD_NO_PROGRESS true -) - -ExternalProject_Add(fds - URL ${OSS_URL_PREFIX}/fds-1.0.0.tar.gz - https://github.com/XiaoMi/galaxy-fds-sdk-cpp/archive/refs/tags/v1.0.0.tar.gz - URL_MD5 f7e0f86534f7b15c2a9b349f5cb45503 - PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_fds_for_macos.patch - CMAKE_ARGS -DPOCO_INCLUDE=${TP_OUTPUT}/include - -DMACOS_OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} - -DPOCO_LIB=${TP_OUTPUT}/lib - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - INSTALL_COMMAND cp libgalaxy-fds-sdk-cpp.a ${TP_OUTPUT}/lib - COMMAND rm -rf ${TP_OUTPUT}/include/fds - COMMAND cp -r include fds-include - COMMAND mv fds-include ${TP_OUTPUT}/include/fds # install fds headers into a stand-alone directory - UPDATE_COMMAND "" # do not update - DEPENDS googletest poco boost - BUILD_IN_SOURCE 1 - DOWNLOAD_EXTRACT_TIMESTAMP true - DOWNLOAD_NO_PROGRESS true -) - ExternalProject_Add(fmt URL https://github.com/fmtlib/fmt/archive/refs/tags/10.1.1.tar.gz URL_MD5 0d41a16f1b3878d44e6fd7ff1f6cc45a diff --git a/thirdparty/fix_fds_for_macos.patch b/thirdparty/fix_fds_for_macos.patch deleted file mode 100644 index 5cebc13e37..0000000000 --- a/thirdparty/fix_fds_for_macos.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index d2d5f09..6de3158 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -11,6 +11,9 @@ add_library(galaxy-fds-sdk-cpp ${DIR_SRCS}) - target_include_directories(galaxy-fds-sdk-cpp PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) - - include_directories(${POCO_INCLUDE}) -+if (APPLE) -+ include_directories(${MACOS_OPENSSL_INCLUDE_DIR}) -+endif() - link_directories(${POCO_LIB}) - - option(WITH_TEST "enable building test" OFF)