From 8ad2418dfa4577e261e51861c3aa93877ff7dd78 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Tue, 17 Dec 2024 19:04:54 +0000 Subject: [PATCH 1/6] refactor bootstrap --- .../counterpartycore/lib/config.py | 31 ++- counterparty-core/counterpartycore/server.py | 197 ++++++++++-------- .../counterpartycore/test/conftest.py | 3 +- 3 files changed, 136 insertions(+), 95 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/config.py b/counterparty-core/counterpartycore/lib/config.py index 47ff48c3d..7ea8e715d 100644 --- a/counterparty-core/counterpartycore/lib/config.py +++ b/counterparty-core/counterpartycore/lib/config.py @@ -173,10 +173,33 @@ PROTOCOL_CHANGES_URL = "https://counterparty.io/protocol_changes.json" # PROTOCOL_CHANGES_URL = "https://raw.githubusercontent.com/CounterpartyXCP/counterparty-core/refs/heads/master/counterparty-core/counterpartycore/protocol_changes.json" -BOOTSTRAP_URL_MAINNET = "https://bootstrap.counterparty.io/counterparty.latest.tar.gz" -BOOTSTRAP_URL_MAINNET_SIG = "https://bootstrap.counterparty.io/counterparty.latest.sig" -BOOTSTRAP_URL_TESTNET = "https://bootstrap.counterparty.io/counterparty-testnet.latest.tar.gz" -BOOTSTRAP_URL_TESTNET_SIG = "https://bootstrap.counterparty.io/counterparty-testnet.latest.sig" +BOOTSTRAP_URL_MAINNET = "https://bootstrap.counterparty.io/counterparty.latest.tar.zst" +BOOTSTRAP_URL_MAINNET_SIG = "https://bootstrap.counterparty.io/counterparty.latest.tar.sig" +BOOTSTRAP_URL_TESTNET = "https://bootstrap.counterparty.io/counterparty-testnet.latest.tar.zst" +BOOTSTRAP_URL_TESTNET_SIG = "https://bootstrap.counterparty.io/counterparty-testnet.latest.tar.sig" + +BOOTSTRAP_URLS = { + "mainnet": [ + ( + "https://storage.googleapis.com/counterparty-bootstrap/counterparty.db.latest.zst", + "https://storage.googleapis.com/counterparty-bootstrap/counterparty.db.sig", + ), + ( + "https://storage.googleapis.com/counterparty-bootstrap/state.db.latest.zst", + "https://storage.googleapis.com/counterparty-bootstrap/state.db.sig", + ), + ], + "testnet": [ + ( + "https://storage.googleapis.com/counterparty-bootstrap/counterparty.testnet.db.latest.zst", + "https://storage.googleapis.com/counterparty-bootstrap/counterparty.testnet.db.sig", + ), + ( + "https://storage.googleapis.com/counterparty-bootstrap/state.testnet.db.latest.zst", + "https://storage.googleapis.com/counterparty-bootstrap/state.testnet.db.sig", + ), + ], +} API_MAX_LOG_SIZE = ( 10 * 1024 * 1024 diff --git a/counterparty-core/counterpartycore/server.py b/counterparty-core/counterpartycore/server.py index f7b932802..1890ddcd7 100755 --- a/counterparty-core/counterpartycore/server.py +++ b/counterparty-core/counterpartycore/server.py @@ -3,19 +3,22 @@ import _thread import binascii import decimal +import glob +import io import logging import multiprocessing import os import sys -import tarfile import tempfile import threading import time import urllib +from multiprocessing import Process from urllib.parse import quote_plus as urlencode import appdirs import bitcoin as bitcoinlib +import pyzstd from termcolor import colored, cprint from counterpartycore.lib import ( @@ -959,7 +962,99 @@ def configure_rpc(rpc_password=None): config.RPC = config.API_ROOT + config.RPC_WEBROOT -def bootstrap(no_confirm=False, snapshot_url=None): +def download_zst(data_dir, zst_url): + print(f"Downloading {zst_url}...") + start_time = time.time() + zst_filename = os.path.basename(zst_url) + zst_filepath = os.path.join(data_dir, zst_filename) + urllib.request.urlretrieve(zst_url, zst_filepath) # nosec B310 # noqa: S310 + print(f"Downloaded {zst_url} in {time.time() - start_time:.2f}s") + return zst_filepath + + +def decompress_zst(zst_filepath): + print(f"Decompressing {zst_filepath}...") + start_time = time.time() + filename = zst_filepath.replace(".latest.zst", "") + filepath = os.path.join(os.path.dirname(zst_filepath), filename) + with io.open(filepath, "wb") as output_file: + with open(zst_filepath, "rb") as input_file: + pyzstd.decompress_stream(input_file, output_file, read_size=16 * 1024) + os.remove(zst_filepath) + os.chmod(filepath, 0o660) + print(f"Decompressed {zst_filepath} in {time.time() - start_time:.2f}s") + return filepath + + +def download_and_decompress(data_dir, zst_url): + # download and decompress .tar.zst file + print(f"Downloading and decompressing {zst_url}...") + start_time = time.time() + response = urllib.request.urlopen(zst_url) # nosec B310 # noqa: S310 + zst_filename = os.path.basename(zst_url) + filename = zst_filename.replace(".latest.zst", "") + filepath = os.path.join(data_dir, filename) + with io.open(filepath, "wb") as output_file: + pyzstd.decompress_stream(response, output_file, read_size=16 * 1024) + os.chmod(filepath, 0o660) + print(f"Downloaded and decompressed {zst_url} in {time.time() - start_time:.2f}s") + return filepath + + +def verify_signature(filepath, sig_url): + sig_filename = os.path.basename(sig_url) + sig_filepath = os.path.join(tempfile.gettempdir(), sig_filename) + urllib.request.urlretrieve(sig_url, sig_filepath) # nosec B310 # noqa: S310 + + print(f"Verifying signature for {filepath}...") + start_time = time.time() + signature_verified = False + for key in PUBLIC_KEYS: + if util.verify_signature(key, sig_filepath, filepath): + signature_verified = True + break + os.remove(sig_filepath) + print(f"Verified signature in {time.time() - start_time:.2f}s") + + if not signature_verified: + print(f"{filepath} was not signed by any trusted keys, deleting...") + os.remove(filepath) + sys.exit(1) + + +def decompress_and_verify(zst_filepath, sig_url): + filepath = decompress_zst(zst_filepath) + verify_signature(filepath, sig_url) + + +def clean_data_dir(): + if not os.path.exists(config.DATA_DIR): + os.makedirs(config.DATA_DIR, mode=0o755) + return + files_to_delete = glob.glob(os.path.join(config.DATA_DIR, "*.db")) + files_to_delete += glob.glob(os.path.join(config.DATA_DIR, "*.db-wal")) + files_to_delete += glob.glob(os.path.join(config.DATA_DIR, "*.db-shm")) + for file in files_to_delete: + os.remove(file) + + +def download_bootstrap_files(): + files = config.BOOTSTRAP_URLS[config.NETWORK_NAME] + decompressors = [] + for zst_url, sig_url in files: + zst_filepath = download_zst(config.DATA_DIR, zst_url) + decompressor = Process( + target=decompress_and_verify, + args=(zst_filepath, sig_url), + ) + decompressor.start() + decompressors.append(decompressor) + + for decompressor in decompressors: + decompressor.join() + + +def confirm_bootstrap(): warning_message = """WARNING: `counterparty-server bootstrap` downloads a recent snapshot of a Counterparty database from a centralized server maintained by the Counterparty Core development team. Because this method does not involve verifying the history of Counterparty transactions yourself, @@ -967,99 +1062,21 @@ def bootstrap(no_confirm=False, snapshot_url=None): """ cprint(warning_message, "yellow") - if not no_confirm: - confirmation_message = colored("Continue? (y/N): ", "magenta") - if input(confirmation_message).lower() != "y": - exit() - - # Set Constants. - if snapshot_url is None: - bootstrap_url = ( - config.BOOTSTRAP_URL_TESTNET if config.TESTNET else config.BOOTSTRAP_URL_MAINNET - ) - bootstrap_sig_url = ( - config.BOOTSTRAP_URL_TESTNET_SIG if config.TESTNET else config.BOOTSTRAP_URL_MAINNET_SIG - ) - else: - bootstrap_url = snapshot_url - bootstrap_sig_url = snapshot_url.replace(".tar.gz", ".sig") - - tar_filename = os.path.basename(bootstrap_url) - sig_filename = os.path.basename(bootstrap_sig_url) - tarball_path = os.path.join(tempfile.gettempdir(), tar_filename) - sig_path = os.path.join(tempfile.gettempdir(), sig_filename) + confirmation_message = colored("Continue? (y/N): ", "magenta") + if input(confirmation_message).lower() != "y": + exit() - ledger_database_path = os.path.join(config.DATA_DIR, config.APP_NAME) - if config.TESTNET: - ledger_database_path += ".testnet" - ledger_database_path += ".db" - - old_api_database_path = ledger_database_path.replace(".db", ".api.db") - if config.TESTNET: - api_database_path = os.path.join(config.DATA_DIR, "state.testnet.db") - else: - api_database_path = os.path.join(config.DATA_DIR, "state.db") +def bootstrap(no_confirm=False, snapshot_url=None): + if not no_confirm: + confirm_bootstrap() - # Prepare Directory. - if not os.path.exists(config.DATA_DIR): - os.makedirs(config.DATA_DIR, mode=0o755) + clean_data_dir() - for database_path in [ledger_database_path, api_database_path, old_api_database_path]: - if os.path.exists(database_path): - os.remove(database_path) - # Delete SQLite Write-Ahead-Log - wal_path = database_path + "-wal" - shm_path = database_path + "-shm" - if os.path.exists(wal_path): - os.remove(wal_path) - if os.path.exists(shm_path): - os.remove(shm_path) - - # Define Progress Bar. - spinner = log.Spinner(f"Downloading database from {bootstrap_url}...") - - def bootstrap_progress(blocknum, blocksize, totalsize): - readsofar = blocknum * blocksize - if totalsize > 0: - percent = readsofar * 1e2 / totalsize - message = f"Downloading database: {percent:5.1f}% {readsofar} / {totalsize}" - spinner.set_messsage(message) - - # Downloading - spinner.start() - urllib.request.urlretrieve(bootstrap_url, tarball_path, bootstrap_progress) # nosec B310 # noqa: S310 - urllib.request.urlretrieve(bootstrap_sig_url, sig_path) # nosec B310 # noqa: S310 - spinner.stop() - - with log.Spinner("Verifying signature..."): - signature_verified = False - for key in PUBLIC_KEYS: - if util.verify_signature(key, sig_path, tarball_path): - signature_verified = True - break - if not signature_verified: - print("Snapshot was not signed by any trusted keys") - sys.exit(1) - - # TODO: check checksum, filenames, etc. - with log.Spinner(f"Extracting database to {config.DATA_DIR}..."): - with tarfile.open(tarball_path, "r:gz") as tar_file: - tar_file.extractall(path=config.DATA_DIR) # nosec B202 # noqa: S202 - - assert os.path.exists(ledger_database_path) - assert os.path.exists(api_database_path) or os.path.exists(old_api_database_path) - # user and group have "rw" access - os.chmod(ledger_database_path, 0o660) # nosec B103 - if os.path.exists(api_database_path): - os.chmod(api_database_path, 0o660) # nosec B103 - if os.path.exists(old_api_database_path): - os.chmod(old_api_database_path, 0o660) # nosec B103 - - with log.Spinner("Cleaning up..."): - os.remove(tarball_path) + with log.Spinner("Downloading and decompressing database..."): + download_bootstrap_files() cprint( - f"Databases have been successfully bootstrapped to {ledger_database_path} and {api_database_path}.", + f"Databases have been successfully bootstrapped to {config.DATA_DIR}.", "green", ) diff --git a/counterparty-core/counterpartycore/test/conftest.py b/counterparty-core/counterpartycore/test/conftest.py index c6b0c64a0..16cbcd68b 100644 --- a/counterparty-core/counterpartycore/test/conftest.py +++ b/counterparty-core/counterpartycore/test/conftest.py @@ -223,11 +223,12 @@ def rawtransactions_db(request): @pytest.fixture(scope="function") def server_db(request, cp_server, api_server): """Enable database access for unit test vectors.""" + config.CACHE_DIR = os.path.dirname(request.module.FIXTURE_DB) + db = database.get_connection(read_only=False) cursor = db.cursor() cursor.execute("""BEGIN""") util_test.reset_current_block_index(db) - config.CACHE_DIR = os.path.dirname(request.module.FIXTURE_DB) request.addfinalizer(lambda: cursor.execute("""ROLLBACK""")) request.addfinalizer(lambda: util_test.reset_current_block_index(db)) From 54fabf723e6927d90122dcb9ed03acaba4a26dc7 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Tue, 17 Dec 2024 21:11:12 +0000 Subject: [PATCH 2/6] verify and decompress --- counterparty-core/counterpartycore/lib/config.py | 8 ++++---- counterparty-core/counterpartycore/server.py | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/config.py b/counterparty-core/counterpartycore/lib/config.py index 7ea8e715d..bf19504e1 100644 --- a/counterparty-core/counterpartycore/lib/config.py +++ b/counterparty-core/counterpartycore/lib/config.py @@ -182,21 +182,21 @@ "mainnet": [ ( "https://storage.googleapis.com/counterparty-bootstrap/counterparty.db.latest.zst", - "https://storage.googleapis.com/counterparty-bootstrap/counterparty.db.sig", + "https://storage.googleapis.com/counterparty-bootstrap/counterparty.db.latest.sig", ), ( "https://storage.googleapis.com/counterparty-bootstrap/state.db.latest.zst", - "https://storage.googleapis.com/counterparty-bootstrap/state.db.sig", + "https://storage.googleapis.com/counterparty-bootstrap/state.db.latest.sig", ), ], "testnet": [ ( "https://storage.googleapis.com/counterparty-bootstrap/counterparty.testnet.db.latest.zst", - "https://storage.googleapis.com/counterparty-bootstrap/counterparty.testnet.db.sig", + "https://storage.googleapis.com/counterparty-bootstrap/counterparty.testnet.db.latest.sig", ), ( "https://storage.googleapis.com/counterparty-bootstrap/state.testnet.db.latest.zst", - "https://storage.googleapis.com/counterparty-bootstrap/state.testnet.db.sig", + "https://storage.googleapis.com/counterparty-bootstrap/state.testnet.db.latest.sig", ), ], } diff --git a/counterparty-core/counterpartycore/server.py b/counterparty-core/counterpartycore/server.py index 1890ddcd7..36f4032fd 100755 --- a/counterparty-core/counterpartycore/server.py +++ b/counterparty-core/counterpartycore/server.py @@ -1027,6 +1027,11 @@ def decompress_and_verify(zst_filepath, sig_url): verify_signature(filepath, sig_url) +def verfif_and_decompress(zst_filepath, sig_url): + verify_signature(zst_filepath, sig_url) + decompress_zst(zst_filepath) + + def clean_data_dir(): if not os.path.exists(config.DATA_DIR): os.makedirs(config.DATA_DIR, mode=0o755) @@ -1044,7 +1049,7 @@ def download_bootstrap_files(): for zst_url, sig_url in files: zst_filepath = download_zst(config.DATA_DIR, zst_url) decompressor = Process( - target=decompress_and_verify, + target=verfif_and_decompress, args=(zst_filepath, sig_url), ) decompressor.start() From 1311b31c63c6582bcc4422ffc7196c1a70858281 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 18 Dec 2024 09:27:11 +0000 Subject: [PATCH 3/6] cleaning --- counterparty-core/counterpartycore/cli.py | 4 +- .../counterpartycore/lib/bootstrap.py | 175 ++++++++++++++++++ .../counterpartycore/lib/config.py | 4 - .../counterpartycore/lib/util.py | 21 --- counterparty-core/counterpartycore/server.py | 134 +------------- 5 files changed, 178 insertions(+), 160 deletions(-) create mode 100644 counterparty-core/counterpartycore/lib/bootstrap.py diff --git a/counterparty-core/counterpartycore/cli.py b/counterparty-core/counterpartycore/cli.py index 156a45c7d..92b46d8ee 100755 --- a/counterparty-core/counterpartycore/cli.py +++ b/counterparty-core/counterpartycore/cli.py @@ -8,7 +8,7 @@ from termcolor import cprint from counterpartycore import server -from counterpartycore.lib import config, sentry, setup +from counterpartycore.lib import bootstrap, config, sentry, setup from counterpartycore.lib.api import dbbuilder logger = logging.getLogger(config.LOGGER_NAME) @@ -504,7 +504,7 @@ def main(): # Bootstrapping if args.action == "bootstrap": - server.bootstrap(no_confirm=args.no_confirm, snapshot_url=args.bootstrap_url) + bootstrap.bootstrap(no_confirm=args.no_confirm, snapshot_url=args.bootstrap_url) # PARSING elif args.action == "reparse": diff --git a/counterparty-core/counterpartycore/lib/bootstrap.py b/counterparty-core/counterpartycore/lib/bootstrap.py new file mode 100644 index 000000000..2b753c8fa --- /dev/null +++ b/counterparty-core/counterpartycore/lib/bootstrap.py @@ -0,0 +1,175 @@ +import glob +import io +import os +import shutil +import sys +import tempfile +import time +import urllib.request +from multiprocessing import Process + +import gnupg +import pyzstd +from termcolor import colored, cprint + +from counterpartycore.lib import config +from counterpartycore.lib.public_keys import PUBLIC_KEYS + + +def download_zst(data_dir, zst_url): + print(f"Downloading {zst_url}...") + start_time = time.time() + zst_filename = os.path.basename(zst_url) + zst_filepath = os.path.join(data_dir, zst_filename) + urllib.request.urlretrieve(zst_url, zst_filepath) # nosec B310 # noqa: S310 + print(f"Downloaded {zst_url} in {time.time() - start_time:.2f}s") + return zst_filepath + + +def decompress_zst(zst_filepath): + print(f"Decompressing {zst_filepath}...") + start_time = time.time() + filename = zst_filepath.replace(".latest.zst", "") + filepath = os.path.join(os.path.dirname(zst_filepath), filename) + with io.open(filepath, "wb") as output_file: + with open(zst_filepath, "rb") as input_file: + pyzstd.decompress_stream(input_file, output_file, read_size=16 * 1024) + os.remove(zst_filepath) + os.chmod(filepath, 0o660) + print(f"Decompressed {zst_filepath} in {time.time() - start_time:.2f}s") + return filepath + + +def download_and_decompress(data_dir, zst_url): + # download and decompress .tar.zst file + print(f"Downloading and decompressing {zst_url}...") + start_time = time.time() + response = urllib.request.urlopen(zst_url) # nosec B310 # noqa: S310 + zst_filename = os.path.basename(zst_url) + filename = zst_filename.replace(".latest.zst", "") + filepath = os.path.join(data_dir, filename) + with io.open(filepath, "wb") as output_file: + pyzstd.decompress_stream(response, output_file, read_size=16 * 1024) + os.chmod(filepath, 0o660) + print(f"Downloaded and decompressed {zst_url} in {time.time() - start_time:.2f}s") + return filepath + + +def verify_signature(public_key_data, signature_path, snapshot_path): + temp_dir = tempfile.mkdtemp() + verified = False + + try: + gpg = gnupg.GPG(gnupghome=temp_dir) + gpg.import_keys(public_key_data) + with open(signature_path, "rb") as s: + verified = gpg.verify_file(s, snapshot_path, close_file=False) + finally: + shutil.rmtree(temp_dir) + + return verified + + +def check_signature(filepath, sig_url): + sig_filename = os.path.basename(sig_url) + sig_filepath = os.path.join(tempfile.gettempdir(), sig_filename) + urllib.request.urlretrieve(sig_url, sig_filepath) # nosec B310 # noqa: S310 + + print(f"Verifying signature for {filepath}...") + start_time = time.time() + signature_verified = False + for key in PUBLIC_KEYS: + if verify_signature(key, sig_filepath, filepath): + signature_verified = True + break + os.remove(sig_filepath) + print(f"Verified signature in {time.time() - start_time:.2f}s") + + if not signature_verified: + print(f"{filepath} was not signed by any trusted keys, deleting...") + os.remove(filepath) + sys.exit(1) + + +def decompress_and_verify(zst_filepath, sig_url): + filepath = decompress_zst(zst_filepath) + check_signature(filepath, sig_url) + + +def verfif_and_decompress(zst_filepath, sig_url): + check_signature(zst_filepath, sig_url) + decompress_zst(zst_filepath) + + +def clean_data_dir(data_dir): + if not os.path.exists(data_dir): + os.makedirs(data_dir, mode=0o755) + return + files_to_delete = glob.glob(os.path.join(data_dir, "*.db")) + files_to_delete += glob.glob(os.path.join(data_dir, "*.db-wal")) + files_to_delete += glob.glob(os.path.join(data_dir, "*.db-shm")) + for file in files_to_delete: + os.remove(file) + + +def download_bootstrap_files(data_dir, files): + decompressors = [] + for zst_url, sig_url in files: + zst_filepath = download_zst(data_dir, zst_url) + decompressor = Process( + target=verfif_and_decompress, + args=(zst_filepath, sig_url), + ) + decompressor.start() + decompressors.append(decompressor) + + for decompressor in decompressors: + decompressor.join() + + +def confirm_bootstrap(): + warning_message = """WARNING: `counterparty-server bootstrap` downloads a recent snapshot of a Counterparty database +from a centralized server maintained by the Counterparty Core development team. +Because this method does not involve verifying the history of Counterparty transactions yourself, +the `bootstrap` command should not be used for mission-critical, commercial or public-facing nodes. + """ + cprint(warning_message, "yellow") + + confirmation_message = colored("Continue? (y/N): ", "magenta") + if input(confirmation_message).lower() != "y": + exit() + + +def generate_urls(counterparty_zst_url): + state_zst_url = counterparty_zst_url.replace("/counterparty.db", "/state.db") + return [ + ( + counterparty_zst_url, + counterparty_zst_url.replace(".zst", ".sig"), + ), + ( + state_zst_url, + state_zst_url.replace(".zst", ".sig"), + ), + ] + + +def bootstrap(no_confirm=False, snapshot_url=None): + if not no_confirm: + confirm_bootstrap() + + start_time = time.time() + + clean_data_dir(config.DATA_DIR) + + if snapshot_url is None: + files = config.BOOTSTRAP_URLS[config.NETWORK_NAME] + else: + files = generate_urls(snapshot_url) + + download_bootstrap_files(config.DATA_DIR, files) + + cprint( + f"Databases have been successfully bootstrapped to {config.DATA_DIR} in {time.time() - start_time:.2f}s.", + "green", + ) diff --git a/counterparty-core/counterpartycore/lib/config.py b/counterparty-core/counterpartycore/lib/config.py index bf19504e1..44b1f9172 100644 --- a/counterparty-core/counterpartycore/lib/config.py +++ b/counterparty-core/counterpartycore/lib/config.py @@ -173,10 +173,6 @@ PROTOCOL_CHANGES_URL = "https://counterparty.io/protocol_changes.json" # PROTOCOL_CHANGES_URL = "https://raw.githubusercontent.com/CounterpartyXCP/counterparty-core/refs/heads/master/counterparty-core/counterpartycore/protocol_changes.json" -BOOTSTRAP_URL_MAINNET = "https://bootstrap.counterparty.io/counterparty.latest.tar.zst" -BOOTSTRAP_URL_MAINNET_SIG = "https://bootstrap.counterparty.io/counterparty.latest.tar.sig" -BOOTSTRAP_URL_TESTNET = "https://bootstrap.counterparty.io/counterparty-testnet.latest.tar.zst" -BOOTSTRAP_URL_TESTNET_SIG = "https://bootstrap.counterparty.io/counterparty-testnet.latest.tar.sig" BOOTSTRAP_URLS = { "mainnet": [ diff --git a/counterparty-core/counterpartycore/lib/util.py b/counterparty-core/counterpartycore/lib/util.py index 2e4f3a4c2..768536e04 100644 --- a/counterparty-core/counterpartycore/lib/util.py +++ b/counterparty-core/counterpartycore/lib/util.py @@ -8,14 +8,11 @@ import os import random import re -import shutil import sys -import tempfile import threading import time from operator import itemgetter -import gnupg import requests from counterparty_rs import utils as pycoin_rs_utils @@ -535,24 +532,6 @@ def clean_url_for_log(url): return url -def verify_signature(public_key_data, signature_path, snapshot_path): - temp_dir = tempfile.mkdtemp() - verified = False - - try: - gpg = gnupg.GPG(gnupghome=temp_dir) - - gpg.import_keys(public_key_data) - - with open(signature_path, "rb") as s: - verified = gpg.verify_file(s, snapshot_path, close_file=False) - - finally: - shutil.rmtree(temp_dir) - - return verified - - # ORACLES def satoshirate_to_fiat(satoshirate): return round(satoshirate / 100.0, 2) diff --git a/counterparty-core/counterpartycore/server.py b/counterparty-core/counterpartycore/server.py index 36f4032fd..b39b513ad 100755 --- a/counterparty-core/counterpartycore/server.py +++ b/counterparty-core/counterpartycore/server.py @@ -3,27 +3,21 @@ import _thread import binascii import decimal -import glob -import io import logging import multiprocessing import os -import sys -import tempfile import threading import time -import urllib -from multiprocessing import Process from urllib.parse import quote_plus as urlencode import appdirs import bitcoin as bitcoinlib -import pyzstd from termcolor import colored, cprint from counterpartycore.lib import ( backend, blocks, + bootstrap, check, config, database, @@ -35,7 +29,6 @@ ) from counterpartycore.lib.api import api_server as api_v2 from counterpartycore.lib.api import api_v1, dbbuilder -from counterpartycore.lib.public_keys import PUBLIC_KEYS logger = logging.getLogger(config.LOGGER_NAME) D = decimal.Decimal @@ -960,128 +953,3 @@ def configure_rpc(rpc_password=None): else: config.API_ROOT = "http://" + config.RPC_HOST + ":" + str(config.RPC_PORT) config.RPC = config.API_ROOT + config.RPC_WEBROOT - - -def download_zst(data_dir, zst_url): - print(f"Downloading {zst_url}...") - start_time = time.time() - zst_filename = os.path.basename(zst_url) - zst_filepath = os.path.join(data_dir, zst_filename) - urllib.request.urlretrieve(zst_url, zst_filepath) # nosec B310 # noqa: S310 - print(f"Downloaded {zst_url} in {time.time() - start_time:.2f}s") - return zst_filepath - - -def decompress_zst(zst_filepath): - print(f"Decompressing {zst_filepath}...") - start_time = time.time() - filename = zst_filepath.replace(".latest.zst", "") - filepath = os.path.join(os.path.dirname(zst_filepath), filename) - with io.open(filepath, "wb") as output_file: - with open(zst_filepath, "rb") as input_file: - pyzstd.decompress_stream(input_file, output_file, read_size=16 * 1024) - os.remove(zst_filepath) - os.chmod(filepath, 0o660) - print(f"Decompressed {zst_filepath} in {time.time() - start_time:.2f}s") - return filepath - - -def download_and_decompress(data_dir, zst_url): - # download and decompress .tar.zst file - print(f"Downloading and decompressing {zst_url}...") - start_time = time.time() - response = urllib.request.urlopen(zst_url) # nosec B310 # noqa: S310 - zst_filename = os.path.basename(zst_url) - filename = zst_filename.replace(".latest.zst", "") - filepath = os.path.join(data_dir, filename) - with io.open(filepath, "wb") as output_file: - pyzstd.decompress_stream(response, output_file, read_size=16 * 1024) - os.chmod(filepath, 0o660) - print(f"Downloaded and decompressed {zst_url} in {time.time() - start_time:.2f}s") - return filepath - - -def verify_signature(filepath, sig_url): - sig_filename = os.path.basename(sig_url) - sig_filepath = os.path.join(tempfile.gettempdir(), sig_filename) - urllib.request.urlretrieve(sig_url, sig_filepath) # nosec B310 # noqa: S310 - - print(f"Verifying signature for {filepath}...") - start_time = time.time() - signature_verified = False - for key in PUBLIC_KEYS: - if util.verify_signature(key, sig_filepath, filepath): - signature_verified = True - break - os.remove(sig_filepath) - print(f"Verified signature in {time.time() - start_time:.2f}s") - - if not signature_verified: - print(f"{filepath} was not signed by any trusted keys, deleting...") - os.remove(filepath) - sys.exit(1) - - -def decompress_and_verify(zst_filepath, sig_url): - filepath = decompress_zst(zst_filepath) - verify_signature(filepath, sig_url) - - -def verfif_and_decompress(zst_filepath, sig_url): - verify_signature(zst_filepath, sig_url) - decompress_zst(zst_filepath) - - -def clean_data_dir(): - if not os.path.exists(config.DATA_DIR): - os.makedirs(config.DATA_DIR, mode=0o755) - return - files_to_delete = glob.glob(os.path.join(config.DATA_DIR, "*.db")) - files_to_delete += glob.glob(os.path.join(config.DATA_DIR, "*.db-wal")) - files_to_delete += glob.glob(os.path.join(config.DATA_DIR, "*.db-shm")) - for file in files_to_delete: - os.remove(file) - - -def download_bootstrap_files(): - files = config.BOOTSTRAP_URLS[config.NETWORK_NAME] - decompressors = [] - for zst_url, sig_url in files: - zst_filepath = download_zst(config.DATA_DIR, zst_url) - decompressor = Process( - target=verfif_and_decompress, - args=(zst_filepath, sig_url), - ) - decompressor.start() - decompressors.append(decompressor) - - for decompressor in decompressors: - decompressor.join() - - -def confirm_bootstrap(): - warning_message = """WARNING: `counterparty-server bootstrap` downloads a recent snapshot of a Counterparty database -from a centralized server maintained by the Counterparty Core development team. -Because this method does not involve verifying the history of Counterparty transactions yourself, -the `bootstrap` command should not be used for mission-critical, commercial or public-facing nodes. - """ - cprint(warning_message, "yellow") - - confirmation_message = colored("Continue? (y/N): ", "magenta") - if input(confirmation_message).lower() != "y": - exit() - - -def bootstrap(no_confirm=False, snapshot_url=None): - if not no_confirm: - confirm_bootstrap() - - clean_data_dir() - - with log.Spinner("Downloading and decompressing database..."): - download_bootstrap_files() - - cprint( - f"Databases have been successfully bootstrapped to {config.DATA_DIR}.", - "green", - ) From 74a2c82f01a234d3f8b0d8e5597025737f28684e Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 18 Dec 2024 09:36:40 +0000 Subject: [PATCH 4/6] fixes --- .../counterpartycore/test/verify_signature_test.py | 2 +- counterparty-core/requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/counterparty-core/counterpartycore/test/verify_signature_test.py b/counterparty-core/counterpartycore/test/verify_signature_test.py index e2fbf1acf..9b5b0b43a 100644 --- a/counterparty-core/counterpartycore/test/verify_signature_test.py +++ b/counterparty-core/counterpartycore/test/verify_signature_test.py @@ -1,6 +1,6 @@ import os -from counterpartycore.lib.util import verify_signature +from counterpartycore.lib.bootstrap import verify_signature def test_verify_signature(): diff --git a/counterparty-core/requirements.txt b/counterparty-core/requirements.txt index a7f36ff46..acc29d391 100644 --- a/counterparty-core/requirements.txt +++ b/counterparty-core/requirements.txt @@ -37,3 +37,4 @@ waitress==3.0.1 hypothesis==6.116.0 bitcoin-utils==0.7.1 counterparty-rs==10.8.0 +pyzstd==0.16.2 From 7063624f3150ed260e0c264fb880a917d1f725ad Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 18 Dec 2024 10:06:58 +0000 Subject: [PATCH 5/6] Update release notes --- release-notes/release-notes-v10.9.0.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index e6d995cbd..70b7d3d3c 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -16,7 +16,8 @@ - Refactor raw mempool parsing; Don't block following - Add a timeout to parse mempool transaction from ZMQ -- Add cache for unsupported transactions when parsing raw mempool +- Add cache for unsupported transactions when parsing raw +- Refactor bootstrap: clean code, use `zstd` instead `gzip`, decompress and verify signature in parallel ## API From bd32bb53891969193fcdff4fddc7e5b6b719a21b Mon Sep 17 00:00:00 2001 From: Adam Krellenstein Date: Wed, 18 Dec 2024 07:46:09 -0500 Subject: [PATCH 6/6] Tweak Logging --- counterparty-core/counterpartycore/lib/bootstrap.py | 2 +- release-notes/release-notes-v10.9.0.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/bootstrap.py b/counterparty-core/counterpartycore/lib/bootstrap.py index 2b753c8fa..77b8c750e 100644 --- a/counterparty-core/counterpartycore/lib/bootstrap.py +++ b/counterparty-core/counterpartycore/lib/bootstrap.py @@ -83,7 +83,7 @@ def check_signature(filepath, sig_url): signature_verified = True break os.remove(sig_filepath) - print(f"Verified signature in {time.time() - start_time:.2f}s") + print(f"Verified signature for {filepath} in {time.time() - start_time:.2f}s") if not signature_verified: print(f"{filepath} was not signed by any trusted keys, deleting...") diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index 70b7d3d3c..d4ca18128 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -17,7 +17,7 @@ - Refactor raw mempool parsing; Don't block following - Add a timeout to parse mempool transaction from ZMQ - Add cache for unsupported transactions when parsing raw -- Refactor bootstrap: clean code, use `zstd` instead `gzip`, decompress and verify signature in parallel +- Refactor and optimize bootstrap process, using `zstd` instead of `gzip` ## API