diff --git a/.github/tests/mix-public.yaml b/.github/tests/mix-public.yaml index 8e23c5bc9..7a3dc6a9f 100644 --- a/.github/tests/mix-public.yaml +++ b/.github/tests/mix-public.yaml @@ -23,6 +23,9 @@ port_publisher: vc: enabled: true public_port_start: 34000 - additional_services: + remote_signer: enabled: true public_port_start: 35000 + additional_services: + enabled: true + public_port_start: 36000 diff --git a/.github/tests/remote-signer.yaml b/.github/tests/remote-signer.yaml new file mode 100644 index 000000000..a4f5d6e62 --- /dev/null +++ b/.github/tests/remote-signer.yaml @@ -0,0 +1,23 @@ +participants: + - el_type: geth + cl_type: lodestar + use_separate_vc: true + use_remote_signer: true + - el_type: geth + cl_type: nimbus + use_separate_vc: true + use_remote_signer: true + - el_type: geth + cl_type: prysm + use_separate_vc: true + use_remote_signer: true + - el_type: geth + cl_type: teku + use_separate_vc: true + use_remote_signer: true + - el_type: geth + cl_type: grandine + use_separate_vc: true + use_remote_signer: true + # Grandine doesn't have a separate VC + vc_type: lodestar diff --git a/README.md b/README.md index 5c05a80ef..a487c5925 100644 --- a/README.md +++ b/README.md @@ -314,7 +314,7 @@ participants: # Defaults to 1 vc_count: 1 - # The log level string that this participant's CL client should log at + # The log level string that this participant's validator client should log at # If this is emptystring then the global `logLevel` parameter's value will be translated into a string appropriate for the client (e.g. if # global `logLevel` = `info` then Teku would receive `INFO`, Prysm would receive `info`, etc.) # If this is not emptystring, then this value will override the global `logLevel` setting to allow for fine-grained control @@ -324,11 +324,11 @@ participants: # A list of optional extra env_vars the vc container should spin up with vc_extra_env_vars: {} - # A list of optional extra labels that will be passed to the CL client validator container. + # A list of optional extra labels that will be passed to the validator client validator container. # Example; vc_extra_labels: {"ethereum-package.partition": "1"} vc_extra_labels: {} - # A list of optional extra params that will be passed to the CL client validator container for modifying its behaviour + # A list of optional extra params that will be passed to the validator client container for modifying its behaviour # If the client combines the Beacon & validator nodes (e.g. Teku, Nimbus), then this list will also be passed to the combined Beacon-validator node vc_extra_params: [] @@ -357,6 +357,51 @@ participants: # network parameter num_validator_keys_per_node validator_count: null + # Whether to use a remote signer instead of the vc directly handling keys + # Note Lighthouse VC does not support this flag + # Defaults to false + use_remote_signer: false + + # Remote signer Specific flags + # The type of remote signer that should be used + # Valid values are web3signer + # Defaults to web3signer + remote_signer_type: "web3signer" + + # The Docker image that should be used for the remote signer + # Defaults to "consensys/web3signer:latest" + remote_signer_image: "consensys/web3signer:latest" + + # A list of optional extra env_vars the remote signer container should spin up with + remote_signer_extra_env_vars: {} + + # A list of optional extra labels that will be passed to the remote signer container. + # Example; remote_signer_extra_labels: {"ethereum-package.partition": "1"} + remote_signer_extra_labels: {} + + # A list of optional extra params that will be passed to the remote signer container for modifying its behaviour + remote_signer_extra_params: [] + + # A list of tolerations that will be passed to the remote signer container + # Only works with Kubernetes + # Example: remote_signer_tolerations: + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + # toleration_seconds: 3600 + # Defaults to empty + remote_signer_tolerations: [] + + # Resource management for remote signer containers + # CPU is milicores + # RAM is in MB + # Defaults to 0, which results in no resource limits + remote_signer_min_cpu: 0 + remote_signer_max_cpu: 0 + remote_signer_min_mem: 0 + remote_signer_max_mem: 0 + # Participant specific flags # Node selector # Only works with Kubernetes @@ -862,13 +907,20 @@ port_publisher: vc: enabled: false public_port_start: 34000 - # Additional services public port exposed to your local machine + # remote signer public port exposed to your local machine # Disabled by default # Public port start defaults to 35000 # You can't run multiple enclaves on the same port settings - additional_services: + remote_signer: enabled: false public_port_start: 35000 + # Additional services public port exposed to your local machine + # Disabled by default + # Public port start defaults to 36000 + # You can't run multiple enclaves on the same port settings + additional_services: + enabled: false + public_port_start: 36000 ``` #### Example configurations diff --git a/main.star b/main.star index 9dce197c9..299930308 100644 --- a/main.star +++ b/main.star @@ -165,12 +165,14 @@ def run(plan, args={}): all_el_contexts = [] all_cl_contexts = [] all_vc_contexts = [] + all_remote_signer_contexts = [] all_ethereum_metrics_exporter_contexts = [] all_xatu_sentry_contexts = [] for participant in all_participants: all_el_contexts.append(participant.el_context) all_cl_contexts.append(participant.cl_context) all_vc_contexts.append(participant.vc_context) + all_remote_signer_contexts.append(participant.remote_signer_context) all_ethereum_metrics_exporter_contexts.append( participant.ethereum_metrics_exporter_context ) @@ -634,6 +636,7 @@ def run(plan, args={}): all_el_contexts, all_cl_contexts, all_vc_contexts, + all_remote_signer_contexts, prometheus_additional_metrics_jobs, all_ethereum_metrics_exporter_contexts, all_xatu_sentry_contexts, diff --git a/network_params.yaml b/network_params.yaml index 21820b93a..39362d01a 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -41,6 +41,18 @@ participants: vc_min_mem: 0 vc_max_mem: 0 validator_count: null + use_remote_signer: false +# Remote signer + remote_signer_type: web3signer + remote_signer_image: consensys/web3signer:latest + remote_signer_extra_env_vars: {} + remote_signer_extra_labels: {} + remote_signer_extra_params: [] + remote_signer_tolerations: [] + remote_signer_min_cpu: 0 + remote_signer_max_cpu: 0 + remote_signer_min_mem: 0 + remote_signer_max_mem: 0 # participant specific node_selectors: {} tolerations: [] @@ -174,6 +186,9 @@ port_publisher: vc: enabled: false public_port_start: 34000 - additional_services: + remote_signer: enabled: false public_port_start: 35000 + additional_services: + enabled: false + public_port_start: 36000 diff --git a/src/package_io/constants.star b/src/package_io/constants.star index e35b52625..148ad5ed5 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -26,6 +26,8 @@ VC_TYPE = struct( teku="teku", ) +REMOTE_SIGNER_TYPE = struct(web3signer="web3signer") + GLOBAL_LOG_LEVEL = struct( info="info", error="error", @@ -38,6 +40,7 @@ CLIENT_TYPES = struct( el="execution", cl="beacon", validator="validator", + remote_signer="remote-signer", ) TCP_DISCOVERY_PORT_ID = "tcp-discovery" diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 7c4b7418e..291544414 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -52,6 +52,10 @@ DEFAULT_VC_IMAGES_MINIMAL = { "grandine": "ethpandaops/grandine:master-minimal", } +DEFAULT_REMOTE_SIGNER_IMAGES = { + "web3signer": "consensys/web3signer:latest", +} + # Placeholder value for the deneb fork epoch if electra is being run # TODO: This is a hack, and should be removed once we electra is rebased on deneb HIGH_DENEB_VALUE_FORK_VERKLE = 2000000000 @@ -213,6 +217,15 @@ def input_parser(plan, input_args): vc_extra_params=participant["vc_extra_params"], vc_extra_env_vars=participant["vc_extra_env_vars"], vc_extra_labels=participant["vc_extra_labels"], + use_remote_signer=participant["use_remote_signer"], + remote_signer_type=participant["remote_signer_type"], + remote_signer_image=participant["remote_signer_image"], + remote_signer_tolerations=participant["remote_signer_tolerations"], + remote_signer_extra_env_vars=participant[ + "remote_signer_extra_env_vars" + ], + remote_signer_extra_params=participant["remote_signer_extra_params"], + remote_signer_extra_labels=participant["remote_signer_extra_labels"], builder_network_params=participant["builder_network_params"], supernode=participant["supernode"], el_min_cpu=participant["el_min_cpu"], @@ -227,6 +240,10 @@ def input_parser(plan, input_args): vc_max_cpu=participant["vc_max_cpu"], vc_min_mem=participant["vc_min_mem"], vc_max_mem=participant["vc_max_mem"], + remote_signer_min_cpu=participant["remote_signer_min_cpu"], + remote_signer_max_cpu=participant["remote_signer_max_cpu"], + remote_signer_min_mem=participant["remote_signer_min_mem"], + remote_signer_max_mem=participant["remote_signer_max_mem"], validator_count=participant["validator_count"], tolerations=participant["tolerations"], node_selectors=participant["node_selectors"], @@ -397,6 +414,10 @@ def input_parser(plan, input_args): el_public_port_start=result["port_publisher"]["el"]["public_port_start"], vc_enabled=result["port_publisher"]["vc"]["enabled"], vc_public_port_start=result["port_publisher"]["vc"]["public_port_start"], + remote_signer_enabled=result["port_publisher"]["remote_signer"]["enabled"], + remote_signer_public_port_start=result["port_publisher"]["remote_signer"][ + "public_port_start" + ], additional_services_enabled=result["port_publisher"]["additional_services"][ "enabled" ], @@ -473,6 +494,7 @@ def parse_network_params(plan, input_args): el_type = participant["el_type"] cl_type = participant["cl_type"] vc_type = participant["vc_type"] + remote_signer_type = participant["remote_signer_type"] if ( cl_type in (constants.CL_TYPE.nimbus) @@ -537,6 +559,9 @@ def parse_network_params(plan, input_args): else: participant["use_separate_vc"] = True + if participant["use_remote_signer"] and not participant["use_separate_vc"]: + fail("`use_remote_signer` requires `use_separate_vc`") + if vc_type == "": # Defaults to matching the chosen CL client vc_type = cl_type @@ -567,6 +592,12 @@ def parse_network_params(plan, input_args): ) participant["vc_image"] = default_image + remote_signer_image = participant["remote_signer_image"] + if remote_signer_image == "": + participant["remote_signer_image"] = DEFAULT_REMOTE_SIGNER_IMAGES.get( + remote_signer_type, "" + ) + if result["parallel_keystore_generation"] and participant["vc_count"] != 1: fail( "parallel_keystore_generation is only supported for 1 validator client per participant (for now)" @@ -635,6 +666,9 @@ def parse_network_params(plan, input_args): vc_extra_params = participant.get("vc_extra_params", []) participant["vc_extra_params"] = vc_extra_params + remote_signer_extra_params = participant.get("remote_signer_extra_params", []) + participant["remote_signer_extra_params"] = remote_signer_extra_params + total_participant_count += participant["count"] if total_participant_count == 1: @@ -784,6 +818,7 @@ def default_input_args(input_args): "apache_port": None, "global_tolerations": [], "global_node_selectors": {}, + "use_remote_signer": False, "keymanager_enabled": False, "checkpoint_sync_enabled": False, "checkpoint_sync_url": "", @@ -902,6 +937,17 @@ def default_participant(): "vc_max_cpu": 0, "vc_min_mem": 0, "vc_max_mem": 0, + "use_remote_signer": None, + "remote_signer_type": "web3signer", + "remote_signer_image": "", + "remote_signer_extra_env_vars": {}, + "remote_signer_extra_labels": {}, + "remote_signer_extra_params": [], + "remote_signer_tolerations": [], + "remote_signer_min_cpu": 0, + "remote_signer_max_cpu": 0, + "remote_signer_min_mem": 0, + "remote_signer_max_mem": 0, "validator_count": None, "node_selectors": {}, "tolerations": [], @@ -1061,7 +1107,8 @@ def get_port_publisher_params(parameter_type, input_args=None): "el": {"enabled": False, "public_port_start": 32000}, "cl": {"enabled": False, "public_port_start": 33000}, "vc": {"enabled": False, "public_port_start": 34000}, - "additional_services": {"enabled": False, "public_port_start": 35000}, + "remote_signer": {"enabled": False, "public_port_start": 35000}, + "additional_services": {"enabled": False, "public_port_start": 36000}, } if parameter_type == "default": return port_publisher_parameters diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index b8d10c8f8..1e1391680 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -39,6 +39,17 @@ PARTICIPANT_CATEGORIES = { "vc_min_mem", "vc_max_mem", "validator_count", + "use_remote_signer", + "remote_signer_type", + "remote_signer_image", + "remote_signer_extra_env_vars", + "remote_signer_extra_labels", + "remote_signer_extra_params", + "remote_signer_tolerations", + "remote_signer_min_cpu", + "remote_signer_max_cpu", + "remote_signer_min_mem", + "remote_signer_max_mem", "node_selectors", "tolerations", "count", @@ -110,6 +121,18 @@ PARTICIPANT_MATRIX_PARAMS = { "vc_min_mem", "vc_max_mem", ], + "remote_signer": [ + "remote_signer_type", + "remote_signer_image", + "remote_signer_extra_env_vars", + "remote_signer_extra_labels", + "remote_signer_extra_params", + "remote_signer_tolerations", + "remote_signer_min_cpu", + "remote_signer_max_cpu", + "remote_signer_min_mem", + "remote_signer_max_mem", + ], }, } @@ -208,6 +231,7 @@ SUBCATEGORY_PARAMS = { "el", "cl", "vc", + "remote_signer", "additional_services", ], } diff --git a/src/participant.star b/src/participant.star index bc87fc977..79d3afc21 100644 --- a/src/participant.star +++ b/src/participant.star @@ -2,9 +2,11 @@ def new_participant( el_type, cl_type, vc_type, + remote_signer_type, el_context, cl_context, vc_context, + remote_signer_context, snooper_engine_context, snooper_beacon_context, ethereum_metrics_exporter_context, @@ -14,9 +16,11 @@ def new_participant( el_type=el_type, cl_type=cl_type, vc_type=vc_type, + remote_signer_type=remote_signer_type, el_context=el_context, cl_context=cl_context, vc_context=vc_context, + remote_signer_context=remote_signer_context, snooper_engine_context=snooper_engine_context, snooper_beacon_context=snooper_beacon_context, ethereum_metrics_exporter_context=ethereum_metrics_exporter_context, diff --git a/src/participant_network.star b/src/participant_network.star index b519dc8d8..14b4f2794 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -23,6 +23,7 @@ launch_shadowfork = import_module("./network_launcher/shadowfork.star") el_client_launcher = import_module("./el/el_launcher.star") cl_client_launcher = import_module("./cl/cl_launcher.star") vc = import_module("./vc/vc_launcher.star") +remote_signer = import_module("./remote_signer/remote_signer_launcher.star") beacon_snooper = import_module("./snooper/snooper_beacon_launcher.star") @@ -186,6 +187,7 @@ def launch_participant_network( all_ethereum_metrics_exporter_contexts = [] all_xatu_sentry_contexts = [] all_vc_contexts = [] + all_remote_signer_contexts = [] all_snooper_beacon_contexts = [] # Some CL clients cannot run validator clients in the same process and need # a separate validator client @@ -200,6 +202,7 @@ def launch_participant_network( el_type = participant.el_type cl_type = participant.cl_type vc_type = participant.vc_type + remote_signer_type = participant.remote_signer_type index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants)))) for sub_index in range(participant.vc_count): vc_index_str = shared_utils.zfill_custom( @@ -270,6 +273,7 @@ def launch_participant_network( # This should only be the case for the MEV participant, # the regular participants default to False/True all_vc_contexts.append(None) + all_remote_signer_contexts.append(None) all_snooper_beacon_contexts.append(None) continue @@ -281,6 +285,7 @@ def launch_participant_network( if not participant.use_separate_vc: all_vc_contexts.append(None) + all_remote_signer_contexts.append(None) all_snooper_beacon_contexts.append(None) continue @@ -298,6 +303,7 @@ def launch_participant_network( ] vc_context = None + remote_signer_context = None snooper_beacon_context = None if participant.snooper_enabled: @@ -336,6 +342,31 @@ def launch_participant_network( ) ) + if participant.use_remote_signer: + remote_signer_context = remote_signer.launch( + plan=plan, + launcher=remote_signer.new_remote_signer_launcher( + el_cl_genesis_data=el_cl_data + ), + service_name="signer-{0}".format(full_name), + remote_signer_type=remote_signer_type, + image=participant.remote_signer_image, + full_name="{0}-remote_signer".format(full_name), + vc_type=vc_type, + node_keystore_files=vc_keystores, + participant=participant, + global_tolerations=global_tolerations, + node_selectors=node_selectors, + port_publisher=port_publisher, + remote_signer_index=current_vc_index, + ) + + all_remote_signer_contexts.append(remote_signer_context) + if remote_signer_context and remote_signer_context.metrics_info: + remote_signer_context.metrics_info[ + "config" + ] = participant.prometheus_config + vc_context = vc.launch( plan=plan, launcher=vc.new_vc_launcher(el_cl_genesis_data=el_cl_data), @@ -346,6 +377,7 @@ def launch_participant_network( global_log_level=global_log_level, cl_context=cl_context, el_context=el_context, + remote_signer_context=remote_signer_context, full_name=full_name, snooper_enabled=participant.snooper_enabled, snooper_beacon_context=snooper_beacon_context, @@ -373,6 +405,7 @@ def launch_participant_network( el_type = participant.el_type cl_type = participant.cl_type vc_type = participant.vc_type + remote_signer_type = participant.remote_signer_type snooper_engine_context = None snooper_beacon_context = None @@ -380,8 +413,10 @@ def launch_participant_network( cl_context = all_cl_contexts[index] if participant.vc_count != 0: vc_context = all_vc_contexts[index] + remote_signer_context = all_remote_signer_contexts[index] else: vc_context = None + remote_signer_context = None if participant.snooper_enabled: snooper_engine_context = all_snooper_engine_contexts[index] @@ -402,9 +437,11 @@ def launch_participant_network( el_type, cl_type, vc_type, + remote_signer_type, el_context, cl_context, vc_context, + remote_signer_context, snooper_engine_context, snooper_beacon_context, ethereum_metrics_exporter_context, diff --git a/src/prometheus/prometheus_launcher.star b/src/prometheus/prometheus_launcher.star index a5ee17701..1abc78cf7 100644 --- a/src/prometheus/prometheus_launcher.star +++ b/src/prometheus/prometheus_launcher.star @@ -3,7 +3,8 @@ prometheus = import_module("github.com/kurtosis-tech/prometheus-package/main.sta EXECUTION_CLIENT_TYPE = "execution" BEACON_CLIENT_TYPE = "beacon" -vc_type = "validator" +VC_TYPE = "validator" +REMOTE_SIGNER_TYPE = "remote-signer" METRICS_INFO_NAME_KEY = "name" METRICS_INFO_URL_KEY = "url" @@ -18,6 +19,7 @@ def launch_prometheus( el_contexts, cl_contexts, vc_contexts, + remote_signer_contexts, additional_metrics_jobs, ethereum_metrics_exporter_contexts, xatu_sentry_contexts, @@ -28,6 +30,7 @@ def launch_prometheus( el_contexts, cl_contexts, vc_contexts, + remote_signer_contexts, additional_metrics_jobs, ethereum_metrics_exporter_contexts, xatu_sentry_contexts, @@ -52,6 +55,7 @@ def get_metrics_jobs( el_contexts, cl_contexts, vc_contexts, + remote_signer_contexts, additional_metrics_jobs, ethereum_metrics_exporter_contexts, xatu_sentry_contexts, @@ -130,7 +134,30 @@ def get_metrics_jobs( scrape_interval = PROMETHEUS_DEFAULT_SCRAPE_INTERVAL labels = { "service": context.service_name, - "client_type": vc_type, + "client_type": VC_TYPE, + "client_name": context.client_name, + } + + metrics_jobs.append( + new_metrics_job( + job_name=metrics_info[METRICS_INFO_NAME_KEY], + endpoint=metrics_info[METRICS_INFO_URL_KEY], + metrics_path=metrics_info[METRICS_INFO_PATH_KEY], + labels=labels, + scrape_interval=scrape_interval, + ) + ) + + # Adding validator clients metrics jobs + for context in remote_signer_contexts: + if context == None: + continue + metrics_info = context.metrics_info + + scrape_interval = PROMETHEUS_DEFAULT_SCRAPE_INTERVAL + labels = { + "service": context.service_name, + "client_type": REMOTE_SIGNER_TYPE, "client_name": context.client_name, } diff --git a/src/remote_signer/remote_signer_context.star b/src/remote_signer/remote_signer_context.star new file mode 100644 index 000000000..a1f9b69ce --- /dev/null +++ b/src/remote_signer/remote_signer_context.star @@ -0,0 +1,12 @@ +def new_remote_signer_context( + http_url, + client_name, + service_name, + metrics_info, +): + return struct( + http_url=http_url, + client_name=client_name, + service_name=service_name, + metrics_info=metrics_info, + ) diff --git a/src/remote_signer/remote_signer_launcher.star b/src/remote_signer/remote_signer_launcher.star new file mode 100644 index 000000000..fd7e9569c --- /dev/null +++ b/src/remote_signer/remote_signer_launcher.star @@ -0,0 +1,187 @@ +constants = import_module("../package_io/constants.star") +input_parser = import_module("../package_io/input_parser.star") +node_metrics = import_module("../node_metrics_info.star") +remote_signer_context = import_module("./remote_signer_context.star") +shared_utils = import_module("../shared_utils/shared_utils.star") + +REMOTE_SIGNER_KEYS_MOUNTPOINT = "/keystores" + +REMOTE_SIGNER_HTTP_PORT_NUM = 9000 +REMOTE_SIGNER_HTTP_PORT_ID = "http" +REMOTE_SIGNER_METRICS_PORT_NUM = 9001 +REMOTE_SIGNER_METRICS_PORT_ID = "metrics" + +METRICS_PATH = "/metrics" + +REMOTE_SIGNER_USED_PORTS = { + REMOTE_SIGNER_HTTP_PORT_ID: shared_utils.new_port_spec( + REMOTE_SIGNER_HTTP_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + REMOTE_SIGNER_METRICS_PORT_ID: shared_utils.new_port_spec( + REMOTE_SIGNER_METRICS_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), +} + +# The min/max CPU/memory that the remote signer can use +MIN_CPU = 50 +MAX_CPU = 300 +MIN_MEMORY = 128 +MAX_MEMORY = 1024 + + +def launch( + plan, + launcher, + service_name, + remote_signer_type, + image, + full_name, + vc_type, + node_keystore_files, + participant, + global_tolerations, + node_selectors, + port_publisher, + remote_signer_index, +): + tolerations = input_parser.get_client_tolerations( + participant.remote_signer_tolerations, + participant.tolerations, + global_tolerations, + ) + + config = get_config( + participant=participant, + el_cl_genesis_data=launcher.el_cl_genesis_data, + image=image, + vc_type=vc_type, + node_keystore_files=node_keystore_files, + tolerations=tolerations, + node_selectors=node_selectors, + port_publisher=port_publisher, + remote_signer_index=remote_signer_index, + ) + + remote_signer_service = plan.add_service(service_name, config) + + remote_signer_http_port = remote_signer_service.ports[REMOTE_SIGNER_HTTP_PORT_ID] + remote_signer_http_url = "http://{0}:{1}".format( + remote_signer_service.ip_address, remote_signer_http_port.number + ) + + remote_signer_metrics_port = remote_signer_service.ports[ + REMOTE_SIGNER_METRICS_PORT_ID + ] + validator_metrics_url = "{0}:{1}".format( + remote_signer_service.ip_address, remote_signer_metrics_port.number + ) + remote_signer_node_metrics_info = node_metrics.new_node_metrics_info( + service_name, METRICS_PATH, validator_metrics_url + ) + + return remote_signer_context.new_remote_signer_context( + http_url=remote_signer_http_url, + client_name=remote_signer_type, + service_name=service_name, + metrics_info=remote_signer_node_metrics_info, + ) + + +def get_config( + participant, + el_cl_genesis_data, + image, + vc_type, + node_keystore_files, + tolerations, + node_selectors, + port_publisher, + remote_signer_index, +): + validator_keys_dirpath = "" + if node_keystore_files != None: + validator_keys_dirpath = shared_utils.path_join( + REMOTE_SIGNER_KEYS_MOUNTPOINT, + node_keystore_files.teku_keys_relative_dirpath, + ) + validator_secrets_dirpath = shared_utils.path_join( + REMOTE_SIGNER_KEYS_MOUNTPOINT, + node_keystore_files.teku_secrets_relative_dirpath, + ) + + cmd = [ + "--http-listen-port={0}".format(REMOTE_SIGNER_HTTP_PORT_NUM), + "--http-host-allowlist=*", + "--metrics-enabled=true", + "--metrics-host-allowlist=*", + "--metrics-host=0.0.0.0", + "--metrics-port={0}".format(REMOTE_SIGNER_METRICS_PORT_NUM), + "eth2", + "--network=" + + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + + "/config.yaml", + "--keystores-path=" + validator_keys_dirpath, + "--keystores-passwords-path=" + validator_secrets_dirpath, + # slashing protection would require a postgres DB, applying DB migrations ... + "--slashing-protection-enabled=false", + ] + + if len(participant.remote_signer_extra_params) > 0: + # this is a repeated, we convert it into Starlark + cmd.extend([param for param in participant.remote_signer_extra_params]) + + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + REMOTE_SIGNER_KEYS_MOUNTPOINT: node_keystore_files.files_artifact_uuid, + } + + public_ports = {} + if port_publisher.remote_signer_enabled: + public_ports_for_component = shared_utils.get_public_ports_for_component( + "remote-signer", port_publisher, remote_signer_index + ) + public_port_assignments = { + constants.METRICS_PORT_ID: public_ports_for_component[0] + } + public_ports = shared_utils.get_port_specs(public_port_assignments) + + ports = {} + ports.update(REMOTE_SIGNER_USED_PORTS) + + config_args = { + "image": image, + "ports": ports, + "public_ports": public_ports, + "cmd": cmd, + "files": files, + "env_vars": participant.remote_signer_extra_env_vars, + "labels": shared_utils.label_maker( + client=constants.REMOTE_SIGNER_TYPE.web3signer, + client_type=constants.CLIENT_TYPES.remote_signer, + image=image, + connected_client=vc_type, + extra_labels=participant.remote_signer_extra_labels, + supernode=participant.supernode, + ), + "tolerations": tolerations, + "node_selectors": node_selectors, + } + + if participant.remote_signer_min_cpu > 0: + config_args["min_cpu"] = participant.remote_signer_min_cpu + if participant.remote_signer_max_cpu > 0: + config_args["max_cpu"] = participant.remote_signer_max_cpu + if participant.remote_signer_min_mem > 0: + config_args["min_memory"] = participant.remote_signer_min_mem + if participant.remote_signer_max_mem > 0: + config_args["max_memory"] = participant.remote_signer_max_mem + + return ServiceConfig(**config_args) + + +def new_remote_signer_launcher(el_cl_genesis_data): + return struct(el_cl_genesis_data=el_cl_genesis_data) diff --git a/src/shared_utils/shared_utils.star b/src/shared_utils/shared_utils.star index 2a7d95d43..7245a8700 100644 --- a/src/shared_utils/shared_utils.star +++ b/src/shared_utils/shared_utils.star @@ -9,6 +9,7 @@ NOT_PROVIDED_WAIT = "not-provided-wait" MAX_PORTS_PER_CL_NODE = 5 MAX_PORTS_PER_EL_NODE = 5 MAX_PORTS_PER_VC_NODE = 3 +MAX_PORTS_PER_REMOTE_SIGNER_NODE = 2 MAX_PORTS_PER_ADDITIONAL_SERVICE = 2 @@ -261,6 +262,12 @@ def get_public_ports_for_component( MAX_PORTS_PER_VC_NODE, participant_index, ) + elif component == "remote-signer": + public_port_range = __get_port_range( + port_publisher_params.remote_signer_public_port_start, + MAX_PORTS_PER_REMOTE_SIGNER_NODE, + participant_index, + ) elif component == "additional_services": public_port_range = __get_port_range( port_publisher_params.additional_services_public_port_start, diff --git a/src/vc/lodestar.star b/src/vc/lodestar.star index 6a41947b8..8bacdf938 100644 --- a/src/vc/lodestar.star +++ b/src/vc/lodestar.star @@ -21,6 +21,7 @@ def get_config( beacon_http_url, cl_context, el_context, + remote_signer_context, full_name, node_keystore_files, tolerations, @@ -51,8 +52,6 @@ def get_config( + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/config.yaml", "--beaconNodes=" + beacon_http_url, - "--keystoresDir=" + validator_keys_dirpath, - "--secretsDir=" + validator_secrets_dirpath, "--suggestedFeeRecipient=" + constants.VALIDATING_REWARDS_ACCOUNT, # vvvvvvvvvvvvvvvvvvv PROMETHEUS CONFIG vvvvvvvvvvvvvvvvvvvvv "--metrics", @@ -64,6 +63,21 @@ def get_config( "--disableKeystoresThreadPool", ] + if remote_signer_context == None: + cmd.extend( + [ + "--keystoresDir=" + validator_keys_dirpath, + "--secretsDir=" + validator_secrets_dirpath, + ] + ) + else: + cmd.extend( + [ + "--externalSigner.url={0}".format(remote_signer_context.http_url), + "--externalSigner.fetch", + ] + ) + keymanager_api_cmd = [ "--keymanager", "--keymanager.authEnabled=true", diff --git a/src/vc/nimbus.star b/src/vc/nimbus.star index 7d525d19c..5887f398d 100644 --- a/src/vc/nimbus.star +++ b/src/vc/nimbus.star @@ -11,6 +11,7 @@ def get_config( beacon_http_url, cl_context, el_context, + remote_signer_context, full_name, node_keystore_files, tolerations, @@ -33,8 +34,6 @@ def get_config( cmd = [ "--beacon-node=" + beacon_http_url, - "--validators-dir=" + validator_keys_dirpath, - "--secrets-dir=" + validator_secrets_dirpath, "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT, # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv "--metrics", @@ -43,6 +42,20 @@ def get_config( "--graffiti=" + full_name, ] + if remote_signer_context == None: + cmd.extend( + [ + "--validators-dir=" + validator_keys_dirpath, + "--secrets-dir=" + validator_secrets_dirpath, + ] + ) + else: + cmd.extend( + [ + "--web3-signer-url={0}".format(remote_signer_context.http_url), + ] + ) + keymanager_api_cmd = [ "--keymanager", "--keymanager-port={0}".format(vc_shared.VALIDATOR_HTTP_PORT_NUM), diff --git a/src/vc/prysm.star b/src/vc/prysm.star index b900b322c..0da059c7c 100644 --- a/src/vc/prysm.star +++ b/src/vc/prysm.star @@ -14,6 +14,7 @@ def get_config( beacon_http_url, cl_context, el_context, + remote_signer_context, full_name, node_keystore_files, prysm_password_relative_filepath, @@ -38,8 +39,6 @@ def get_config( "--chain-config-file=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/config.yaml", - "--wallet-dir=" + validator_keys_dirpath, - "--wallet-password-file=" + validator_secrets_dirpath, "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT, # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv "--disable-monitoring=false", @@ -49,6 +48,23 @@ def get_config( "--graffiti=" + full_name, ] + if remote_signer_context == None: + cmd.extend( + [ + "--wallet-dir=" + validator_keys_dirpath, + "--wallet-password-file=" + validator_secrets_dirpath, + ] + ) + else: + cmd.extend( + [ + "--remote-signer-url={0}".format(remote_signer_context.http_url), + "--remote-signer-keys={0}/api/v1/eth2/publicKeys".format( + remote_signer_context.http_url + ), + ] + ) + keymanager_api_cmd = [ "--rpc", "--http-port={0}".format(vc_shared.VALIDATOR_HTTP_PORT_NUM), diff --git a/src/vc/teku.star b/src/vc/teku.star index dcdbf29a0..7f5e56172 100644 --- a/src/vc/teku.star +++ b/src/vc/teku.star @@ -11,6 +11,7 @@ def get_config( beacon_http_url, cl_context, el_context, + remote_signer_context, full_name, node_keystore_files, tolerations, @@ -37,10 +38,6 @@ def get_config( + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/config.yaml", "--beacon-node-api-endpoint=" + beacon_http_url, - "--validator-keys={0}:{1}".format( - validator_keys_dirpath, - validator_secrets_dirpath, - ), "--validators-proposer-default-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT, "--validators-graffiti=" + full_name, @@ -51,6 +48,25 @@ def get_config( "--metrics-port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), ] + if remote_signer_context == None: + cmd.extend( + [ + "--validator-keys={0}:{1}".format( + validator_keys_dirpath, + validator_secrets_dirpath, + ), + ] + ) + else: + cmd.extend( + [ + "--validators-external-signer-url={0}".format( + remote_signer_context.http_url + ), + "--validators-external-signer-public-keys=external-signer", + ] + ) + keymanager_api_cmd = [ "--validator-api-enabled=true", "--validator-api-host-allowlist=*", diff --git a/src/vc/vc_launcher.star b/src/vc/vc_launcher.star index 3b54c4023..3d3fe43f0 100644 --- a/src/vc/vc_launcher.star +++ b/src/vc/vc_launcher.star @@ -22,6 +22,7 @@ def launch( global_log_level, cl_context, el_context, + remote_signer_context, full_name, snooper_enabled, snooper_beacon_context, @@ -56,6 +57,8 @@ def launch( keymanager_enabled = participant.keymanager_enabled if vc_type == constants.VC_TYPE.lighthouse: + if remote_signer_context != None: + fail("`use_remote_signer` flag not supported for lighthouse VC") config = lighthouse.get_config( participant=participant, el_cl_genesis_data=launcher.el_cl_genesis_data, @@ -84,6 +87,7 @@ def launch( beacon_http_url=beacon_http_url, cl_context=cl_context, el_context=el_context, + remote_signer_context=remote_signer_context, full_name=full_name, node_keystore_files=node_keystore_files, tolerations=tolerations, @@ -102,6 +106,7 @@ def launch( beacon_http_url=beacon_http_url, cl_context=cl_context, el_context=el_context, + remote_signer_context=remote_signer_context, full_name=full_name, node_keystore_files=node_keystore_files, tolerations=tolerations, @@ -119,6 +124,7 @@ def launch( beacon_http_url=beacon_http_url, cl_context=cl_context, el_context=el_context, + remote_signer_context=remote_signer_context, full_name=full_name, node_keystore_files=node_keystore_files, tolerations=tolerations, @@ -136,6 +142,7 @@ def launch( beacon_http_url=beacon_http_url, cl_context=cl_context, el_context=el_context, + remote_signer_context=remote_signer_context, full_name=full_name, node_keystore_files=node_keystore_files, prysm_password_relative_filepath=prysm_password_relative_filepath,