Skip to content

Commit

Permalink
feat: External prover API (#2538)
Browse files Browse the repository at this point in the history
## What ❔

Adds prover API component, which allows to get proof generation data and
verify final proofs against generated by our prover subsystem.

## Why ❔

First step of prover decentralization.

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.

---------

Co-authored-by: EmilLuta <[email protected]>
  • Loading branch information
Artemka374 and EmilLuta authored Aug 8, 2024
1 parent e23e661 commit 129a181
Show file tree
Hide file tree
Showing 30 changed files with 712 additions and 43 deletions.
17 changes: 17 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ members = [
"core/node/api_server",
"core/node/tee_verifier_input_producer",
"core/node/base_token_adjuster",
"core/node/external_proof_integration_api",
# Libraries
"core/lib/db_connection",
"core/lib/zksync_core_leftovers",
Expand Down Expand Up @@ -77,7 +78,6 @@ members = [
"core/tests/loadnext",
"core/tests/vm-benchmark",
"core/tests/vm-benchmark/harness",

# Parts of prover workspace that are needed for Core workspace
"prover/crates/lib/prover_dal",
]
Expand Down Expand Up @@ -282,6 +282,7 @@ zksync_eth_sender = { version = "0.1.0", path = "core/node/eth_sender" }
zksync_node_db_pruner = { version = "0.1.0", path = "core/node/db_pruner" }
zksync_node_fee_model = { version = "0.1.0", path = "core/node/fee_model" }
zksync_vm_runner = { version = "0.1.0", path = "core/node/vm_runner" }
zksync_external_proof_integration_api = { version = "0.1.0", path = "core/node/external_proof_integration_api" }
zksync_node_test_utils = { version = "0.1.0", path = "core/node/test_utils" }
zksync_state_keeper = { version = "0.1.0", path = "core/node/state_keeper" }
zksync_reorg_detector = { version = "0.1.0", path = "core/node/reorg_detector" }
Expand Down
5 changes: 3 additions & 2 deletions core/bin/zksync_server/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ use zksync_config::{
ProtectiveReadsWriterConfig, Secrets,
},
ApiConfig, BaseTokenAdjusterConfig, ContractVerifierConfig, DADispatcherConfig, DBConfig,
EthConfig, EthWatchConfig, GasAdjusterConfig, GenesisConfig, ObjectStoreConfig, PostgresConfig,
SnapshotsCreatorConfig,
EthConfig, EthWatchConfig, ExternalProofIntegrationApiConfig, GasAdjusterConfig, GenesisConfig,
ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig,
};
use zksync_core_leftovers::{
temp_config_store::{decode_yaml_repr, TempConfigStore},
Expand Down Expand Up @@ -208,5 +208,6 @@ fn load_env_config() -> anyhow::Result<TempConfigStore> {
pruning: None,
snapshot_recovery: None,
external_price_api_client_config: ExternalPriceApiClientConfig::from_env().ok(),
external_proof_integration_api_config: ExternalProofIntegrationApiConfig::from_env().ok(),
})
}
14 changes: 14 additions & 0 deletions core/bin/zksync_server/src/node_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ use zksync_node_framework::{
da_dispatcher::DataAvailabilityDispatcherLayer,
eth_sender::{EthTxAggregatorLayer, EthTxManagerLayer},
eth_watch::EthWatchLayer,
external_proof_integration_api::ExternalProofIntegrationApiLayer,
gas_adjuster::GasAdjusterLayer,
healtcheck_server::HealthCheckLayer,
house_keeper::HouseKeeperLayer,
Expand Down Expand Up @@ -574,6 +575,16 @@ impl MainNodeBuilder {
Ok(self)
}

fn add_external_proof_integration_api_layer(mut self) -> anyhow::Result<Self> {
let config = try_load_config!(self.configs.external_proof_integration_api_config);
self.node.add_layer(ExternalProofIntegrationApiLayer::new(
config,
self.genesis_config.l1_batch_commit_data_generator_mode,
));

Ok(self)
}

/// This layer will make sure that the database is initialized correctly,
/// e.g. genesis will be performed if it's required.
///
Expand Down Expand Up @@ -718,6 +729,9 @@ impl MainNodeBuilder {
Component::VmRunnerBwip => {
self = self.add_vm_runner_bwip_layer()?;
}
Component::ExternalProofIntegrationApi => {
self = self.add_external_proof_integration_api_layer()?;
}
}
}
Ok(self.node.build())
Expand Down
6 changes: 6 additions & 0 deletions core/lib/config/src/configs/external_proof_integration_api.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
use serde::Deserialize;

#[derive(Debug, Deserialize, Clone, PartialEq)]
pub struct ExternalProofIntegrationApiConfig {
pub http_port: u16,
}
5 changes: 3 additions & 2 deletions core/lib/config/src/configs/general.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ use crate::{
FriWitnessVectorGeneratorConfig, ObservabilityConfig, PrometheusConfig,
ProofDataHandlerConfig,
},
ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, ObjectStoreConfig, PostgresConfig,
SnapshotsCreatorConfig,
ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, ExternalProofIntegrationApiConfig,
ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig,
};

#[derive(Debug, Clone, PartialEq)]
Expand Down Expand Up @@ -50,4 +50,5 @@ pub struct GeneralConfig {
pub base_token_adjuster: Option<BaseTokenAdjusterConfig>,
pub external_price_api_client_config: Option<ExternalPriceApiClientConfig>,
pub consensus_config: Option<ConsensusConfig>,
pub external_proof_integration_api_config: Option<ExternalProofIntegrationApiConfig>,
}
2 changes: 2 additions & 0 deletions core/lib/config/src/configs/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ pub use self::{
eth_watch::EthWatchConfig,
experimental::ExperimentalDBConfig,
external_price_api_client::ExternalPriceApiClientConfig,
external_proof_integration_api::ExternalProofIntegrationApiConfig,
fri_proof_compressor::FriProofCompressorConfig,
fri_prover::FriProverConfig,
fri_prover_gateway::FriProverGatewayConfig,
Expand Down Expand Up @@ -43,6 +44,7 @@ pub mod eth_sender;
pub mod eth_watch;
mod experimental;
pub mod external_price_api_client;
pub mod external_proof_integration_api;
pub mod fri_proof_compressor;
pub mod fri_prover;
pub mod fri_prover_gateway;
Expand Down
4 changes: 2 additions & 2 deletions core/lib/config/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

pub use crate::configs::{
ApiConfig, BaseTokenAdjusterConfig, ContractVerifierConfig, ContractsConfig,
DADispatcherConfig, DBConfig, EthConfig, EthWatchConfig, GasAdjusterConfig, GenesisConfig,
ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig,
DADispatcherConfig, DBConfig, EthConfig, EthWatchConfig, ExternalProofIntegrationApiConfig,
GasAdjusterConfig, GenesisConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig,
};

pub mod configs;
Expand Down
14 changes: 14 additions & 0 deletions core/lib/config/src/testonly.rs
Original file line number Diff line number Diff line change
Expand Up @@ -995,6 +995,19 @@ impl Distribution<configs::base_token_adjuster::BaseTokenAdjusterConfig> for Enc
}
}

impl Distribution<configs::external_proof_integration_api::ExternalProofIntegrationApiConfig>
for EncodeDist
{
fn sample<R: Rng + ?Sized>(
&self,
rng: &mut R,
) -> configs::external_proof_integration_api::ExternalProofIntegrationApiConfig {
configs::external_proof_integration_api::ExternalProofIntegrationApiConfig {
http_port: self.sample(rng),
}
}
}

impl Distribution<configs::external_price_api_client::ExternalPriceApiClientConfig> for EncodeDist {
fn sample<R: Rng + ?Sized>(
&self,
Expand Down Expand Up @@ -1044,6 +1057,7 @@ impl Distribution<configs::GeneralConfig> for EncodeDist {
base_token_adjuster: self.sample(rng),
external_price_api_client_config: self.sample(rng),
consensus_config: self.sample(rng),
external_proof_integration_api_config: self.sample(rng),
}
}
}

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

23 changes: 23 additions & 0 deletions core/lib/dal/src/proof_generation_dal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,29 @@ impl ProofGenerationDal<'_, '_> {
Ok(result)
}

pub async fn get_available_batch(&mut self) -> DalResult<L1BatchNumber> {
let result = sqlx::query!(
r#"
SELECT
l1_batch_number
FROM
proof_generation_details
WHERE
proof_blob_url IS NOT NULL
ORDER BY
l1_batch_number ASC
LIMIT
1
"#,
)
.instrument("get_available batch")
.fetch_one(self.storage)
.await?
.l1_batch_number as u32;

Ok(L1BatchNumber(result))
}

/// Marks a previously locked batch as 'unpicked', allowing it to be picked without having
/// to wait for the processing timeout.
pub async fn unlock_batch(&mut self, l1_batch_number: L1BatchNumber) -> DalResult<()> {
Expand Down
35 changes: 35 additions & 0 deletions core/lib/env_config/src/external_proof_integration_api.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
use zksync_config::configs::ExternalProofIntegrationApiConfig;

use crate::{envy_load, FromEnv};

impl FromEnv for ExternalProofIntegrationApiConfig {
fn from_env() -> anyhow::Result<Self> {
envy_load(
"external_proof_integration_api",
"EXTERNAL_PROOF_INTEGRATION_API_",
)
}
}

#[cfg(test)]
mod tests {
use super::*;
use crate::test_utils::EnvMutex;

static MUTEX: EnvMutex = EnvMutex::new();

fn expected_config() -> ExternalProofIntegrationApiConfig {
ExternalProofIntegrationApiConfig { http_port: 3320 }
}

#[test]
fn from_env() {
let config = r#"
EXTERNAL_PROOF_INTEGRATION_API_HTTP_PORT="3320"
"#;
let mut lock = MUTEX.lock();
lock.set_env(config);
let actual = ExternalProofIntegrationApiConfig::from_env().unwrap();
assert_eq!(actual, expected_config());
}
}
1 change: 1 addition & 0 deletions core/lib/env_config/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ mod utils;
mod base_token_adjuster;
mod da_dispatcher;
mod external_price_api_client;
mod external_proof_integration_api;
mod genesis;
#[cfg(test)]
mod test_utils;
Expand Down
22 changes: 22 additions & 0 deletions core/lib/protobuf_config/src/external_proof_integration_api.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
use anyhow::Context;
use zksync_config::ExternalProofIntegrationApiConfig;
use zksync_protobuf::{required, ProtoRepr};

use crate::proto::external_proof_integration_api as proto;

impl ProtoRepr for proto::ExternalProofIntegrationApi {
type Type = ExternalProofIntegrationApiConfig;
fn read(&self) -> anyhow::Result<Self::Type> {
Ok(Self::Type {
http_port: required(&self.http_port)
.and_then(|p| Ok((*p).try_into()?))
.context("http_port")?,
})
}

fn build(this: &Self::Type) -> Self {
Self {
http_port: Some(this.http_port.into()),
}
}
}
7 changes: 7 additions & 0 deletions core/lib/protobuf_config/src/general.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ impl ProtoRepr for proto::GeneralConfig {
snapshot_recovery: read_optional_repr(&self.snapshot_recovery),
external_price_api_client_config: read_optional_repr(&self.external_price_api_client),
consensus_config: read_optional_repr(&self.consensus),
external_proof_integration_api_config: read_optional_repr(
&self.external_proof_integration_api,
),
})
}

Expand Down Expand Up @@ -90,6 +93,10 @@ impl ProtoRepr for proto::GeneralConfig {
.as_ref()
.map(ProtoRepr::build),
consensus: this.consensus_config.as_ref().map(ProtoRepr::build),
external_proof_integration_api: this
.external_proof_integration_api_config
.as_ref()
.map(ProtoRepr::build),
}
}
}
1 change: 1 addition & 0 deletions core/lib/protobuf_config/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ mod secrets;
mod snapshots_creator;

mod external_price_api_client;
mod external_proof_integration_api;
mod snapshot_recovery;
#[cfg(test)]
mod tests;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
syntax = "proto3";

package zksync.config.external_proof_integration_api;

message ExternalProofIntegrationApi {
optional uint32 http_port = 1;
}
Loading

0 comments on commit 129a181

Please sign in to comment.