diff --git a/Cargo.lock b/Cargo.lock index 98dca2e4b7814..bfe00ed19d795 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14590,83 +14590,6 @@ dependencies = [ "uuid 1.2.2", ] -[[package]] -name = "sui-mvr-indexer" -version = "1.40.0" -dependencies = [ - "anyhow", - "async-trait", - "axum 0.7.5", - "backoff", - "bb8", - "bcs", - "bytes", - "cached", - "chrono", - "clap", - "criterion", - "csv", - "dashmap", - "diesel", - "diesel-async", - "diesel_migrations", - "fastcrypto", - "futures", - "hex", - "indicatif", - "itertools 0.13.0", - "jsonrpsee", - "move-binary-format", - "move-bytecode-utils", - "move-core-types", - "mysten-metrics", - "ntest", - "object_store", - "prometheus", - "rand 0.8.5", - "rayon", - "regex", - "serde", - "serde_json", - "serde_with 3.9.0", - "simulacrum", - "strum 0.24.1", - "strum_macros 0.24.3", - "sui-archival", - "sui-config", - "sui-core", - "sui-data-ingestion-core", - "sui-json", - "sui-json-rpc", - "sui-json-rpc-api", - "sui-json-rpc-types", - "sui-keys", - "sui-move-build", - "sui-open-rpc", - "sui-package-resolver", - "sui-protocol-config", - "sui-rpc-api", - "sui-sdk", - "sui-snapshot", - "sui-storage", - "sui-swarm-config", - "sui-synthetic-ingestion", - "sui-test-transaction-builder", - "sui-transaction-builder", - "sui-types", - "tap", - "telemetry-subscribers", - "tempfile", - "test-cluster", - "thiserror", - "tokio", - "tokio-stream", - "tokio-util 0.7.10", - "toml 0.7.4", - "tracing", - "url", -] - [[package]] name = "sui-network" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index f0a5b4a984bc6..9a278559323be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,7 +135,6 @@ members = [ "crates/sui-move-build", "crates/sui-move-lsp", "crates/sui-mvr-graphql-rpc", - "crates/sui-mvr-indexer", "crates/sui-network", "crates/sui-node", "crates/sui-open-rpc", @@ -657,7 +656,6 @@ sui-move = { path = "crates/sui-move" } sui-move-build = { path = "crates/sui-move-build" } sui-move-lsp = { path = "crates/sui-move-lsp" } sui-mvr-graphql-rpc = { path = "crates/sui-mvr-graphql-rpc" } -sui-mvr-indexer = { path = "crates/sui-mvr-indexer" } sui-network = { path = "crates/sui-network" } sui-node = { path = "crates/sui-node" } sui-open-rpc = { path = "crates/sui-open-rpc" } diff --git a/crates/sui-mvr-indexer/Cargo.toml b/crates/sui-mvr-indexer/Cargo.toml deleted file mode 100644 index b592ede0bbd12..0000000000000 --- a/crates/sui-mvr-indexer/Cargo.toml +++ /dev/null @@ -1,89 +0,0 @@ -[package] -name = "sui-mvr-indexer" -version.workspace = true -authors = ["Mysten Labs "] -license = "Apache-2.0" -publish = false -edition = "2021" - -[dependencies] -anyhow.workspace = true -rand = "0.8.5" -async-trait.workspace = true -axum.workspace = true -backoff.workspace = true -bb8 = "0.8.5" -bcs.workspace = true -bytes.workspace = true -chrono.workspace = true -clap = { workspace = true, features = ["env"] } -csv.workspace = true -diesel = { workspace = true, features = ["chrono", "serde_json"] } -diesel-async = { workspace = true, features = ["bb8", "postgres", "async-connection-wrapper"] } -futures.workspace = true -hex.workspace = true -indicatif.workspace = true -itertools.workspace = true -jsonrpsee.workspace = true -object_store.workspace = true -prometheus.workspace = true -rayon.workspace = true -regex.workspace = true -serde.workspace = true -serde_json.workspace = true -serde_with.workspace = true -strum.workspace = true -strum_macros.workspace = true -tap.workspace = true -tempfile.workspace = true -thiserror.workspace = true -tokio = { workspace = true, features = ["full"] } -tokio-util = { workspace = true, features = ["rt"] } -toml.workspace = true -tracing.workspace = true -url.workspace = true - -fastcrypto = { workspace = true, features = ["copy_key"] } -mysten-metrics.workspace = true -simulacrum.workspace = true -sui-config.workspace = true -sui-archival.workspace = true -sui-core.workspace = true -sui-data-ingestion-core.workspace = true -sui-json.workspace = true -sui-json-rpc.workspace = true -sui-json-rpc-api.workspace = true -sui-json-rpc-types.workspace = true -sui-open-rpc.workspace = true -sui-sdk.workspace = true -sui-snapshot.workspace = true -sui-storage.workspace = true -sui-types.workspace = true -sui-package-resolver.workspace = true -sui-protocol-config.workspace = true -telemetry-subscribers.workspace = true -sui-rpc-api.workspace = true -sui-transaction-builder.workspace = true -sui-synthetic-ingestion.workspace = true - -move-core-types.workspace = true -move-bytecode-utils.workspace = true -move-binary-format.workspace = true - -diesel_migrations.workspace = true -cached.workspace = true -tokio-stream.workspace = true -dashmap.workspace = true - -[dev-dependencies] -sui-keys.workspace = true -sui-move-build.workspace = true -sui-swarm-config.workspace = true -sui-test-transaction-builder.workspace = true -test-cluster.workspace = true -ntest.workspace = true -criterion.workspace = true - -[[bin]] -name = "sui-mvr-indexer" -path = "src/main.rs" diff --git a/crates/sui-mvr-indexer/README.md b/crates/sui-mvr-indexer/README.md deleted file mode 100644 index e579bc76ac3ad..0000000000000 --- a/crates/sui-mvr-indexer/README.md +++ /dev/null @@ -1,27 +0,0 @@ -The MVR indexer is a spin-off of the Sui indexer. It has a subset of the full indexer schema, limited to just the tables needed to support MVR. The required tables are `epochs`, `checkpoints`, `packages`, `objects_snapshot`, and `objects_history`. This enables the custom indexer to support the `package_by_name` and `type_by_name` queries on GraphQL. - -# Running this indexer -## Start the Postgres Service - -Postgres must run as a service in the background for other tools to communicate with. If it was installed using homebrew, it can be started as a service with: - -``` sh -brew services start postgresql@version -``` - -## DB reset -When making db-related changes, you may find yourself having to run migrations and reset dbs often. The commands below are how you can invoke these actions. -```sh -cargo run --bin sui-mvr-indexer -- --database-url "" reset-database --force -``` - -## Start the indexer -```SH -cargo run --bin sui-mvr-indexer -- --db-url "" indexer --rpc-client-url "https://fullnode.devnet.sui.io:443" --remote-store-url http://lax-suifn-t99eb.devnet.sui.io:9000/rest -``` - -## Migrations - -To add a new table, run `diesel migration generate your_table_name`, and modify the newly created `up.sql` and `down.sql` files. - -You would apply the migration with `diesel migration run`, and run the script in `./scripts/generate_indexer_schema.sh` to update the `schema.rs` file. diff --git a/crates/sui-mvr-indexer/diesel.toml b/crates/sui-mvr-indexer/diesel.toml deleted file mode 100644 index 4430344705b42..0000000000000 --- a/crates/sui-mvr-indexer/diesel.toml +++ /dev/null @@ -1,8 +0,0 @@ -# For documentation on how to configure this file, -# see https://diesel.rs/guides/configuring-diesel-cli - -[print_schema] -file = "src/schema/pg.rs" - -[migrations_directory] -dir = "migrations/pg" diff --git a/crates/sui-mvr-indexer/migrations/pg/00000000000000_diesel_initial_setup/down.sql b/crates/sui-mvr-indexer/migrations/pg/00000000000000_diesel_initial_setup/down.sql deleted file mode 100644 index a9f526091194b..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/00000000000000_diesel_initial_setup/down.sql +++ /dev/null @@ -1,6 +0,0 @@ --- This file was automatically created by Diesel to setup helper functions --- and other internal bookkeeping. This file is safe to edit, any future --- changes will be added to existing projects as new migrations. - -DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); -DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/crates/sui-mvr-indexer/migrations/pg/00000000000000_diesel_initial_setup/up.sql b/crates/sui-mvr-indexer/migrations/pg/00000000000000_diesel_initial_setup/up.sql deleted file mode 100644 index d68895b1a7b7d..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/00000000000000_diesel_initial_setup/up.sql +++ /dev/null @@ -1,36 +0,0 @@ --- This file was automatically created by Diesel to setup helper functions --- and other internal bookkeeping. This file is safe to edit, any future --- changes will be added to existing projects as new migrations. - - - - --- Sets up a trigger for the given table to automatically set a column called --- `updated_at` whenever the row is modified (unless `updated_at` was included --- in the modified columns) --- --- # Example --- --- ```sql --- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); --- --- SELECT diesel_manage_updated_at('users'); --- ``` -CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ -BEGIN - EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s - FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ -BEGIN - IF ( - NEW IS DISTINCT FROM OLD AND - NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at - ) THEN - NEW.updated_at := current_timestamp; - END IF; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044020_events/down.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044020_events/down.sql deleted file mode 100644 index 6cd33fe82b338..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044020_events/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -DROP TABLE IF EXISTS events; diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044020_events/up.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044020_events/up.sql deleted file mode 100644 index 14aa6a098161f..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044020_events/up.sql +++ /dev/null @@ -1,26 +0,0 @@ --- TODO: modify queries in indexer reader to take advantage of the new indices -CREATE TABLE events -( - tx_sequence_number BIGINT NOT NULL, - event_sequence_number BIGINT NOT NULL, - transaction_digest bytea NOT NULL, - -- array of SuiAddress in bytes. All signers of the transaction. - senders bytea[] NOT NULL, - -- bytes of the entry package ID. Notice that the package and module here - -- are the package and module of the function that emitted the event, diffrent - -- from the package and module of the event type. - package bytea NOT NULL, - -- entry module name - module text NOT NULL, - -- StructTag in Display format, fully qualified including type parameters - event_type text NOT NULL, - -- timestamp of the checkpoint when the event was emitted - timestamp_ms BIGINT NOT NULL, - -- bcs of the Event contents (Event.contents) - bcs BYTEA NOT NULL, - PRIMARY KEY(tx_sequence_number, event_sequence_number) -) PARTITION BY RANGE (tx_sequence_number); -CREATE TABLE events_partition_0 PARTITION OF events FOR VALUES FROM (0) TO (MAXVALUE); -CREATE INDEX events_package ON events (package, tx_sequence_number, event_sequence_number); -CREATE INDEX events_package_module ON events (package, module, tx_sequence_number, event_sequence_number); -CREATE INDEX events_event_type ON events (event_type text_pattern_ops, tx_sequence_number, event_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044023_objects/down.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044023_objects/down.sql deleted file mode 100644 index edea7960b79d7..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044023_objects/down.sql +++ /dev/null @@ -1,3 +0,0 @@ -DROP TABLE IF EXISTS objects; -DROP TABLE IF EXISTS objects_history; -DROP TABLE IF EXISTS objects_snapshot; diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044023_objects/up.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044023_objects/up.sql deleted file mode 100644 index 54854fabf4359..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044023_objects/up.sql +++ /dev/null @@ -1,95 +0,0 @@ -CREATE TABLE objects ( - object_id bytea PRIMARY KEY, - object_version bigint NOT NULL, - object_digest bytea NOT NULL, - checkpoint_sequence_number bigint NOT NULL, - -- Immutable/Address/Object/Shared, see types.rs - owner_type smallint NOT NULL, - -- bytes of SuiAddress/ObjectID of the owner ID. - -- Non-null for objects with an owner: Addresso or Objects - owner_id bytea, - -- Object type - object_type text, - -- Components of the StructTag: package, module, name (name of the struct, without type parameters) - object_type_package bytea, - object_type_module text, - object_type_name text, - -- bcs serialized Object - serialized_object bytea NOT NULL, - -- Non-null when the object is a coin. - -- e.g. `0x2::sui::SUI` - coin_type text, - -- Non-null when the object is a coin. - coin_balance bigint, - -- DynamicField/DynamicObject, see types.rs - -- Non-null when the object is a dynamic field - df_kind smallint, - -- bcs serialized DynamicFieldName - -- Non-null when the object is a dynamic field - df_name bytea, - -- object_type in DynamicFieldInfo. - df_object_type text, - -- object_id in DynamicFieldInfo. - df_object_id bytea -); - --- OwnerType: 1: Address, 2: Object, see types.rs -CREATE INDEX objects_owner ON objects (owner_type, owner_id) WHERE owner_type BETWEEN 1 AND 2 AND owner_id IS NOT NULL; -CREATE INDEX objects_coin ON objects (owner_id, coin_type) WHERE coin_type IS NOT NULL AND owner_type = 1; -CREATE INDEX objects_checkpoint_sequence_number ON objects (checkpoint_sequence_number); -CREATE INDEX objects_package_module_name_full_type ON objects (object_type_package, object_type_module, object_type_name, object_type); -CREATE INDEX objects_owner_package_module_name_full_type ON objects (owner_id, object_type_package, object_type_module, object_type_name, object_type); - --- similar to objects table, except that --- 1. the primary key to store multiple object versions and partitions by checkpoint_sequence_number --- 2. allow null values in some columns for deleted / wrapped objects --- 3. object_status to mark the status of the object, which is either Active or WrappedOrDeleted -CREATE TABLE objects_history ( - object_id bytea NOT NULL, - object_version bigint NOT NULL, - object_status smallint NOT NULL, - object_digest bytea, - checkpoint_sequence_number bigint NOT NULL, - owner_type smallint, - owner_id bytea, - object_type text, - object_type_package bytea, - object_type_module text, - object_type_name text, - serialized_object bytea, - coin_type text, - coin_balance bigint, - df_kind smallint, - df_name bytea, - df_object_type text, - df_object_id bytea, - CONSTRAINT objects_history_pk PRIMARY KEY (checkpoint_sequence_number, object_id, object_version) -) PARTITION BY RANGE (checkpoint_sequence_number); -CREATE INDEX objects_history_id_version ON objects_history (object_id, object_version, checkpoint_sequence_number); --- init with first partition of the history table -CREATE TABLE objects_history_partition_0 PARTITION OF objects_history FOR VALUES FROM (0) TO (MAXVALUE); - --- snapshot table by folding objects_history table until certain checkpoint, --- effectively the snapshot of objects at the same checkpoint, --- except that it also includes deleted or wrapped objects with the corresponding object_status. -CREATE TABLE objects_snapshot ( - object_id bytea PRIMARY KEY, - object_version bigint NOT NULL, - object_status smallint NOT NULL, - object_digest bytea, - checkpoint_sequence_number bigint NOT NULL, - owner_type smallint, - owner_id bytea, - object_type text, - object_type_package bytea, - object_type_module text, - object_type_name text, - serialized_object bytea, - coin_type text, - coin_balance bigint, - df_kind smallint, - df_name bytea, - df_object_type text, - df_object_id bytea -); -CREATE INDEX objects_snapshot_checkpoint_sequence_number ON objects_snapshot (checkpoint_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044026_transactions/down.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044026_transactions/down.sql deleted file mode 100644 index 15e9dc9f1cb82..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044026_transactions/down.sql +++ /dev/null @@ -1,3 +0,0 @@ --- This file should undo anything in `up.sql` -DROP TABLE IF EXISTS transactions; -DROP TABLE IF EXISTS transactions_partition_0; diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044026_transactions/up.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044026_transactions/up.sql deleted file mode 100644 index f5404e3610751..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044026_transactions/up.sql +++ /dev/null @@ -1,23 +0,0 @@ -CREATE TABLE transactions ( - tx_sequence_number BIGINT NOT NULL, - transaction_digest bytea NOT NULL, - -- bcs serialized SenderSignedData bytes - raw_transaction bytea NOT NULL, - -- bcs serialized TransactionEffects bytes - raw_effects bytea NOT NULL, - checkpoint_sequence_number BIGINT NOT NULL, - timestamp_ms BIGINT NOT NULL, - -- array of bcs serialized IndexedObjectChange bytes - object_changes bytea[] NOT NULL, - -- array of bcs serialized BalanceChange bytes - balance_changes bytea[] NOT NULL, - -- array of bcs serialized StoredEvent bytes - events bytea[] NOT NULL, - -- SystemTransaction/ProgrammableTransaction. See types.rs - transaction_kind smallint NOT NULL, - -- number of successful commands in this transaction, bound by number of command - -- in a programmaable transaction. - success_command_count smallint NOT NULL, - PRIMARY KEY (tx_sequence_number) -) PARTITION BY RANGE (tx_sequence_number); -CREATE TABLE transactions_partition_0 PARTITION OF transactions FOR VALUES FROM (0) TO (MAXVALUE); diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044044_checkpoints/down.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044044_checkpoints/down.sql deleted file mode 100644 index fba5a8b5468c6..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044044_checkpoints/down.sql +++ /dev/null @@ -1,3 +0,0 @@ --- This file should undo anything in `up.sql` -DROP TABLE IF EXISTS checkpoints; -DROP TABLE IF EXISTS pruner_cp_watermark; diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044044_checkpoints/up.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044044_checkpoints/up.sql deleted file mode 100644 index ddb63b020de70..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044044_checkpoints/up.sql +++ /dev/null @@ -1,36 +0,0 @@ -CREATE TABLE checkpoints -( - sequence_number BIGINT PRIMARY KEY, - checkpoint_digest BYTEA NOT NULL, - epoch BIGINT NOT NULL, - -- total transactions in the network at the end of this checkpoint (including itself) - network_total_transactions BIGINT NOT NULL, - previous_checkpoint_digest BYTEA, - -- if this checkpoitn is the last checkpoint of an epoch - end_of_epoch boolean NOT NULL, - -- array of TranscationDigest in bytes included in this checkpoint - tx_digests BYTEA[] NOT NULL, - timestamp_ms BIGINT NOT NULL, - total_gas_cost BIGINT NOT NULL, - computation_cost BIGINT NOT NULL, - storage_cost BIGINT NOT NULL, - storage_rebate BIGINT NOT NULL, - non_refundable_storage_fee BIGINT NOT NULL, - -- bcs serialized Vec bytes - checkpoint_commitments BYTEA NOT NULL, - -- bcs serialized AggregateAuthoritySignature bytes - validator_signature BYTEA NOT NULL, - -- bcs serialzied EndOfEpochData bytes, if the checkpoint marks end of an epoch - end_of_epoch_data BYTEA, - min_tx_sequence_number BIGINT, - max_tx_sequence_number BIGINT -); - -CREATE INDEX checkpoints_epoch ON checkpoints (epoch, sequence_number); -CREATE INDEX checkpoints_digest ON checkpoints USING HASH (checkpoint_digest); - -CREATE TABLE pruner_cp_watermark ( - checkpoint_sequence_number BIGINT PRIMARY KEY, - min_tx_sequence_number BIGINT NOT NULL, - max_tx_sequence_number BIGINT NOT NULL -) diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044052_epochs/down.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044052_epochs/down.sql deleted file mode 100644 index ddb05ac2ebe8b..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044052_epochs/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -DROP TABLE IF EXISTS epochs; diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044052_epochs/up.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044052_epochs/up.sql deleted file mode 100644 index 5b540121cb849..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-044052_epochs/up.sql +++ /dev/null @@ -1,47 +0,0 @@ -CREATE TABLE epochs -( - epoch BIGINT PRIMARY KEY, - first_checkpoint_id BIGINT NOT NULL, - epoch_start_timestamp BIGINT NOT NULL, - reference_gas_price BIGINT NOT NULL, - protocol_version BIGINT NOT NULL, - total_stake BIGINT NOT NULL, - storage_fund_balance BIGINT NOT NULL, - system_state bytea NOT NULL, - -- The following fields are nullable because they are filled in - -- only at the end of an epoch. - epoch_total_transactions BIGINT, - last_checkpoint_id BIGINT, - epoch_end_timestamp BIGINT, - -- The following fields are from SystemEpochInfoEvent emitted - -- **after** advancing to the next epoch - storage_fund_reinvestment BIGINT, - storage_charge BIGINT, - storage_rebate BIGINT, - stake_subsidy_amount BIGINT, - total_gas_fees BIGINT, - total_stake_rewards_distributed BIGINT, - leftover_storage_fund_inflow BIGINT, - -- bcs serialized Vec bytes, found in last CheckpointSummary - -- of the epoch - epoch_commitments bytea -); - --- Table storing the protocol configs for each protocol version. --- Examples include gas schedule, transaction limits, etc. -CREATE TABLE protocol_configs -( - protocol_version BIGINT NOT NULL, - config_name TEXT NOT NULL, - config_value TEXT, - PRIMARY KEY(protocol_version, config_name) -); - --- Table storing the feature flags for each protocol version. -CREATE TABLE feature_flags -( - protocol_version BIGINT NOT NULL, - flag_name TEXT NOT NULL, - flag_value BOOLEAN NOT NULL, - PRIMARY KEY(protocol_version, flag_name) -); diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-060729_packages/down.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-060729_packages/down.sql deleted file mode 100644 index 6b473dc06f4a2..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-060729_packages/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -DROP TABLE IF EXISTS packages; diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-060729_packages/up.sql b/crates/sui-mvr-indexer/migrations/pg/2023-08-19-060729_packages/up.sql deleted file mode 100644 index f08a5549608eb..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-08-19-060729_packages/up.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE TABLE packages -( - package_id bytea NOT NULL, - original_id bytea NOT NULL, - package_version bigint NOT NULL, - -- bcs serialized MovePackage - move_package bytea NOT NULL, - checkpoint_sequence_number bigint NOT NULL, - CONSTRAINT packages_pkey PRIMARY KEY (package_id, original_id, package_version), - CONSTRAINT packages_unique_package_id UNIQUE (package_id) -); - -CREATE INDEX packages_cp_id_version ON packages (checkpoint_sequence_number, original_id, package_version); -CREATE INDEX packages_id_version_cp ON packages (original_id, package_version, checkpoint_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-10-06-204335_tx_indices/down.sql b/crates/sui-mvr-indexer/migrations/pg/2023-10-06-204335_tx_indices/down.sql deleted file mode 100644 index f5604c0db5357..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-10-06-204335_tx_indices/down.sql +++ /dev/null @@ -1,9 +0,0 @@ -DROP TABLE IF EXISTS tx_senders; -DROP TABLE IF EXISTS tx_recipients; -DROP TABLE IF EXISTS tx_input_objects; -DROP TABLE IF EXISTS tx_changed_objects; -DROP TABLE IF EXISTS tx_calls_pkg; -DROP TABLE IF EXISTS tx_calls_mod; -DROP TABLE IF EXISTS tx_calls_fun; -DROP TABLE IF EXISTS tx_digests; -DROP TABLE IF EXISTS tx_kinds; diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-10-06-204335_tx_indices/up.sql b/crates/sui-mvr-indexer/migrations/pg/2023-10-06-204335_tx_indices/up.sql deleted file mode 100644 index 563df854b97ef..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-10-06-204335_tx_indices/up.sql +++ /dev/null @@ -1,67 +0,0 @@ -CREATE TABLE tx_senders ( - tx_sequence_number BIGINT NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(sender, tx_sequence_number) -); - -CREATE TABLE tx_recipients ( - tx_sequence_number BIGINT NOT NULL, - recipient BYTEA NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(recipient, tx_sequence_number) -); -CREATE INDEX tx_recipients_sender ON tx_recipients (sender, recipient, tx_sequence_number); - -CREATE TABLE tx_input_objects ( - tx_sequence_number BIGINT NOT NULL, - object_id BYTEA NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(object_id, tx_sequence_number) -); -CREATE INDEX tx_input_objects_sender ON tx_input_objects (sender, object_id, tx_sequence_number); - -CREATE TABLE tx_changed_objects ( - tx_sequence_number BIGINT NOT NULL, - object_id BYTEA NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(object_id, tx_sequence_number) -); -CREATE INDEX tx_changed_objects_sender ON tx_changed_objects (sender, object_id, tx_sequence_number); - -CREATE TABLE tx_calls_pkg ( - tx_sequence_number BIGINT NOT NULL, - package BYTEA NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(package, tx_sequence_number) -); -CREATE INDEX tx_calls_pkg_sender ON tx_calls_pkg (sender, package, tx_sequence_number); - -CREATE TABLE tx_calls_mod ( - tx_sequence_number BIGINT NOT NULL, - package BYTEA NOT NULL, - module TEXT NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(package, module, tx_sequence_number) -); -CREATE INDEX tx_calls_mod_sender ON tx_calls_mod (sender, package, module, tx_sequence_number); - -CREATE TABLE tx_calls_fun ( - tx_sequence_number BIGINT NOT NULL, - package BYTEA NOT NULL, - module TEXT NOT NULL, - func TEXT NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(package, module, func, tx_sequence_number) -); -CREATE INDEX tx_calls_fun_sender ON tx_calls_fun (sender, package, module, func, tx_sequence_number); - -CREATE TABLE tx_digests ( - tx_digest BYTEA PRIMARY KEY, - tx_sequence_number BIGINT NOT NULL -); - -CREATE TABLE tx_kinds ( - tx_sequence_number BIGINT NOT NULL, - tx_kind SMALLINT NOT NULL, - PRIMARY KEY(tx_kind, tx_sequence_number) -); diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-10-07-160139_display/down.sql b/crates/sui-mvr-indexer/migrations/pg/2023-10-07-160139_display/down.sql deleted file mode 100644 index f73e497c406d3..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-10-07-160139_display/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -DROP TABLE IF EXISTS display; diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-10-07-160139_display/up.sql b/crates/sui-mvr-indexer/migrations/pg/2023-10-07-160139_display/up.sql deleted file mode 100644 index c82918e253c8c..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-10-07-160139_display/up.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE display -( - object_type text PRIMARY KEY, - id BYTEA NOT NULL, - version SMALLINT NOT NULL, - bcs BYTEA NOT NULL -); diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-11-29-193859_advance_partition/down.sql b/crates/sui-mvr-indexer/migrations/pg/2023-11-29-193859_advance_partition/down.sql deleted file mode 100644 index bab0311186e1d..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-11-29-193859_advance_partition/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP PROCEDURE IF EXISTS advance_partition; -DROP PROCEDURE IF EXISTS drop_partition; diff --git a/crates/sui-mvr-indexer/migrations/pg/2023-11-29-193859_advance_partition/up.sql b/crates/sui-mvr-indexer/migrations/pg/2023-11-29-193859_advance_partition/up.sql deleted file mode 100644 index 8ca64b86a7081..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2023-11-29-193859_advance_partition/up.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE OR REPLACE PROCEDURE advance_partition(table_name TEXT, last_epoch BIGINT, new_epoch BIGINT, last_epoch_start BIGINT, new_epoch_start BIGINT) -LANGUAGE plpgsql -AS $$ -BEGIN - EXECUTE format('ALTER TABLE %I DETACH PARTITION %I_partition_%s', table_name, table_name, last_epoch); - EXECUTE format('ALTER TABLE %I ATTACH PARTITION %I_partition_%s FOR VALUES FROM (%L) TO (%L)', table_name, table_name, last_epoch, last_epoch_start, new_epoch_start); - EXECUTE format('CREATE TABLE IF NOT EXISTS %I_partition_%s PARTITION OF %I FOR VALUES FROM (%L) TO (MAXVALUE)', table_name, new_epoch, table_name, new_epoch_start); -END; -$$; - -CREATE OR REPLACE PROCEDURE drop_partition(table_name TEXT, epoch BIGINT) -LANGUAGE plpgsql -AS $$ -BEGIN - EXECUTE format('DROP TABLE IF EXISTS %I_partition_%s', table_name, epoch); -END; -$$; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-05-05-155158_obj_indices/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-05-05-155158_obj_indices/down.sql deleted file mode 100644 index 7a3a7670f24c2..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-05-05-155158_obj_indices/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE IF EXISTS objects_version; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-05-05-155158_obj_indices/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-05-05-155158_obj_indices/up.sql deleted file mode 100644 index 666e5a2423319..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-05-05-155158_obj_indices/up.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Indexing table mapping an object's ID and version to its checkpoint --- sequence number, partitioned by the first byte of its Object ID. -CREATE TABLE objects_version ( - object_id bytea NOT NULL, - object_version bigint NOT NULL, - cp_sequence_number bigint NOT NULL, - PRIMARY KEY (object_id, object_version) -) PARTITION BY RANGE (object_id); - --- Create a partition for each first byte value. -DO $$ -DECLARE - lo text; - hi text; -BEGIN - FOR i IN 0..254 LOOP - lo := LPAD(TO_HEX(i), 2, '0'); - hi := LPAD(TO_HEX(i + 1), 2, '0'); - EXECUTE FORMAT($F$ - CREATE TABLE objects_version_%1$s PARTITION OF objects_version FOR VALUES - FROM (E'\\x%1$s00000000000000000000000000000000000000000000000000000000000000') - TO (E'\\x%2$s00000000000000000000000000000000000000000000000000000000000000'); - $F$, lo, hi); - END LOOP; -END; -$$ LANGUAGE plpgsql; - --- Special case for the last partition, because of the upper bound. -CREATE TABLE objects_version_ff PARTITION OF objects_version FOR VALUES -FROM (E'\\xff00000000000000000000000000000000000000000000000000000000000000') -TO (MAXVALUE); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-06-14-045801_event_indices/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-06-14-045801_event_indices/down.sql deleted file mode 100644 index 3583887435168..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-06-14-045801_event_indices/down.sql +++ /dev/null @@ -1,7 +0,0 @@ -DROP TABLE IF EXISTS event_emit_package; -DROP TABLE IF EXISTS event_emit_module; -DROP TABLE IF EXISTS event_struct_package; -DROP TABLE IF EXISTS event_struct_module; -DROP TABLE IF EXISTS event_struct_name; -DROP TABLE IF EXISTS event_struct_instantiation; -DROP TABLE IF EXISTS event_senders; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-06-14-045801_event_indices/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-06-14-045801_event_indices/up.sql deleted file mode 100644 index a89625146a9fd..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-06-14-045801_event_indices/up.sql +++ /dev/null @@ -1,74 +0,0 @@ -CREATE TABLE event_emit_package -( - package BYTEA NOT NULL, - tx_sequence_number BIGINT NOT NULL, - event_sequence_number BIGINT NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(package, tx_sequence_number, event_sequence_number) -); -CREATE INDEX event_emit_package_sender ON event_emit_package (sender, package, tx_sequence_number, event_sequence_number); - -CREATE TABLE event_emit_module -( - package BYTEA NOT NULL, - module TEXT NOT NULL, - tx_sequence_number BIGINT NOT NULL, - event_sequence_number BIGINT NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(package, module, tx_sequence_number, event_sequence_number) -); -CREATE INDEX event_emit_module_sender ON event_emit_module (sender, package, module, tx_sequence_number, event_sequence_number); - -CREATE TABLE event_struct_package -( - package BYTEA NOT NULL, - tx_sequence_number BIGINT NOT NULL, - event_sequence_number BIGINT NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(package, tx_sequence_number, event_sequence_number) -); -CREATE INDEX event_struct_package_sender ON event_struct_package (sender, package, tx_sequence_number, event_sequence_number); - - -CREATE TABLE event_struct_module -( - package BYTEA NOT NULL, - module TEXT NOT NULL, - tx_sequence_number BIGINT NOT NULL, - event_sequence_number BIGINT NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(package, module, tx_sequence_number, event_sequence_number) -); -CREATE INDEX event_struct_module_sender ON event_struct_module (sender, package, module, tx_sequence_number, event_sequence_number); - -CREATE TABLE event_struct_name -( - package BYTEA NOT NULL, - module TEXT NOT NULL, - type_name TEXT NOT NULL, - tx_sequence_number BIGINT NOT NULL, - event_sequence_number BIGINT NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(package, module, type_name, tx_sequence_number, event_sequence_number) -); -CREATE INDEX event_struct_name_sender ON event_struct_name (sender, package, module, type_name, tx_sequence_number, event_sequence_number); - -CREATE TABLE event_struct_instantiation -( - package BYTEA NOT NULL, - module TEXT NOT NULL, - type_instantiation TEXT NOT NULL, - tx_sequence_number BIGINT NOT NULL, - event_sequence_number BIGINT NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(package, module, type_instantiation, tx_sequence_number, event_sequence_number) -); -CREATE INDEX event_struct_instantiation_sender ON event_struct_instantiation (sender, package, module, type_instantiation, tx_sequence_number, event_sequence_number); - -CREATE TABLE event_senders -( - sender BYTEA NOT NULL, - tx_sequence_number BIGINT NOT NULL, - event_sequence_number BIGINT NOT NULL, - PRIMARY KEY(sender, tx_sequence_number, event_sequence_number) -); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-07-13-003534_chain_identifier/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-07-13-003534_chain_identifier/down.sql deleted file mode 100644 index 57f1de973b1d2..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-07-13-003534_chain_identifier/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -DROP TABLE IF EXISTS chain_identifier; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-07-13-003534_chain_identifier/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-07-13-003534_chain_identifier/up.sql deleted file mode 100644 index 205e3a89f63e5..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-07-13-003534_chain_identifier/up.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Your SQL goes here -CREATE TABLE chain_identifier -( - checkpoint_digest BYTEA NOT NULL, - PRIMARY KEY(checkpoint_digest) -); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-05-164455_full_objects_history/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-05-164455_full_objects_history/down.sql deleted file mode 100644 index 619fc41782e68..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-05-164455_full_objects_history/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE IF EXISTS full_objects_history; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-05-164455_full_objects_history/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-05-164455_full_objects_history/up.sql deleted file mode 100644 index 1504a21e51658..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-05-164455_full_objects_history/up.sql +++ /dev/null @@ -1,10 +0,0 @@ --- This table will store every history version of each object, and never get pruned. --- Since it can grow indefinitely, we keep minimum amount of information in this table for the purpose --- of point lookups. -CREATE TABLE full_objects_history -( - object_id bytea NOT NULL, - object_version bigint NOT NULL, - serialized_object bytea, - PRIMARY KEY (object_id, object_version) -); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-10-195655_drop-df-columns/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-10-195655_drop-df-columns/down.sql deleted file mode 100644 index 8490a091b30f4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-10-195655_drop-df-columns/down.sql +++ /dev/null @@ -1,15 +0,0 @@ -ALTER TABLE objects -ADD COLUMN df_name bytea, -ADD COLUMN df_object_type text, -ADD COLUMN df_object_id bytea, -ADD COLUMN checkpoint_sequence_number bigint; - -ALTER TABLE objects_snapshot -ADD COLUMN df_name bytea, -ADD COLUMN df_object_type text, -ADD COLUMN df_object_id bytea; - -ALTER TABLE objects_history -ADD COLUMN df_name bytea, -ADD COLUMN df_object_type text, -ADD COLUMN df_object_id bytea; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-10-195655_drop-df-columns/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-10-195655_drop-df-columns/up.sql deleted file mode 100644 index 4782193c4edc9..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-10-195655_drop-df-columns/up.sql +++ /dev/null @@ -1,15 +0,0 @@ -ALTER TABLE objects -DROP COLUMN df_name, -DROP COLUMN df_object_type, -DROP COLUMN df_object_id, -DROP COLUMN checkpoint_sequence_number; - -ALTER TABLE objects_snapshot -DROP COLUMN df_name, -DROP COLUMN df_object_type, -DROP COLUMN df_object_id; - -ALTER TABLE objects_history -DROP COLUMN df_name, -DROP COLUMN df_object_type, -DROP COLUMN df_object_id; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-12-150939_tx_affected/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-12-150939_tx_affected/down.sql deleted file mode 100644 index 98cc9c0a36ce9..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-12-150939_tx_affected/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE IF EXISTS tx_affected_addresses; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-12-150939_tx_affected/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-12-150939_tx_affected/up.sql deleted file mode 100644 index 4f71554f1394a..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-12-150939_tx_affected/up.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE tx_affected_addresses ( - tx_sequence_number BIGINT NOT NULL, - affected BYTEA NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(affected, tx_sequence_number) -); - -CREATE INDEX tx_affected_addresses_tx_sequence_number_index ON tx_affected_addresses (tx_sequence_number); -CREATE INDEX tx_affected_addresses_sender ON tx_affected_addresses (sender, affected, tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-12-213234_watermarks/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-12-213234_watermarks/down.sql deleted file mode 100644 index e9de336153f62..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-12-213234_watermarks/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE IF EXISTS watermarks; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-12-213234_watermarks/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-12-213234_watermarks/up.sql deleted file mode 100644 index 73bdc70055246..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-12-213234_watermarks/up.sql +++ /dev/null @@ -1,34 +0,0 @@ -CREATE TABLE IF NOT EXISTS watermarks -( - -- The pipeline governed by this watermark, i.e `epochs`, `checkpoints`, - -- `transactions`. - pipeline TEXT PRIMARY KEY, - -- Inclusive upper epoch bound for this entity's data. Committer updates - -- this field. Pruner uses this to determine if pruning is necessary based - -- on the retention policy. - epoch_hi_inclusive BIGINT NOT NULL, - -- Inclusive upper checkpoint bound for this entity's data. Committer - -- updates this field. All data of this entity in the checkpoint must be - -- persisted before advancing this watermark. The committer refers to this - -- on disaster recovery to resume writing. - checkpoint_hi_inclusive BIGINT NOT NULL, - -- Exclusive upper transaction sequence number bound for this entity's - -- data. Committer updates this field. - tx_hi BIGINT NOT NULL, - -- Inclusive lower epoch bound for this entity's data. Pruner updates this - -- field when the epoch range exceeds the retention policy. - epoch_lo BIGINT NOT NULL, - -- Inclusive low watermark that the pruner advances. Corresponds to the - -- epoch id, checkpoint sequence number, or tx sequence number depending on - -- the entity. Data before this watermark is considered pruned by a reader. - -- The underlying data may still exist in the db instance. - reader_lo BIGINT NOT NULL, - -- Updated using the database's current timestamp when the pruner sees that - -- some data needs to be dropped. The pruner uses this column to determine - -- whether to prune or wait long enough that all in-flight reads complete - -- or timeout before it acts on an updated watermark. - timestamp_ms BIGINT NOT NULL, - -- Column used by the pruner to track its true progress. Data below this - -- watermark can be immediately pruned. - pruner_hi BIGINT NOT NULL -); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-18-003318_epochs_add_json_system_state/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-18-003318_epochs_add_json_system_state/down.sql deleted file mode 100644 index 9de241cfe20dc..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-18-003318_epochs_add_json_system_state/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE epochs DROP COLUMN system_state_summary_json; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-18-003318_epochs_add_json_system_state/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-18-003318_epochs_add_json_system_state/up.sql deleted file mode 100644 index 4dce2a5a9ecfd..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-18-003318_epochs_add_json_system_state/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE epochs ADD COLUMN system_state_summary_json JSONB; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-19-011238_raw_checkpoints/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-19-011238_raw_checkpoints/down.sql deleted file mode 100644 index 9cfef48c9b5f6..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-19-011238_raw_checkpoints/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE raw_checkpoints; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-19-011238_raw_checkpoints/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-19-011238_raw_checkpoints/up.sql deleted file mode 100644 index 26791856ff4c9..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-19-011238_raw_checkpoints/up.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE raw_checkpoints -( - sequence_number BIGINT PRIMARY KEY, - certified_checkpoint BYTEA NOT NULL, - checkpoint_contents BYTEA NOT NULL -); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-19-121113_tx_affected_objects/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-19-121113_tx_affected_objects/down.sql deleted file mode 100644 index b0868da73b0f2..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-19-121113_tx_affected_objects/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE IF EXISTS tx_affected_objects; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-19-121113_tx_affected_objects/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-19-121113_tx_affected_objects/up.sql deleted file mode 100644 index 146f78b2f5063..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-19-121113_tx_affected_objects/up.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE tx_affected_objects ( - tx_sequence_number BIGINT NOT NULL, - affected BYTEA NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(affected, tx_sequence_number) -); - -CREATE INDEX tx_affected_objects_tx_sequence_number_index ON tx_affected_objects (tx_sequence_number); -CREATE INDEX tx_affected_objects_sender ON tx_affected_objects (sender, affected, tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-24-213054_epochs_system_state_nullable/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-24-213054_epochs_system_state_nullable/down.sql deleted file mode 100644 index e6697b4849a4e..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-24-213054_epochs_system_state_nullable/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE epochs ALTER COLUMN system_state SET NOT NULL; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-24-213054_epochs_system_state_nullable/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-24-213054_epochs_system_state_nullable/up.sql deleted file mode 100644 index a6e7f167c48cc..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-24-213054_epochs_system_state_nullable/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE epochs ALTER COLUMN system_state DROP NOT NULL; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135801_event_emit_module_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135801_event_emit_module_pruning_index/down.sql deleted file mode 100644 index 825855f3d700b..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135801_event_emit_module_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS event_emit_module_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135801_event_emit_module_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135801_event_emit_module_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135801_event_emit_module_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135801_event_emit_module_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135801_event_emit_module_pruning_index/up.sql deleted file mode 100644 index ac69bc8488758..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135801_event_emit_module_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - event_emit_module_tx_sequence_number -ON event_emit_module (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135802_event_emit_package_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135802_event_emit_package_pruning_index/down.sql deleted file mode 100644 index 30b5fdb6cead6..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135802_event_emit_package_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS event_emit_package_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135802_event_emit_package_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135802_event_emit_package_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135802_event_emit_package_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135802_event_emit_package_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135802_event_emit_package_pruning_index/up.sql deleted file mode 100644 index 231ab598b4766..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135802_event_emit_package_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - event_emit_package_tx_sequence_number -ON event_emit_package (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135803_event_senders_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135803_event_senders_pruning_index/down.sql deleted file mode 100644 index e9b5b0b903ed5..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135803_event_senders_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS event_senders_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135803_event_senders_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135803_event_senders_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135803_event_senders_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135803_event_senders_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135803_event_senders_pruning_index/up.sql deleted file mode 100644 index b5883b8a3a4ce..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135803_event_senders_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - event_senders_tx_sequence_number -ON event_senders (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135804_event_struct_instantiation_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135804_event_struct_instantiation_pruning_index/down.sql deleted file mode 100644 index 43b1d27d9ed2e..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135804_event_struct_instantiation_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS event_struct_instantiation_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135804_event_struct_instantiation_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135804_event_struct_instantiation_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135804_event_struct_instantiation_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135804_event_struct_instantiation_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135804_event_struct_instantiation_pruning_index/up.sql deleted file mode 100644 index 7847620e936f3..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135804_event_struct_instantiation_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - event_struct_instantiation_tx_sequence_number -ON event_struct_instantiation (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135805_event_struct_module_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135805_event_struct_module_pruning_index/down.sql deleted file mode 100644 index 76606ab0400a6..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135805_event_struct_module_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS event_struct_module_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135805_event_struct_module_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135805_event_struct_module_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135805_event_struct_module_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135805_event_struct_module_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135805_event_struct_module_pruning_index/up.sql deleted file mode 100644 index 748a4095da169..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135805_event_struct_module_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - event_struct_module_tx_sequence_number -ON event_struct_module (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135806_event_struct_name_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135806_event_struct_name_pruning_index/down.sql deleted file mode 100644 index 944405cf172e3..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135806_event_struct_name_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS event_struct_name_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135806_event_struct_name_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135806_event_struct_name_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135806_event_struct_name_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135806_event_struct_name_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135806_event_struct_name_pruning_index/up.sql deleted file mode 100644 index 2ca251c139af9..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135806_event_struct_name_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - event_struct_name_tx_sequence_number -ON event_struct_name (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135807_event_struct_package_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135807_event_struct_package_pruning_index/down.sql deleted file mode 100644 index 40fde7e4578b6..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135807_event_struct_package_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS event_struct_package_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135807_event_struct_package_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135807_event_struct_package_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135807_event_struct_package_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135807_event_struct_package_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135807_event_struct_package_pruning_index/up.sql deleted file mode 100644 index 00e88fcfb5273..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135807_event_struct_package_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - event_struct_package_tx_sequence_number -ON event_struct_package (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135808_tx_calls_fun_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135808_tx_calls_fun_pruning_index/down.sql deleted file mode 100644 index da1519d208f7a..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135808_tx_calls_fun_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS tx_calls_fun_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135808_tx_calls_fun_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135808_tx_calls_fun_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135808_tx_calls_fun_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135808_tx_calls_fun_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135808_tx_calls_fun_pruning_index/up.sql deleted file mode 100644 index c868f6e55a66f..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135808_tx_calls_fun_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - tx_calls_fun_tx_sequence_number -ON tx_calls_fun (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135809_tx_calls_mod_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135809_tx_calls_mod_pruning_index/down.sql deleted file mode 100644 index 16bf8eb87dbef..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135809_tx_calls_mod_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS tx_calls_mod_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135809_tx_calls_mod_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135809_tx_calls_mod_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135809_tx_calls_mod_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135809_tx_calls_mod_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135809_tx_calls_mod_pruning_index/up.sql deleted file mode 100644 index debc152d98f2d..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135809_tx_calls_mod_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - tx_calls_mod_tx_sequence_number -ON tx_calls_mod (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135810_tx_calls_pkg_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135810_tx_calls_pkg_pruning_index/down.sql deleted file mode 100644 index f6ef795109c61..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135810_tx_calls_pkg_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS tx_calls_pkg_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135810_tx_calls_pkg_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135810_tx_calls_pkg_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135810_tx_calls_pkg_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135810_tx_calls_pkg_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135810_tx_calls_pkg_pruning_index/up.sql deleted file mode 100644 index 0e6c1f1bf7d30..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135810_tx_calls_pkg_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - tx_calls_pkg_tx_sequence_number -ON tx_calls_pkg (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135811_tx_changed_objects_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135811_tx_changed_objects_pruning_index/down.sql deleted file mode 100644 index 1dfcf480b9e86..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135811_tx_changed_objects_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS tx_changed_objects_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135811_tx_changed_objects_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135811_tx_changed_objects_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135811_tx_changed_objects_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135811_tx_changed_objects_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135811_tx_changed_objects_pruning_index/up.sql deleted file mode 100644 index 4ef5b459dbf05..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135811_tx_changed_objects_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - tx_changed_objects_tx_sequence_number -ON tx_changed_objects (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135812_tx_digests_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135812_tx_digests_pruning_index/down.sql deleted file mode 100644 index d0bd714bc60b2..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135812_tx_digests_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS tx_digests_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135812_tx_digests_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135812_tx_digests_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135812_tx_digests_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135812_tx_digests_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135812_tx_digests_pruning_index/up.sql deleted file mode 100644 index efdff9cbe7a56..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135812_tx_digests_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - tx_digests_tx_sequence_number -ON tx_digests (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135813_tx_input_objects_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135813_tx_input_objects_pruning_index/down.sql deleted file mode 100644 index 5061884270f6b..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135813_tx_input_objects_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS tx_input_objects_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135813_tx_input_objects_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135813_tx_input_objects_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135813_tx_input_objects_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135813_tx_input_objects_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135813_tx_input_objects_pruning_index/up.sql deleted file mode 100644 index 39d01c598a4a5..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135813_tx_input_objects_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - tx_input_objects_tx_sequence_number -ON tx_input_objects (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135814_tx_kinds_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135814_tx_kinds_pruning_index/down.sql deleted file mode 100644 index 10f5e96b1ce17..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135814_tx_kinds_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS tx_kinds_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135814_tx_kinds_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135814_tx_kinds_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135814_tx_kinds_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135814_tx_kinds_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135814_tx_kinds_pruning_index/up.sql deleted file mode 100644 index 6227a18f8eb46..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135814_tx_kinds_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - tx_kinds_tx_sequence_number -ON tx_kinds (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135815_tx_recipients_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135815_tx_recipients_pruning_index/down.sql deleted file mode 100644 index 138ba02741229..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135815_tx_recipients_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS tx_recipients_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135815_tx_recipients_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135815_tx_recipients_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135815_tx_recipients_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135815_tx_recipients_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135815_tx_recipients_pruning_index/up.sql deleted file mode 100644 index d2294ac2561ed..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135815_tx_recipients_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - tx_recipients_tx_sequence_number -ON tx_recipients (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135816_tx_senders_pruning_index/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135816_tx_senders_pruning_index/down.sql deleted file mode 100644 index c09d44eb4660d..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135816_tx_senders_pruning_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX CONCURRENTLY IF EXISTS tx_senders_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135816_tx_senders_pruning_index/metadata.toml b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135816_tx_senders_pruning_index/metadata.toml deleted file mode 100644 index 79e9221c1f2a4..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135816_tx_senders_pruning_index/metadata.toml +++ /dev/null @@ -1 +0,0 @@ -run_in_transaction = false diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135816_tx_senders_pruning_index/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135816_tx_senders_pruning_index/up.sql deleted file mode 100644 index f22a06e2bb548..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-25-135816_tx_senders_pruning_index/up.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX CONCURRENTLY IF NOT EXISTS - tx_senders_tx_sequence_number -ON tx_senders (tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-30-153705_add_event_sender/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-30-153705_add_event_sender/down.sql deleted file mode 100644 index 89e6710d92f1b..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-30-153705_add_event_sender/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE events DROP COLUMN sender; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-09-30-153705_add_event_sender/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-09-30-153705_add_event_sender/up.sql deleted file mode 100644 index 7ea312c09453c..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-09-30-153705_add_event_sender/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE events ADD COLUMN sender BYTEA; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-10-08-025030_partial_index_instead/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-10-08-025030_partial_index_instead/down.sql deleted file mode 100644 index 82659b80658c0..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-10-08-025030_partial_index_instead/down.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Drop the new partial indices -DROP INDEX IF EXISTS objects_history_owner_partial; -DROP INDEX IF EXISTS objects_history_coin_owner_partial; -DROP INDEX IF EXISTS objects_history_coin_only_partial; -DROP INDEX IF EXISTS objects_history_type_partial; -DROP INDEX IF EXISTS objects_history_package_module_name_full_type_partial; -DROP INDEX IF EXISTS objects_history_owner_package_module_name_full_type_partial; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-10-08-025030_partial_index_instead/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-10-08-025030_partial_index_instead/up.sql deleted file mode 100644 index 800f77b3f540b..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-10-08-025030_partial_index_instead/up.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Create new partial indices with object_status = 0 condition -CREATE INDEX IF NOT EXISTS objects_history_owner_partial ON objects_history (checkpoint_sequence_number, owner_type, owner_id) -WHERE owner_type BETWEEN 1 AND 2 AND owner_id IS NOT NULL AND object_status = 0; - -CREATE INDEX IF NOT EXISTS objects_history_coin_owner_partial ON objects_history (checkpoint_sequence_number, owner_id, coin_type, object_id) -WHERE coin_type IS NOT NULL AND owner_type = 1 AND object_status = 0; - -CREATE INDEX IF NOT EXISTS objects_history_coin_only_partial ON objects_history (checkpoint_sequence_number, coin_type, object_id) -WHERE coin_type IS NOT NULL AND object_status = 0; - -CREATE INDEX IF NOT EXISTS objects_history_type_partial ON objects_history (checkpoint_sequence_number, object_type) -WHERE object_status = 0; - -CREATE INDEX IF NOT EXISTS objects_history_package_module_name_full_type_partial ON objects_history (checkpoint_sequence_number, object_type_package, object_type_module, object_type_name, object_type) -WHERE object_status = 0; - -CREATE INDEX IF NOT EXISTS objects_history_owner_package_module_name_full_type_partial ON objects_history (checkpoint_sequence_number, owner_id, object_type_package, object_type_module, object_type_name, object_type) -WHERE object_status = 0; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-10-09-180628_add_network_total_transactions_to_epochs/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-10-09-180628_add_network_total_transactions_to_epochs/down.sql deleted file mode 100644 index e088120452e58..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-10-09-180628_add_network_total_transactions_to_epochs/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE epochs DROP COLUMN first_tx_sequence_number; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-10-09-180628_add_network_total_transactions_to_epochs/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-10-09-180628_add_network_total_transactions_to_epochs/up.sql deleted file mode 100644 index becdb61fe5e83..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-10-09-180628_add_network_total_transactions_to_epochs/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE epochs ADD COLUMN first_tx_sequence_number bigint; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-10-25-153629_remove_objects_history_full_indices/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-10-25-153629_remove_objects_history_full_indices/down.sql deleted file mode 100644 index 807c01dca462d..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-10-25-153629_remove_objects_history_full_indices/down.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE INDEX IF NOT EXISTS objects_history_owner ON objects_history (checkpoint_sequence_number, owner_type, owner_id) WHERE owner_type BETWEEN 1 AND 2 AND owner_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS objects_history_coin_owner ON objects_history (checkpoint_sequence_number, owner_id, coin_type, object_id) WHERE coin_type IS NOT NULL AND owner_type = 1; -CREATE INDEX IF NOT EXISTS objects_history_coin_only ON objects_history (checkpoint_sequence_number, coin_type, object_id) WHERE coin_type IS NOT NULL; -CREATE INDEX IF NOT EXISTS objects_history_type ON objects_history (checkpoint_sequence_number, object_type); -CREATE INDEX IF NOT EXISTS objects_history_package_module_name_full_type ON objects_history (checkpoint_sequence_number, object_type_package, object_type_module, object_type_name, object_type); -CREATE INDEX IF NOT EXISTS objects_history_owner_package_module_name_full_type ON objects_history (checkpoint_sequence_number, owner_id, object_type_package, object_type_module, object_type_name, object_type); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-10-25-153629_remove_objects_history_full_indices/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-10-25-153629_remove_objects_history_full_indices/up.sql deleted file mode 100644 index 754e719819f1e..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-10-25-153629_remove_objects_history_full_indices/up.sql +++ /dev/null @@ -1,6 +0,0 @@ -DROP INDEX IF EXISTS objects_history_owner; -DROP INDEX IF EXISTS objects_history_coin_owner; -DROP INDEX IF EXISTS objects_history_coin_only; -DROP INDEX IF EXISTS objects_history_type; -DROP INDEX IF EXISTS objects_history_package_module_name_full_type; -DROP INDEX IF EXISTS objects_history_owner_package_module_name_full_type; diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-10-30-005153_drop_tx_sender_and_recipient/down.sql b/crates/sui-mvr-indexer/migrations/pg/2024-10-30-005153_drop_tx_sender_and_recipient/down.sql deleted file mode 100644 index b9fcef3e1f439..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-10-30-005153_drop_tx_sender_and_recipient/down.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE TABLE tx_senders ( - tx_sequence_number BIGINT NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(sender, tx_sequence_number) -); - -CREATE INDEX IF NOT EXISTS tx_senders_tx_sequence_number - ON tx_senders (tx_sequence_number); - -CREATE TABLE tx_recipients ( - tx_sequence_number BIGINT NOT NULL, - recipient BYTEA NOT NULL, - sender BYTEA NOT NULL, - PRIMARY KEY(recipient, tx_sequence_number) -); - -CREATE INDEX IF NOT EXISTS tx_recipients_sender - ON tx_recipients (sender, recipient, tx_sequence_number); diff --git a/crates/sui-mvr-indexer/migrations/pg/2024-10-30-005153_drop_tx_sender_and_recipient/up.sql b/crates/sui-mvr-indexer/migrations/pg/2024-10-30-005153_drop_tx_sender_and_recipient/up.sql deleted file mode 100644 index fb259ea615d84..0000000000000 --- a/crates/sui-mvr-indexer/migrations/pg/2024-10-30-005153_drop_tx_sender_and_recipient/up.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE IF EXISTS tx_senders; -DROP TABLE IF EXISTS tx_recipients; diff --git a/crates/sui-mvr-indexer/src/apis/coin_api.rs b/crates/sui-mvr-indexer/src/apis/coin_api.rs deleted file mode 100644 index 13a5a2c55a819..0000000000000 --- a/crates/sui-mvr-indexer/src/apis/coin_api.rs +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::indexer_reader::IndexerReader; -use async_trait::async_trait; -use jsonrpsee::core::RpcResult; -use jsonrpsee::RpcModule; -use sui_json_rpc::coin_api::{parse_to_struct_tag, parse_to_type_tag}; -use sui_json_rpc::SuiRpcModule; -use sui_json_rpc_api::{cap_page_limit, CoinReadApiServer}; -use sui_json_rpc_types::{Balance, CoinPage, Page, SuiCoinMetadata}; -use sui_open_rpc::Module; -use sui_types::balance::Supply; -use sui_types::base_types::{ObjectID, SuiAddress}; -use sui_types::gas_coin::{GAS, TOTAL_SUPPLY_MIST}; - -pub(crate) struct CoinReadApi { - inner: IndexerReader, -} - -impl CoinReadApi { - pub fn new(inner: IndexerReader) -> Self { - Self { inner } - } -} - -#[async_trait] -impl CoinReadApiServer for CoinReadApi { - async fn get_coins( - &self, - owner: SuiAddress, - coin_type: Option, - cursor: Option, - limit: Option, - ) -> RpcResult { - let limit = cap_page_limit(limit); - if limit == 0 { - return Ok(CoinPage::empty()); - } - - // Normalize coin type tag and default to Gas - let coin_type = - parse_to_type_tag(coin_type)?.to_canonical_string(/* with_prefix */ true); - - let cursor = match cursor { - Some(c) => c, - // If cursor is not specified, we need to start from the beginning of the coin type, which is the minimal possible ObjectID. - None => ObjectID::ZERO, - }; - let mut results = self - .inner - .get_owned_coins(owner, Some(coin_type), cursor, limit + 1) - .await?; - - let has_next_page = results.len() > limit; - results.truncate(limit); - let next_cursor = results.last().map(|o| o.coin_object_id); - Ok(Page { - data: results, - next_cursor, - has_next_page, - }) - } - - async fn get_all_coins( - &self, - owner: SuiAddress, - cursor: Option, - limit: Option, - ) -> RpcResult { - let limit = cap_page_limit(limit); - if limit == 0 { - return Ok(CoinPage::empty()); - } - - let cursor = match cursor { - Some(c) => c, - // If cursor is not specified, we need to start from the beginning of the coin type, which is the minimal possible ObjectID. - None => ObjectID::ZERO, - }; - let mut results = self - .inner - .get_owned_coins(owner, None, cursor, limit + 1) - .await?; - - let has_next_page = results.len() > limit; - results.truncate(limit); - let next_cursor = results.last().map(|o| o.coin_object_id); - Ok(Page { - data: results, - next_cursor, - has_next_page, - }) - } - - async fn get_balance( - &self, - owner: SuiAddress, - coin_type: Option, - ) -> RpcResult { - // Normalize coin type tag and default to Gas - let coin_type = - parse_to_type_tag(coin_type)?.to_canonical_string(/* with_prefix */ true); - - let mut results = self - .inner - .get_coin_balances(owner, Some(coin_type.clone())) - .await?; - if results.is_empty() { - return Ok(Balance::zero(coin_type)); - } - Ok(results.swap_remove(0)) - } - - async fn get_all_balances(&self, owner: SuiAddress) -> RpcResult> { - self.inner - .get_coin_balances(owner, None) - .await - .map_err(Into::into) - } - - async fn get_coin_metadata(&self, coin_type: String) -> RpcResult> { - let coin_struct = parse_to_struct_tag(&coin_type)?; - self.inner - .get_coin_metadata(coin_struct) - .await - .map_err(Into::into) - } - - async fn get_total_supply(&self, coin_type: String) -> RpcResult { - let coin_struct = parse_to_struct_tag(&coin_type)?; - if GAS::is_gas(&coin_struct) { - Ok(Supply { - value: TOTAL_SUPPLY_MIST, - }) - } else { - self.inner - .get_total_supply(coin_struct) - .await - .map_err(Into::into) - } - } -} - -impl SuiRpcModule for CoinReadApi { - fn rpc(self) -> RpcModule { - self.into_rpc() - } - - fn rpc_doc_module() -> Module { - sui_json_rpc_api::CoinReadApiOpenRpc::module_doc() - } -} diff --git a/crates/sui-mvr-indexer/src/apis/extended_api.rs b/crates/sui-mvr-indexer/src/apis/extended_api.rs deleted file mode 100644 index 9b9827ea2bbe1..0000000000000 --- a/crates/sui-mvr-indexer/src/apis/extended_api.rs +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::indexer_reader::IndexerReader; -use jsonrpsee::{core::RpcResult, RpcModule}; -use sui_json_rpc::SuiRpcModule; -use sui_json_rpc_api::{validate_limit, ExtendedApiServer, QUERY_MAX_RESULT_LIMIT_CHECKPOINTS}; -use sui_json_rpc_types::{ - CheckpointedObjectID, EpochInfo, EpochPage, Page, QueryObjectsPage, SuiObjectResponseQuery, -}; -use sui_open_rpc::Module; -use sui_types::sui_serde::BigInt; - -pub(crate) struct ExtendedApi { - inner: IndexerReader, -} - -impl ExtendedApi { - pub fn new(inner: IndexerReader) -> Self { - Self { inner } - } -} - -#[async_trait::async_trait] -impl ExtendedApiServer for ExtendedApi { - async fn get_epochs( - &self, - cursor: Option>, - limit: Option, - descending_order: Option, - ) -> RpcResult { - let limit = validate_limit(limit, QUERY_MAX_RESULT_LIMIT_CHECKPOINTS)?; - let mut epochs = self - .inner - .get_epochs( - cursor.map(|x| *x), - limit + 1, - descending_order.unwrap_or(false), - ) - .await?; - - let has_next_page = epochs.len() > limit; - epochs.truncate(limit); - let next_cursor = epochs.last().map(|e| e.epoch); - Ok(Page { - data: epochs, - next_cursor: next_cursor.map(|id| id.into()), - has_next_page, - }) - } - - async fn get_current_epoch(&self) -> RpcResult { - let stored_epoch = self.inner.get_latest_epoch_info_from_db().await?; - EpochInfo::try_from(stored_epoch).map_err(Into::into) - } - - async fn query_objects( - &self, - _query: SuiObjectResponseQuery, - _cursor: Option, - _limit: Option, - ) -> RpcResult { - Err(jsonrpsee::types::error::CallError::Custom( - jsonrpsee::types::error::ErrorCode::MethodNotFound.into(), - ) - .into()) - } - - async fn get_total_transactions(&self) -> RpcResult> { - let latest_checkpoint = self.inner.get_latest_checkpoint().await?; - Ok(latest_checkpoint.network_total_transactions.into()) - } -} - -impl SuiRpcModule for ExtendedApi { - fn rpc(self) -> RpcModule { - self.into_rpc() - } - - fn rpc_doc_module() -> Module { - sui_json_rpc_api::ExtendedApiOpenRpc::module_doc() - } -} diff --git a/crates/sui-mvr-indexer/src/apis/governance_api.rs b/crates/sui-mvr-indexer/src/apis/governance_api.rs deleted file mode 100644 index 0cb52dc8e3a11..0000000000000 --- a/crates/sui-mvr-indexer/src/apis/governance_api.rs +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::BTreeMap; - -use crate::{errors::IndexerError, indexer_reader::IndexerReader}; -use async_trait::async_trait; -use jsonrpsee::{core::RpcResult, RpcModule}; - -use cached::{proc_macro::cached, SizedCache}; -use sui_json_rpc::{governance_api::ValidatorExchangeRates, SuiRpcModule}; -use sui_json_rpc_api::GovernanceReadApiServer; -use sui_json_rpc_types::{ - DelegatedStake, EpochInfo, StakeStatus, SuiCommittee, SuiObjectDataFilter, ValidatorApys, -}; -use sui_open_rpc::Module; -use sui_types::{ - base_types::{MoveObjectType, ObjectID, SuiAddress}, - committee::EpochId, - governance::StakedSui, - sui_serde::BigInt, - sui_system_state::{sui_system_state_summary::SuiSystemStateSummary, PoolTokenExchangeRate}, -}; - -#[derive(Clone)] -pub struct GovernanceReadApi { - inner: IndexerReader, -} - -impl GovernanceReadApi { - pub fn new(inner: IndexerReader) -> Self { - Self { inner } - } - - pub async fn get_epoch_info(&self, epoch: Option) -> Result { - match self.inner.get_epoch_info(epoch).await { - Ok(Some(epoch_info)) => Ok(epoch_info), - Ok(None) => Err(IndexerError::InvalidArgumentError(format!( - "Missing epoch {epoch:?}" - ))), - Err(e) => Err(e), - } - } - - async fn get_latest_sui_system_state(&self) -> Result { - self.inner.get_latest_sui_system_state().await - } - - async fn get_stakes_by_ids( - &self, - ids: Vec, - ) -> Result, IndexerError> { - let mut stakes = vec![]; - for stored_object in self.inner.multi_get_objects(ids).await? { - let object = sui_types::object::Object::try_from(stored_object)?; - let stake_object = StakedSui::try_from(&object)?; - stakes.push(stake_object); - } - - self.get_delegated_stakes(stakes).await - } - - async fn get_staked_by_owner( - &self, - owner: SuiAddress, - ) -> Result, IndexerError> { - let mut stakes = vec![]; - for stored_object in self - .inner - .get_owned_objects( - owner, - Some(SuiObjectDataFilter::StructType( - MoveObjectType::staked_sui().into(), - )), - None, - // Allow querying for up to 1000 staked objects - 1000, - ) - .await? - { - let object = sui_types::object::Object::try_from(stored_object)?; - let stake_object = StakedSui::try_from(&object)?; - stakes.push(stake_object); - } - - self.get_delegated_stakes(stakes).await - } - - pub async fn get_delegated_stakes( - &self, - stakes: Vec, - ) -> Result, IndexerError> { - let pools = stakes - .into_iter() - .fold(BTreeMap::<_, Vec<_>>::new(), |mut pools, stake| { - pools.entry(stake.pool_id()).or_default().push(stake); - pools - }); - - let system_state_summary = self.get_latest_sui_system_state().await?; - let epoch = system_state_summary.epoch; - - let rates = exchange_rates(self, &system_state_summary) - .await? - .into_iter() - .map(|rates| (rates.pool_id, rates)) - .collect::>(); - - let mut delegated_stakes = vec![]; - for (pool_id, stakes) in pools { - // Rate table and rate can be null when the pool is not active - let rate_table = rates.get(&pool_id).ok_or_else(|| { - IndexerError::InvalidArgumentError( - "Cannot find rates for staking pool {pool_id}".to_string(), - ) - })?; - let current_rate = rate_table.rates.first().map(|(_, rate)| rate); - - let mut delegations = vec![]; - for stake in stakes { - let status = if epoch >= stake.activation_epoch() { - let estimated_reward = if let Some(current_rate) = current_rate { - let stake_rate = rate_table - .rates - .iter() - .find_map(|(epoch, rate)| { - if *epoch == stake.activation_epoch() { - Some(rate.clone()) - } else { - None - } - }) - .unwrap_or_default(); - let estimated_reward = ((stake_rate.rate() / current_rate.rate()) - 1.0) - * stake.principal() as f64; - std::cmp::max(0, estimated_reward.round() as u64) - } else { - 0 - }; - StakeStatus::Active { estimated_reward } - } else { - StakeStatus::Pending - }; - delegations.push(sui_json_rpc_types::Stake { - staked_sui_id: stake.id(), - // TODO: this might change when we implement warm up period. - stake_request_epoch: stake.activation_epoch().saturating_sub(1), - stake_active_epoch: stake.activation_epoch(), - principal: stake.principal(), - status, - }) - } - delegated_stakes.push(DelegatedStake { - validator_address: rate_table.address, - staking_pool: pool_id, - stakes: delegations, - }) - } - Ok(delegated_stakes) - } -} - -/// Cached exchange rates for validators for the given epoch, the cache size is 1, it will be cleared when the epoch changes. -/// rates are in descending order by epoch. -#[cached( - type = "SizedCache>", - create = "{ SizedCache::with_size(1) }", - convert = " { system_state_summary.epoch } ", - result = true -)] -pub async fn exchange_rates( - state: &GovernanceReadApi, - system_state_summary: &SuiSystemStateSummary, -) -> Result, IndexerError> { - // Get validator rate tables - let mut tables = vec![]; - - for validator in &system_state_summary.active_validators { - tables.push(( - validator.sui_address, - validator.staking_pool_id, - validator.exchange_rates_id, - validator.exchange_rates_size, - true, - )); - } - - // Get inactive validator rate tables - for df in state - .inner - .get_dynamic_fields( - system_state_summary.inactive_pools_id, - None, - system_state_summary.inactive_pools_size as usize, - ) - .await? - { - let pool_id: sui_types::id::ID = bcs::from_bytes(&df.bcs_name).map_err(|e| { - sui_types::error::SuiError::ObjectDeserializationError { - error: e.to_string(), - } - })?; - let inactive_pools_id = system_state_summary.inactive_pools_id; - let validator = state - .inner - .get_validator_from_table(inactive_pools_id, pool_id) - .await?; - tables.push(( - validator.sui_address, - validator.staking_pool_id, - validator.exchange_rates_id, - validator.exchange_rates_size, - false, - )); - } - - let mut exchange_rates = vec![]; - // Get exchange rates for each validator - for (address, pool_id, exchange_rates_id, exchange_rates_size, active) in tables { - let mut rates = vec![]; - for df in state - .inner - .get_dynamic_fields_raw(exchange_rates_id, None, exchange_rates_size as usize) - .await? - { - let dynamic_field = df - .to_dynamic_field::() - .ok_or_else(|| sui_types::error::SuiError::ObjectDeserializationError { - error: "dynamic field malformed".to_owned(), - })?; - - rates.push((dynamic_field.name, dynamic_field.value)); - } - - rates.sort_by(|(a, _), (b, _)| a.cmp(b).reverse()); - - exchange_rates.push(ValidatorExchangeRates { - address, - pool_id, - active, - rates, - }); - } - Ok(exchange_rates) -} - -#[async_trait] -impl GovernanceReadApiServer for GovernanceReadApi { - async fn get_stakes_by_ids( - &self, - staked_sui_ids: Vec, - ) -> RpcResult> { - self.get_stakes_by_ids(staked_sui_ids) - .await - .map_err(Into::into) - } - - async fn get_stakes(&self, owner: SuiAddress) -> RpcResult> { - self.get_staked_by_owner(owner).await.map_err(Into::into) - } - - async fn get_committee_info(&self, epoch: Option>) -> RpcResult { - let epoch = self.get_epoch_info(epoch.as_deref().copied()).await?; - Ok(epoch.committee().map_err(IndexerError::from)?.into()) - } - - async fn get_latest_sui_system_state(&self) -> RpcResult { - self.get_latest_sui_system_state().await.map_err(Into::into) - } - - async fn get_reference_gas_price(&self) -> RpcResult> { - let epoch = self.get_epoch_info(None).await?; - Ok(BigInt::from(epoch.reference_gas_price.ok_or_else( - || { - IndexerError::PersistentStorageDataCorruptionError( - "missing latest reference gas price".to_owned(), - ) - }, - )?)) - } - - async fn get_validators_apy(&self) -> RpcResult { - Ok(self.get_validators_apy().await?) - } -} - -impl SuiRpcModule for GovernanceReadApi { - fn rpc(self) -> RpcModule { - self.into_rpc() - } - - fn rpc_doc_module() -> Module { - sui_json_rpc_api::GovernanceReadApiOpenRpc::module_doc() - } -} diff --git a/crates/sui-mvr-indexer/src/apis/mod.rs b/crates/sui-mvr-indexer/src/apis/mod.rs deleted file mode 100644 index e797c7ecdc239..0000000000000 --- a/crates/sui-mvr-indexer/src/apis/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -pub(crate) use coin_api::CoinReadApi; -pub(crate) use extended_api::ExtendedApi; -pub use governance_api::GovernanceReadApi; -pub(crate) use indexer_api::IndexerApi; -pub(crate) use move_utils::MoveUtilsApi; -pub(crate) use read_api::ReadApi; -pub(crate) use transaction_builder_api::TransactionBuilderApi; -pub(crate) use write_api::WriteApi; - -mod coin_api; -mod extended_api; -pub mod governance_api; -mod indexer_api; -mod move_utils; -pub mod read_api; -mod transaction_builder_api; -mod write_api; diff --git a/crates/sui-mvr-indexer/src/apis/move_utils.rs b/crates/sui-mvr-indexer/src/apis/move_utils.rs deleted file mode 100644 index 2bb75b9831dbf..0000000000000 --- a/crates/sui-mvr-indexer/src/apis/move_utils.rs +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::BTreeMap; - -use async_trait::async_trait; -use jsonrpsee::core::RpcResult; -use jsonrpsee::RpcModule; -use move_binary_format::normalized::Module as NormalizedModule; - -use sui_json_rpc::error::SuiRpcInputError; -use sui_json_rpc::SuiRpcModule; -use sui_json_rpc_api::MoveUtilsServer; -use sui_json_rpc_types::ObjectValueKind; -use sui_json_rpc_types::SuiMoveNormalizedType; -use sui_json_rpc_types::{ - MoveFunctionArgType, SuiMoveNormalizedFunction, SuiMoveNormalizedModule, - SuiMoveNormalizedStruct, -}; -use sui_open_rpc::Module; -use sui_types::base_types::ObjectID; - -use crate::indexer_reader::IndexerReader; - -pub struct MoveUtilsApi { - inner: IndexerReader, -} - -impl MoveUtilsApi { - pub fn new(inner: IndexerReader) -> Self { - Self { inner } - } -} - -#[async_trait] -impl MoveUtilsServer for MoveUtilsApi { - async fn get_normalized_move_modules_by_package( - &self, - package_id: ObjectID, - ) -> RpcResult> { - let resolver_modules = self.inner.get_package(package_id).await?.modules().clone(); - let sui_normalized_modules = resolver_modules - .into_iter() - .map(|(k, v)| (k, NormalizedModule::new(v.bytecode()).into())) - .collect::>(); - Ok(sui_normalized_modules) - } - - async fn get_normalized_move_module( - &self, - package: ObjectID, - module_name: String, - ) -> RpcResult { - let mut modules = self.get_normalized_move_modules_by_package(package).await?; - let module = modules.remove(&module_name).ok_or_else(|| { - SuiRpcInputError::GenericNotFound(format!( - "No module was found with name {module_name}", - )) - })?; - Ok(module) - } - - async fn get_normalized_move_struct( - &self, - package: ObjectID, - module_name: String, - struct_name: String, - ) -> RpcResult { - let mut module = self - .get_normalized_move_module(package, module_name) - .await?; - module - .structs - .remove(&struct_name) - .ok_or_else(|| { - SuiRpcInputError::GenericNotFound(format!( - "No struct was found with struct name {struct_name}" - )) - }) - .map_err(Into::into) - } - - async fn get_normalized_move_function( - &self, - package: ObjectID, - module_name: String, - function_name: String, - ) -> RpcResult { - let mut module = self - .get_normalized_move_module(package, module_name) - .await?; - module - .exposed_functions - .remove(&function_name) - .ok_or_else(|| { - SuiRpcInputError::GenericNotFound(format!( - "No function was found with function name {function_name}", - )) - }) - .map_err(Into::into) - } - - async fn get_move_function_arg_types( - &self, - package: ObjectID, - module: String, - function: String, - ) -> RpcResult> { - let function = self - .get_normalized_move_function(package, module, function) - .await?; - let args = function - .parameters - .iter() - .map(|p| match p { - SuiMoveNormalizedType::Struct { .. } => { - MoveFunctionArgType::Object(ObjectValueKind::ByValue) - } - SuiMoveNormalizedType::Vector(_) => { - MoveFunctionArgType::Object(ObjectValueKind::ByValue) - } - SuiMoveNormalizedType::Reference(_) => { - MoveFunctionArgType::Object(ObjectValueKind::ByImmutableReference) - } - SuiMoveNormalizedType::MutableReference(_) => { - MoveFunctionArgType::Object(ObjectValueKind::ByMutableReference) - } - _ => MoveFunctionArgType::Pure, - }) - .collect::>(); - Ok(args) - } -} - -impl SuiRpcModule for MoveUtilsApi { - fn rpc(self) -> RpcModule { - self.into_rpc() - } - - fn rpc_doc_module() -> Module { - sui_json_rpc_api::MoveUtilsOpenRpc::module_doc() - } -} diff --git a/crates/sui-mvr-indexer/src/apis/read_api.rs b/crates/sui-mvr-indexer/src/apis/read_api.rs deleted file mode 100644 index 3e3de5343869d..0000000000000 --- a/crates/sui-mvr-indexer/src/apis/read_api.rs +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use async_trait::async_trait; -use jsonrpsee::core::RpcResult; -use jsonrpsee::RpcModule; -use sui_json_rpc::error::SuiRpcInputError; -use sui_types::error::SuiObjectResponseError; -use sui_types::object::ObjectRead; - -use crate::errors::IndexerError; -use crate::indexer_reader::IndexerReader; -use sui_json_rpc::SuiRpcModule; -use sui_json_rpc_api::{ReadApiServer, QUERY_MAX_RESULT_LIMIT}; -use sui_json_rpc_types::{ - Checkpoint, CheckpointId, CheckpointPage, ProtocolConfigResponse, SuiEvent, - SuiGetPastObjectRequest, SuiObjectDataOptions, SuiObjectResponse, SuiPastObjectResponse, - SuiTransactionBlockResponse, SuiTransactionBlockResponseOptions, -}; -use sui_open_rpc::Module; -use sui_protocol_config::{ProtocolConfig, ProtocolVersion}; -use sui_types::base_types::{ObjectID, SequenceNumber}; -use sui_types::digests::{ChainIdentifier, TransactionDigest}; -use sui_types::sui_serde::BigInt; - -#[derive(Clone)] -pub struct ReadApi { - inner: IndexerReader, -} - -impl ReadApi { - pub fn new(inner: IndexerReader) -> Self { - Self { inner } - } - - async fn get_checkpoint(&self, id: CheckpointId) -> Result { - match self.inner.get_checkpoint(id).await { - Ok(Some(epoch_info)) => Ok(epoch_info), - Ok(None) => Err(IndexerError::InvalidArgumentError(format!( - "Checkpoint {id:?} not found" - ))), - Err(e) => Err(e), - } - } - - async fn get_latest_checkpoint(&self) -> Result { - self.inner.get_latest_checkpoint().await - } - - async fn get_chain_identifier(&self) -> RpcResult { - let genesis_checkpoint = self.get_checkpoint(CheckpointId::SequenceNumber(0)).await?; - Ok(ChainIdentifier::from(genesis_checkpoint.digest)) - } -} - -#[async_trait] -impl ReadApiServer for ReadApi { - async fn get_object( - &self, - object_id: ObjectID, - options: Option, - ) -> RpcResult { - let object_read = self.inner.get_object_read(object_id).await?; - object_read_to_object_response(&self.inner, object_read, options.unwrap_or_default()).await - } - - // For ease of implementation we just forward to the single object query, although in the - // future we may want to improve the performance by having a more naitive multi_get - // functionality - async fn multi_get_objects( - &self, - object_ids: Vec, - options: Option, - ) -> RpcResult> { - if object_ids.len() > *QUERY_MAX_RESULT_LIMIT { - return Err( - SuiRpcInputError::SizeLimitExceeded(QUERY_MAX_RESULT_LIMIT.to_string()).into(), - ); - } - let stored_objects = self.inner.multi_get_objects(object_ids).await?; - let options = options.unwrap_or_default(); - - let futures = stored_objects.into_iter().map(|stored_object| async { - let object_read = stored_object - .try_into_object_read(self.inner.package_resolver()) - .await?; - object_read_to_object_response(&self.inner, object_read, options.clone()).await - }); - - let mut objects = futures::future::try_join_all(futures).await?; - // Resort the objects by the order of the object id. - objects.sort_by_key(|obj| obj.data.as_ref().map(|data| data.object_id)); - - Ok(objects) - } - - async fn get_total_transaction_blocks(&self) -> RpcResult> { - let checkpoint = self.get_latest_checkpoint().await?; - Ok(BigInt::from(checkpoint.network_total_transactions)) - } - - async fn get_transaction_block( - &self, - digest: TransactionDigest, - options: Option, - ) -> RpcResult { - let mut txn = self - .multi_get_transaction_blocks(vec![digest], options) - .await?; - - let txn = txn.pop().ok_or_else(|| { - IndexerError::InvalidArgumentError(format!("Transaction {digest} not found")) - })?; - - Ok(txn) - } - - async fn multi_get_transaction_blocks( - &self, - digests: Vec, - options: Option, - ) -> RpcResult> { - let num_digests = digests.len(); - if num_digests > *QUERY_MAX_RESULT_LIMIT { - Err(SuiRpcInputError::SizeLimitExceeded( - QUERY_MAX_RESULT_LIMIT.to_string(), - ))? - } - - let options = options.unwrap_or_default(); - let txns = self - .inner - .multi_get_transaction_block_response_in_blocking_task(digests, options) - .await?; - - Ok(txns) - } - - async fn try_get_past_object( - &self, - _object_id: ObjectID, - _version: SequenceNumber, - _options: Option, - ) -> RpcResult { - Err(jsonrpsee::types::error::CallError::Custom( - jsonrpsee::types::error::ErrorCode::MethodNotFound.into(), - ) - .into()) - } - - async fn try_get_object_before_version( - &self, - _: ObjectID, - _: SequenceNumber, - ) -> RpcResult { - Err(jsonrpsee::types::error::CallError::Custom( - jsonrpsee::types::error::ErrorCode::MethodNotFound.into(), - ) - .into()) - } - - async fn try_multi_get_past_objects( - &self, - _past_objects: Vec, - _options: Option, - ) -> RpcResult> { - Err(jsonrpsee::types::error::CallError::Custom( - jsonrpsee::types::error::ErrorCode::MethodNotFound.into(), - ) - .into()) - } - - async fn get_latest_checkpoint_sequence_number(&self) -> RpcResult> { - let checkpoint = self.get_latest_checkpoint().await?; - Ok(BigInt::from(checkpoint.sequence_number)) - } - - async fn get_checkpoint(&self, id: CheckpointId) -> RpcResult { - self.get_checkpoint(id).await.map_err(Into::into) - } - - async fn get_checkpoints( - &self, - cursor: Option>, - limit: Option, - descending_order: bool, - ) -> RpcResult { - let cursor = cursor.map(BigInt::into_inner); - let limit = sui_json_rpc_api::validate_limit( - limit, - sui_json_rpc_api::QUERY_MAX_RESULT_LIMIT_CHECKPOINTS, - ) - .map_err(SuiRpcInputError::from)?; - - let mut checkpoints = self - .inner - .get_checkpoints(cursor, limit + 1, descending_order) - .await?; - - let has_next_page = checkpoints.len() > limit; - checkpoints.truncate(limit); - - let next_cursor = checkpoints.last().map(|d| d.sequence_number.into()); - - Ok(CheckpointPage { - data: checkpoints, - next_cursor, - has_next_page, - }) - } - - async fn get_checkpoints_deprecated_limit( - &self, - cursor: Option>, - limit: Option>, - descending_order: bool, - ) -> RpcResult { - self.get_checkpoints( - cursor, - limit.map(|l| l.into_inner() as usize), - descending_order, - ) - .await - } - - async fn get_events(&self, transaction_digest: TransactionDigest) -> RpcResult> { - self.inner - .get_transaction_events(transaction_digest) - .await - .map_err(Into::into) - } - - async fn get_protocol_config( - &self, - version: Option>, - ) -> RpcResult { - let chain = self.get_chain_identifier().await?.chain(); - let version = if let Some(version) = version { - (*version).into() - } else { - let latest_epoch = self.inner.get_latest_epoch_info_from_db().await?; - (latest_epoch.protocol_version as u64).into() - }; - - ProtocolConfig::get_for_version_if_supported(version, chain) - .ok_or(SuiRpcInputError::ProtocolVersionUnsupported( - ProtocolVersion::MIN.as_u64(), - ProtocolVersion::MAX.as_u64(), - )) - .map_err(Into::into) - .map(ProtocolConfigResponse::from) - } - - async fn get_chain_identifier(&self) -> RpcResult { - self.get_chain_identifier().await.map(|id| id.to_string()) - } -} - -impl SuiRpcModule for ReadApi { - fn rpc(self) -> RpcModule { - self.into_rpc() - } - - fn rpc_doc_module() -> Module { - sui_json_rpc_api::ReadApiOpenRpc::module_doc() - } -} - -async fn object_read_to_object_response( - indexer_reader: &IndexerReader, - object_read: ObjectRead, - options: SuiObjectDataOptions, -) -> RpcResult { - match object_read { - ObjectRead::NotExists(id) => Ok(SuiObjectResponse::new_with_error( - SuiObjectResponseError::NotExists { object_id: id }, - )), - ObjectRead::Exists(object_ref, o, layout) => { - let mut display_fields = None; - if options.show_display { - match indexer_reader.get_display_fields(&o, &layout).await { - Ok(rendered_fields) => display_fields = Some(rendered_fields), - Err(e) => { - return Ok(SuiObjectResponse::new( - Some((object_ref, o, layout, options, None).try_into()?), - Some(SuiObjectResponseError::DisplayError { - error: e.to_string(), - }), - )); - } - } - } - Ok(SuiObjectResponse::new_with_data( - (object_ref, o, layout, options, display_fields).try_into()?, - )) - } - ObjectRead::Deleted((object_id, version, digest)) => Ok(SuiObjectResponse::new_with_error( - SuiObjectResponseError::Deleted { - object_id, - version, - digest, - }, - )), - } -} diff --git a/crates/sui-mvr-indexer/src/apis/transaction_builder_api.rs b/crates/sui-mvr-indexer/src/apis/transaction_builder_api.rs deleted file mode 100644 index c98ce9c371c10..0000000000000 --- a/crates/sui-mvr-indexer/src/apis/transaction_builder_api.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use super::governance_api::GovernanceReadApi; -use crate::indexer_reader::IndexerReader; -use async_trait::async_trait; -use move_core_types::language_storage::StructTag; -use sui_json_rpc::transaction_builder_api::TransactionBuilderApi as SuiTransactionBuilderApi; -use sui_json_rpc_types::{SuiObjectDataFilter, SuiObjectDataOptions, SuiObjectResponse}; -use sui_transaction_builder::DataReader; -use sui_types::base_types::{ObjectID, ObjectInfo, SuiAddress}; -use sui_types::object::Object; - -pub(crate) struct TransactionBuilderApi { - inner: IndexerReader, -} - -impl TransactionBuilderApi { - #[allow(clippy::new_ret_no_self)] - pub fn new(inner: IndexerReader) -> SuiTransactionBuilderApi { - SuiTransactionBuilderApi::new_with_data_reader(std::sync::Arc::new(Self { inner })) - } -} - -#[async_trait] -impl DataReader for TransactionBuilderApi { - async fn get_owned_objects( - &self, - address: SuiAddress, - object_type: StructTag, - ) -> Result, anyhow::Error> { - let stored_objects = self - .inner - .get_owned_objects( - address, - Some(SuiObjectDataFilter::StructType(object_type)), - None, - 50, // Limit the number of objects returned to 50 - ) - .await?; - - stored_objects - .into_iter() - .map(|object| { - let object = Object::try_from(object)?; - let object_ref = object.compute_object_reference(); - let info = ObjectInfo::new(&object_ref, &object); - Ok(info) - }) - .collect::, _>>() - } - - async fn get_object_with_options( - &self, - object_id: ObjectID, - options: SuiObjectDataOptions, - ) -> Result { - let result = self.inner.get_object_read(object_id).await?; - Ok((result, options).try_into()?) - } - - async fn get_reference_gas_price(&self) -> Result { - let epoch_info = GovernanceReadApi::new(self.inner.clone()) - .get_epoch_info(None) - .await?; - Ok(epoch_info - .reference_gas_price - .ok_or_else(|| anyhow::anyhow!("missing latest reference_gas_price"))?) - } -} diff --git a/crates/sui-mvr-indexer/src/apis/write_api.rs b/crates/sui-mvr-indexer/src/apis/write_api.rs deleted file mode 100644 index 71a54c356635b..0000000000000 --- a/crates/sui-mvr-indexer/src/apis/write_api.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use async_trait::async_trait; -use fastcrypto::encoding::Base64; -use jsonrpsee::core::RpcResult; -use jsonrpsee::http_client::HttpClient; -use jsonrpsee::RpcModule; - -use sui_json_rpc::SuiRpcModule; -use sui_json_rpc_api::{WriteApiClient, WriteApiServer}; -use sui_json_rpc_types::{ - DevInspectArgs, DevInspectResults, DryRunTransactionBlockResponse, SuiTransactionBlockResponse, - SuiTransactionBlockResponseOptions, -}; -use sui_open_rpc::Module; -use sui_types::base_types::SuiAddress; -use sui_types::quorum_driver_types::ExecuteTransactionRequestType; -use sui_types::sui_serde::BigInt; - -use crate::types::SuiTransactionBlockResponseWithOptions; - -pub(crate) struct WriteApi { - fullnode: HttpClient, -} - -impl WriteApi { - pub fn new(fullnode_client: HttpClient) -> Self { - Self { - fullnode: fullnode_client, - } - } -} - -#[async_trait] -impl WriteApiServer for WriteApi { - async fn execute_transaction_block( - &self, - tx_bytes: Base64, - signatures: Vec, - options: Option, - request_type: Option, - ) -> RpcResult { - let sui_transaction_response = self - .fullnode - .execute_transaction_block(tx_bytes, signatures, options.clone(), request_type) - .await?; - Ok(SuiTransactionBlockResponseWithOptions { - response: sui_transaction_response, - options: options.unwrap_or_default(), - } - .into()) - } - - async fn dev_inspect_transaction_block( - &self, - sender_address: SuiAddress, - tx_bytes: Base64, - gas_price: Option>, - epoch: Option>, - additional_args: Option, - ) -> RpcResult { - self.fullnode - .dev_inspect_transaction_block( - sender_address, - tx_bytes, - gas_price, - epoch, - additional_args, - ) - .await - } - - async fn dry_run_transaction_block( - &self, - tx_bytes: Base64, - ) -> RpcResult { - self.fullnode.dry_run_transaction_block(tx_bytes).await - } -} - -impl SuiRpcModule for WriteApi { - fn rpc(self) -> RpcModule { - self.into_rpc() - } - - fn rpc_doc_module() -> Module { - sui_json_rpc_api::WriteApiOpenRpc::module_doc() - } -} diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/digest_task.rs b/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/digest_task.rs deleted file mode 100644 index 8273bcdaa3b7b..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/digest_task.rs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::backfill::backfill_instances::ingestion_backfills::IngestionBackfillTrait; -use crate::database::ConnectionPool; -use sui_types::full_checkpoint_content::CheckpointData; -use tracing::info; - -/// Dummy backfill that only prints the sequence number and checkpoint of the digest. Intended to -/// benchmark backfill performance. -pub struct DigestBackfill; - -#[async_trait::async_trait] -impl IngestionBackfillTrait for DigestBackfill { - type ProcessedType = (); - - fn process_checkpoint(checkpoint: &CheckpointData) -> Vec { - let cp = checkpoint.checkpoint_summary.sequence_number; - let digest = checkpoint.checkpoint_summary.content_digest; - info!("{cp}: {digest}"); - - vec![] - } - - async fn commit_chunk(_pool: ConnectionPool, _processed_data: Vec) {} -} diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/ingestion_backfill_task.rs b/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/ingestion_backfill_task.rs deleted file mode 100644 index 2702f755c0842..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/ingestion_backfill_task.rs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::backfill::backfill_instances::ingestion_backfills::IngestionBackfillTrait; -use crate::backfill::backfill_task::BackfillTask; -use crate::database::ConnectionPool; -use dashmap::DashMap; -use std::ops::RangeInclusive; -use std::sync::Arc; -use sui_data_ingestion_core::{setup_single_workflow, ReaderOptions, Worker}; -use sui_types::full_checkpoint_content::CheckpointData; -use sui_types::messages_checkpoint::CheckpointSequenceNumber; -use tokio::sync::Notify; - -pub struct IngestionBackfillTask { - ready_checkpoints: Arc>>, - notify: Arc, - _exit_sender: tokio::sync::oneshot::Sender<()>, -} - -impl IngestionBackfillTask { - pub async fn new(remote_store_url: String, start_checkpoint: CheckpointSequenceNumber) -> Self { - let ready_checkpoints = Arc::new(DashMap::new()); - let notify = Arc::new(Notify::new()); - let adapter: Adapter = Adapter { - ready_checkpoints: ready_checkpoints.clone(), - notify: notify.clone(), - }; - let reader_options = ReaderOptions { - batch_size: 200, - ..Default::default() - }; - let (executor, _exit_sender) = setup_single_workflow( - adapter, - remote_store_url, - start_checkpoint, - 200, - Some(reader_options), - ) - .await - .unwrap(); - tokio::task::spawn(async move { - executor.await.unwrap(); - }); - Self { - ready_checkpoints, - notify, - _exit_sender, - } - } -} - -pub struct Adapter { - ready_checkpoints: Arc>>, - notify: Arc, -} - -#[async_trait::async_trait] -impl Worker for Adapter { - type Result = (); - async fn process_checkpoint(&self, checkpoint: &CheckpointData) -> anyhow::Result<()> { - let processed = T::process_checkpoint(checkpoint); - self.ready_checkpoints - .insert(checkpoint.checkpoint_summary.sequence_number, processed); - self.notify.notify_waiters(); - Ok(()) - } -} - -#[async_trait::async_trait] -impl BackfillTask for IngestionBackfillTask { - async fn backfill_range(&self, pool: ConnectionPool, range: &RangeInclusive) { - let mut processed_data = vec![]; - let mut start = *range.start(); - let end = *range.end(); - loop { - while start <= end { - if let Some((_, processed)) = self - .ready_checkpoints - .remove(&(start as CheckpointSequenceNumber)) - { - processed_data.extend(processed); - start += 1; - } else { - break; - } - } - if start <= end { - self.notify.notified().await; - } else { - break; - } - } - // TODO: Limit the size of each chunk. - // postgres has a parameter limit of 65535, meaning that row_count * col_count <= 65536. - T::commit_chunk(pool.clone(), processed_data).await; - } -} diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/mod.rs b/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/mod.rs deleted file mode 100644 index 935ba5562bd9c..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -pub(crate) mod digest_task; -pub(crate) mod ingestion_backfill_task; -pub(crate) mod raw_checkpoints; -pub(crate) mod tx_affected_objects; - -use crate::database::ConnectionPool; -use sui_types::full_checkpoint_content::CheckpointData; - -#[async_trait::async_trait] -pub trait IngestionBackfillTrait: Send + Sync { - type ProcessedType: Send + Sync; - - fn process_checkpoint(checkpoint: &CheckpointData) -> Vec; - async fn commit_chunk(pool: ConnectionPool, processed_data: Vec); -} diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/raw_checkpoints.rs b/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/raw_checkpoints.rs deleted file mode 100644 index aec4f0263ee80..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/raw_checkpoints.rs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::backfill::backfill_instances::ingestion_backfills::IngestionBackfillTrait; -use crate::database::ConnectionPool; -use crate::models::raw_checkpoints::StoredRawCheckpoint; -use crate::schema::raw_checkpoints::dsl::raw_checkpoints; -use diesel_async::RunQueryDsl; -use sui_types::full_checkpoint_content::CheckpointData; - -pub struct RawCheckpointsBackFill; - -#[async_trait::async_trait] -impl IngestionBackfillTrait for RawCheckpointsBackFill { - type ProcessedType = StoredRawCheckpoint; - - fn process_checkpoint(checkpoint: &CheckpointData) -> Vec { - vec![StoredRawCheckpoint { - sequence_number: checkpoint.checkpoint_summary.sequence_number as i64, - certified_checkpoint: bcs::to_bytes(&checkpoint.checkpoint_summary).unwrap(), - checkpoint_contents: bcs::to_bytes(&checkpoint.checkpoint_contents).unwrap(), - }] - } - - async fn commit_chunk(pool: ConnectionPool, processed_data: Vec) { - let mut conn = pool.get().await.unwrap(); - diesel::insert_into(raw_checkpoints) - .values(processed_data) - .on_conflict_do_nothing() - .execute(&mut conn) - .await - .unwrap(); - } -} diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/tx_affected_objects.rs b/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/tx_affected_objects.rs deleted file mode 100644 index 4e6f6efa6a897..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/ingestion_backfills/tx_affected_objects.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::backfill::backfill_instances::ingestion_backfills::IngestionBackfillTrait; -use crate::database::ConnectionPool; -use crate::models::tx_indices::StoredTxAffectedObjects; -use crate::schema::tx_affected_objects; -use diesel_async::RunQueryDsl; -use sui_types::effects::TransactionEffectsAPI; -use sui_types::full_checkpoint_content::CheckpointData; - -pub struct TxAffectedObjectsBackfill; - -#[async_trait::async_trait] -impl IngestionBackfillTrait for TxAffectedObjectsBackfill { - type ProcessedType = StoredTxAffectedObjects; - - fn process_checkpoint(checkpoint: &CheckpointData) -> Vec { - let first_tx = checkpoint.checkpoint_summary.network_total_transactions as usize - - checkpoint.transactions.len(); - - checkpoint - .transactions - .iter() - .enumerate() - .flat_map(|(i, tx)| { - tx.effects - .object_changes() - .into_iter() - .map(move |change| StoredTxAffectedObjects { - tx_sequence_number: (first_tx + i) as i64, - affected: change.id.to_vec(), - sender: tx.transaction.sender_address().to_vec(), - }) - }) - .collect() - } - - async fn commit_chunk(pool: ConnectionPool, processed_data: Vec) { - let mut conn = pool.get().await.unwrap(); - diesel::insert_into(tx_affected_objects::table) - .values(processed_data) - .on_conflict_do_nothing() - .execute(&mut conn) - .await - .unwrap(); - } -} diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/mod.rs b/crates/sui-mvr-indexer/src/backfill/backfill_instances/mod.rs deleted file mode 100644 index 304ed4e715e1d..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/mod.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::backfill::backfill_instances::ingestion_backfills::digest_task::DigestBackfill; -use crate::backfill::backfill_instances::ingestion_backfills::ingestion_backfill_task::IngestionBackfillTask; -use crate::backfill::backfill_instances::ingestion_backfills::raw_checkpoints::RawCheckpointsBackFill; -use crate::backfill::backfill_instances::ingestion_backfills::tx_affected_objects::TxAffectedObjectsBackfill; -use crate::backfill::backfill_task::BackfillTask; -use crate::backfill::{BackfillTaskKind, IngestionBackfillKind}; -use std::sync::Arc; -use sui_types::messages_checkpoint::CheckpointSequenceNumber; - -mod ingestion_backfills; -mod sql_backfill; -mod system_state_summary_json; - -pub async fn get_backfill_task( - kind: BackfillTaskKind, - range_start: usize, -) -> Arc { - match kind { - BackfillTaskKind::SystemStateSummaryJson => { - Arc::new(system_state_summary_json::SystemStateSummaryJsonBackfill) - } - BackfillTaskKind::Sql { sql, key_column } => { - Arc::new(sql_backfill::SqlBackFill::new(sql, key_column)) - } - BackfillTaskKind::Ingestion { - kind, - remote_store_url, - } => match kind { - IngestionBackfillKind::Digest => Arc::new( - IngestionBackfillTask::::new( - remote_store_url, - range_start as CheckpointSequenceNumber, - ) - .await, - ), - IngestionBackfillKind::RawCheckpoints => Arc::new( - IngestionBackfillTask::::new( - remote_store_url, - range_start as CheckpointSequenceNumber, - ) - .await, - ), - IngestionBackfillKind::TxAffectedObjects => Arc::new( - IngestionBackfillTask::::new( - remote_store_url, - range_start as CheckpointSequenceNumber, - ) - .await, - ), - }, - } -} diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfill.rs b/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfill.rs deleted file mode 100644 index 543f077e2ba3b..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfill.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::backfill::backfill_task::BackfillTask; -use crate::database::ConnectionPool; -use async_trait::async_trait; -use diesel_async::RunQueryDsl; -use std::ops::RangeInclusive; - -pub struct SqlBackFill { - sql: String, - key_column: String, -} - -impl SqlBackFill { - pub fn new(sql: String, key_column: String) -> Self { - Self { sql, key_column } - } -} - -#[async_trait] -impl BackfillTask for SqlBackFill { - async fn backfill_range(&self, pool: ConnectionPool, range: &RangeInclusive) { - let mut conn = pool.get().await.unwrap(); - - let query = format!( - "{} WHERE {} BETWEEN {} AND {} ON CONFLICT DO NOTHING", - self.sql, - self.key_column, - *range.start(), - *range.end() - ); - - diesel::sql_query(query).execute(&mut conn).await.unwrap(); - } -} diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/event_sender.sh b/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/event_sender.sh deleted file mode 100644 index ea883107b31ca..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/event_sender.sh +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Mysten Labs, Inc. -# SPDX-License-Identifier: Apache-2.0 - -INDEXER=${INDEXER:-"sui-mvr-indexer"} -DB=${DB:-"postgres://postgres:postgrespw@localhost:5432/postgres"} -"$INDEXER" --database-url "$DB" run-back-fill "$1" "$2" sql "UPDATE events SET sender = CASE WHEN cardinality(senders) > 0 THEN senders[1] ELSE NULL END" checkpoint_sequence_number diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/full_objects_history.sh b/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/full_objects_history.sh deleted file mode 100644 index 18a0e3b9e84de..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/full_objects_history.sh +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Mysten Labs, Inc. -# SPDX-License-Identifier: Apache-2.0 - -INDEXER=${INDEXER:-"sui-mvr-indexer"} -DB=${DB:-"postgres://postgres:postgrespw@localhost:5432/postgres"} -"$INDEXER" --database-url "$DB" run-back-fill "$1" "$2" sql "INSERT INTO full_objects_history (object_id, object_version, serialized_object) SELECT object_id, object_version, serialized_object FROM objects_history" checkpoint_sequence_number diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/tx_affected_addresses.sh b/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/tx_affected_addresses.sh deleted file mode 100644 index da0dc0915a0b4..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/sql_backfills/tx_affected_addresses.sh +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) Mysten Labs, Inc. -# SPDX-License-Identifier: Apache-2.0 - -INDEXER=${INDEXER:-"sui-mvr-indexer"} -DB=${DB:-"postgres://postgres:postgrespw@localhost:5432/postgres"} -"$INDEXER" --database-url "$DB" run-back-fill "$1" "$2" sql "INSERT INTO tx_affected_addresses SELECT tx_sequence_number, sender AS affected, sender FROM tx_senders" tx_sequence_number -"$INDEXER" --database-url "$DB" run-back-fill "$1" "$2" sql "INSERT INTO tx_affected_addresses SELECT tx_sequence_number, recipient AS affected, sender FROM tx_recipients" tx_sequence_number diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_instances/system_state_summary_json.rs b/crates/sui-mvr-indexer/src/backfill/backfill_instances/system_state_summary_json.rs deleted file mode 100644 index 912abdd871a1c..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_instances/system_state_summary_json.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::backfill::backfill_task::BackfillTask; -use crate::database::ConnectionPool; -use crate::schema::epochs; -use async_trait::async_trait; -use diesel::{ExpressionMethods, QueryDsl}; -use diesel_async::{AsyncConnection, RunQueryDsl}; -use std::ops::RangeInclusive; -use sui_types::sui_system_state::sui_system_state_summary::SuiSystemStateSummary; - -pub struct SystemStateSummaryJsonBackfill; - -#[async_trait] -impl BackfillTask for SystemStateSummaryJsonBackfill { - async fn backfill_range(&self, pool: ConnectionPool, range: &RangeInclusive) { - let mut conn = pool.get().await.unwrap(); - - let results: Vec>> = epochs::table - .select(epochs::system_state) - .filter(epochs::epoch.between(*range.start() as i64, *range.end() as i64)) - .load(&mut conn) - .await - .unwrap(); - - let mut system_states = vec![]; - for bytes in results { - let Some(bytes) = bytes else { - continue; - }; - let system_state_summary: SuiSystemStateSummary = bcs::from_bytes(&bytes).unwrap(); - let json_ser = serde_json::to_value(&system_state_summary).unwrap(); - if system_state_summary.epoch == 1 { - // Each existing system state's epoch is off by 1. - // This means there won't be any row with a system state summary for epoch 0. - // We need to manually insert a row for epoch 0. - system_states.push((0, json_ser.clone())); - } - system_states.push((system_state_summary.epoch, json_ser)); - } - conn.transaction::<_, diesel::result::Error, _>(|conn| { - Box::pin(async move { - for (epoch, json_ser) in system_states { - diesel::update(epochs::table.filter(epochs::epoch.eq(epoch as i64))) - .set(epochs::system_state_summary_json.eq(Some(json_ser))) - .execute(conn) - .await?; - } - Ok(()) - }) - }) - .await - .unwrap(); - } -} diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_runner.rs b/crates/sui-mvr-indexer/src/backfill/backfill_runner.rs deleted file mode 100644 index 3126dc90fe35f..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_runner.rs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::backfill::backfill_instances::get_backfill_task; -use crate::backfill::backfill_task::BackfillTask; -use crate::backfill::BackfillTaskKind; -use crate::config::BackFillConfig; -use crate::database::ConnectionPool; -use futures::StreamExt; -use std::collections::BTreeSet; -use std::ops::RangeInclusive; -use std::sync::Arc; -use std::time::Instant; -use tokio::sync::{mpsc, Mutex}; -use tokio_stream::wrappers::ReceiverStream; - -pub struct BackfillRunner {} - -impl BackfillRunner { - pub async fn run( - runner_kind: BackfillTaskKind, - pool: ConnectionPool, - backfill_config: BackFillConfig, - total_range: RangeInclusive, - ) { - let task = get_backfill_task(runner_kind, *total_range.start()).await; - Self::run_impl(pool, backfill_config, total_range, task).await; - } - - /// Main function to run the parallel queries and batch processing. - async fn run_impl( - pool: ConnectionPool, - config: BackFillConfig, - total_range: RangeInclusive, - task: Arc, - ) { - let cur_time = Instant::now(); - // Keeps track of the checkpoint ranges (using starting checkpoint number) - // that are in progress. - let in_progress = Arc::new(Mutex::new(BTreeSet::new())); - - let concurrency = config.max_concurrency; - let (tx, rx) = mpsc::channel(concurrency * 10); - // Spawn a task to send chunks lazily over the channel - tokio::spawn(async move { - for chunk in create_chunk_iter(total_range, config.chunk_size) { - if tx.send(chunk).await.is_err() { - // Channel closed, stop producing chunks - break; - } - } - }); - // Convert the receiver into a stream - let stream = ReceiverStream::new(rx); - - // Process chunks in parallel, limiting the number of concurrent query tasks - stream - .for_each_concurrent(concurrency, move |range| { - let pool_clone = pool.clone(); - let in_progress_clone = in_progress.clone(); - let task = task.clone(); - - async move { - in_progress_clone.lock().await.insert(*range.start()); - task.backfill_range(pool_clone, &range).await; - println!("Finished range: {:?}.", range); - in_progress_clone.lock().await.remove(range.start()); - let cur_min_in_progress = in_progress_clone.lock().await.iter().next().cloned(); - if let Some(cur_min_in_progress) = cur_min_in_progress { - println!( - "Average backfill speed: {} checkpoints/s. Minimum range start number still in progress: {:?}.", - cur_min_in_progress as f64 / cur_time.elapsed().as_secs_f64(), - cur_min_in_progress - ); - } - } - }) - .await; - - println!("Finished backfilling in {:?}", cur_time.elapsed()); - } -} - -/// Creates chunks based on the total range and chunk size. -fn create_chunk_iter( - total_range: RangeInclusive, - chunk_size: usize, -) -> impl Iterator> { - let end = *total_range.end(); - total_range.step_by(chunk_size).map(move |chunk_start| { - let chunk_end = std::cmp::min(chunk_start + chunk_size - 1, end); - chunk_start..=chunk_end - }) -} diff --git a/crates/sui-mvr-indexer/src/backfill/backfill_task.rs b/crates/sui-mvr-indexer/src/backfill/backfill_task.rs deleted file mode 100644 index 008bfa5b482c0..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/backfill_task.rs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::database::ConnectionPool; -use async_trait::async_trait; -use std::ops::RangeInclusive; - -#[async_trait] -pub trait BackfillTask: Send + Sync { - /// Backfill the database for a specific range. - async fn backfill_range(&self, pool: ConnectionPool, range: &RangeInclusive); -} diff --git a/crates/sui-mvr-indexer/src/backfill/mod.rs b/crates/sui-mvr-indexer/src/backfill/mod.rs deleted file mode 100644 index e17ba40628ef1..0000000000000 --- a/crates/sui-mvr-indexer/src/backfill/mod.rs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use clap::{Subcommand, ValueEnum}; - -pub mod backfill_instances; -pub mod backfill_runner; -pub mod backfill_task; - -#[derive(Subcommand, Clone, Debug)] -pub enum BackfillTaskKind { - SystemStateSummaryJson, - /// \sql is the SQL string to run, appended with the range between the start and end, - /// as well as conflict resolution (see sql_backfill.rs). - /// \key_column is the primary key column to use for the range. - Sql { - sql: String, - key_column: String, - }, - /// Starts a backfill pipeline from the ingestion engine. - /// \remote_store_url is the URL of the remote store to ingest from. - /// Any `IngestionBackfillKind` will need to map to a type that - /// implements `IngestionBackfillTrait`. - Ingestion { - kind: IngestionBackfillKind, - remote_store_url: String, - }, -} - -#[derive(ValueEnum, Clone, Debug)] -pub enum IngestionBackfillKind { - Digest, - RawCheckpoints, - TxAffectedObjects, -} diff --git a/crates/sui-mvr-indexer/src/config.rs b/crates/sui-mvr-indexer/src/config.rs deleted file mode 100644 index dceead17bfebb..0000000000000 --- a/crates/sui-mvr-indexer/src/config.rs +++ /dev/null @@ -1,632 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::db::ConnectionPoolConfig; -use crate::{backfill::BackfillTaskKind, handlers::pruner::PrunableTable}; -use clap::{Args, Parser, Subcommand}; -use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, net::SocketAddr, path::PathBuf}; -use strum::IntoEnumIterator; -use sui_json_rpc::name_service::NameServiceConfig; -use sui_types::base_types::{ObjectID, SuiAddress}; -use url::Url; - -/// The primary purpose of objects_history is to serve consistency query. -/// A short retention is sufficient. -const OBJECTS_HISTORY_EPOCHS_TO_KEEP: u64 = 2; - -#[derive(Parser, Clone, Debug)] -#[clap( - name = "Sui indexer", - about = "An off-fullnode service serving data from Sui protocol" -)] -pub struct IndexerConfig { - #[clap(long, alias = "db-url")] - pub database_url: Url, - - #[clap(flatten)] - pub connection_pool_config: ConnectionPoolConfig, - - #[clap(long, default_value = "0.0.0.0:9184")] - pub metrics_address: SocketAddr, - - #[command(subcommand)] - pub command: Command, -} - -#[derive(Args, Debug, Clone)] -pub struct NameServiceOptions { - #[arg(default_value_t = NameServiceConfig::default().package_address)] - #[arg(long = "name-service-package-address")] - pub package_address: SuiAddress, - #[arg(default_value_t = NameServiceConfig::default().registry_id)] - #[arg(long = "name-service-registry-id")] - pub registry_id: ObjectID, - #[arg(default_value_t = NameServiceConfig::default().reverse_registry_id)] - #[arg(long = "name-service-reverse-registry-id")] - pub reverse_registry_id: ObjectID, -} - -impl NameServiceOptions { - pub fn to_config(&self) -> NameServiceConfig { - let Self { - package_address, - registry_id, - reverse_registry_id, - } = self.clone(); - NameServiceConfig { - package_address, - registry_id, - reverse_registry_id, - } - } -} - -impl Default for NameServiceOptions { - fn default() -> Self { - let NameServiceConfig { - package_address, - registry_id, - reverse_registry_id, - } = NameServiceConfig::default(); - Self { - package_address, - registry_id, - reverse_registry_id, - } - } -} - -#[derive(Args, Debug, Clone)] -pub struct JsonRpcConfig { - #[command(flatten)] - pub name_service_options: NameServiceOptions, - - #[clap(long, default_value = "0.0.0.0:9000")] - pub rpc_address: SocketAddr, - - #[clap(long)] - pub rpc_client_url: String, -} - -#[derive(Args, Debug, Default, Clone)] -#[group(required = true, multiple = true)] -pub struct IngestionSources { - #[arg(long)] - pub data_ingestion_path: Option, - - #[arg(long)] - pub remote_store_url: Option, - - #[arg(long)] - pub rpc_client_url: Option, -} - -#[derive(Args, Debug, Clone)] -pub struct IngestionConfig { - #[clap(flatten)] - pub sources: IngestionSources, - - #[arg( - long, - default_value_t = Self::DEFAULT_CHECKPOINT_DOWNLOAD_QUEUE_SIZE, - env = "DOWNLOAD_QUEUE_SIZE", - )] - pub checkpoint_download_queue_size: usize, - - /// Start checkpoint to ingest from, this is optional and if not provided, the ingestion will - /// start from the next checkpoint after the latest committed checkpoint. - #[arg(long, env = "START_CHECKPOINT")] - pub start_checkpoint: Option, - - /// End checkpoint to ingest until, this is optional and if not provided, the ingestion will - /// continue until u64::MAX. - #[arg(long, env = "END_CHECKPOINT")] - pub end_checkpoint: Option, - - #[arg( - long, - default_value_t = Self::DEFAULT_CHECKPOINT_DOWNLOAD_TIMEOUT, - env = "INGESTION_READER_TIMEOUT_SECS", - )] - pub checkpoint_download_timeout: u64, - - /// Limit indexing parallelism on big checkpoints to avoid OOMing by limiting the total size of - /// the checkpoint download queue. - #[arg( - long, - default_value_t = Self::DEFAULT_CHECKPOINT_DOWNLOAD_QUEUE_SIZE_BYTES, - env = "CHECKPOINT_PROCESSING_BATCH_DATA_LIMIT", - )] - pub checkpoint_download_queue_size_bytes: usize, - - /// Whether to delete processed checkpoint files from the local directory, - /// when running Fullnode-colocated indexer. - #[arg(long, default_value_t = true)] - pub gc_checkpoint_files: bool, -} - -impl IngestionConfig { - const DEFAULT_CHECKPOINT_DOWNLOAD_QUEUE_SIZE: usize = 200; - const DEFAULT_CHECKPOINT_DOWNLOAD_QUEUE_SIZE_BYTES: usize = 20_000_000; - const DEFAULT_CHECKPOINT_DOWNLOAD_TIMEOUT: u64 = 20; -} - -impl Default for IngestionConfig { - fn default() -> Self { - Self { - sources: Default::default(), - start_checkpoint: None, - end_checkpoint: None, - checkpoint_download_queue_size: Self::DEFAULT_CHECKPOINT_DOWNLOAD_QUEUE_SIZE, - checkpoint_download_timeout: Self::DEFAULT_CHECKPOINT_DOWNLOAD_TIMEOUT, - checkpoint_download_queue_size_bytes: - Self::DEFAULT_CHECKPOINT_DOWNLOAD_QUEUE_SIZE_BYTES, - gc_checkpoint_files: true, - } - } -} - -#[derive(Args, Debug, Clone)] -pub struct BackFillConfig { - /// Maximum number of concurrent tasks to run. - #[arg( - long, - default_value_t = Self::DEFAULT_MAX_CONCURRENCY, - )] - pub max_concurrency: usize, - /// Number of checkpoints to backfill in a single SQL command. - #[arg( - long, - default_value_t = Self::DEFAULT_CHUNK_SIZE, - )] - pub chunk_size: usize, -} - -impl BackFillConfig { - const DEFAULT_MAX_CONCURRENCY: usize = 10; - const DEFAULT_CHUNK_SIZE: usize = 1000; -} - -#[derive(Subcommand, Clone, Debug)] -pub enum Command { - Indexer { - #[command(flatten)] - ingestion_config: IngestionConfig, - #[command(flatten)] - snapshot_config: SnapshotLagConfig, - #[command(flatten)] - pruning_options: PruningOptions, - #[command(flatten)] - upload_options: UploadOptions, - }, - JsonRpcService(JsonRpcConfig), - ResetDatabase { - #[clap(long)] - force: bool, - /// If true, only drop all tables but do not run the migrations. - /// That is, no tables will exist in the DB after the reset. - #[clap(long, default_value_t = false)] - skip_migrations: bool, - }, - /// Run through the migration scripts. - RunMigrations, - /// Backfill DB tables for some ID range [\start, \end]. - /// The tool will automatically slice it into smaller ranges and for each range, - /// it first makes a read query to the DB to get data needed for backfil if needed, - /// which then can be processed and written back to the DB. - /// To add a new backfill, add a new module and implement the `BackfillTask` trait. - /// full_objects_history.rs provides an example to do SQL-only backfills. - /// system_state_summary_json.rs provides an example to do SQL + processing backfills. - RunBackFill { - /// Start of the range to backfill, inclusive. - /// It can be a checkpoint number or an epoch or any other identifier that can be used to - /// slice the backfill range. - start: usize, - /// End of the range to backfill, inclusive. - end: usize, - #[clap(subcommand)] - runner_kind: BackfillTaskKind, - #[command(flatten)] - backfill_config: BackFillConfig, - }, - /// Restore the database from formal snaphots. - Restore(RestoreConfig), -} - -#[derive(Args, Default, Debug, Clone)] -pub struct PruningOptions { - /// Path to TOML file containing configuration for retention policies. - #[arg(long)] - pub pruning_config_path: Option, -} - -/// Represents the default retention policy and overrides for prunable tables. Instantiated only if -/// `PruningOptions` is provided on indexer start. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RetentionConfig { - /// Default retention policy for all tables. - pub epochs_to_keep: u64, - /// A map of tables to their respective retention policies that will override the default. - /// Prunable tables not named here will use the default retention policy. - #[serde(default)] - pub overrides: HashMap, -} - -impl PruningOptions { - /// Load default retention policy and overrides from file. - pub fn load_from_file(&self) -> Option { - let config_path = self.pruning_config_path.as_ref()?; - - let contents = std::fs::read_to_string(config_path) - .expect("Failed to read default retention policy and overrides from file"); - let retention_with_overrides = toml::de::from_str::(&contents) - .expect("Failed to parse into RetentionConfig struct"); - - let default_retention = retention_with_overrides.epochs_to_keep; - - assert!( - default_retention > 0, - "Default retention must be greater than 0" - ); - assert!( - retention_with_overrides - .overrides - .values() - .all(|&policy| policy > 0), - "All retention overrides must be greater than 0" - ); - - Some(retention_with_overrides) - } -} - -impl RetentionConfig { - /// Create a new `RetentionConfig` with the specified default retention and overrides. Call - /// `finalize()` on the instance to update the `policies` field with the default retention - /// policy for all tables that do not have an override specified. - pub fn new(epochs_to_keep: u64, overrides: HashMap) -> Self { - Self { - epochs_to_keep, - overrides, - } - } - - pub fn new_with_default_retention_only_for_testing(epochs_to_keep: u64) -> Self { - let mut overrides = HashMap::new(); - overrides.insert( - PrunableTable::ObjectsHistory, - OBJECTS_HISTORY_EPOCHS_TO_KEEP, - ); - - Self::new(epochs_to_keep, HashMap::new()) - } - - /// Consumes this struct to produce a full mapping of every prunable table and its retention - /// policy. By default, every prunable table will have the default retention policy from - /// `epochs_to_keep`. Some tables like `objects_history` will observe a different default - /// retention policy. These default values are overridden by any entries in `overrides`. - pub fn retention_policies(self) -> HashMap { - let RetentionConfig { - epochs_to_keep, - mut overrides, - } = self; - - for table in PrunableTable::iter() { - let default_retention = match table { - PrunableTable::ObjectsHistory => OBJECTS_HISTORY_EPOCHS_TO_KEEP, - _ => epochs_to_keep, - }; - - overrides.entry(table).or_insert(default_retention); - } - - overrides - } -} - -#[derive(Args, Debug, Clone)] -pub struct SnapshotLagConfig { - #[arg( - long = "objects-snapshot-min-checkpoint-lag", - default_value_t = Self::DEFAULT_MIN_LAG, - env = "OBJECTS_SNAPSHOT_MIN_CHECKPOINT_LAG", - )] - pub snapshot_min_lag: usize, - - #[arg( - long = "objects-snapshot-sleep-duration", - default_value_t = Self::DEFAULT_SLEEP_DURATION_SEC, - )] - pub sleep_duration: u64, -} - -impl SnapshotLagConfig { - const DEFAULT_MIN_LAG: usize = 300; - const DEFAULT_SLEEP_DURATION_SEC: u64 = 5; -} - -impl Default for SnapshotLagConfig { - fn default() -> Self { - SnapshotLagConfig { - snapshot_min_lag: Self::DEFAULT_MIN_LAG, - sleep_duration: Self::DEFAULT_SLEEP_DURATION_SEC, - } - } -} - -#[derive(Args, Debug, Clone, Default)] -pub struct UploadOptions { - #[arg(long, env = "GCS_DISPLAY_BUCKET")] - pub gcs_display_bucket: Option, - #[arg(long, env = "GCS_CRED_PATH")] - pub gcs_cred_path: Option, -} - -#[derive(Args, Debug, Clone)] -pub struct RestoreConfig { - #[arg(long, env = "START_EPOCH", required = true)] - pub start_epoch: u64, - #[arg(long, env = "SNAPSHOT_ENDPOINT")] - pub snapshot_endpoint: String, - #[arg(long, env = "SNAPSHOT_BUCKET")] - pub snapshot_bucket: String, - #[arg(long, env = "SNAPSHOT_DOWNLOAD_DIR", required = true)] - pub snapshot_download_dir: String, - - #[arg(long, env = "GCS_ARCHIVE_BUCKET")] - pub gcs_archive_bucket: String, - #[arg(long, env = "GCS_DISPLAY_BUCKET")] - pub gcs_display_bucket: String, - - #[arg(env = "OBJECT_STORE_CONCURRENT_LIMIT")] - pub object_store_concurrent_limit: usize, - #[arg(env = "OBJECT_STORE_MAX_TIMEOUT_SECS")] - pub object_store_max_timeout_secs: u64, -} - -impl Default for RestoreConfig { - fn default() -> Self { - Self { - start_epoch: 0, // not used b/c it's required - snapshot_endpoint: "https://formal-snapshot.mainnet.sui.io".to_string(), - snapshot_bucket: "mysten-mainnet-formal".to_string(), - snapshot_download_dir: "".to_string(), // not used b/c it's required - gcs_archive_bucket: "mysten-mainnet-archives".to_string(), - gcs_display_bucket: "mysten-mainnet-display-table".to_string(), - object_store_concurrent_limit: 50, - object_store_max_timeout_secs: 512, - } - } -} - -#[derive(Args, Debug, Clone)] -pub struct BenchmarkConfig { - #[arg( - long, - default_value_t = 200, - help = "Number of transactions in a checkpoint." - )] - pub checkpoint_size: u64, - #[arg( - long, - default_value_t = 2000, - help = "Total number of synthetic checkpoints to generate." - )] - pub num_checkpoints: u64, - #[arg( - long, - default_value_t = 1, - help = "Customize the first checkpoint sequence number to be committed, must be non-zero." - )] - pub starting_checkpoint: u64, - #[arg( - long, - default_value_t = false, - help = "Whether to reset the database before running." - )] - pub reset_db: bool, - #[arg( - long, - help = "Path to workload directory. If not provided, a temporary directory will be created.\ - If provided, synthetic workload generator will either load data from it if it exists or generate new data.\ - This avoids repeat generation of the same data." - )] - pub workload_dir: Option, -} - -#[cfg(test)] -mod test { - use super::*; - use std::io::Write; - use tap::Pipe; - use tempfile::NamedTempFile; - - fn parse_args<'a, T>(args: impl IntoIterator) -> Result - where - T: clap::Args + clap::FromArgMatches, - { - clap::Command::new("test") - .no_binary_name(true) - .pipe(T::augment_args) - .try_get_matches_from(args) - .and_then(|matches| T::from_arg_matches(&matches)) - } - - #[test] - fn name_service() { - parse_args::(["--name-service-registry-id=0x1"]).unwrap(); - parse_args::([ - "--name-service-package-address", - "0x0000000000000000000000000000000000000000000000000000000000000001", - ]) - .unwrap(); - parse_args::(["--name-service-reverse-registry-id=0x1"]).unwrap(); - parse_args::([ - "--name-service-registry-id=0x1", - "--name-service-package-address", - "0x0000000000000000000000000000000000000000000000000000000000000002", - "--name-service-reverse-registry-id=0x3", - ]) - .unwrap(); - parse_args::([]).unwrap(); - } - - #[test] - fn ingestion_sources() { - parse_args::(["--data-ingestion-path=/tmp/foo"]).unwrap(); - parse_args::(["--remote-store-url=http://example.com"]).unwrap(); - parse_args::(["--rpc-client-url=http://example.com"]).unwrap(); - - parse_args::([ - "--data-ingestion-path=/tmp/foo", - "--remote-store-url=http://example.com", - "--rpc-client-url=http://example.com", - ]) - .unwrap(); - - // At least one must be present - parse_args::([]).unwrap_err(); - } - - #[test] - fn json_rpc_config() { - parse_args::(["--rpc-client-url=http://example.com"]).unwrap(); - - // Can include name service options and bind address - parse_args::([ - "--rpc-address=127.0.0.1:8080", - "--name-service-registry-id=0x1", - "--rpc-client-url=http://example.com", - ]) - .unwrap(); - - // fullnode rpc url must be present - parse_args::([]).unwrap_err(); - } - - #[test] - fn pruning_options_with_objects_history_override() { - let mut temp_file = NamedTempFile::new().unwrap(); - let toml_content = r#" - epochs_to_keep = 5 - - [overrides] - objects_history = 10 - transactions = 20 - "#; - temp_file.write_all(toml_content.as_bytes()).unwrap(); - let temp_path: PathBuf = temp_file.path().to_path_buf(); - let pruning_options = PruningOptions { - pruning_config_path: Some(temp_path.clone()), - }; - let retention_config = pruning_options.load_from_file().unwrap(); - - // Assert the parsed values - assert_eq!(retention_config.epochs_to_keep, 5); - assert_eq!( - retention_config - .overrides - .get(&PrunableTable::ObjectsHistory) - .copied(), - Some(10) - ); - assert_eq!( - retention_config - .overrides - .get(&PrunableTable::Transactions) - .copied(), - Some(20) - ); - assert_eq!(retention_config.overrides.len(), 2); - - let retention_policies = retention_config.retention_policies(); - - for table in PrunableTable::iter() { - let Some(retention) = retention_policies.get(&table).copied() else { - panic!("Expected a retention policy for table {:?}", table); - }; - - match table { - PrunableTable::ObjectsHistory => assert_eq!(retention, 10), - PrunableTable::Transactions => assert_eq!(retention, 20), - _ => assert_eq!(retention, 5), - }; - } - } - - #[test] - fn pruning_options_no_objects_history_override() { - let mut temp_file = NamedTempFile::new().unwrap(); - let toml_content = r#" - epochs_to_keep = 5 - - [overrides] - tx_affected_addresses = 10 - transactions = 20 - "#; - temp_file.write_all(toml_content.as_bytes()).unwrap(); - let temp_path: PathBuf = temp_file.path().to_path_buf(); - let pruning_options = PruningOptions { - pruning_config_path: Some(temp_path.clone()), - }; - let retention_config = pruning_options.load_from_file().unwrap(); - - // Assert the parsed values - assert_eq!(retention_config.epochs_to_keep, 5); - assert_eq!( - retention_config - .overrides - .get(&PrunableTable::TxAffectedAddresses) - .copied(), - Some(10) - ); - assert_eq!( - retention_config - .overrides - .get(&PrunableTable::Transactions) - .copied(), - Some(20) - ); - assert_eq!(retention_config.overrides.len(), 2); - - let retention_policies = retention_config.retention_policies(); - - for table in PrunableTable::iter() { - let Some(retention) = retention_policies.get(&table).copied() else { - panic!("Expected a retention policy for table {:?}", table); - }; - - match table { - PrunableTable::ObjectsHistory => { - assert_eq!(retention, OBJECTS_HISTORY_EPOCHS_TO_KEEP) - } - PrunableTable::TxAffectedAddresses => assert_eq!(retention, 10), - PrunableTable::Transactions => assert_eq!(retention, 20), - _ => assert_eq!(retention, 5), - }; - } - } - - #[test] - fn test_invalid_pruning_config_file() { - let toml_str = r#" - epochs_to_keep = 5 - - [overrides] - objects_history = 10 - transactions = 20 - invalid_table = 30 - "#; - - let result = toml::from_str::(toml_str); - assert!(result.is_err(), "Expected an error, but parsing succeeded"); - - if let Err(e) = result { - assert!( - e.to_string().contains("unknown variant `invalid_table`"), - "Error message doesn't mention the invalid table" - ); - } - } -} diff --git a/crates/sui-mvr-indexer/src/database.rs b/crates/sui-mvr-indexer/src/database.rs deleted file mode 100644 index 9c1446ff9c8ed..0000000000000 --- a/crates/sui-mvr-indexer/src/database.rs +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::Arc; - -use diesel::prelude::ConnectionError; -use diesel_async::pooled_connection::bb8::Pool; -use diesel_async::pooled_connection::bb8::PooledConnection; -use diesel_async::pooled_connection::bb8::RunError; -use diesel_async::pooled_connection::AsyncDieselConnectionManager; -use diesel_async::pooled_connection::PoolError; -use diesel_async::RunQueryDsl; -use diesel_async::{AsyncConnection, AsyncPgConnection}; -use futures::FutureExt; -use url::Url; - -use crate::db::ConnectionConfig; -use crate::db::ConnectionPoolConfig; - -#[derive(Clone, Debug)] -pub struct ConnectionPool { - database_url: Arc, - pool: Pool, -} - -impl ConnectionPool { - pub async fn new(database_url: Url, config: ConnectionPoolConfig) -> Result { - let database_url = Arc::new(database_url); - let connection_config = config.connection_config(); - let mut manager_config = diesel_async::pooled_connection::ManagerConfig::default(); - manager_config.custom_setup = - Box::new(move |url| establish_connection(url, connection_config).boxed()); - let manager = - AsyncDieselConnectionManager::new_with_config(database_url.as_str(), manager_config); - - Pool::builder() - .max_size(config.pool_size) - .connection_timeout(config.connection_timeout) - .build(manager) - .await - .map(|pool| Self { database_url, pool }) - } - - /// Retrieves a connection from the pool. - pub async fn get(&self) -> Result, RunError> { - self.pool.get().await.map(Connection::PooledConnection) - } - - /// Get a new dedicated connection that will not be managed by the pool. - /// An application may want a persistent connection (e.g. to do a - /// postgres LISTEN) that will not be closed or repurposed by the pool. - /// - /// This method allows reusing the manager's configuration but otherwise - /// bypassing the pool - pub async fn dedicated_connection(&self) -> Result, PoolError> { - self.pool - .dedicated_connection() - .await - .map(Connection::Dedicated) - } - - /// Returns information about the current state of the pool. - pub fn state(&self) -> bb8::State { - self.pool.state() - } - - /// Returns the database url that this pool is configured with - pub fn url(&self) -> &Url { - &self.database_url - } -} - -pub enum Connection<'a> { - PooledConnection(PooledConnection<'a, AsyncPgConnection>), - Dedicated(AsyncPgConnection), -} - -impl Connection<'static> { - pub async fn dedicated(database_url: &Url) -> Result { - AsyncPgConnection::establish(database_url.as_str()) - .await - .map(Connection::Dedicated) - } - - /// Run the provided Migrations - pub async fn run_pending_migrations( - self, - migrations: M, - ) -> diesel::migration::Result>> - where - M: diesel::migration::MigrationSource + Send + 'static, - { - use diesel::migration::MigrationVersion; - use diesel_migrations::MigrationHarness; - - let mut connection = - diesel_async::async_connection_wrapper::AsyncConnectionWrapper::::from(self); - - tokio::task::spawn_blocking(move || { - connection - .run_pending_migrations(migrations) - .map(|versions| versions.iter().map(MigrationVersion::as_owned).collect()) - }) - .await - .unwrap() - } -} - -impl<'a> std::ops::Deref for Connection<'a> { - type Target = AsyncPgConnection; - - fn deref(&self) -> &Self::Target { - match self { - Connection::PooledConnection(pooled) => pooled.deref(), - Connection::Dedicated(dedicated) => dedicated, - } - } -} - -impl<'a> std::ops::DerefMut for Connection<'a> { - fn deref_mut(&mut self) -> &mut AsyncPgConnection { - match self { - Connection::PooledConnection(pooled) => pooled.deref_mut(), - Connection::Dedicated(dedicated) => dedicated, - } - } -} - -impl ConnectionConfig { - async fn apply(&self, connection: &mut AsyncPgConnection) -> Result<(), diesel::result::Error> { - diesel::sql_query(format!( - "SET statement_timeout = {}", - self.statement_timeout.as_millis(), - )) - .execute(connection) - .await?; - - if self.read_only { - diesel::sql_query("SET default_transaction_read_only = 'on'") - .execute(connection) - .await?; - } - - Ok(()) - } -} - -/// Function used by the Connection Pool Manager to establish and setup new connections -async fn establish_connection( - url: &str, - config: ConnectionConfig, -) -> Result { - let mut connection = AsyncPgConnection::establish(url).await?; - - config - .apply(&mut connection) - .await - .map_err(ConnectionError::CouldntSetupConfiguration)?; - - Ok(connection) -} diff --git a/crates/sui-mvr-indexer/src/db.rs b/crates/sui-mvr-indexer/src/db.rs deleted file mode 100644 index 4a2893603bb10..0000000000000 --- a/crates/sui-mvr-indexer/src/db.rs +++ /dev/null @@ -1,395 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::database::Connection; -use crate::errors::IndexerError; -use crate::handlers::pruner::PrunableTable; -use clap::Args; -use diesel::migration::{Migration, MigrationSource, MigrationVersion}; -use diesel::pg::Pg; -use diesel::prelude::QueryableByName; -use diesel::table; -use diesel::QueryDsl; -use diesel_migrations::{embed_migrations, EmbeddedMigrations}; -use std::collections::{BTreeSet, HashSet}; -use std::time::Duration; -use strum::IntoEnumIterator; -use tracing::info; - -table! { - __diesel_schema_migrations (version) { - version -> VarChar, - run_on -> Timestamp, - } -} - -const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations/pg"); - -#[derive(Args, Debug, Clone)] -pub struct ConnectionPoolConfig { - #[arg(long, default_value_t = 100)] - #[arg(env = "DB_POOL_SIZE")] - pub pool_size: u32, - #[arg(long, value_parser = parse_duration, default_value = "30")] - #[arg(env = "DB_CONNECTION_TIMEOUT")] - pub connection_timeout: Duration, - #[arg(long, value_parser = parse_duration, default_value = "3600")] - #[arg(env = "DB_STATEMENT_TIMEOUT")] - pub statement_timeout: Duration, -} - -fn parse_duration(arg: &str) -> Result { - let seconds = arg.parse()?; - Ok(std::time::Duration::from_secs(seconds)) -} - -impl ConnectionPoolConfig { - const DEFAULT_POOL_SIZE: u32 = 100; - const DEFAULT_CONNECTION_TIMEOUT: u64 = 30; - const DEFAULT_STATEMENT_TIMEOUT: u64 = 3600; - - pub(crate) fn connection_config(&self) -> ConnectionConfig { - ConnectionConfig { - statement_timeout: self.statement_timeout, - read_only: false, - } - } - - pub fn set_pool_size(&mut self, size: u32) { - self.pool_size = size; - } - - pub fn set_connection_timeout(&mut self, timeout: Duration) { - self.connection_timeout = timeout; - } - - pub fn set_statement_timeout(&mut self, timeout: Duration) { - self.statement_timeout = timeout; - } -} - -impl Default for ConnectionPoolConfig { - fn default() -> Self { - Self { - pool_size: Self::DEFAULT_POOL_SIZE, - connection_timeout: Duration::from_secs(Self::DEFAULT_CONNECTION_TIMEOUT), - statement_timeout: Duration::from_secs(Self::DEFAULT_STATEMENT_TIMEOUT), - } - } -} - -#[derive(Debug, Clone, Copy)] -pub struct ConnectionConfig { - pub statement_timeout: Duration, - pub read_only: bool, -} - -/// Checks that the local migration scripts is a prefix of the records in the database. -/// This allows us run migration scripts against a DB at anytime, without worrying about -/// existing readers fail over. -/// We do however need to make sure that whenever we are deploying a new version of either reader or writer, -/// we must first run migration scripts to ensure that there is not more local scripts than in the DB record. -pub async fn check_db_migration_consistency(conn: &mut Connection<'_>) -> Result<(), IndexerError> { - info!("Starting compatibility check"); - let migrations: Vec>> = MIGRATIONS.migrations().map_err(|err| { - IndexerError::DbMigrationError(format!( - "Failed to fetch local migrations from schema: {err}" - )) - })?; - let local_migrations: Vec<_> = migrations - .into_iter() - .map(|m| m.name().version().as_owned()) - .collect(); - check_db_migration_consistency_impl(conn, local_migrations).await?; - info!("Compatibility check passed"); - Ok(()) -} - -async fn check_db_migration_consistency_impl( - conn: &mut Connection<'_>, - local_migrations: Vec>, -) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - // Unfortunately we cannot call applied_migrations() directly on the connection, - // since it implicitly creates the __diesel_schema_migrations table if it doesn't exist, - // which is a write operation that we don't want to do in this function. - let applied_migrations: BTreeSet> = BTreeSet::from_iter( - __diesel_schema_migrations::table - .select(__diesel_schema_migrations::version) - .load(conn) - .await?, - ); - - // We check that the local migrations is a subset of the applied migrations. - let unapplied_migrations: Vec<_> = local_migrations - .into_iter() - .filter(|m| !applied_migrations.contains(m)) - .collect(); - - if unapplied_migrations.is_empty() { - return Ok(()); - } - - Err(IndexerError::DbMigrationError(format!( - "This binary expected the following migrations to have been run, and they were not: {:?}", - unapplied_migrations - ))) -} - -/// Check that prunable tables exist in the database. -pub async fn check_prunable_tables_valid(conn: &mut Connection<'_>) -> Result<(), IndexerError> { - info!("Starting compatibility check"); - - use diesel_async::RunQueryDsl; - - let select_parent_tables = r#" - SELECT c.relname AS table_name - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - LEFT JOIN pg_partitioned_table pt ON pt.partrelid = c.oid - WHERE c.relkind IN ('r', 'p') -- 'r' for regular tables, 'p' for partitioned tables - AND n.nspname = 'public' - AND ( - pt.partrelid IS NOT NULL -- This is a partitioned (parent) table - OR NOT EXISTS ( -- This is not a partition (child table) - SELECT 1 - FROM pg_inherits i - WHERE i.inhrelid = c.oid - ) - ); - "#; - - #[derive(QueryableByName)] - struct TableName { - #[diesel(sql_type = diesel::sql_types::Text)] - table_name: String, - } - - let result: Vec = diesel::sql_query(select_parent_tables) - .load(conn) - .await - .map_err(|e| IndexerError::DbMigrationError(format!("Failed to fetch tables: {e}")))?; - - let parent_tables_from_db: HashSet<_> = result.into_iter().map(|t| t.table_name).collect(); - - for key in PrunableTable::iter() { - if !parent_tables_from_db.contains(key.as_ref()) { - return Err(IndexerError::GenericError(format!( - "Invalid retention policy override provided for table {}: does not exist in the database", - key - ))); - } - } - - info!("Compatibility check passed"); - Ok(()) -} - -pub use setup_postgres::{reset_database, run_migrations}; - -pub mod setup_postgres { - use crate::{database::Connection, db::MIGRATIONS}; - use anyhow::anyhow; - use diesel_async::RunQueryDsl; - use tracing::info; - - pub async fn reset_database(mut conn: Connection<'static>) -> Result<(), anyhow::Error> { - info!("Resetting PG database ..."); - clear_database(&mut conn).await?; - run_migrations(conn).await?; - info!("Reset database complete."); - Ok(()) - } - - pub async fn clear_database(conn: &mut Connection<'static>) -> Result<(), anyhow::Error> { - info!("Clearing the database..."); - let drop_all_tables = " - DO $$ DECLARE - r RECORD; - BEGIN - FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = 'public') - LOOP - EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; - END LOOP; - END $$;"; - diesel::sql_query(drop_all_tables).execute(conn).await?; - info!("Dropped all tables."); - - let drop_all_procedures = " - DO $$ DECLARE - r RECORD; - BEGIN - FOR r IN (SELECT proname, oidvectortypes(proargtypes) as argtypes - FROM pg_proc INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid) - WHERE ns.nspname = 'public' AND prokind = 'p') - LOOP - EXECUTE 'DROP PROCEDURE IF EXISTS ' || quote_ident(r.proname) || '(' || r.argtypes || ') CASCADE'; - END LOOP; - END $$;"; - diesel::sql_query(drop_all_procedures).execute(conn).await?; - info!("Dropped all procedures."); - - let drop_all_functions = " - DO $$ DECLARE - r RECORD; - BEGIN - FOR r IN (SELECT proname, oidvectortypes(proargtypes) as argtypes - FROM pg_proc INNER JOIN pg_namespace ON (pg_proc.pronamespace = pg_namespace.oid) - WHERE pg_namespace.nspname = 'public' AND prokind = 'f') - LOOP - EXECUTE 'DROP FUNCTION IF EXISTS ' || quote_ident(r.proname) || '(' || r.argtypes || ') CASCADE'; - END LOOP; - END $$;"; - diesel::sql_query(drop_all_functions).execute(conn).await?; - info!("Database cleared."); - Ok(()) - } - - pub async fn run_migrations(conn: Connection<'static>) -> Result<(), anyhow::Error> { - info!("Running migrations ..."); - conn.run_pending_migrations(MIGRATIONS) - .await - .map_err(|e| anyhow!("Failed to run migrations {e}"))?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use crate::database::{Connection, ConnectionPool}; - use crate::db::{ - check_db_migration_consistency, check_db_migration_consistency_impl, reset_database, - ConnectionPoolConfig, MIGRATIONS, - }; - use crate::tempdb::TempDb; - use diesel::migration::{Migration, MigrationSource}; - use diesel::pg::Pg; - use diesel_migrations::MigrationHarness; - - // Check that the migration records in the database created from the local schema - // pass the consistency check. - #[tokio::test] - async fn db_migration_consistency_smoke_test() { - let database = TempDb::new().unwrap(); - let pool = ConnectionPool::new( - database.database().url().to_owned(), - ConnectionPoolConfig { - pool_size: 2, - ..Default::default() - }, - ) - .await - .unwrap(); - - reset_database(pool.dedicated_connection().await.unwrap()) - .await - .unwrap(); - check_db_migration_consistency(&mut pool.get().await.unwrap()) - .await - .unwrap(); - } - - #[tokio::test] - async fn db_migration_consistency_non_prefix_test() { - let database = TempDb::new().unwrap(); - let pool = ConnectionPool::new( - database.database().url().to_owned(), - ConnectionPoolConfig { - pool_size: 2, - ..Default::default() - }, - ) - .await - .unwrap(); - - reset_database(pool.dedicated_connection().await.unwrap()) - .await - .unwrap(); - let mut connection = pool.get().await.unwrap(); - - let mut sync_connection_wrapper = - diesel_async::async_connection_wrapper::AsyncConnectionWrapper::::from( - pool.dedicated_connection().await.unwrap(), - ); - - tokio::task::spawn_blocking(move || { - sync_connection_wrapper - .revert_migration(MIGRATIONS.migrations().unwrap().last().unwrap()) - .unwrap(); - }) - .await - .unwrap(); - // Local migrations is one record more than the applied migrations. - // This will fail the consistency check since it's not a prefix. - assert!(check_db_migration_consistency(&mut connection) - .await - .is_err()); - - pool.dedicated_connection() - .await - .unwrap() - .run_pending_migrations(MIGRATIONS) - .await - .unwrap(); - // After running pending migrations they should be consistent. - check_db_migration_consistency(&mut connection) - .await - .unwrap(); - } - - #[tokio::test] - async fn db_migration_consistency_prefix_test() { - let database = TempDb::new().unwrap(); - let pool = ConnectionPool::new( - database.database().url().to_owned(), - ConnectionPoolConfig { - pool_size: 2, - ..Default::default() - }, - ) - .await - .unwrap(); - - reset_database(pool.dedicated_connection().await.unwrap()) - .await - .unwrap(); - - let migrations: Vec>> = MIGRATIONS.migrations().unwrap(); - let mut local_migrations: Vec<_> = migrations.iter().map(|m| m.name().version()).collect(); - local_migrations.pop(); - // Local migrations is one record less than the applied migrations. - // This should pass the consistency check since it's still a prefix. - check_db_migration_consistency_impl(&mut pool.get().await.unwrap(), local_migrations) - .await - .unwrap(); - } - - #[tokio::test] - async fn db_migration_consistency_subset_test() { - let database = TempDb::new().unwrap(); - let pool = ConnectionPool::new( - database.database().url().to_owned(), - ConnectionPoolConfig { - pool_size: 2, - ..Default::default() - }, - ) - .await - .unwrap(); - - reset_database(pool.dedicated_connection().await.unwrap()) - .await - .unwrap(); - - let migrations: Vec>> = MIGRATIONS.migrations().unwrap(); - let mut local_migrations: Vec<_> = migrations.iter().map(|m| m.name().version()).collect(); - local_migrations.remove(2); - - // Local migrations are missing one record compared to the applied migrations, which should - // still be okay. - check_db_migration_consistency_impl(&mut pool.get().await.unwrap(), local_migrations) - .await - .unwrap(); - } -} diff --git a/crates/sui-mvr-indexer/src/errors.rs b/crates/sui-mvr-indexer/src/errors.rs deleted file mode 100644 index c8971e39781ad..0000000000000 --- a/crates/sui-mvr-indexer/src/errors.rs +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use fastcrypto::error::FastCryptoError; -use jsonrpsee::core::Error as RpcError; -use jsonrpsee::types::error::CallError; -use sui_json_rpc::name_service::NameServiceError; -use thiserror::Error; - -use sui_types::base_types::ObjectIDParseError; -use sui_types::error::{SuiError, SuiObjectResponseError, UserInputError}; - -#[derive(Debug, Error)] -pub struct DataDownloadError { - pub error: IndexerError, - pub next_checkpoint_sequence_number: u64, -} - -impl std::fmt::Display for DataDownloadError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "next_checkpoint_seq: {}, error: {}", - self.next_checkpoint_sequence_number, self.error - ) - } -} - -#[derive(Debug, Error)] -pub enum IndexerError { - #[error("Indexer failed to read from archives store with error: `{0}`")] - ArchiveReaderError(String), - - #[error("Stream closed unexpectedly with error: `{0}`")] - ChannelClosed(String), - - #[error("Indexer failed to convert timestamp to NaiveDateTime with error: `{0}`")] - DateTimeParsingError(String), - - #[error("Indexer failed to deserialize event from events table with error: `{0}`")] - EventDeserializationError(String), - - #[error("Fullnode returns unexpected responses, which may block indexers from proceeding, with error: `{0}`")] - UnexpectedFullnodeResponseError(String), - - #[error("Indexer failed to transform data with error: `{0}`")] - DataTransformationError(String), - - #[error("Indexer failed to read fullnode with error: `{0}`")] - FullNodeReadingError(String), - - #[error("Indexer failed to convert structs to diesel Insertable with error: `{0}`")] - InsertableParsingError(String), - - #[error("Indexer failed to build JsonRpcServer with error: `{0}`")] - JsonRpcServerError(#[from] sui_json_rpc::error::Error), - - #[error("Indexer failed to find object mutations, which should never happen.")] - ObjectMutationNotAvailable, - - #[error("Indexer failed to build PG connection pool with error: `{0}`")] - PgConnectionPoolInitError(String), - - #[error("Indexer failed to get a pool connection from PG connection pool with error: `{0}`")] - PgPoolConnectionError(String), - - #[error("Indexer failed to read PostgresDB with error: `{0}`")] - PostgresReadError(String), - - #[error("Indexer failed to reset PostgresDB with error: `{0}`")] - PostgresResetError(String), - - #[error("Indexer failed to commit changes to PostgresDB with error: `{0}`")] - PostgresWriteError(String), - - #[error(transparent)] - PostgresError(#[from] diesel::result::Error), - - #[error("Indexer failed to initialize fullnode Http client with error: `{0}`")] - HttpClientInitError(String), - - #[error("Indexer failed to serialize/deserialize with error: `{0}`")] - SerdeError(String), - - #[error("Indexer error related to dynamic field: `{0}`")] - DynamicFieldError(String), - - #[error("Indexer does not support the feature with error: `{0}`")] - NotSupportedError(String), - - #[error("Indexer read corrupted/incompatible data from persistent storage: `{0}`")] - PersistentStorageDataCorruptionError(String), - - #[error("Indexer generic error: `{0}`")] - GenericError(String), - - #[error("GCS error: `{0}`")] - GcsError(String), - - #[error("Indexer failed to resolve object to move struct with error: `{0}`")] - ResolveMoveStructError(String), - - #[error(transparent)] - UncategorizedError(#[from] anyhow::Error), - - #[error(transparent)] - ObjectIdParseError(#[from] ObjectIDParseError), - - #[error("Invalid transaction digest with error: `{0}`")] - InvalidTransactionDigestError(String), - - #[error(transparent)] - SuiError(#[from] SuiError), - - #[error(transparent)] - BcsError(#[from] bcs::Error), - - #[error("Invalid argument with error: `{0}`")] - InvalidArgumentError(String), - - #[error(transparent)] - UserInputError(#[from] UserInputError), - - #[error("Indexer failed to resolve module with error: `{0}`")] - ModuleResolutionError(String), - - #[error(transparent)] - ObjectResponseError(#[from] SuiObjectResponseError), - - #[error(transparent)] - FastCryptoError(#[from] FastCryptoError), - - #[error("`{0}`: `{1}`")] - ErrorWithContext(String, Box), - - #[error("Indexer failed to send item to channel with error: `{0}`")] - MpscChannelError(String), - - #[error(transparent)] - NameServiceError(#[from] NameServiceError), - - #[error("Inconsistent migration records: {0}")] - DbMigrationError(String), -} - -pub trait Context { - fn context(self, context: &str) -> Result; -} - -impl Context for Result { - fn context(self, context: &str) -> Result { - self.map_err(|e| IndexerError::ErrorWithContext(context.to_string(), Box::new(e))) - } -} - -impl From for RpcError { - fn from(e: IndexerError) -> Self { - RpcError::Call(CallError::Failed(e.into())) - } -} - -impl From for IndexerError { - fn from(value: tokio::task::JoinError) -> Self { - IndexerError::UncategorizedError(anyhow::Error::from(value)) - } -} - -impl From for IndexerError { - fn from(value: diesel_async::pooled_connection::bb8::RunError) -> Self { - Self::PgPoolConnectionError(value.to_string()) - } -} diff --git a/crates/sui-mvr-indexer/src/handlers/checkpoint_handler.rs b/crates/sui-mvr-indexer/src/handlers/checkpoint_handler.rs deleted file mode 100644 index e7b0e55c7c4d1..0000000000000 --- a/crates/sui-mvr-indexer/src/handlers/checkpoint_handler.rs +++ /dev/null @@ -1,649 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::BTreeMap; -use std::sync::Arc; - -use async_trait::async_trait; -use itertools::Itertools; -use sui_types::dynamic_field::DynamicFieldInfo; -use tokio_util::sync::CancellationToken; -use tracing::{info, warn}; - -use move_core_types::language_storage::{StructTag, TypeTag}; -use mysten_metrics::{get_metrics, spawn_monitored_task}; -use sui_data_ingestion_core::Worker; -use sui_rpc_api::{CheckpointData, CheckpointTransaction}; -use sui_types::dynamic_field::DynamicFieldType; -use sui_types::effects::{ObjectChange, TransactionEffectsAPI}; -use sui_types::event::SystemEpochInfoEvent; -use sui_types::messages_checkpoint::{ - CertifiedCheckpointSummary, CheckpointContents, CheckpointSequenceNumber, -}; -use sui_types::object::Object; -use sui_types::object::Owner; -use sui_types::sui_system_state::{get_sui_system_state, SuiSystemStateTrait}; -use sui_types::transaction::TransactionDataAPI; - -use crate::errors::IndexerError; -use crate::handlers::committer::start_tx_checkpoint_commit_task; -use crate::metrics::IndexerMetrics; -use crate::models::display::StoredDisplay; -use crate::models::epoch::{EndOfEpochUpdate, EpochEndInfo, EpochStartInfo, StartOfEpochUpdate}; -use crate::models::obj_indices::StoredObjectVersion; -use crate::store::{IndexerStore, PgIndexerStore}; -use crate::types::{ - EventIndex, IndexedCheckpoint, IndexedDeletedObject, IndexedEvent, IndexedObject, - IndexedPackage, IndexedTransaction, IndexerResult, TransactionKind, TxIndex, -}; - -use super::tx_processor::EpochEndIndexingObjectStore; -use super::tx_processor::TxChangesProcessor; -use super::CheckpointDataToCommit; -use super::EpochToCommit; -use super::TransactionObjectChangesToCommit; - -const CHECKPOINT_QUEUE_SIZE: usize = 100; - -pub async fn new_handlers( - state: PgIndexerStore, - metrics: IndexerMetrics, - cancel: CancellationToken, - start_checkpoint_opt: Option, - end_checkpoint_opt: Option, -) -> Result<(CheckpointHandler, u64), IndexerError> { - let start_checkpoint = match start_checkpoint_opt { - Some(start_checkpoint) => start_checkpoint, - None => state - .get_latest_checkpoint_sequence_number() - .await? - .map(|seq| seq.saturating_add(1)) - .unwrap_or_default(), - }; - - let checkpoint_queue_size = std::env::var("CHECKPOINT_QUEUE_SIZE") - .unwrap_or(CHECKPOINT_QUEUE_SIZE.to_string()) - .parse::() - .unwrap(); - let global_metrics = get_metrics().unwrap(); - let (indexed_checkpoint_sender, indexed_checkpoint_receiver) = - mysten_metrics::metered_channel::channel( - checkpoint_queue_size, - &global_metrics - .channel_inflight - .with_label_values(&["checkpoint_indexing"]), - ); - - let state_clone = state.clone(); - let metrics_clone = metrics.clone(); - spawn_monitored_task!(start_tx_checkpoint_commit_task( - state_clone, - metrics_clone, - indexed_checkpoint_receiver, - cancel.clone(), - start_checkpoint, - end_checkpoint_opt, - )); - Ok(( - CheckpointHandler::new(state, metrics, indexed_checkpoint_sender), - start_checkpoint, - )) -} - -pub struct CheckpointHandler { - state: PgIndexerStore, - metrics: IndexerMetrics, - indexed_checkpoint_sender: mysten_metrics::metered_channel::Sender, -} - -#[async_trait] -impl Worker for CheckpointHandler { - type Result = (); - async fn process_checkpoint(&self, checkpoint: &CheckpointData) -> anyhow::Result<()> { - let time_now_ms = chrono::Utc::now().timestamp_millis(); - let cp_download_lag = time_now_ms - checkpoint.checkpoint_summary.timestamp_ms as i64; - info!( - "checkpoint download lag for cp {}: {} ms", - checkpoint.checkpoint_summary.sequence_number, cp_download_lag - ); - self.metrics.download_lag_ms.set(cp_download_lag); - self.metrics - .max_downloaded_checkpoint_sequence_number - .set(checkpoint.checkpoint_summary.sequence_number as i64); - self.metrics - .downloaded_checkpoint_timestamp_ms - .set(checkpoint.checkpoint_summary.timestamp_ms as i64); - info!( - "Indexer lag: downloaded checkpoint {} with time now {} and checkpoint time {}", - checkpoint.checkpoint_summary.sequence_number, - time_now_ms, - checkpoint.checkpoint_summary.timestamp_ms - ); - let checkpoint_data = Self::index_checkpoint( - &self.state, - checkpoint, - Arc::new(self.metrics.clone()), - Self::index_packages(std::slice::from_ref(checkpoint), &self.metrics), - ) - .await?; - self.indexed_checkpoint_sender.send(checkpoint_data).await?; - Ok(()) - } -} - -impl CheckpointHandler { - fn new( - state: PgIndexerStore, - metrics: IndexerMetrics, - indexed_checkpoint_sender: mysten_metrics::metered_channel::Sender, - ) -> Self { - Self { - state, - metrics, - indexed_checkpoint_sender, - } - } - - async fn index_epoch( - state: &PgIndexerStore, - data: &CheckpointData, - ) -> Result, IndexerError> { - let checkpoint_object_store = EpochEndIndexingObjectStore::new(data); - - let CheckpointData { - transactions, - checkpoint_summary, - checkpoint_contents: _, - } = data; - - // Genesis epoch - if *checkpoint_summary.sequence_number() == 0 { - info!("Processing genesis epoch"); - let system_state_summary = - get_sui_system_state(&checkpoint_object_store)?.into_sui_system_state_summary(); - return Ok(Some(EpochToCommit { - last_epoch: None, - new_epoch: StartOfEpochUpdate::new(system_state_summary, EpochStartInfo::default()), - })); - } - - // If not end of epoch, return - if checkpoint_summary.end_of_epoch_data.is_none() { - return Ok(None); - } - - let system_state_summary = - get_sui_system_state(&checkpoint_object_store)?.into_sui_system_state_summary(); - - let epoch_event_opt = transactions - .iter() - .find_map(|t| { - t.events.as_ref()?.data.iter().find_map(|ev| { - if ev.is_system_epoch_info_event() { - Some(bcs::from_bytes::(&ev.contents)) - } else { - None - } - }) - }) - .transpose()?; - if epoch_event_opt.is_none() { - warn!( - "No SystemEpochInfoEvent found at end of epoch {}, some epoch data will be set to default.", - checkpoint_summary.epoch, - ); - assert!( - system_state_summary.safe_mode, - "Sui is not in safe mode but no SystemEpochInfoEvent found at end of epoch {}", - checkpoint_summary.epoch - ); - } - - // At some point while committing data in epoch X - 1, we will encounter a new epoch X. We - // want to retrieve X - 2's network total transactions to calculate the number of - // transactions that occurred in epoch X - 1. - let first_tx_sequence_number = match system_state_summary.epoch { - // If first epoch change, this number is 0 - 1 => Ok(0), - _ => { - let last_epoch = system_state_summary.epoch - 2; - state - .get_network_total_transactions_by_end_of_epoch(last_epoch) - .await? - .ok_or_else(|| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Network total transactions for epoch {} not found", - last_epoch - )) - }) - } - }?; - - let epoch_end_info = EpochEndInfo::new(epoch_event_opt.as_ref()); - let epoch_start_info = EpochStartInfo::new( - checkpoint_summary.sequence_number.saturating_add(1), - checkpoint_summary.network_total_transactions, - epoch_event_opt.as_ref(), - ); - - Ok(Some(EpochToCommit { - last_epoch: Some(EndOfEpochUpdate::new( - checkpoint_summary, - first_tx_sequence_number, - epoch_end_info, - )), - new_epoch: StartOfEpochUpdate::new(system_state_summary, epoch_start_info), - })) - } - - fn derive_object_versions( - object_history_changes: &TransactionObjectChangesToCommit, - ) -> Vec { - let mut object_versions = vec![]; - for changed_obj in object_history_changes.changed_objects.iter() { - object_versions.push(StoredObjectVersion { - object_id: changed_obj.object.id().to_vec(), - object_version: changed_obj.object.version().value() as i64, - cp_sequence_number: changed_obj.checkpoint_sequence_number as i64, - }); - } - for deleted_obj in object_history_changes.deleted_objects.iter() { - object_versions.push(StoredObjectVersion { - object_id: deleted_obj.object_id.to_vec(), - object_version: deleted_obj.object_version as i64, - cp_sequence_number: deleted_obj.checkpoint_sequence_number as i64, - }); - } - object_versions - } - - async fn index_checkpoint( - state: &PgIndexerStore, - data: &CheckpointData, - metrics: Arc, - packages: Vec, - ) -> Result { - let checkpoint_seq = data.checkpoint_summary.sequence_number; - info!(checkpoint_seq, "Indexing checkpoint data blob"); - - // Index epoch - let epoch = Self::index_epoch(state, data).await?; - - // Index Objects - let object_changes: TransactionObjectChangesToCommit = - Self::index_objects(data, &metrics).await?; - let object_history_changes: TransactionObjectChangesToCommit = - Self::index_objects_history(data).await?; - let object_versions = Self::derive_object_versions(&object_history_changes); - - let (checkpoint, db_transactions, db_events, db_tx_indices, db_event_indices, db_displays) = { - let CheckpointData { - transactions, - checkpoint_summary, - checkpoint_contents, - } = data; - - let (db_transactions, db_events, db_tx_indices, db_event_indices, db_displays) = - Self::index_transactions( - transactions, - checkpoint_summary, - checkpoint_contents, - &metrics, - ) - .await?; - - let successful_tx_num: u64 = db_transactions.iter().map(|t| t.successful_tx_num).sum(); - ( - IndexedCheckpoint::from_sui_checkpoint( - checkpoint_summary, - checkpoint_contents, - successful_tx_num as usize, - ), - db_transactions, - db_events, - db_tx_indices, - db_event_indices, - db_displays, - ) - }; - let time_now_ms = chrono::Utc::now().timestamp_millis(); - metrics - .index_lag_ms - .set(time_now_ms - checkpoint.timestamp_ms as i64); - metrics - .max_indexed_checkpoint_sequence_number - .set(checkpoint.sequence_number as i64); - metrics - .indexed_checkpoint_timestamp_ms - .set(checkpoint.timestamp_ms as i64); - info!( - "Indexer lag: indexed checkpoint {} with time now {} and checkpoint time {}", - checkpoint.sequence_number, time_now_ms, checkpoint.timestamp_ms - ); - - Ok(CheckpointDataToCommit { - checkpoint, - transactions: db_transactions, - events: db_events, - tx_indices: db_tx_indices, - event_indices: db_event_indices, - display_updates: db_displays, - object_changes, - object_history_changes, - object_versions, - packages, - epoch, - }) - } - - async fn index_transactions( - transactions: &[CheckpointTransaction], - checkpoint_summary: &CertifiedCheckpointSummary, - checkpoint_contents: &CheckpointContents, - metrics: &IndexerMetrics, - ) -> IndexerResult<( - Vec, - Vec, - Vec, - Vec, - BTreeMap, - )> { - let checkpoint_seq = checkpoint_summary.sequence_number(); - - let mut tx_seq_num_iter = checkpoint_contents - .enumerate_transactions(checkpoint_summary) - .map(|(seq, execution_digest)| (execution_digest.transaction, seq)); - - if checkpoint_contents.size() != transactions.len() { - return Err(IndexerError::FullNodeReadingError(format!( - "CheckpointContents has different size {} compared to Transactions {} for checkpoint {}", - checkpoint_contents.size(), - transactions.len(), - checkpoint_seq - ))); - } - - let mut db_transactions = Vec::new(); - let mut db_events = Vec::new(); - let mut db_displays = BTreeMap::new(); - let mut db_tx_indices = Vec::new(); - let mut db_event_indices = Vec::new(); - - for tx in transactions { - let CheckpointTransaction { - transaction: sender_signed_data, - effects: fx, - events, - input_objects, - output_objects, - } = tx; - // Unwrap safe - we checked they have equal length above - let (tx_digest, tx_sequence_number) = tx_seq_num_iter.next().unwrap(); - if tx_digest != *sender_signed_data.digest() { - return Err(IndexerError::FullNodeReadingError(format!( - "Transactions has different ordering from CheckpointContents, for checkpoint {}, Mismatch found at {} v.s. {}", - checkpoint_seq, tx_digest, sender_signed_data.digest() - ))); - } - - let tx = sender_signed_data.transaction_data(); - let events = events - .as_ref() - .map(|events| events.data.clone()) - .unwrap_or_default(); - - let transaction_kind = if tx.is_system_tx() { - TransactionKind::SystemTransaction - } else { - TransactionKind::ProgrammableTransaction - }; - - db_events.extend(events.iter().enumerate().map(|(idx, event)| { - IndexedEvent::from_event( - tx_sequence_number, - idx as u64, - *checkpoint_seq, - tx_digest, - event, - checkpoint_summary.timestamp_ms, - ) - })); - - db_event_indices.extend( - events.iter().enumerate().map(|(idx, event)| { - EventIndex::from_event(tx_sequence_number, idx as u64, event) - }), - ); - - db_displays.extend( - events - .iter() - .flat_map(StoredDisplay::try_from_event) - .map(|display| (display.object_type.clone(), display)), - ); - - let objects: Vec<_> = input_objects.iter().chain(output_objects.iter()).collect(); - - let (balance_change, object_changes) = - TxChangesProcessor::new(&objects, metrics.clone()) - .get_changes(tx, fx, &tx_digest) - .await?; - - let db_txn = IndexedTransaction { - tx_sequence_number, - tx_digest, - checkpoint_sequence_number: *checkpoint_summary.sequence_number(), - timestamp_ms: checkpoint_summary.timestamp_ms, - sender_signed_data: sender_signed_data.data().clone(), - effects: fx.clone(), - object_changes, - balance_change, - events, - transaction_kind: transaction_kind.clone(), - successful_tx_num: if fx.status().is_ok() { - tx.kind().tx_count() as u64 - } else { - 0 - }, - }; - - db_transactions.push(db_txn); - - // Input Objects - let input_objects = tx - .input_objects() - .expect("committed txns have been validated") - .into_iter() - .map(|obj_kind| obj_kind.object_id()) - .collect(); - - // Changed Objects - let changed_objects = fx - .all_changed_objects() - .into_iter() - .map(|(object_ref, _owner, _write_kind)| object_ref.0) - .collect(); - - // Affected Objects - let affected_objects = fx - .object_changes() - .into_iter() - .map(|ObjectChange { id, .. }| id) - .collect(); - - // Payers - let payers = vec![tx.gas_owner()]; - - // Sender - let sender = tx.sender(); - - // Recipients - let recipients = fx - .all_changed_objects() - .into_iter() - .filter_map(|(_object_ref, owner, _write_kind)| match owner { - Owner::AddressOwner(address) => Some(address), - _ => None, - }) - .unique() - .collect(); - - // Move Calls - let move_calls = tx - .move_calls() - .into_iter() - .map(|(p, m, f)| (*p, m.to_string(), f.to_string())) - .collect(); - - db_tx_indices.push(TxIndex { - tx_sequence_number, - transaction_digest: tx_digest, - checkpoint_sequence_number: *checkpoint_seq, - input_objects, - changed_objects, - affected_objects, - sender, - payers, - recipients, - move_calls, - tx_kind: transaction_kind, - }); - } - Ok(( - db_transactions, - db_events, - db_tx_indices, - db_event_indices, - db_displays, - )) - } - - pub(crate) async fn index_objects( - data: &CheckpointData, - metrics: &IndexerMetrics, - ) -> Result { - let _timer = metrics.indexing_objects_latency.start_timer(); - let checkpoint_seq = data.checkpoint_summary.sequence_number; - - let eventually_removed_object_refs_post_version = - data.eventually_removed_object_refs_post_version(); - let indexed_eventually_removed_objects = eventually_removed_object_refs_post_version - .into_iter() - .map(|obj_ref| IndexedDeletedObject { - object_id: obj_ref.0, - object_version: obj_ref.1.into(), - checkpoint_sequence_number: checkpoint_seq, - }) - .collect(); - - let latest_live_output_objects = data.latest_live_output_objects(); - let changed_objects = latest_live_output_objects - .into_iter() - .map(|o| { - try_extract_df_kind(o) - .map(|df_kind| IndexedObject::from_object(checkpoint_seq, o.clone(), df_kind)) - }) - .collect::, _>>()?; - - Ok(TransactionObjectChangesToCommit { - changed_objects, - deleted_objects: indexed_eventually_removed_objects, - }) - } - - // similar to index_objects, but objects_history keeps all versions of objects - async fn index_objects_history( - data: &CheckpointData, - ) -> Result { - let checkpoint_seq = data.checkpoint_summary.sequence_number; - let deleted_objects = data - .transactions - .iter() - .flat_map(|tx| tx.removed_object_refs_post_version()) - .collect::>(); - let indexed_deleted_objects: Vec = deleted_objects - .into_iter() - .map(|obj_ref| IndexedDeletedObject { - object_id: obj_ref.0, - object_version: obj_ref.1.into(), - checkpoint_sequence_number: checkpoint_seq, - }) - .collect(); - - let output_objects: Vec<_> = data - .transactions - .iter() - .flat_map(|tx| &tx.output_objects) - .collect(); - - // TODO(gegaowp): the current df_info implementation is not correct, - // but we have decided remove all df_* except df_kind. - let changed_objects = output_objects - .into_iter() - .map(|o| { - try_extract_df_kind(o) - .map(|df_kind| IndexedObject::from_object(checkpoint_seq, o.clone(), df_kind)) - }) - .collect::, _>>()?; - - Ok(TransactionObjectChangesToCommit { - changed_objects, - deleted_objects: indexed_deleted_objects, - }) - } - - fn index_packages( - checkpoint_data: &[CheckpointData], - metrics: &IndexerMetrics, - ) -> Vec { - let _timer = metrics.indexing_packages_latency.start_timer(); - checkpoint_data - .iter() - .flat_map(|data| { - let checkpoint_sequence_number = data.checkpoint_summary.sequence_number; - data.transactions - .iter() - .flat_map(|tx| &tx.output_objects) - .filter_map(|o| { - if let sui_types::object::Data::Package(p) = &o.data { - Some(IndexedPackage { - package_id: o.id(), - move_package: p.clone(), - checkpoint_sequence_number, - }) - } else { - None - } - }) - .collect::>() - }) - .collect() - } -} - -/// If `o` is a dynamic `Field`, determine whether it represents a Dynamic Field or a Dynamic -/// Object Field based on its type. -fn try_extract_df_kind(o: &Object) -> IndexerResult> { - // Skip if not a move object - let Some(move_object) = o.data.try_as_move() else { - return Ok(None); - }; - - if !move_object.type_().is_dynamic_field() { - return Ok(None); - } - - let type_: StructTag = move_object.type_().clone().into(); - let [name, _] = type_.type_params.as_slice() else { - return Ok(None); - }; - - Ok(Some( - if matches!(name, TypeTag::Struct(s) if DynamicFieldInfo::is_dynamic_object_field_wrapper(s)) - { - DynamicFieldType::DynamicObject - } else { - DynamicFieldType::DynamicField - }, - )) -} diff --git a/crates/sui-mvr-indexer/src/handlers/committer.rs b/crates/sui-mvr-indexer/src/handlers/committer.rs deleted file mode 100644 index cd9f9a0385174..0000000000000 --- a/crates/sui-mvr-indexer/src/handlers/committer.rs +++ /dev/null @@ -1,257 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::{BTreeMap, HashMap}; - -use sui_types::messages_checkpoint::CheckpointSequenceNumber; -use tap::tap::TapFallible; -use tokio_util::sync::CancellationToken; -use tracing::instrument; -use tracing::{error, info}; - -use crate::metrics::IndexerMetrics; -use crate::store::IndexerStore; -use crate::types::IndexerResult; - -use super::{CheckpointDataToCommit, CommitterTables, CommitterWatermark, EpochToCommit}; - -pub(crate) const CHECKPOINT_COMMIT_BATCH_SIZE: usize = 100; - -pub async fn start_tx_checkpoint_commit_task( - state: S, - metrics: IndexerMetrics, - tx_indexing_receiver: mysten_metrics::metered_channel::Receiver, - cancel: CancellationToken, - mut next_checkpoint_sequence_number: CheckpointSequenceNumber, - end_checkpoint_opt: Option, -) -> IndexerResult<()> -where - S: IndexerStore + Clone + Sync + Send + 'static, -{ - use futures::StreamExt; - - info!("Indexer checkpoint commit task started..."); - let checkpoint_commit_batch_size = std::env::var("CHECKPOINT_COMMIT_BATCH_SIZE") - .unwrap_or(CHECKPOINT_COMMIT_BATCH_SIZE.to_string()) - .parse::() - .unwrap(); - info!("Using checkpoint commit batch size {checkpoint_commit_batch_size}"); - - let mut stream = mysten_metrics::metered_channel::ReceiverStream::new(tx_indexing_receiver) - .ready_chunks(checkpoint_commit_batch_size); - - let mut unprocessed = HashMap::new(); - let mut batch = vec![]; - - while let Some(indexed_checkpoint_batch) = stream.next().await { - if cancel.is_cancelled() { - break; - } - - // split the batch into smaller batches per epoch to handle partitioning - for checkpoint in indexed_checkpoint_batch { - unprocessed.insert(checkpoint.checkpoint.sequence_number, checkpoint); - } - while let Some(checkpoint) = unprocessed.remove(&next_checkpoint_sequence_number) { - let epoch = checkpoint.epoch.clone(); - batch.push(checkpoint); - next_checkpoint_sequence_number += 1; - let epoch_number_option = epoch.as_ref().map(|epoch| epoch.new_epoch_id()); - // The batch will consist of contiguous checkpoints and at most one epoch boundary at - // the end. - if batch.len() == checkpoint_commit_batch_size || epoch.is_some() { - commit_checkpoints(&state, batch, epoch, &metrics).await; - batch = vec![]; - } - if let Some(epoch_number) = epoch_number_option { - state.upload_display(epoch_number).await.tap_err(|e| { - error!( - "Failed to upload display table before epoch {} with error: {}", - epoch_number, - e.to_string() - ); - })?; - } - // stop adding to the commit batch if we've reached the end checkpoint - if let Some(end_checkpoint_sequence_number) = end_checkpoint_opt { - if next_checkpoint_sequence_number > end_checkpoint_sequence_number { - break; - } - } - } - if !batch.is_empty() { - commit_checkpoints(&state, batch, None, &metrics).await; - batch = vec![]; - } - - // stop the commit task if we've reached the end checkpoint - if let Some(end_checkpoint_sequence_number) = end_checkpoint_opt { - if next_checkpoint_sequence_number > end_checkpoint_sequence_number { - break; - } - } - } - Ok(()) -} - -/// Writes indexed checkpoint data to the database, and then update watermark upper bounds and -/// metrics. Expects `indexed_checkpoint_batch` to be non-empty, and contain contiguous checkpoints. -/// There can be at most one epoch boundary at the end. If an epoch boundary is detected, -/// epoch-partitioned tables must be advanced. -// Unwrap: Caller needs to make sure indexed_checkpoint_batch is not empty -#[instrument(skip_all, fields( - first = indexed_checkpoint_batch.first().as_ref().unwrap().checkpoint.sequence_number, - last = indexed_checkpoint_batch.last().as_ref().unwrap().checkpoint.sequence_number -))] -async fn commit_checkpoints( - state: &S, - indexed_checkpoint_batch: Vec, - epoch: Option, - metrics: &IndexerMetrics, -) where - S: IndexerStore + Clone + Sync + Send + 'static, -{ - let mut checkpoint_batch = vec![]; - let mut tx_batch = vec![]; - let mut events_batch = vec![]; - let mut tx_indices_batch = vec![]; - let mut event_indices_batch = vec![]; - let mut display_updates_batch = BTreeMap::new(); - let mut object_changes_batch = vec![]; - let mut object_history_changes_batch = vec![]; - let mut object_versions_batch = vec![]; - let mut packages_batch = vec![]; - - for indexed_checkpoint in indexed_checkpoint_batch { - let CheckpointDataToCommit { - checkpoint, - transactions, - events, - event_indices, - tx_indices, - display_updates, - object_changes, - object_history_changes, - object_versions, - packages, - epoch: _, - } = indexed_checkpoint; - checkpoint_batch.push(checkpoint); - tx_batch.push(transactions); - events_batch.push(events); - tx_indices_batch.push(tx_indices); - event_indices_batch.push(event_indices); - display_updates_batch.extend(display_updates.into_iter()); - object_changes_batch.push(object_changes); - object_history_changes_batch.push(object_history_changes); - object_versions_batch.push(object_versions); - packages_batch.push(packages); - } - - let first_checkpoint_seq = checkpoint_batch.first().unwrap().sequence_number; - let last_checkpoint = checkpoint_batch.last().unwrap(); - let committer_watermark = CommitterWatermark::from(last_checkpoint); - - let guard = metrics.checkpoint_db_commit_latency.start_timer(); - let tx_batch = tx_batch.into_iter().flatten().collect::>(); - let packages_batch = packages_batch.into_iter().flatten().collect::>(); - let checkpoint_num = checkpoint_batch.len(); - let tx_count = tx_batch.len(); - - { - let _step_1_guard = metrics.checkpoint_db_commit_latency_step_1.start_timer(); - let mut persist_tasks = vec![ - state.persist_packages(packages_batch), - state.persist_object_history(object_history_changes_batch.clone()), - ]; - if let Some(epoch_data) = epoch.clone() { - persist_tasks.push(state.persist_epoch(epoch_data)); - } - futures::future::join_all(persist_tasks) - .await - .into_iter() - .map(|res| { - if res.is_err() { - error!("Failed to persist data with error: {:?}", res); - } - res - }) - .collect::>>() - .expect("Persisting data into DB should not fail."); - } - - let is_epoch_end = epoch.is_some(); - - // On epoch boundary, we need to modify the existing partitions' upper bound, and introduce a - // new partition for incoming data for the upcoming epoch. - if let Some(epoch_data) = epoch { - state - .advance_epoch(epoch_data) - .await - .tap_err(|e| { - error!("Failed to advance epoch with error: {}", e.to_string()); - }) - .expect("Advancing epochs in DB should not fail."); - metrics.total_epoch_committed.inc(); - } - - state - .persist_checkpoints(checkpoint_batch) - .await - .tap_err(|e| { - error!( - "Failed to persist checkpoint data with error: {}", - e.to_string() - ); - }) - .expect("Persisting data into DB should not fail."); - - if is_epoch_end { - // The epoch has advanced so we update the configs for the new protocol version, if it has changed. - let chain_id = state - .get_chain_identifier() - .await - .expect("Failed to get chain identifier") - .expect("Chain identifier should have been indexed at this point"); - let _ = state - .persist_protocol_configs_and_feature_flags(chain_id) - .await; - } - - state - .update_watermarks_upper_bound::(committer_watermark) - .await - .tap_err(|e| { - error!( - "Failed to update watermark upper bound with error: {}", - e.to_string() - ); - }) - .expect("Updating watermark upper bound in DB should not fail."); - - let elapsed = guard.stop_and_record(); - - info!( - elapsed, - "Checkpoint {}-{} committed with {} transactions.", - first_checkpoint_seq, - committer_watermark.checkpoint_hi_inclusive, - tx_count, - ); - metrics - .latest_tx_checkpoint_sequence_number - .set(committer_watermark.checkpoint_hi_inclusive as i64); - metrics - .total_tx_checkpoint_committed - .inc_by(checkpoint_num as u64); - metrics.total_transaction_committed.inc_by(tx_count as u64); - metrics.transaction_per_checkpoint.observe( - tx_count as f64 - / (committer_watermark.checkpoint_hi_inclusive - first_checkpoint_seq + 1) as f64, - ); - // 1000.0 is not necessarily the batch size, it's to roughly map average tx commit latency to [0.1, 1] seconds, - // which is well covered by DB_COMMIT_LATENCY_SEC_BUCKETS. - metrics - .thousand_transaction_avg_db_commit_latency - .observe(elapsed * 1000.0 / tx_count as f64); -} diff --git a/crates/sui-mvr-indexer/src/handlers/mod.rs b/crates/sui-mvr-indexer/src/handlers/mod.rs deleted file mode 100644 index 5859b72616911..0000000000000 --- a/crates/sui-mvr-indexer/src/handlers/mod.rs +++ /dev/null @@ -1,316 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::BTreeMap; - -use async_trait::async_trait; -use futures::{FutureExt, StreamExt}; - -use serde::{Deserialize, Serialize}; -use sui_rpc_api::CheckpointData; -use tokio_util::sync::CancellationToken; - -use crate::{ - errors::IndexerError, - models::{ - display::StoredDisplay, - epoch::{EndOfEpochUpdate, StartOfEpochUpdate}, - obj_indices::StoredObjectVersion, - }, - types::{ - EventIndex, IndexedCheckpoint, IndexedDeletedObject, IndexedEvent, IndexedObject, - IndexedPackage, IndexedTransaction, IndexerResult, TxIndex, - }, -}; - -pub mod checkpoint_handler; -pub mod committer; -pub mod objects_snapshot_handler; -pub mod pruner; -pub mod tx_processor; - -pub(crate) const CHECKPOINT_COMMIT_BATCH_SIZE: usize = 100; -pub(crate) const UNPROCESSED_CHECKPOINT_SIZE_LIMIT: usize = 1000; - -#[derive(Debug)] -pub struct CheckpointDataToCommit { - pub checkpoint: IndexedCheckpoint, - pub transactions: Vec, - pub events: Vec, - pub event_indices: Vec, - pub tx_indices: Vec, - pub display_updates: BTreeMap, - pub object_changes: TransactionObjectChangesToCommit, - pub object_history_changes: TransactionObjectChangesToCommit, - pub object_versions: Vec, - pub packages: Vec, - pub epoch: Option, -} - -#[derive(Clone, Debug)] -pub struct TransactionObjectChangesToCommit { - pub changed_objects: Vec, - pub deleted_objects: Vec, -} - -#[derive(Clone, Debug)] -pub struct EpochToCommit { - pub last_epoch: Option, - pub new_epoch: StartOfEpochUpdate, -} - -impl EpochToCommit { - pub fn new_epoch_id(&self) -> u64 { - self.new_epoch.epoch as u64 - } - - pub fn new_epoch_first_checkpoint_id(&self) -> u64 { - self.new_epoch.first_checkpoint_id as u64 - } - - pub fn last_epoch_total_transactions(&self) -> Option { - self.last_epoch - .as_ref() - .map(|e| e.epoch_total_transactions as u64) - } - - pub fn new_epoch_first_tx_sequence_number(&self) -> u64 { - self.new_epoch.first_tx_sequence_number as u64 - } -} - -pub struct CommonHandler { - handler: Box>, -} - -impl CommonHandler { - pub fn new(handler: Box>) -> Self { - Self { handler } - } - - async fn start_transform_and_load( - &self, - cp_receiver: mysten_metrics::metered_channel::Receiver<(CommitterWatermark, T)>, - cancel: CancellationToken, - start_checkpoint: u64, - end_checkpoint_opt: Option, - ) -> IndexerResult<()> { - let checkpoint_commit_batch_size = std::env::var("CHECKPOINT_COMMIT_BATCH_SIZE") - .unwrap_or(CHECKPOINT_COMMIT_BATCH_SIZE.to_string()) - .parse::() - .unwrap(); - let mut stream = mysten_metrics::metered_channel::ReceiverStream::new(cp_receiver) - .ready_chunks(checkpoint_commit_batch_size); - - // Mapping of ordered checkpoint data to ensure that we process them in order. The key is - // just the checkpoint sequence number, and the tuple is (CommitterWatermark, T). - let mut unprocessed: BTreeMap = BTreeMap::new(); - let mut tuple_batch = vec![]; - let mut next_cp_to_process = start_checkpoint; - - loop { - if cancel.is_cancelled() { - return Ok(()); - } - - // Try to fetch new data tuple from the stream - if unprocessed.len() >= UNPROCESSED_CHECKPOINT_SIZE_LIMIT { - tracing::info!( - "Unprocessed checkpoint size reached limit {}, skip reading from stream...", - UNPROCESSED_CHECKPOINT_SIZE_LIMIT - ); - } else { - // Try to fetch new data tuple from the stream - match stream.next().now_or_never() { - Some(Some(tuple_chunk)) => { - if cancel.is_cancelled() { - return Ok(()); - } - for tuple in tuple_chunk { - unprocessed.insert(tuple.0.checkpoint_hi_inclusive, tuple); - } - } - Some(None) => break, // Stream has ended - None => {} // No new data tuple available right now - } - } - - // Process unprocessed checkpoints, even no new checkpoints from stream - let checkpoint_lag_limiter = self.handler.get_max_committable_checkpoint().await?; - let max_commitable_cp = std::cmp::min( - checkpoint_lag_limiter, - end_checkpoint_opt.unwrap_or(u64::MAX), - ); - // Stop pushing to tuple_batch if we've reached the end checkpoint. - while next_cp_to_process <= max_commitable_cp { - if let Some(data_tuple) = unprocessed.remove(&next_cp_to_process) { - tuple_batch.push(data_tuple); - next_cp_to_process += 1; - } else { - break; - } - } - - if !tuple_batch.is_empty() { - let committer_watermark = tuple_batch.last().unwrap().0; - let batch = tuple_batch.into_iter().map(|t| t.1).collect(); - self.handler.load(batch).await.map_err(|e| { - IndexerError::PostgresWriteError(format!( - "Failed to load transformed data into DB for handler {}: {}", - self.handler.name(), - e - )) - })?; - self.handler.set_watermark_hi(committer_watermark).await?; - tuple_batch = vec![]; - } - - if let Some(end_checkpoint) = end_checkpoint_opt { - if next_cp_to_process > end_checkpoint { - tracing::info!( - "Reached end checkpoint, stopping handler {}...", - self.handler.name() - ); - return Ok(()); - } - } - } - Err(IndexerError::ChannelClosed(format!( - "Checkpoint channel is closed unexpectedly for handler {}", - self.handler.name() - ))) - } -} - -#[async_trait] -pub trait Handler: Send + Sync { - /// return handler name - fn name(&self) -> String; - - /// commit batch of transformed data to DB - async fn load(&self, batch: Vec) -> IndexerResult<()>; - - /// read high watermark of the table DB - async fn get_watermark_hi(&self) -> IndexerResult>; - - /// Updates the relevant entries on the `watermarks` table with the full `CommitterWatermark`, - /// which tracks the latest epoch, cp, and tx sequence number of the committed batch. - async fn set_watermark_hi(&self, watermark: CommitterWatermark) -> IndexerResult<()>; - - /// By default, return u64::MAX, which means no extra waiting is needed before commiting; - /// get max committable checkpoint, for handlers that want to wait for some condition before commiting, - /// one use-case is the objects snapshot handler, - /// which waits for the lag between snapshot and latest checkpoint to reach a certain threshold. - async fn get_max_committable_checkpoint(&self) -> IndexerResult { - Ok(u64::MAX) - } -} - -/// The indexer writer operates on checkpoint data, which contains information on the current epoch, -/// checkpoint, and transaction. These three numbers form the watermark upper bound for each -/// committed table. The reader and pruner are responsible for determining which of the three units -/// will be used for a particular table. -#[derive(Clone, Copy, Ord, PartialOrd, Eq, PartialEq)] -pub struct CommitterWatermark { - pub epoch_hi_inclusive: u64, - pub checkpoint_hi_inclusive: u64, - pub tx_hi: u64, -} - -impl From<&IndexedCheckpoint> for CommitterWatermark { - fn from(checkpoint: &IndexedCheckpoint) -> Self { - Self { - epoch_hi_inclusive: checkpoint.epoch, - checkpoint_hi_inclusive: checkpoint.sequence_number, - tx_hi: checkpoint.network_total_transactions, - } - } -} - -impl From<&CheckpointData> for CommitterWatermark { - fn from(checkpoint: &CheckpointData) -> Self { - Self { - epoch_hi_inclusive: checkpoint.checkpoint_summary.epoch, - checkpoint_hi_inclusive: checkpoint.checkpoint_summary.sequence_number, - tx_hi: checkpoint.checkpoint_summary.network_total_transactions, - } - } -} - -/// Enum representing tables that the committer handler writes to. -#[derive( - Debug, - Eq, - PartialEq, - strum_macros::Display, - strum_macros::EnumString, - strum_macros::EnumIter, - strum_macros::AsRefStr, - Hash, - Serialize, - Deserialize, - Clone, -)] -#[strum(serialize_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum CommitterTables { - // Unpruned tables - ChainIdentifier, - Display, - Epochs, - FeatureFlags, - FullObjectsHistory, - Objects, - ObjectsVersion, - Packages, - ProtocolConfigs, - RawCheckpoints, - - // Prunable tables - ObjectsHistory, - Transactions, - Events, - - EventEmitPackage, - EventEmitModule, - EventSenders, - EventStructInstantiation, - EventStructModule, - EventStructName, - EventStructPackage, - - TxAffectedAddresses, - TxAffectedObjects, - TxCallsPkg, - TxCallsMod, - TxCallsFun, - TxChangedObjects, - TxDigests, - TxInputObjects, - TxKinds, - TxRecipients, - TxSenders, - - Checkpoints, - PrunerCpWatermark, -} - -/// Enum representing tables that the objects snapshot handler writes to. -#[derive( - Debug, - Eq, - PartialEq, - strum_macros::Display, - strum_macros::EnumString, - strum_macros::EnumIter, - strum_macros::AsRefStr, - Hash, - Serialize, - Deserialize, - Clone, -)] -#[strum(serialize_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum ObjectsSnapshotHandlerTables { - ObjectsSnapshot, -} diff --git a/crates/sui-mvr-indexer/src/handlers/objects_snapshot_handler.rs b/crates/sui-mvr-indexer/src/handlers/objects_snapshot_handler.rs deleted file mode 100644 index cdf4a2234e059..0000000000000 --- a/crates/sui-mvr-indexer/src/handlers/objects_snapshot_handler.rs +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use async_trait::async_trait; -use mysten_metrics::get_metrics; -use mysten_metrics::metered_channel::Sender; -use mysten_metrics::spawn_monitored_task; -use sui_data_ingestion_core::Worker; -use sui_rpc_api::CheckpointData; -use tokio_util::sync::CancellationToken; -use tracing::info; - -use crate::config::SnapshotLagConfig; -use crate::store::PgIndexerStore; -use crate::types::IndexerResult; -use crate::{metrics::IndexerMetrics, store::IndexerStore}; - -use super::checkpoint_handler::CheckpointHandler; -use super::{CommitterWatermark, ObjectsSnapshotHandlerTables, TransactionObjectChangesToCommit}; -use super::{CommonHandler, Handler}; - -#[derive(Clone)] -pub struct ObjectsSnapshotHandler { - pub store: PgIndexerStore, - pub sender: Sender<(CommitterWatermark, TransactionObjectChangesToCommit)>, - snapshot_config: SnapshotLagConfig, - metrics: IndexerMetrics, -} - -pub struct CheckpointObjectChanges { - pub checkpoint_sequence_number: u64, - pub object_changes: TransactionObjectChangesToCommit, -} - -#[async_trait] -impl Worker for ObjectsSnapshotHandler { - type Result = (); - async fn process_checkpoint(&self, checkpoint: &CheckpointData) -> anyhow::Result<()> { - let transformed_data = CheckpointHandler::index_objects(checkpoint, &self.metrics).await?; - self.sender - .send((CommitterWatermark::from(checkpoint), transformed_data)) - .await?; - Ok(()) - } -} - -#[async_trait] -impl Handler for ObjectsSnapshotHandler { - fn name(&self) -> String { - "objects_snapshot_handler".to_string() - } - - async fn load( - &self, - transformed_data: Vec, - ) -> IndexerResult<()> { - self.store - .persist_objects_snapshot(transformed_data) - .await?; - Ok(()) - } - - async fn get_watermark_hi(&self) -> IndexerResult> { - self.store - .get_latest_object_snapshot_checkpoint_sequence_number() - .await - } - - async fn set_watermark_hi(&self, watermark: CommitterWatermark) -> IndexerResult<()> { - self.store - .update_watermarks_upper_bound::(watermark) - .await?; - - self.metrics - .latest_object_snapshot_sequence_number - .set(watermark.checkpoint_hi_inclusive as i64); - Ok(()) - } - - async fn get_max_committable_checkpoint(&self) -> IndexerResult { - let latest_checkpoint = self.store.get_latest_checkpoint_sequence_number().await?; - Ok(latest_checkpoint - .map(|seq| seq.saturating_sub(self.snapshot_config.snapshot_min_lag as u64)) - .unwrap_or_default()) // hold snapshot handler until at least one checkpoint is in DB - } -} - -pub async fn start_objects_snapshot_handler( - store: PgIndexerStore, - metrics: IndexerMetrics, - snapshot_config: SnapshotLagConfig, - cancel: CancellationToken, - start_checkpoint_opt: Option, - end_checkpoint_opt: Option, -) -> IndexerResult<(ObjectsSnapshotHandler, u64)> { - info!("Starting object snapshot handler..."); - - let global_metrics = get_metrics().unwrap(); - let (sender, receiver) = mysten_metrics::metered_channel::channel( - 600, - &global_metrics - .channel_inflight - .with_label_values(&["objects_snapshot_handler_checkpoint_data"]), - ); - - let objects_snapshot_handler = - ObjectsSnapshotHandler::new(store.clone(), sender, metrics.clone(), snapshot_config); - - let next_cp_from_db = objects_snapshot_handler - .get_watermark_hi() - .await? - .map(|cp| cp.saturating_add(1)) - .unwrap_or_default(); - let start_checkpoint = start_checkpoint_opt.unwrap_or(next_cp_from_db); - let common_handler = CommonHandler::new(Box::new(objects_snapshot_handler.clone())); - spawn_monitored_task!(common_handler.start_transform_and_load( - receiver, - cancel, - start_checkpoint, - end_checkpoint_opt, - )); - Ok((objects_snapshot_handler, start_checkpoint)) -} - -impl ObjectsSnapshotHandler { - pub fn new( - store: PgIndexerStore, - sender: Sender<(CommitterWatermark, TransactionObjectChangesToCommit)>, - metrics: IndexerMetrics, - snapshot_config: SnapshotLagConfig, - ) -> ObjectsSnapshotHandler { - Self { - store, - sender, - metrics, - snapshot_config, - } - } -} diff --git a/crates/sui-mvr-indexer/src/handlers/pruner.rs b/crates/sui-mvr-indexer/src/handlers/pruner.rs deleted file mode 100644 index 228196f83bbc2..0000000000000 --- a/crates/sui-mvr-indexer/src/handlers/pruner.rs +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use mysten_metrics::spawn_monitored_task; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::time::Duration; -use strum_macros; -use tokio_util::sync::CancellationToken; -use tracing::{error, info}; - -use crate::config::RetentionConfig; -use crate::errors::IndexerError; -use crate::store::pg_partition_manager::PgPartitionManager; -use crate::store::PgIndexerStore; -use crate::{metrics::IndexerMetrics, store::IndexerStore, types::IndexerResult}; - -pub struct Pruner { - pub store: PgIndexerStore, - pub partition_manager: PgPartitionManager, - // TODO: (wlmyng) - we can remove this when pruner logic is updated to use `retention_policies`. - pub epochs_to_keep: u64, - pub retention_policies: HashMap, - pub metrics: IndexerMetrics, -} - -/// Enum representing tables that the pruner is allowed to prune. This corresponds to table names in -/// the database, and should be used in lieu of string literals. This enum is also meant to -/// facilitate the process of determining which unit (epoch, cp, or tx) should be used for the -/// table's range. Pruner will ignore any table that is not listed here. -#[derive( - Debug, - Eq, - PartialEq, - strum_macros::Display, - strum_macros::EnumString, - strum_macros::EnumIter, - strum_macros::AsRefStr, - Hash, - Serialize, - Deserialize, - Clone, -)] -#[strum(serialize_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum PrunableTable { - ObjectsHistory, - Transactions, - Events, - - EventEmitPackage, - EventEmitModule, - EventSenders, - EventStructInstantiation, - EventStructModule, - EventStructName, - EventStructPackage, - - TxAffectedAddresses, - TxAffectedObjects, - TxCallsPkg, - TxCallsMod, - TxCallsFun, - TxChangedObjects, - TxDigests, - TxInputObjects, - TxKinds, - - Checkpoints, - PrunerCpWatermark, -} - -impl PrunableTable { - pub fn select_reader_lo(&self, cp: u64, tx: u64) -> u64 { - match self { - PrunableTable::ObjectsHistory => cp, - PrunableTable::Transactions => tx, - PrunableTable::Events => tx, - - PrunableTable::EventEmitPackage => tx, - PrunableTable::EventEmitModule => tx, - PrunableTable::EventSenders => tx, - PrunableTable::EventStructInstantiation => tx, - PrunableTable::EventStructModule => tx, - PrunableTable::EventStructName => tx, - PrunableTable::EventStructPackage => tx, - - PrunableTable::TxAffectedAddresses => tx, - PrunableTable::TxAffectedObjects => tx, - PrunableTable::TxCallsPkg => tx, - PrunableTable::TxCallsMod => tx, - PrunableTable::TxCallsFun => tx, - PrunableTable::TxChangedObjects => tx, - PrunableTable::TxDigests => tx, - PrunableTable::TxInputObjects => tx, - PrunableTable::TxKinds => tx, - - PrunableTable::Checkpoints => cp, - PrunableTable::PrunerCpWatermark => cp, - } - } -} - -impl Pruner { - /// Instantiates a pruner with default retention and overrides. Pruner will finalize the - /// retention policies so there is a value for every prunable table. - pub fn new( - store: PgIndexerStore, - retention_config: RetentionConfig, - metrics: IndexerMetrics, - ) -> Result { - let partition_manager = PgPartitionManager::new(store.pool())?; - let epochs_to_keep = retention_config.epochs_to_keep; - let retention_policies = retention_config.retention_policies(); - - Ok(Self { - store, - epochs_to_keep, - partition_manager, - retention_policies, - metrics, - }) - } - - /// Given a table name, return the number of epochs to keep for that table. Return `None` if the - /// table is not prunable. - fn table_retention(&self, table_name: &str) -> Option { - if let Ok(variant) = table_name.parse::() { - self.retention_policies.get(&variant).copied() - } else { - None - } - } - - pub async fn start(&self, cancel: CancellationToken) -> IndexerResult<()> { - let store_clone = self.store.clone(); - let retention_policies = self.retention_policies.clone(); - let cancel_clone = cancel.clone(); - spawn_monitored_task!(update_watermarks_lower_bounds_task( - store_clone, - retention_policies, - cancel_clone - )); - - let mut last_seen_max_epoch = 0; - // The first epoch that has not yet been pruned. - let mut next_prune_epoch = None; - while !cancel.is_cancelled() { - let (min_epoch, max_epoch) = self.store.get_available_epoch_range().await?; - if max_epoch == last_seen_max_epoch { - tokio::time::sleep(Duration::from_secs(5)).await; - continue; - } - last_seen_max_epoch = max_epoch; - - // Not all partitioned tables are epoch-partitioned, so we need to filter them out. - let table_partitions: HashMap<_, _> = self - .partition_manager - .get_table_partitions() - .await? - .into_iter() - .filter(|(table_name, _)| { - self.partition_manager - .get_strategy(table_name) - .is_epoch_partitioned() - }) - .collect(); - - for (table_name, (min_partition, max_partition)) in &table_partitions { - if let Some(epochs_to_keep) = self.table_retention(table_name) { - if last_seen_max_epoch != *max_partition { - error!( - "Epochs are out of sync for table {}: max_epoch={}, max_partition={}", - table_name, last_seen_max_epoch, max_partition - ); - } - - for epoch in - *min_partition..last_seen_max_epoch.saturating_sub(epochs_to_keep - 1) - { - if cancel.is_cancelled() { - info!("Pruner task cancelled."); - return Ok(()); - } - self.partition_manager - .drop_table_partition(table_name.clone(), epoch) - .await?; - info!( - "Batch dropped table partition {} epoch {}", - table_name, epoch - ); - } - } - } - - // TODO: (wlmyng) Once we have the watermarks table, we can iterate through each row - // returned from `watermarks`, look it up against `retention_policies`, and process them - // independently. This also means that pruning overrides will only apply for - // epoch-partitioned tables right now. - let prune_to_epoch = last_seen_max_epoch.saturating_sub(self.epochs_to_keep - 1); - let prune_start_epoch = next_prune_epoch.unwrap_or(min_epoch); - for epoch in prune_start_epoch..prune_to_epoch { - if cancel.is_cancelled() { - info!("Pruner task cancelled."); - return Ok(()); - } - info!("Pruning epoch {}", epoch); - if let Err(err) = self.store.prune_epoch(epoch).await { - error!("Failed to prune epoch {}: {}", epoch, err); - break; - }; - self.metrics.last_pruned_epoch.set(epoch as i64); - info!("Pruned epoch {}", epoch); - next_prune_epoch = Some(epoch + 1); - } - } - info!("Pruner task cancelled."); - Ok(()) - } -} - -/// Task to periodically query the `watermarks` table and update the lower bounds for all watermarks -/// if the entry exceeds epoch-level retention policy. -async fn update_watermarks_lower_bounds_task( - store: PgIndexerStore, - retention_policies: HashMap, - cancel: CancellationToken, -) -> IndexerResult<()> { - let mut interval = tokio::time::interval(Duration::from_secs(5)); - loop { - tokio::select! { - _ = cancel.cancelled() => { - info!("Pruner watermark lower bound update task cancelled."); - return Ok(()); - } - _ = interval.tick() => { - update_watermarks_lower_bounds(&store, &retention_policies, &cancel).await?; - } - } - } -} - -/// Fetches all entries from the `watermarks` table, and updates the `reader_lo` for each entry if -/// its epoch range exceeds the respective retention policy. -async fn update_watermarks_lower_bounds( - store: &PgIndexerStore, - retention_policies: &HashMap, - cancel: &CancellationToken, -) -> IndexerResult<()> { - let (watermarks, _) = store.get_watermarks().await?; - let mut lower_bound_updates = vec![]; - - for watermark in watermarks.iter() { - if cancel.is_cancelled() { - info!("Pruner watermark lower bound update task cancelled."); - return Ok(()); - } - - let Some(prunable_table) = watermark.entity() else { - continue; - }; - - let Some(epochs_to_keep) = retention_policies.get(&prunable_table) else { - error!( - "No retention policy found for prunable table {}", - prunable_table - ); - continue; - }; - - if let Some(new_epoch_lo) = watermark.new_epoch_lo(*epochs_to_keep) { - lower_bound_updates.push((prunable_table, new_epoch_lo)); - }; - } - - if !lower_bound_updates.is_empty() { - store - .update_watermarks_lower_bound(lower_bound_updates) - .await?; - info!("Finished updating lower bounds for watermarks"); - } - - Ok(()) -} diff --git a/crates/sui-mvr-indexer/src/handlers/tx_processor.rs b/crates/sui-mvr-indexer/src/handlers/tx_processor.rs deleted file mode 100644 index cbfd8fa0416b2..0000000000000 --- a/crates/sui-mvr-indexer/src/handlers/tx_processor.rs +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::HashMap; - -use async_trait::async_trait; -use sui_json_rpc::get_balance_changes_from_effect; -use sui_json_rpc::get_object_changes; -use sui_json_rpc::ObjectProvider; -use sui_rpc_api::CheckpointData; -use sui_types::base_types::ObjectID; -use sui_types::base_types::SequenceNumber; -use sui_types::digests::TransactionDigest; -use sui_types::effects::{TransactionEffects, TransactionEffectsAPI}; -use sui_types::object::Object; -use sui_types::transaction::{TransactionData, TransactionDataAPI}; - -use crate::errors::IndexerError; -use crate::metrics::IndexerMetrics; -use crate::types::{IndexedObjectChange, IndexerResult}; - -pub struct InMemObjectCache { - id_map: HashMap, - seq_map: HashMap<(ObjectID, SequenceNumber), Object>, -} - -impl InMemObjectCache { - pub fn new() -> Self { - Self { - id_map: HashMap::new(), - seq_map: HashMap::new(), - } - } - - pub fn insert_object(&mut self, obj: Object) { - self.id_map.insert(obj.id(), obj.clone()); - self.seq_map.insert((obj.id(), obj.version()), obj); - } - - pub fn get(&self, id: &ObjectID, version: Option<&SequenceNumber>) -> Option<&Object> { - if let Some(version) = version { - self.seq_map.get(&(*id, *version)) - } else { - self.id_map.get(id) - } - } -} - -impl Default for InMemObjectCache { - fn default() -> Self { - Self::new() - } -} - -/// Along with InMemObjectCache, TxChangesProcessor implements ObjectProvider -/// so it can be used in indexing write path to get object/balance changes. -/// Its lifetime is per checkpoint. -pub struct TxChangesProcessor { - object_cache: InMemObjectCache, - metrics: IndexerMetrics, -} - -impl TxChangesProcessor { - pub fn new(objects: &[&Object], metrics: IndexerMetrics) -> Self { - let mut object_cache = InMemObjectCache::new(); - for obj in objects { - object_cache.insert_object(<&Object>::clone(obj).clone()); - } - Self { - object_cache, - metrics, - } - } - - pub(crate) async fn get_changes( - &self, - tx: &TransactionData, - effects: &TransactionEffects, - tx_digest: &TransactionDigest, - ) -> IndexerResult<( - Vec, - Vec, - )> { - let _timer = self - .metrics - .indexing_tx_object_changes_latency - .start_timer(); - let object_change: Vec<_> = get_object_changes( - self, - effects, - tx.sender(), - effects.modified_at_versions(), - effects.all_changed_objects(), - effects.all_removed_objects(), - ) - .await? - .into_iter() - .map(IndexedObjectChange::from) - .collect(); - let balance_change = get_balance_changes_from_effect( - self, - effects, - tx.input_objects().unwrap_or_else(|e| { - panic!( - "Checkpointed tx {:?} has inavlid input objects: {e}", - tx_digest, - ) - }), - None, - ) - .await?; - Ok((balance_change, object_change)) - } -} - -#[async_trait] -impl ObjectProvider for TxChangesProcessor { - type Error = IndexerError; - - async fn get_object( - &self, - id: &ObjectID, - version: &SequenceNumber, - ) -> Result { - let object = self - .object_cache - .get(id, Some(version)) - .as_ref() - .map(|o| <&Object>::clone(o).clone()); - if let Some(o) = object { - self.metrics.indexing_get_object_in_mem_hit.inc(); - return Ok(o); - } - - panic!( - "Object {} is not found in TxChangesProcessor as an ObjectProvider (fn get_object)", - id - ); - } - - async fn find_object_lt_or_eq_version( - &self, - id: &ObjectID, - version: &SequenceNumber, - ) -> Result, Self::Error> { - // First look up the exact version in object_cache. - let object = self - .object_cache - .get(id, Some(version)) - .as_ref() - .map(|o| <&Object>::clone(o).clone()); - if let Some(o) = object { - self.metrics.indexing_get_object_in_mem_hit.inc(); - return Ok(Some(o)); - } - - // Second look up the latest version in object_cache. This may be - // called when the object is deleted hence the version at deletion - // is given. - let object = self - .object_cache - .get(id, None) - .as_ref() - .map(|o| <&Object>::clone(o).clone()); - if let Some(o) = object { - if o.version() > *version { - panic!( - "Found a higher version {} for object {}, expected lt_or_eq {}", - o.version(), - id, - *version - ); - } - if o.version() <= *version { - self.metrics.indexing_get_object_in_mem_hit.inc(); - return Ok(Some(o)); - } - } - - panic!("Object {} is not found in TxChangesProcessor as an ObjectProvider (fn find_object_lt_or_eq_version)", id); - } -} - -// This is a struct that is used to extract SuiSystemState and its dynamic children -// for end-of-epoch indexing. -pub(crate) struct EpochEndIndexingObjectStore<'a> { - objects: Vec<&'a Object>, -} - -impl<'a> EpochEndIndexingObjectStore<'a> { - pub fn new(data: &'a CheckpointData) -> Self { - Self { - objects: data.latest_live_output_objects(), - } - } -} - -impl<'a> sui_types::storage::ObjectStore for EpochEndIndexingObjectStore<'a> { - fn get_object(&self, object_id: &ObjectID) -> Option { - self.objects - .iter() - .find(|o| o.id() == *object_id) - .cloned() - .cloned() - } - - fn get_object_by_key( - &self, - object_id: &ObjectID, - version: sui_types::base_types::VersionNumber, - ) -> Option { - self.objects - .iter() - .find(|o| o.id() == *object_id && o.version() == version) - .cloned() - .cloned() - } -} diff --git a/crates/sui-mvr-indexer/src/indexer.rs b/crates/sui-mvr-indexer/src/indexer.rs deleted file mode 100644 index 4033469930ba1..0000000000000 --- a/crates/sui-mvr-indexer/src/indexer.rs +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::HashMap; -use std::env; - -use anyhow::Result; -use prometheus::Registry; -use tokio::sync::oneshot; -use tokio_util::sync::CancellationToken; -use tracing::info; - -use async_trait::async_trait; -use futures::future::try_join_all; -use mysten_metrics::spawn_monitored_task; -use sui_data_ingestion_core::{ - DataIngestionMetrics, IndexerExecutor, ProgressStore, ReaderOptions, WorkerPool, -}; -use sui_types::messages_checkpoint::CheckpointSequenceNumber; - -use crate::build_json_rpc_server; -use crate::config::{IngestionConfig, JsonRpcConfig, RetentionConfig, SnapshotLagConfig}; -use crate::database::ConnectionPool; -use crate::errors::IndexerError; -use crate::handlers::checkpoint_handler::new_handlers; -use crate::handlers::objects_snapshot_handler::start_objects_snapshot_handler; -use crate::handlers::pruner::Pruner; -use crate::indexer_reader::IndexerReader; -use crate::metrics::IndexerMetrics; -use crate::store::{IndexerStore, PgIndexerStore}; - -pub struct Indexer; - -impl Indexer { - pub async fn start_writer( - config: IngestionConfig, - store: PgIndexerStore, - metrics: IndexerMetrics, - snapshot_config: SnapshotLagConfig, - retention_config: Option, - cancel: CancellationToken, - ) -> Result<(), IndexerError> { - info!( - "Sui Indexer Writer (version {:?}) started...", - env!("CARGO_PKG_VERSION") - ); - info!("Sui Indexer Writer config: {config:?}",); - - let extra_reader_options = ReaderOptions { - batch_size: config.checkpoint_download_queue_size, - timeout_secs: config.checkpoint_download_timeout, - data_limit: config.checkpoint_download_queue_size_bytes, - gc_checkpoint_files: config.gc_checkpoint_files, - ..Default::default() - }; - - // Start objects snapshot processor, which is a separate pipeline with its ingestion pipeline. - let (object_snapshot_worker, object_snapshot_watermark) = start_objects_snapshot_handler( - store.clone(), - metrics.clone(), - snapshot_config, - cancel.clone(), - config.start_checkpoint, - config.end_checkpoint, - ) - .await?; - - if let Some(retention_config) = retention_config { - let pruner = Pruner::new(store.clone(), retention_config, metrics.clone())?; - let cancel_clone = cancel.clone(); - spawn_monitored_task!(pruner.start(cancel_clone)); - } - - // If we already have chain identifier indexed (i.e. the first checkpoint has been indexed), - // then we persist protocol configs for protocol versions not yet in the db. - // Otherwise, we would do the persisting in `commit_checkpoint` while the first cp is - // being indexed. - if let Some(chain_id) = IndexerStore::get_chain_identifier(&store).await? { - store - .persist_protocol_configs_and_feature_flags(chain_id) - .await?; - } - - let mut exit_senders = vec![]; - let mut executors = vec![]; - - let (worker, primary_watermark) = new_handlers( - store, - metrics, - cancel.clone(), - config.start_checkpoint, - config.end_checkpoint, - ) - .await?; - // Ingestion task watermarks are snapshotted once on indexer startup based on the - // corresponding watermark table before being handed off to the ingestion task. - let progress_store = ShimIndexerProgressStore::new(vec![ - ("primary".to_string(), primary_watermark), - ("object_snapshot".to_string(), object_snapshot_watermark), - ]); - let mut executor = IndexerExecutor::new( - progress_store.clone(), - 2, - DataIngestionMetrics::new(&Registry::new()), - ); - - let worker_pool = WorkerPool::new( - worker, - "primary".to_string(), - config.checkpoint_download_queue_size, - ); - executor.register(worker_pool).await?; - let (exit_sender, exit_receiver) = oneshot::channel(); - executors.push((executor, exit_receiver)); - exit_senders.push(exit_sender); - - // in a non-colocated setup, start a separate indexer for processing object snapshots - if config.sources.data_ingestion_path.is_none() { - let executor = IndexerExecutor::new( - progress_store, - 1, - DataIngestionMetrics::new(&Registry::new()), - ); - let (exit_sender, exit_receiver) = oneshot::channel(); - exit_senders.push(exit_sender); - executors.push((executor, exit_receiver)); - } - - let worker_pool = WorkerPool::new( - object_snapshot_worker, - "object_snapshot".to_string(), - config.checkpoint_download_queue_size, - ); - let executor = executors.last_mut().expect("executors is not empty"); - executor.0.register(worker_pool).await?; - - // Spawn a task that links the cancellation token to the exit sender - spawn_monitored_task!(async move { - cancel.cancelled().await; - for exit_sender in exit_senders { - let _ = exit_sender.send(()); - } - }); - - info!("Starting data ingestion executor..."); - let futures = executors.into_iter().map(|(executor, exit_receiver)| { - executor.run( - config - .sources - .data_ingestion_path - .clone() - .unwrap_or(tempfile::tempdir().unwrap().into_path()), - config - .sources - .remote_store_url - .as_ref() - .map(|url| url.as_str().to_owned()), - vec![], - extra_reader_options.clone(), - exit_receiver, - ) - }); - try_join_all(futures).await?; - Ok(()) - } - - pub async fn start_reader( - config: &JsonRpcConfig, - registry: &Registry, - pool: ConnectionPool, - cancel: CancellationToken, - ) -> Result<(), IndexerError> { - info!( - "Sui Indexer Reader (version {:?}) started...", - env!("CARGO_PKG_VERSION") - ); - let indexer_reader = IndexerReader::new(pool); - let handle = build_json_rpc_server(registry, indexer_reader, config, cancel) - .await - .expect("Json rpc server should not run into errors upon start."); - tokio::spawn(async move { handle.stopped().await }) - .await - .expect("Rpc server task failed"); - - Ok(()) - } -} - -#[derive(Clone)] -struct ShimIndexerProgressStore { - watermarks: HashMap, -} - -impl ShimIndexerProgressStore { - fn new(watermarks: Vec<(String, CheckpointSequenceNumber)>) -> Self { - Self { - watermarks: watermarks.into_iter().collect(), - } - } -} - -#[async_trait] -impl ProgressStore for ShimIndexerProgressStore { - async fn load(&mut self, task_name: String) -> Result { - Ok(*self.watermarks.get(&task_name).expect("missing watermark")) - } - - async fn save(&mut self, _: String, _: CheckpointSequenceNumber) -> Result<()> { - Ok(()) - } -} diff --git a/crates/sui-mvr-indexer/src/indexer_reader.rs b/crates/sui-mvr-indexer/src/indexer_reader.rs deleted file mode 100644 index 510e02821543b..0000000000000 --- a/crates/sui-mvr-indexer/src/indexer_reader.rs +++ /dev/null @@ -1,1508 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use anyhow::anyhow; -use anyhow::Result; -use diesel::{ - dsl::sql, sql_types::Bool, ExpressionMethods, JoinOnDsl, NullableExpressionMethods, - OptionalExtension, QueryDsl, SelectableHelper, TextExpressionMethods, -}; -use itertools::Itertools; -use std::sync::Arc; -use sui_types::dynamic_field::visitor as DFV; -use sui_types::object::bounded_visitor::BoundedVisitor; -use tap::{Pipe, TapFallible}; -use tracing::{debug, error, warn}; - -use fastcrypto::encoding::Encoding; -use fastcrypto::encoding::Hex; -use move_core_types::annotated_value::MoveStructLayout; -use move_core_types::language_storage::{StructTag, TypeTag}; -use sui_json_rpc_types::DisplayFieldsResponse; -use sui_json_rpc_types::{Balance, Coin as SuiCoin, SuiCoinMetadata, SuiMoveValue}; -use sui_json_rpc_types::{ - CheckpointId, EpochInfo, EventFilter, SuiEvent, SuiObjectDataFilter, - SuiTransactionBlockResponse, TransactionFilter, -}; -use sui_package_resolver::Package; -use sui_package_resolver::PackageStore; -use sui_package_resolver::{PackageStoreWithLruCache, Resolver}; -use sui_types::effects::TransactionEvents; -use sui_types::{balance::Supply, coin::TreasuryCap, dynamic_field::DynamicFieldName}; -use sui_types::{ - base_types::{ObjectID, SuiAddress, VersionNumber}, - committee::EpochId, - digests::TransactionDigest, - dynamic_field::DynamicFieldInfo, - object::{Object, ObjectRead}, - sui_system_state::{sui_system_state_summary::SuiSystemStateSummary, SuiSystemStateTrait}, -}; -use sui_types::{coin::CoinMetadata, event::EventID}; - -use crate::database::ConnectionPool; -use crate::db::ConnectionPoolConfig; -use crate::models::transactions::{stored_events_to_events, StoredTransactionEvents}; -use crate::schema::pruner_cp_watermark; -use crate::schema::tx_digests; -use crate::{ - errors::IndexerError, - models::{ - checkpoints::StoredCheckpoint, - display::StoredDisplay, - epoch::StoredEpochInfo, - events::StoredEvent, - objects::{CoinBalance, StoredObject}, - transactions::{tx_events_to_sui_tx_events, StoredTransaction}, - tx_indices::TxSequenceNumber, - }, - schema::{checkpoints, display, epochs, events, objects, transactions}, - store::package_resolver::IndexerStorePackageResolver, - types::{IndexerResult, OwnerType}, -}; - -pub const TX_SEQUENCE_NUMBER_STR: &str = "tx_sequence_number"; -pub const TRANSACTION_DIGEST_STR: &str = "transaction_digest"; -pub const EVENT_SEQUENCE_NUMBER_STR: &str = "event_sequence_number"; - -#[derive(Clone)] -pub struct IndexerReader { - pool: ConnectionPool, - package_resolver: PackageResolver, -} - -pub type PackageResolver = Arc>>; - -// Impl for common initialization and utilities -impl IndexerReader { - pub fn new(pool: ConnectionPool) -> Self { - let indexer_store_pkg_resolver = IndexerStorePackageResolver::new(pool.clone()); - let package_cache = PackageStoreWithLruCache::new(indexer_store_pkg_resolver); - let package_resolver = Arc::new(Resolver::new(package_cache)); - Self { - pool, - package_resolver, - } - } - - pub async fn new_with_config>( - db_url: T, - config: ConnectionPoolConfig, - ) -> Result { - let db_url = db_url.into(); - - let pool = ConnectionPool::new(db_url.parse()?, config).await?; - - let indexer_store_pkg_resolver = IndexerStorePackageResolver::new(pool.clone()); - let package_cache = PackageStoreWithLruCache::new(indexer_store_pkg_resolver); - let package_resolver = Arc::new(Resolver::new(package_cache)); - Ok(Self { - pool, - package_resolver, - }) - } - - pub fn pool(&self) -> &ConnectionPool { - &self.pool - } -} - -// Impl for reading data from the DB -impl IndexerReader { - async fn get_object_from_db( - &self, - object_id: &ObjectID, - version: Option, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let mut query = objects::table - .filter(objects::object_id.eq(object_id.to_vec())) - .into_boxed(); - if let Some(version) = version { - query = query.filter(objects::object_version.eq(version.value() as i64)) - } - - query - .first::(&mut connection) - .await - .optional() - .map_err(Into::into) - } - - pub async fn get_object( - &self, - object_id: &ObjectID, - version: Option, - ) -> Result, IndexerError> { - let Some(stored_package) = self.get_object_from_db(object_id, version).await? else { - return Ok(None); - }; - - let object = stored_package.try_into()?; - Ok(Some(object)) - } - - pub async fn get_object_read(&self, object_id: ObjectID) -> Result { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let stored_object = objects::table - .filter(objects::object_id.eq(object_id.to_vec())) - .first::(&mut connection) - .await - .optional()?; - - if let Some(object) = stored_object { - object - .try_into_object_read(self.package_resolver.clone()) - .await - } else { - Ok(ObjectRead::NotExists(object_id)) - } - } - - pub async fn get_package(&self, package_id: ObjectID) -> Result { - let store = self.package_resolver.package_store(); - let pkg = store - .fetch(package_id.into()) - .await - .map_err(|e| { - IndexerError::PostgresReadError(format!( - "Fail to fetch package from package store with error {:?}", - e - )) - })? - .as_ref() - .clone(); - Ok(pkg) - } - - async fn get_epoch_info_from_db( - &self, - epoch: Option, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let stored_epoch = epochs::table - .into_boxed() - .pipe(|query| { - if let Some(epoch) = epoch { - query.filter(epochs::epoch.eq(epoch as i64)) - } else { - query.order_by(epochs::epoch.desc()) - } - }) - .first::(&mut connection) - .await - .optional()?; - - Ok(stored_epoch) - } - - pub async fn get_latest_epoch_info_from_db(&self) -> Result { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let stored_epoch = epochs::table - .order_by(epochs::epoch.desc()) - .first::(&mut connection) - .await?; - - Ok(stored_epoch) - } - - pub async fn get_epoch_info( - &self, - epoch: Option, - ) -> Result, IndexerError> { - let stored_epoch = self.get_epoch_info_from_db(epoch).await?; - - let stored_epoch = match stored_epoch { - Some(stored_epoch) => stored_epoch, - None => return Ok(None), - }; - - let epoch_info = EpochInfo::try_from(stored_epoch)?; - Ok(Some(epoch_info)) - } - - async fn get_epochs_from_db( - &self, - cursor: Option, - limit: usize, - descending_order: bool, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let mut query = epochs::table.into_boxed(); - - if let Some(cursor) = cursor { - if descending_order { - query = query.filter(epochs::epoch.lt(cursor as i64)); - } else { - query = query.filter(epochs::epoch.gt(cursor as i64)); - } - } - - if descending_order { - query = query.order_by(epochs::epoch.desc()); - } else { - query = query.order_by(epochs::epoch.asc()); - } - - query - .limit(limit as i64) - .load(&mut connection) - .await - .map_err(Into::into) - } - - pub async fn get_epochs( - &self, - cursor: Option, - limit: usize, - descending_order: bool, - ) -> Result, IndexerError> { - self.get_epochs_from_db(cursor, limit, descending_order) - .await? - .into_iter() - .map(EpochInfo::try_from) - .collect::, _>>() - .map_err(Into::into) - } - - pub async fn get_latest_sui_system_state(&self) -> Result { - let object_store = ConnectionAsObjectStore::from_pool(&self.pool) - .await - .map_err(|e| IndexerError::PgPoolConnectionError(e.to_string()))?; - - let system_state = tokio::task::spawn_blocking(move || { - sui_types::sui_system_state::get_sui_system_state(&object_store) - }) - .await - .unwrap()? - .into_sui_system_state_summary(); - - Ok(system_state) - } - - pub async fn get_validator_from_table( - &self, - table_id: ObjectID, - pool_id: sui_types::id::ID, - ) -> Result< - sui_types::sui_system_state::sui_system_state_summary::SuiValidatorSummary, - IndexerError, - > { - let object_store = ConnectionAsObjectStore::from_pool(&self.pool) - .await - .map_err(|e| IndexerError::PgPoolConnectionError(e.to_string()))?; - - let validator = tokio::task::spawn_blocking(move || { - sui_types::sui_system_state::get_validator_from_table(&object_store, table_id, &pool_id) - }) - .await - .unwrap()?; - Ok(validator) - } - - /// Retrieve the system state data for the given epoch. If no epoch is given, - /// it will retrieve the latest epoch's data and return the system state. - /// System state of the an epoch is written at the end of the epoch, so system state - /// of the current epoch is empty until the epoch ends. You can call - /// `get_latest_sui_system_state` for current epoch instead. - pub async fn get_epoch_sui_system_state( - &self, - epoch: Option, - ) -> Result { - let stored_epoch = self.get_epoch_info_from_db(epoch).await?; - let stored_epoch = match stored_epoch { - Some(stored_epoch) => stored_epoch, - None => return Err(IndexerError::InvalidArgumentError("Invalid epoch".into())), - }; - stored_epoch.get_json_system_state_summary() - } - - async fn get_checkpoint_from_db( - &self, - checkpoint_id: CheckpointId, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - let stored_checkpoint = checkpoints::table - .into_boxed() - .pipe(|query| match checkpoint_id { - CheckpointId::SequenceNumber(seq) => { - query.filter(checkpoints::sequence_number.eq(seq as i64)) - } - CheckpointId::Digest(digest) => { - query.filter(checkpoints::checkpoint_digest.eq(digest.into_inner().to_vec())) - } - }) - .first::(&mut connection) - .await - .optional()?; - - Ok(stored_checkpoint) - } - - async fn get_latest_checkpoint_from_db(&self) -> Result { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - checkpoints::table - .order_by(checkpoints::sequence_number.desc()) - .first::(&mut connection) - .await - .map_err(Into::into) - } - - pub async fn get_checkpoint( - &self, - checkpoint_id: CheckpointId, - ) -> Result, IndexerError> { - let stored_checkpoint = match self.get_checkpoint_from_db(checkpoint_id).await? { - Some(stored_checkpoint) => stored_checkpoint, - None => return Ok(None), - }; - - let checkpoint = sui_json_rpc_types::Checkpoint::try_from(stored_checkpoint)?; - Ok(Some(checkpoint)) - } - - pub async fn get_latest_checkpoint( - &self, - ) -> Result { - let stored_checkpoint = self.get_latest_checkpoint_from_db().await?; - - sui_json_rpc_types::Checkpoint::try_from(stored_checkpoint) - } - - async fn get_checkpoints_from_db( - &self, - cursor: Option, - limit: usize, - descending_order: bool, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let mut query = checkpoints::table.into_boxed(); - if let Some(cursor) = cursor { - if descending_order { - query = query.filter(checkpoints::sequence_number.lt(cursor as i64)); - } else { - query = query.filter(checkpoints::sequence_number.gt(cursor as i64)); - } - } - if descending_order { - query = query.order_by(checkpoints::sequence_number.desc()); - } else { - query = query.order_by(checkpoints::sequence_number.asc()); - } - - query - .limit(limit as i64) - .load::(&mut connection) - .await - .map_err(Into::into) - } - - pub async fn get_checkpoints( - &self, - cursor: Option, - limit: usize, - descending_order: bool, - ) -> Result, IndexerError> { - self.get_checkpoints_from_db(cursor, limit, descending_order) - .await? - .into_iter() - .map(sui_json_rpc_types::Checkpoint::try_from) - .collect() - } - - async fn multi_get_transactions( - &self, - digests: &[TransactionDigest], - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let digests = digests - .iter() - .map(|digest| digest.inner().to_vec()) - .collect::>(); - - transactions::table - .inner_join( - tx_digests::table - .on(transactions::tx_sequence_number.eq(tx_digests::tx_sequence_number)), - ) - .filter(tx_digests::tx_digest.eq_any(digests)) - .select(StoredTransaction::as_select()) - .load::(&mut connection) - .await - .map_err(Into::into) - } - - async fn stored_transaction_to_transaction_block( - &self, - stored_txes: Vec, - options: sui_json_rpc_types::SuiTransactionBlockResponseOptions, - ) -> IndexerResult> { - let mut tx_block_responses_futures = vec![]; - for stored_tx in stored_txes { - let package_resolver_clone = self.package_resolver(); - let options_clone = options.clone(); - tx_block_responses_futures.push(tokio::task::spawn( - stored_tx - .try_into_sui_transaction_block_response(options_clone, package_resolver_clone), - )); - } - - let tx_blocks = futures::future::join_all(tx_block_responses_futures) - .await - .into_iter() - .collect::, _>>() - .tap_err(|e| error!("Failed to join all tx block futures: {}", e))? - .into_iter() - .collect::, _>>() - .tap_err(|e| error!("Failed to collect tx block futures: {}", e))?; - Ok(tx_blocks) - } - - async fn multi_get_transactions_with_sequence_numbers( - &self, - tx_sequence_numbers: Vec, - // Some(true) for desc, Some(false) for asc, None for undefined order - is_descending: Option, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let mut query = transactions::table - .filter(transactions::tx_sequence_number.eq_any(tx_sequence_numbers)) - .into_boxed(); - match is_descending { - Some(true) => { - query = query.order(transactions::dsl::tx_sequence_number.desc()); - } - Some(false) => { - query = query.order(transactions::dsl::tx_sequence_number.asc()); - } - None => (), - } - - query - .load::(&mut connection) - .await - .map_err(Into::into) - } - - pub async fn get_owned_objects( - &self, - address: SuiAddress, - filter: Option, - cursor: Option, - limit: usize, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let mut query = objects::table - .filter(objects::owner_type.eq(OwnerType::Address as i16)) - .filter(objects::owner_id.eq(address.to_vec())) - .order(objects::object_id.asc()) - .limit(limit as i64) - .into_boxed(); - if let Some(filter) = filter { - match filter { - SuiObjectDataFilter::StructType(struct_tag) => { - let object_type = struct_tag.to_canonical_string(/* with_prefix */ true); - query = query.filter(objects::object_type.like(format!("{}%", object_type))); - } - SuiObjectDataFilter::MatchAny(filters) => { - let mut condition = "(".to_string(); - for (i, filter) in filters.iter().enumerate() { - if let SuiObjectDataFilter::StructType(struct_tag) = filter { - let object_type = - struct_tag.to_canonical_string(/* with_prefix */ true); - if i == 0 { - condition += - format!("objects.object_type LIKE '{}%'", object_type).as_str(); - } else { - condition += - format!(" OR objects.object_type LIKE '{}%'", object_type) - .as_str(); - } - } else { - return Err(IndexerError::InvalidArgumentError( - "Invalid filter type. Only struct, MatchAny and MatchNone of struct filters are supported.".into(), - )); - } - } - condition += ")"; - query = query.filter(sql::(&condition)); - } - SuiObjectDataFilter::MatchNone(filters) => { - for filter in filters { - if let SuiObjectDataFilter::StructType(struct_tag) = filter { - let object_type = - struct_tag.to_canonical_string(/* with_prefix */ true); - query = query - .filter(objects::object_type.not_like(format!("{}%", object_type))); - } else { - return Err(IndexerError::InvalidArgumentError( - "Invalid filter type. Only struct, MatchAny and MatchNone of struct filters are supported.".into(), - )); - } - } - } - _ => { - return Err(IndexerError::InvalidArgumentError( - "Invalid filter type. Only struct, MatchAny and MatchNone of struct filters are supported.".into(), - )); - } - } - } - - if let Some(object_cursor) = cursor { - query = query.filter(objects::object_id.gt(object_cursor.to_vec())); - } - - query - .load::(&mut connection) - .await - .map_err(|e| IndexerError::PostgresReadError(e.to_string())) - } - - pub async fn multi_get_objects( - &self, - object_ids: Vec, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - let object_ids = object_ids.into_iter().map(|id| id.to_vec()).collect_vec(); - - objects::table - .filter(objects::object_id.eq_any(object_ids)) - .load::(&mut connection) - .await - .map_err(Into::into) - } - - async fn query_transaction_blocks_by_checkpoint( - &self, - checkpoint_seq: u64, - options: sui_json_rpc_types::SuiTransactionBlockResponseOptions, - cursor_tx_seq: Option, - limit: usize, - is_descending: bool, - ) -> IndexerResult> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let tx_range: (i64, i64) = pruner_cp_watermark::dsl::pruner_cp_watermark - .select(( - pruner_cp_watermark::min_tx_sequence_number, - pruner_cp_watermark::max_tx_sequence_number, - )) - .filter(pruner_cp_watermark::checkpoint_sequence_number.eq(checkpoint_seq as i64)) - .first::<(i64, i64)>(&mut connection) - .await?; - - let mut query = transactions::table - .filter(transactions::tx_sequence_number.between(tx_range.0, tx_range.1)) - .into_boxed(); - - if let Some(cursor_tx_seq) = cursor_tx_seq { - if is_descending { - query = query.filter(transactions::tx_sequence_number.lt(cursor_tx_seq)); - } else { - query = query.filter(transactions::tx_sequence_number.gt(cursor_tx_seq)); - } - } - if is_descending { - query = query.order(transactions::tx_sequence_number.desc()); - } else { - query = query.order(transactions::tx_sequence_number.asc()); - } - let stored_txes = query - .limit(limit as i64) - .load::(&mut connection) - .await?; - self.stored_transaction_to_transaction_block(stored_txes, options) - .await - } - - pub async fn query_transaction_blocks( - &self, - filter: Option, - options: sui_json_rpc_types::SuiTransactionBlockResponseOptions, - cursor: Option, - limit: usize, - is_descending: bool, - ) -> IndexerResult> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let cursor_tx_seq = if let Some(cursor) = cursor { - let tx_seq = tx_digests::table - .select(tx_digests::tx_sequence_number) - .filter(tx_digests::tx_digest.eq(cursor.into_inner().to_vec())) - .first::(&mut connection) - .await?; - Some(tx_seq) - } else { - None - }; - let cursor_clause = if let Some(cursor_tx_seq) = cursor_tx_seq { - if is_descending { - format!("AND {TX_SEQUENCE_NUMBER_STR} < {}", cursor_tx_seq) - } else { - format!("AND {TX_SEQUENCE_NUMBER_STR} > {}", cursor_tx_seq) - } - } else { - "".to_string() - }; - let order_str = if is_descending { "DESC" } else { "ASC" }; - let (table_name, main_where_clause) = match filter { - // Processed above - Some(TransactionFilter::Checkpoint(seq)) => { - return self - .query_transaction_blocks_by_checkpoint( - seq, - options, - cursor_tx_seq, - limit, - is_descending, - ) - .await - } - // FIXME: sanitize module & function - Some(TransactionFilter::MoveFunction { - package, - module, - function, - }) => { - let package = Hex::encode(package.to_vec()); - match (module, function) { - (Some(module), Some(function)) => ( - "tx_calls_fun".to_owned(), - format!( - "package = '\\x{package}'::bytea AND module = '{module}' AND func = '{function}'", - ), - ), - (Some(module), None) => ( - "tx_calls_mod".to_owned(), - format!( - "package = '\\x{package}'::bytea AND module = '{module}'", - ), - ), - (None, Some(_)) => { - return Err(IndexerError::InvalidArgumentError( - "Function cannot be present without Module.".into(), - )); - } - (None, None) => ( - "tx_calls_pkg".to_owned(), - format!("package = '\\x{package}'::bytea"), - ), - } - } - Some(TransactionFilter::AffectedObject(object_id)) => { - let object_id = Hex::encode(object_id.to_vec()); - ( - "tx_affected_objects".to_owned(), - format!("affected = '\\x{object_id}'::bytea"), - ) - } - Some(TransactionFilter::FromAddress(from_address)) => { - let from_address = Hex::encode(from_address.to_vec()); - ( - "tx_affected_addresses".to_owned(), - format!("sender = '\\x{from_address}'::bytea AND affected = '\\x{from_address}'::bytea"), - ) - } - Some(TransactionFilter::FromAndToAddress { from, to }) => { - let from_address = Hex::encode(from.to_vec()); - let to_address = Hex::encode(to.to_vec()); - ( - "tx_affected_addresses".to_owned(), - format!("sender = '\\x{from_address}'::bytea AND affected = '\\x{to_address}'::bytea"), - ) - } - Some(TransactionFilter::FromOrToAddress { addr }) => { - let address = Hex::encode(addr.to_vec()); - ( - "tx_affected_addresses".to_owned(), - format!("affected = '\\x{address}'::bytea"), - ) - } - Some( - TransactionFilter::TransactionKind(_) | TransactionFilter::TransactionKindIn(_), - ) => { - return Err(IndexerError::NotSupportedError( - "TransactionKind filter is not supported.".into(), - )); - } - Some(TransactionFilter::InputObject(_) | TransactionFilter::ChangedObject(_)) => { - return Err(IndexerError::NotSupportedError( - "InputObject and OutputObject filters are not supported, please use AffectedObject instead.".into() - )) - } - Some(TransactionFilter::ToAddress(_)) => { - return Err(IndexerError::NotSupportedError( - "ToAddress filter is not supported, please use FromOrToAddress instead.".into() - )) - } - None => { - // apply no filter - ("transactions".to_owned(), "1 = 1".into()) - } - }; - - let query = format!( - "SELECT {TX_SEQUENCE_NUMBER_STR} FROM {} WHERE {} {} ORDER BY {TX_SEQUENCE_NUMBER_STR} {} LIMIT {}", - table_name, - main_where_clause, - cursor_clause, - order_str, - limit, - ); - - debug!("query transaction blocks: {}", query); - let tx_sequence_numbers = diesel::sql_query(query.clone()) - .load::(&mut connection) - .await? - .into_iter() - .map(|tsn| tsn.tx_sequence_number) - .collect::>(); - self.multi_get_transaction_block_response_by_sequence_numbers( - tx_sequence_numbers, - options, - Some(is_descending), - ) - .await - } - - async fn multi_get_transaction_block_response_in_blocking_task_impl( - &self, - digests: &[TransactionDigest], - options: sui_json_rpc_types::SuiTransactionBlockResponseOptions, - ) -> Result, IndexerError> { - let stored_txes = self.multi_get_transactions(digests).await?; - self.stored_transaction_to_transaction_block(stored_txes, options) - .await - } - - async fn multi_get_transaction_block_response_by_sequence_numbers( - &self, - tx_sequence_numbers: Vec, - options: sui_json_rpc_types::SuiTransactionBlockResponseOptions, - // Some(true) for desc, Some(false) for asc, None for undefined order - is_descending: Option, - ) -> Result, IndexerError> { - let stored_txes: Vec = self - .multi_get_transactions_with_sequence_numbers(tx_sequence_numbers, is_descending) - .await?; - self.stored_transaction_to_transaction_block(stored_txes, options) - .await - } - - pub async fn multi_get_transaction_block_response_in_blocking_task( - &self, - digests: Vec, - options: sui_json_rpc_types::SuiTransactionBlockResponseOptions, - ) -> Result, IndexerError> { - self.multi_get_transaction_block_response_in_blocking_task_impl(&digests, options) - .await - } - - pub async fn get_transaction_events( - &self, - digest: TransactionDigest, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - // Use the tx_digests lookup table for the corresponding tx_sequence_number, and then fetch - // event-relevant data from the entry on the transactions table. - let (timestamp_ms, serialized_events) = transactions::table - .filter( - transactions::tx_sequence_number - .nullable() - .eq(tx_digests::table - .select(tx_digests::tx_sequence_number) - .filter(tx_digests::tx_digest.eq(digest.into_inner().to_vec())) - .single_value()), - ) - .select((transactions::timestamp_ms, transactions::events)) - .first::<(i64, StoredTransactionEvents)>(&mut connection) - .await?; - - let events = stored_events_to_events(serialized_events)?; - let tx_events = TransactionEvents { data: events }; - - let sui_tx_events = tx_events_to_sui_tx_events( - tx_events, - self.package_resolver(), - digest, - timestamp_ms as u64, - ) - .await?; - Ok(sui_tx_events.map_or(vec![], |ste| ste.data)) - } - - async fn query_events_by_tx_digest( - &self, - tx_digest: TransactionDigest, - cursor: Option, - cursor_tx_seq: i64, - limit: usize, - descending_order: bool, - ) -> IndexerResult> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let mut query = events::table.into_boxed(); - - if let Some(cursor) = cursor { - if cursor.tx_digest != tx_digest { - return Err(IndexerError::InvalidArgumentError( - "Cursor tx_digest does not match the tx_digest in the query.".into(), - )); - } - if descending_order { - query = query.filter(events::event_sequence_number.lt(cursor.event_seq as i64)); - } else { - query = query.filter(events::event_sequence_number.gt(cursor.event_seq as i64)); - } - } else if descending_order { - query = query.filter(events::event_sequence_number.le(i64::MAX)); - } else { - query = query.filter(events::event_sequence_number.ge(0)); - }; - - if descending_order { - query = query.order(events::event_sequence_number.desc()); - } else { - query = query.order(events::event_sequence_number.asc()); - } - - // If the cursor is provided and matches tx_digest, we've already fetched the - // tx_sequence_number and can query events table directly. Otherwise, we can just consult - // the tx_digests table for the tx_sequence_number to key into events table. - if cursor.is_some() { - query = query.filter(events::tx_sequence_number.eq(cursor_tx_seq)); - } else { - query = query.filter( - events::tx_sequence_number.nullable().eq(tx_digests::table - .select(tx_digests::tx_sequence_number) - .filter(tx_digests::tx_digest.eq(tx_digest.into_inner().to_vec())) - .single_value()), - ); - } - - let stored_events = query - .limit(limit as i64) - .load::(&mut connection) - .await?; - - let mut sui_event_futures = vec![]; - for stored_event in stored_events { - sui_event_futures.push(tokio::task::spawn( - stored_event.try_into_sui_event(self.package_resolver.clone()), - )); - } - - let sui_events = futures::future::join_all(sui_event_futures) - .await - .into_iter() - .collect::, _>>() - .tap_err(|e| error!("Failed to join sui event futures: {}", e))? - .into_iter() - .collect::, _>>() - .tap_err(|e| error!("Failed to collect sui event futures: {}", e))?; - Ok(sui_events) - } - - pub async fn query_events( - &self, - filter: EventFilter, - cursor: Option, - limit: usize, - descending_order: bool, - ) -> IndexerResult> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let (tx_seq, event_seq) = if let Some(cursor) = cursor { - let EventID { - tx_digest, - event_seq, - } = cursor; - let tx_seq = transactions::table - .select(transactions::tx_sequence_number) - .filter( - transactions::tx_sequence_number - .nullable() - .eq(tx_digests::table - .select(tx_digests::tx_sequence_number) - .filter(tx_digests::tx_digest.eq(tx_digest.into_inner().to_vec())) - .single_value()), - ) - .first::(&mut connection) - .await?; - (tx_seq, event_seq as i64) - } else if descending_order { - (i64::MAX, i64::MAX) - } else { - (-1, 0) - }; - - let query = if let EventFilter::Sender(sender) = &filter { - // Need to remove ambiguities for tx_sequence_number column - let cursor_clause = if descending_order { - format!("(e.{TX_SEQUENCE_NUMBER_STR} < {} OR (e.{TX_SEQUENCE_NUMBER_STR} = {} AND e.{EVENT_SEQUENCE_NUMBER_STR} < {}))", tx_seq, tx_seq, event_seq) - } else { - format!("(e.{TX_SEQUENCE_NUMBER_STR} > {} OR (e.{TX_SEQUENCE_NUMBER_STR} = {} AND e.{EVENT_SEQUENCE_NUMBER_STR} > {}))", tx_seq, tx_seq, event_seq) - }; - let order_clause = if descending_order { - format!("e.{TX_SEQUENCE_NUMBER_STR} DESC, e.{EVENT_SEQUENCE_NUMBER_STR} DESC") - } else { - format!("e.{TX_SEQUENCE_NUMBER_STR} ASC, e.{EVENT_SEQUENCE_NUMBER_STR} ASC") - }; - format!( - "( \ - SELECT * - FROM event_senders s - JOIN events e - USING (tx_sequence_number, event_sequence_number) - WHERE s.sender = '\\x{}'::bytea AND {} \ - ORDER BY {} \ - LIMIT {} - )", - Hex::encode(sender.to_vec()), - cursor_clause, - order_clause, - limit, - ) - } else if let EventFilter::Transaction(tx_digest) = filter { - return self - .query_events_by_tx_digest(tx_digest, cursor, tx_seq, limit, descending_order) - .await; - } else { - let main_where_clause = match filter { - EventFilter::All([]) => { - // No filter - "1 = 1".to_string() - } - EventFilter::MoveModule { package, module } => { - format!( - "package = '\\x{}'::bytea AND module = '{}'", - package.to_hex(), - module, - ) - } - EventFilter::MoveEventType(struct_tag) => { - format!("event_type = '{}'", struct_tag) - } - EventFilter::MoveEventModule { package, module } => { - let package_module_prefix = format!("{}::{}", package.to_hex_literal(), module); - format!("event_type LIKE '{package_module_prefix}::%'") - } - EventFilter::Sender(_) => { - // Processed above - unreachable!() - } - EventFilter::Transaction(_) => { - // Processed above - unreachable!() - } - EventFilter::TimeRange { .. } | EventFilter::Any(_) => { - return Err(IndexerError::NotSupportedError( - "This type of EventFilter is not supported.".to_owned(), - )); - } - }; - - let cursor_clause = if descending_order { - format!("AND ({TX_SEQUENCE_NUMBER_STR} < {} OR ({TX_SEQUENCE_NUMBER_STR} = {} AND {EVENT_SEQUENCE_NUMBER_STR} < {}))", tx_seq, tx_seq, event_seq) - } else { - format!("AND ({TX_SEQUENCE_NUMBER_STR} > {} OR ({TX_SEQUENCE_NUMBER_STR} = {} AND {EVENT_SEQUENCE_NUMBER_STR} > {}))", tx_seq, tx_seq, event_seq) - }; - let order_clause = if descending_order { - format!("{TX_SEQUENCE_NUMBER_STR} DESC, {EVENT_SEQUENCE_NUMBER_STR} DESC") - } else { - format!("{TX_SEQUENCE_NUMBER_STR} ASC, {EVENT_SEQUENCE_NUMBER_STR} ASC") - }; - - format!( - " - SELECT * FROM events \ - WHERE {} {} \ - ORDER BY {} \ - LIMIT {} - ", - main_where_clause, cursor_clause, order_clause, limit, - ) - }; - debug!("query events: {}", query); - let stored_events = diesel::sql_query(query) - .load::(&mut connection) - .await?; - - let mut sui_event_futures = vec![]; - for stored_event in stored_events { - sui_event_futures.push(tokio::task::spawn( - stored_event.try_into_sui_event(self.package_resolver.clone()), - )); - } - - let sui_events = futures::future::join_all(sui_event_futures) - .await - .into_iter() - .collect::, _>>() - .tap_err(|e| error!("Failed to join sui event futures: {}", e))? - .into_iter() - .collect::, _>>() - .tap_err(|e| error!("Failed to collect sui event futures: {}", e))?; - Ok(sui_events) - } - - pub async fn get_dynamic_fields( - &self, - parent_object_id: ObjectID, - cursor: Option, - limit: usize, - ) -> Result, IndexerError> { - let stored_objects = self - .get_dynamic_fields_raw(parent_object_id, cursor, limit) - .await?; - let mut df_futures = vec![]; - let indexer_reader_arc = Arc::new(self.clone()); - for stored_object in stored_objects { - let indexer_reader_arc_clone = Arc::clone(&indexer_reader_arc); - df_futures.push(tokio::task::spawn(async move { - indexer_reader_arc_clone - .try_create_dynamic_field_info(stored_object) - .await - })); - } - let df_infos = futures::future::join_all(df_futures) - .await - .into_iter() - .collect::, _>>() - .tap_err(|e| error!("Error joining DF futures: {:?}", e))? - .into_iter() - .collect::, _>>() - .tap_err(|e| error!("Error calling try_create_dynamic_field_info: {:?}", e))? - .into_iter() - .flatten() - .collect::>(); - Ok(df_infos) - } - - pub async fn get_dynamic_fields_raw( - &self, - parent_object_id: ObjectID, - cursor: Option, - limit: usize, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let mut query = objects::table - .filter(objects::owner_type.eq(OwnerType::Object as i16)) - .filter(objects::owner_id.eq(parent_object_id.to_vec())) - .order(objects::object_id.asc()) - .limit(limit as i64) - .into_boxed(); - - if let Some(object_cursor) = cursor { - query = query.filter(objects::object_id.gt(object_cursor.to_vec())); - } - - query - .load::(&mut connection) - .await - .map_err(Into::into) - } - - async fn try_create_dynamic_field_info( - &self, - stored_object: StoredObject, - ) -> Result, IndexerError> { - if stored_object.df_kind.is_none() { - return Ok(None); - } - - let object: Object = stored_object.try_into()?; - let move_object = match object.data.try_as_move().cloned() { - Some(move_object) => move_object, - None => { - return Err(IndexerError::ResolveMoveStructError( - "Object is not a MoveObject".to_string(), - )); - } - }; - let type_tag: TypeTag = move_object.type_().clone().into(); - let layout = self - .package_resolver - .type_layout(type_tag.clone()) - .await - .map_err(|e| { - IndexerError::ResolveMoveStructError(format!( - "Failed to get type layout for type {}: {e}", - type_tag.to_canonical_display(/* with_prefix */ true), - )) - })?; - - let field = DFV::FieldVisitor::deserialize(move_object.contents(), &layout) - .tap_err(|e| warn!("{e}"))?; - - let type_ = field.kind; - let name_type: TypeTag = field.name_layout.into(); - let bcs_name = field.name_bytes.to_owned(); - - let name_value = BoundedVisitor::deserialize_value(field.name_bytes, field.name_layout) - .tap_err(|e| warn!("{e}"))?; - - let name = DynamicFieldName { - type_: name_type, - value: SuiMoveValue::from(name_value).to_json_value(), - }; - - let value_metadata = field.value_metadata().map_err(|e| { - warn!("{e}"); - IndexerError::UncategorizedError(anyhow!(e)) - })?; - - Ok(Some(match value_metadata { - DFV::ValueMetadata::DynamicField(object_type) => DynamicFieldInfo { - name, - bcs_name, - type_, - object_type: object_type.to_canonical_string(/* with_prefix */ true), - object_id: object.id(), - version: object.version(), - digest: object.digest(), - }, - - DFV::ValueMetadata::DynamicObjectField(object_id) => { - let object = self.get_object(&object_id, None).await?.ok_or_else(|| { - IndexerError::UncategorizedError(anyhow!( - "Failed to find object_id {} when trying to create dynamic field info", - object_id.to_canonical_display(/* with_prefix */ true), - )) - })?; - - let object_type = object.data.type_().unwrap().clone(); - DynamicFieldInfo { - name, - bcs_name, - type_, - object_type: object_type.to_canonical_string(/* with_prefix */ true), - object_id, - version: object.version(), - digest: object.digest(), - } - } - })) - } - - pub async fn bcs_name_from_dynamic_field_name( - &self, - name: &DynamicFieldName, - ) -> Result, IndexerError> { - let move_type_layout = self - .package_resolver() - .type_layout(name.type_.clone()) - .await - .map_err(|e| { - IndexerError::ResolveMoveStructError(format!( - "Failed to get type layout for type {}: {}", - name.type_, e - )) - })?; - let sui_json_value = sui_json::SuiJsonValue::new(name.value.clone())?; - let name_bcs_value = sui_json_value.to_bcs_bytes(&move_type_layout)?; - Ok(name_bcs_value) - } - - async fn get_display_object_by_type( - &self, - object_type: &move_core_types::language_storage::StructTag, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let object_type = object_type.to_canonical_string(/* with_prefix */ true); - let stored_display = display::table - .filter(display::object_type.eq(object_type)) - .first::(&mut connection) - .await - .optional()?; - - let stored_display = match stored_display { - Some(display) => display, - None => return Ok(None), - }; - - let display_update = stored_display.to_display_update_event()?; - - Ok(Some(display_update)) - } - - pub async fn get_owned_coins( - &self, - owner: SuiAddress, - // If coin_type is None, look for all coins. - coin_type: Option, - cursor: ObjectID, - limit: usize, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - let mut query = objects::dsl::objects - .filter(objects::dsl::owner_type.eq(OwnerType::Address as i16)) - .filter(objects::dsl::owner_id.eq(owner.to_vec())) - .filter(objects::dsl::object_id.gt(cursor.to_vec())) - .into_boxed(); - if let Some(coin_type) = coin_type { - query = query.filter(objects::dsl::coin_type.eq(Some(coin_type))); - } else { - query = query.filter(objects::dsl::coin_type.is_not_null()); - } - - query - .order((objects::dsl::coin_type.asc(), objects::dsl::object_id.asc())) - .limit(limit as i64) - .load::(&mut connection) - .await? - .into_iter() - .map(|o| o.try_into()) - .collect::>>() - } - - pub async fn get_coin_balances( - &self, - owner: SuiAddress, - // If coin_type is None, look for all coins. - coin_type: Option, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let coin_type_filter = if let Some(coin_type) = coin_type { - format!("= '{}'", coin_type) - } else { - "IS NOT NULL".to_string() - }; - // Note: important to cast to BIGINT to avoid deserialize confusion - let query = format!( - " - SELECT coin_type, \ - CAST(COUNT(*) AS BIGINT) AS coin_num, \ - CAST(SUM(coin_balance) AS BIGINT) AS coin_balance \ - FROM objects \ - WHERE owner_type = {} \ - AND owner_id = '\\x{}'::BYTEA \ - AND coin_type {} \ - GROUP BY coin_type \ - ORDER BY coin_type ASC - ", - OwnerType::Address as i16, - Hex::encode(owner.to_vec()), - coin_type_filter, - ); - - debug!("get coin balances query: {query}"); - diesel::sql_query(query) - .load::(&mut connection) - .await? - .into_iter() - .map(|cb| cb.try_into()) - .collect::>>() - } - - pub(crate) async fn get_display_fields( - &self, - original_object: &sui_types::object::Object, - original_layout: &Option, - ) -> Result { - let (object_type, layout) = if let Some((object_type, layout)) = - sui_json_rpc::read_api::get_object_type_and_struct(original_object, original_layout) - .map_err(|e| IndexerError::GenericError(e.to_string()))? - { - (object_type, layout) - } else { - return Ok(DisplayFieldsResponse { - data: None, - error: None, - }); - }; - - if let Some(display_object) = self.get_display_object_by_type(&object_type).await? { - return sui_json_rpc::read_api::get_rendered_fields(display_object.fields, &layout) - .map_err(|e| IndexerError::GenericError(e.to_string())); - } - Ok(DisplayFieldsResponse { - data: None, - error: None, - }) - } - - pub async fn get_singleton_object(&self, type_: &StructTag) -> Result> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let object = match objects::table - .filter(objects::object_type_package.eq(type_.address.to_vec())) - .filter(objects::object_type_module.eq(type_.module.to_string())) - .filter(objects::object_type_name.eq(type_.name.to_string())) - .filter(objects::object_type.eq(type_.to_canonical_string(/* with_prefix */ true))) - .first::(&mut connection) - .await - .optional()? - { - Some(object) => object, - None => return Ok(None), - } - .try_into()?; - - Ok(Some(object)) - } - - pub async fn get_coin_metadata( - &self, - coin_struct: StructTag, - ) -> Result, IndexerError> { - let coin_metadata_type = CoinMetadata::type_(coin_struct); - - self.get_singleton_object(&coin_metadata_type) - .await? - .and_then(|o| SuiCoinMetadata::try_from(o).ok()) - .pipe(Ok) - } - - pub async fn get_total_supply(&self, coin_struct: StructTag) -> Result { - let treasury_cap_type = TreasuryCap::type_(coin_struct); - - self.get_singleton_object(&treasury_cap_type) - .await? - .and_then(|o| TreasuryCap::try_from(o).ok()) - .ok_or(IndexerError::GenericError(format!( - "Cannot find treasury cap object with type {}", - treasury_cap_type - )))? - .total_supply - .pipe(Ok) - } - - pub fn package_resolver(&self) -> PackageResolver { - self.package_resolver.clone() - } -} - -// NOTE: Do not make this public and easily accessible as we need to be careful that it is only -// used in non-async contexts via the use of tokio::task::spawn_blocking in order to avoid blocking -// the async runtime. -// -// Maybe we should look into introducing an async object store trait... -struct ConnectionAsObjectStore { - inner: std::sync::Mutex< - diesel_async::async_connection_wrapper::AsyncConnectionWrapper< - crate::database::Connection<'static>, - >, - >, -} - -impl ConnectionAsObjectStore { - async fn from_pool( - pool: &ConnectionPool, - ) -> Result { - let connection = std::sync::Mutex::new(pool.dedicated_connection().await?.into()); - - Ok(Self { inner: connection }) - } - - fn get_object_from_db( - &self, - object_id: &ObjectID, - version: Option, - ) -> Result, IndexerError> { - use diesel::RunQueryDsl; - - let mut guard = self.inner.lock().unwrap(); - let connection: &mut diesel_async::async_connection_wrapper::AsyncConnectionWrapper<_> = - &mut guard; - - let mut query = objects::table - .filter(objects::object_id.eq(object_id.to_vec())) - .into_boxed(); - if let Some(version) = version { - query = query.filter(objects::object_version.eq(version.value() as i64)) - } - - query - .first::(connection) - .optional() - .map_err(Into::into) - } - - fn get_object( - &self, - object_id: &ObjectID, - version: Option, - ) -> Result, IndexerError> { - let Some(stored_package) = self.get_object_from_db(object_id, version)? else { - return Ok(None); - }; - - let object = stored_package.try_into()?; - Ok(Some(object)) - } -} - -impl sui_types::storage::ObjectStore for ConnectionAsObjectStore { - fn get_object(&self, object_id: &ObjectID) -> Option { - self.get_object(object_id, None) - .expect("Error getting object") - } - - fn get_object_by_key( - &self, - object_id: &ObjectID, - version: sui_types::base_types::VersionNumber, - ) -> Option { - self.get_object(object_id, Some(version)) - .expect("Error getting object by key") - } -} diff --git a/crates/sui-mvr-indexer/src/lib.rs b/crates/sui-mvr-indexer/src/lib.rs deleted file mode 100644 index e759370c72798..0000000000000 --- a/crates/sui-mvr-indexer/src/lib.rs +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 -#![recursion_limit = "256"] - -use std::time::Duration; - -use anyhow::Result; -use config::JsonRpcConfig; -use jsonrpsee::http_client::{HeaderMap, HeaderValue, HttpClient, HttpClientBuilder}; -use metrics::IndexerMetrics; -use mysten_metrics::spawn_monitored_task; -use prometheus::Registry; -use system_package_task::SystemPackageTask; -use tokio_util::sync::CancellationToken; -use tracing::warn; - -use sui_json_rpc::ServerType; -use sui_json_rpc::{JsonRpcServerBuilder, ServerHandle}; -use sui_json_rpc_api::CLIENT_SDK_TYPE_HEADER; - -use crate::apis::{ - CoinReadApi, ExtendedApi, GovernanceReadApi, IndexerApi, MoveUtilsApi, ReadApi, - TransactionBuilderApi, WriteApi, -}; -use crate::indexer_reader::IndexerReader; -use errors::IndexerError; - -pub mod apis; -pub mod backfill; -pub mod config; -pub mod database; -pub mod db; -pub mod errors; -pub mod handlers; -pub mod indexer; -pub mod indexer_reader; -pub mod metrics; -pub mod models; -pub mod restorer; -pub mod schema; -pub mod store; -pub mod system_package_task; -pub mod tempdb; -pub mod test_utils; -pub mod types; - -pub async fn build_json_rpc_server( - prometheus_registry: &Registry, - reader: IndexerReader, - config: &JsonRpcConfig, - cancel: CancellationToken, -) -> Result { - let mut builder = - JsonRpcServerBuilder::new(env!("CARGO_PKG_VERSION"), prometheus_registry, None, None); - let http_client = crate::get_http_client(&config.rpc_client_url)?; - - builder.register_module(WriteApi::new(http_client.clone()))?; - builder.register_module(IndexerApi::new( - reader.clone(), - config.name_service_options.to_config(), - ))?; - builder.register_module(TransactionBuilderApi::new(reader.clone()))?; - builder.register_module(MoveUtilsApi::new(reader.clone()))?; - builder.register_module(GovernanceReadApi::new(reader.clone()))?; - builder.register_module(ReadApi::new(reader.clone()))?; - builder.register_module(CoinReadApi::new(reader.clone()))?; - builder.register_module(ExtendedApi::new(reader.clone()))?; - - let system_package_task = - SystemPackageTask::new(reader.clone(), cancel.clone(), Duration::from_secs(10)); - - tracing::info!("Starting system package task"); - spawn_monitored_task!(async move { system_package_task.run().await }); - - Ok(builder - .start(config.rpc_address, None, ServerType::Http, Some(cancel)) - .await?) -} - -fn get_http_client(rpc_client_url: &str) -> Result { - let mut headers = HeaderMap::new(); - headers.insert(CLIENT_SDK_TYPE_HEADER, HeaderValue::from_static("indexer")); - - HttpClientBuilder::default() - .max_request_body_size(2 << 30) - .max_concurrent_requests(usize::MAX) - .set_headers(headers.clone()) - .build(rpc_client_url) - .map_err(|e| { - warn!("Failed to get new Http client with error: {:?}", e); - IndexerError::HttpClientInitError(format!( - "Failed to initialize fullnode RPC client with error: {:?}", - e - )) - }) -} diff --git a/crates/sui-mvr-indexer/src/main.rs b/crates/sui-mvr-indexer/src/main.rs deleted file mode 100644 index 3bd9fb1e05e45..0000000000000 --- a/crates/sui-mvr-indexer/src/main.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use clap::Parser; -use sui_mvr_indexer::backfill::backfill_runner::BackfillRunner; -use sui_mvr_indexer::config::{Command, UploadOptions}; -use sui_mvr_indexer::database::ConnectionPool; -use sui_mvr_indexer::db::setup_postgres::clear_database; -use sui_mvr_indexer::db::{ - check_db_migration_consistency, check_prunable_tables_valid, reset_database, run_migrations, -}; -use sui_mvr_indexer::indexer::Indexer; -use sui_mvr_indexer::metrics::{ - spawn_connection_pool_metric_collector, start_prometheus_server, IndexerMetrics, -}; -use sui_mvr_indexer::restorer::formal_snapshot::IndexerFormalSnapshotRestorer; -use sui_mvr_indexer::store::PgIndexerStore; -use tokio_util::sync::CancellationToken; -use tracing::warn; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let opts = sui_mvr_indexer::config::IndexerConfig::parse(); - - // NOTE: this is to print out tracing like info, warn & error. - let _guard = telemetry_subscribers::TelemetryConfig::new() - .with_env() - .init(); - warn!("WARNING: Sui indexer is still experimental and we expect occasional breaking changes that require backfills."); - - let (_registry_service, registry) = start_prometheus_server(opts.metrics_address)?; - mysten_metrics::init_metrics(®istry); - let indexer_metrics = IndexerMetrics::new(®istry); - - let pool = ConnectionPool::new( - opts.database_url.clone(), - opts.connection_pool_config.clone(), - ) - .await?; - spawn_connection_pool_metric_collector(indexer_metrics.clone(), pool.clone()); - - match opts.command { - Command::Indexer { - ingestion_config, - snapshot_config, - pruning_options, - upload_options, - } => { - // Make sure to run all migrations on startup, and also serve as a compatibility check. - run_migrations(pool.dedicated_connection().await?).await?; - let retention_config = pruning_options.load_from_file(); - if retention_config.is_some() { - check_prunable_tables_valid(&mut pool.get().await?).await?; - } - - let store = PgIndexerStore::new(pool, upload_options, indexer_metrics.clone()); - - Indexer::start_writer( - ingestion_config, - store, - indexer_metrics, - snapshot_config, - retention_config, - CancellationToken::new(), - ) - .await?; - } - Command::JsonRpcService(json_rpc_config) => { - check_db_migration_consistency(&mut pool.get().await?).await?; - - Indexer::start_reader(&json_rpc_config, ®istry, pool, CancellationToken::new()) - .await?; - } - Command::ResetDatabase { - force, - skip_migrations, - } => { - if !force { - return Err(anyhow::anyhow!( - "Resetting the DB requires use of the `--force` flag", - )); - } - - if skip_migrations { - clear_database(&mut pool.dedicated_connection().await?).await?; - } else { - reset_database(pool.dedicated_connection().await?).await?; - } - } - Command::RunMigrations => { - run_migrations(pool.dedicated_connection().await?).await?; - } - Command::RunBackFill { - start, - end, - runner_kind, - backfill_config, - } => { - let total_range = start..=end; - BackfillRunner::run(runner_kind, pool, backfill_config, total_range).await; - } - Command::Restore(restore_config) => { - let store = - PgIndexerStore::new(pool, UploadOptions::default(), indexer_metrics.clone()); - let mut formal_restorer = - IndexerFormalSnapshotRestorer::new(store, restore_config).await?; - formal_restorer.restore().await?; - } - } - - Ok(()) -} diff --git a/crates/sui-mvr-indexer/src/metrics.rs b/crates/sui-mvr-indexer/src/metrics.rs deleted file mode 100644 index 0b1f8c1e5bed5..0000000000000 --- a/crates/sui-mvr-indexer/src/metrics.rs +++ /dev/null @@ -1,813 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use axum::{extract::Extension, http::StatusCode, routing::get, Router}; -use mysten_metrics::RegistryService; -use prometheus::{ - register_histogram_with_registry, register_int_counter_with_registry, - register_int_gauge_with_registry, Histogram, IntCounter, IntGauge, -}; -use prometheus::{Registry, TextEncoder}; -use std::net::SocketAddr; -use tracing::info; - -const METRICS_ROUTE: &str = "/metrics"; - -pub fn start_prometheus_server( - addr: SocketAddr, -) -> Result<(RegistryService, Registry), anyhow::Error> { - info!(address =% addr, "Starting prometheus server"); - let registry = Registry::new_custom(Some("indexer".to_string()), None)?; - let registry_service = RegistryService::new(registry.clone()); - - let app = Router::new() - .route(METRICS_ROUTE, get(metrics)) - .layer(Extension(registry_service.clone())); - - tokio::spawn(async move { - let listener = tokio::net::TcpListener::bind(&addr).await.unwrap(); - axum::serve(listener, app).await.unwrap(); - }); - Ok((registry_service, registry)) -} - -async fn metrics(Extension(registry_service): Extension) -> (StatusCode, String) { - let metrics_families = registry_service.gather_all(); - match TextEncoder.encode_to_string(&metrics_families) { - Ok(metrics) => (StatusCode::OK, metrics), - Err(error) => ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("unable to encode metrics: {error}"), - ), - } -} - -/// NOTE: for various data ingestion steps, which are expected to be within [0.001, 100] seconds, -/// and high double digits usually means something is broken. -const DATA_INGESTION_LATENCY_SEC_BUCKETS: &[f64] = &[ - 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, -]; -/// NOTE: for objects_snapshot update and advance_epoch, which are expected to be within [0.1, 100] seconds, -/// and can go up to high hundreds of seconds when things go wrong. -const DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS: &[f64] = &[ - 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0, 5000.0, - 10000.0, -]; -/// NOTE: for json_rpc calls, which are expected to be within [0.01, 100] seconds, -/// high hundreds of seconds usually means something is broken. -const JSON_RPC_LATENCY_SEC_BUCKETS: &[f64] = &[ - 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, -]; - -#[derive(Clone)] -pub struct IndexerMetrics { - pub total_checkpoint_received: IntCounter, - pub total_tx_checkpoint_committed: IntCounter, - pub total_object_checkpoint_committed: IntCounter, - pub total_transaction_committed: IntCounter, - pub total_object_change_committed: IntCounter, - pub total_transaction_chunk_committed: IntCounter, - pub total_object_change_chunk_committed: IntCounter, - pub total_epoch_committed: IntCounter, - pub latest_fullnode_checkpoint_sequence_number: IntGauge, - pub latest_tx_checkpoint_sequence_number: IntGauge, - pub latest_indexer_object_checkpoint_sequence_number: IntGauge, - pub latest_object_snapshot_sequence_number: IntGauge, - // max checkpoint sequence numbers on various stages of indexer data ingestion - pub max_downloaded_checkpoint_sequence_number: IntGauge, - pub max_indexed_checkpoint_sequence_number: IntGauge, - pub max_committed_checkpoint_sequence_number: IntGauge, - // the related timestamps of max checkpoint ^ on various stages - pub downloaded_checkpoint_timestamp_ms: IntGauge, - pub indexed_checkpoint_timestamp_ms: IntGauge, - pub committed_checkpoint_timestamp_ms: IntGauge, - // lag starting from the timestamp of the latest checkpoint to the current time - pub download_lag_ms: IntGauge, - pub index_lag_ms: IntGauge, - pub db_commit_lag_ms: IntGauge, - // latencies of various steps of data ingestion. - // checkpoint E2E latency is: fullnode_download_latency + checkpoint_index_latency + db_commit_latency - pub checkpoint_download_bytes_size: IntGauge, - pub tokio_blocking_task_wait_latency: Histogram, - pub fullnode_checkpoint_data_download_latency: Histogram, - pub fullnode_checkpoint_wait_and_download_latency: Histogram, - pub fullnode_transaction_download_latency: Histogram, - pub fullnode_object_download_latency: Histogram, - pub checkpoint_index_latency: Histogram, - pub indexing_batch_size: IntGauge, - pub indexing_tx_object_changes_latency: Histogram, - pub indexing_objects_latency: Histogram, - pub indexing_get_object_in_mem_hit: IntCounter, - pub indexing_get_object_db_hit: IntCounter, - pub indexing_module_resolver_in_mem_hit: IntCounter, - pub indexing_package_resolver_in_mem_hit: IntCounter, - pub indexing_packages_latency: Histogram, - pub checkpoint_objects_index_latency: Histogram, - pub checkpoint_db_commit_latency: Histogram, - pub checkpoint_db_commit_latency_step_1: Histogram, - pub checkpoint_db_commit_latency_transactions: Histogram, - pub checkpoint_db_commit_latency_transactions_chunks: Histogram, - pub checkpoint_db_commit_latency_transactions_chunks_transformation: Histogram, - pub checkpoint_db_commit_latency_objects: Histogram, - pub checkpoint_db_commit_latency_objects_snapshot: Histogram, - pub checkpoint_db_commit_latency_objects_version: Histogram, - pub checkpoint_db_commit_latency_objects_history: Histogram, - pub checkpoint_db_commit_latency_full_objects_history: Histogram, - pub checkpoint_db_commit_latency_objects_chunks: Histogram, - pub checkpoint_db_commit_latency_objects_snapshot_chunks: Histogram, - pub checkpoint_db_commit_latency_objects_version_chunks: Histogram, - pub checkpoint_db_commit_latency_objects_history_chunks: Histogram, - pub checkpoint_db_commit_latency_full_objects_history_chunks: Histogram, - pub checkpoint_db_commit_latency_events: Histogram, - pub checkpoint_db_commit_latency_events_chunks: Histogram, - pub checkpoint_db_commit_latency_event_indices: Histogram, - pub checkpoint_db_commit_latency_event_indices_chunks: Histogram, - pub checkpoint_db_commit_latency_packages: Histogram, - pub checkpoint_db_commit_latency_tx_indices: Histogram, - pub checkpoint_db_commit_latency_tx_indices_chunks: Histogram, - pub checkpoint_db_commit_latency_checkpoints: Histogram, - pub checkpoint_db_commit_latency_epoch: Histogram, - pub checkpoint_db_commit_latency_watermarks: Histogram, - pub thousand_transaction_avg_db_commit_latency: Histogram, - pub object_db_commit_latency: Histogram, - pub object_mutation_db_commit_latency: Histogram, - pub object_deletion_db_commit_latency: Histogram, - pub epoch_db_commit_latency: Histogram, - // latencies of slow DB update queries, now only advance epoch and objects_snapshot update - pub advance_epoch_latency: Histogram, - // latencies of RPC endpoints in read.rs - pub get_transaction_block_latency: Histogram, - pub multi_get_transaction_blocks_latency: Histogram, - pub get_object_latency: Histogram, - pub multi_get_objects_latency: Histogram, - pub try_get_past_object_latency: Histogram, - pub try_multi_get_past_objects_latency: Histogram, - pub get_checkpoint_latency: Histogram, - pub get_checkpoints_latency: Histogram, - pub get_events_latency: Histogram, - pub get_loaded_child_objects_latency: Histogram, - pub get_total_transaction_blocks_latency: Histogram, - pub get_latest_checkpoint_sequence_number_latency: Histogram, - // latencies of RPC endpoints in indexer.rs - pub get_owned_objects_latency: Histogram, - pub query_transaction_blocks_latency: Histogram, - pub query_events_latency: Histogram, - pub get_dynamic_fields_latency: Histogram, - pub get_dynamic_field_object_latency: Histogram, - pub get_protocol_config_latency: Histogram, - // latency of event websocket subscription - pub subscription_process_latency: Histogram, - pub transaction_per_checkpoint: Histogram, - // indexer state metrics - pub db_conn_pool_size: IntGauge, - pub idle_db_conn: IntGauge, - pub address_processor_failure: IntCounter, - pub checkpoint_metrics_processor_failure: IntCounter, - // pruner metrics - pub last_pruned_epoch: IntGauge, - pub last_pruned_checkpoint: IntGauge, - pub last_pruned_transaction: IntGauge, - pub epoch_pruning_latency: Histogram, -} - -impl IndexerMetrics { - pub fn new(registry: &Registry) -> Self { - Self { - total_checkpoint_received: register_int_counter_with_registry!( - "total_checkpoint_received", - "Total number of checkpoint received", - registry, - ) - .unwrap(), - total_tx_checkpoint_committed: register_int_counter_with_registry!( - "total_checkpoint_committed", - "Total number of checkpoint committed", - registry, - ) - .unwrap(), - total_object_checkpoint_committed: register_int_counter_with_registry!( - "total_object_checkpoint_committed", - "Total number of object checkpoint committed", - registry, - ) - .unwrap(), - total_transaction_committed: register_int_counter_with_registry!( - "total_transaction_committed", - "Total number of transaction committed", - registry, - ) - .unwrap(), - total_object_change_committed: register_int_counter_with_registry!( - "total_object_change_committed", - "Total number of object change committed", - registry, - ) - .unwrap(), - total_transaction_chunk_committed: register_int_counter_with_registry!( - "total_transaction_chunk_committed", - "Total number of transaction chunk committed", - registry, - ) - .unwrap(), - total_object_change_chunk_committed: register_int_counter_with_registry!( - "total_object_change_chunk_committed", - "Total number of object change chunk committed", - registry, - ) - .unwrap(), - total_epoch_committed: register_int_counter_with_registry!( - "total_epoch_committed", - "Total number of epoch committed", - registry, - ) - .unwrap(), - latest_fullnode_checkpoint_sequence_number: register_int_gauge_with_registry!( - "latest_fullnode_checkpoint_sequence_number", - "Latest checkpoint sequence number from the Full Node", - registry, - ) - .unwrap(), - latest_tx_checkpoint_sequence_number: register_int_gauge_with_registry!( - "latest_indexer_checkpoint_sequence_number", - "Latest checkpoint sequence number from the Indexer", - registry, - ) - .unwrap(), - latest_indexer_object_checkpoint_sequence_number: register_int_gauge_with_registry!( - "latest_indexer_object_checkpoint_sequence_number", - "Latest object checkpoint sequence number from the Indexer", - registry, - ) - .unwrap(), - latest_object_snapshot_sequence_number: register_int_gauge_with_registry!( - "latest_object_snapshot_sequence_number", - "Latest object snapshot sequence number from the Indexer", - registry, - ).unwrap(), - max_downloaded_checkpoint_sequence_number: register_int_gauge_with_registry!( - "max_downloaded_checkpoint_sequence_number", - "Max downloaded checkpoint sequence number", - registry, - ).unwrap(), - max_indexed_checkpoint_sequence_number: register_int_gauge_with_registry!( - "max_indexed_checkpoint_sequence_number", - "Max indexed checkpoint sequence number", - registry, - ).unwrap(), - max_committed_checkpoint_sequence_number: register_int_gauge_with_registry!( - "max_committed_checkpoint_sequence_number", - "Max committed checkpoint sequence number", - registry, - ).unwrap(), - downloaded_checkpoint_timestamp_ms: register_int_gauge_with_registry!( - "downloaded_checkpoint_timestamp_ms", - "Timestamp of the downloaded checkpoint", - registry, - ).unwrap(), - indexed_checkpoint_timestamp_ms: register_int_gauge_with_registry!( - "indexed_checkpoint_timestamp_ms", - "Timestamp of the indexed checkpoint", - registry, - ).unwrap(), - committed_checkpoint_timestamp_ms: register_int_gauge_with_registry!( - "committed_checkpoint_timestamp_ms", - "Timestamp of the committed checkpoint", - registry, - ).unwrap(), - download_lag_ms: register_int_gauge_with_registry!( - "download_lag_ms", - "Lag of the latest checkpoint in milliseconds", - registry, - ).unwrap(), - index_lag_ms: register_int_gauge_with_registry!( - "index_lag_ms", - "Lag of the latest checkpoint in milliseconds", - registry, - ).unwrap(), - db_commit_lag_ms: register_int_gauge_with_registry!( - "db_commit_lag_ms", - "Lag of the latest checkpoint in milliseconds", - registry, - ).unwrap(), - checkpoint_download_bytes_size: register_int_gauge_with_registry!( - "checkpoint_download_bytes_size", - "Size of the downloaded checkpoint in bytes", - registry, - ).unwrap(), - fullnode_checkpoint_data_download_latency: register_histogram_with_registry!( - "fullnode_checkpoint_data_download_latency", - "Time spent in downloading checkpoint and transaction for a new checkpoint from the Full Node", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - fullnode_checkpoint_wait_and_download_latency: register_histogram_with_registry!( - "fullnode_checkpoint_wait_and_download_latency", - "Time spent in waiting for a new checkpoint from the Full Node", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - - fullnode_transaction_download_latency: register_histogram_with_registry!( - "fullnode_transaction_download_latency", - "Time spent in waiting for a new transaction from the Full Node", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - fullnode_object_download_latency: register_histogram_with_registry!( - "fullnode_object_download_latency", - "Time spent in waiting for a new epoch from the Full Node", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_index_latency: register_histogram_with_registry!( - "checkpoint_index_latency", - "Time spent in indexing a checkpoint", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - indexing_batch_size: register_int_gauge_with_registry!( - "indexing_batch_size", - "Size of the indexing batch", - registry, - ).unwrap(), - indexing_tx_object_changes_latency: register_histogram_with_registry!( - "indexing_tx_object_changes_latency", - "Time spent in indexing object changes for a transaction", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - indexing_objects_latency: register_histogram_with_registry!( - "indexing_objects_latency", - "Time spent in indexing objects", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - indexing_packages_latency: register_histogram_with_registry!( - "indexing_packages_latency", - "Time spent in indexing packages", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - indexing_get_object_in_mem_hit: register_int_counter_with_registry!( - "indexing_get_object_in_mem_hit", - "Total number get object hit in mem", - registry, - ) - .unwrap(), - indexing_get_object_db_hit: register_int_counter_with_registry!( - "indexing_get_object_db_hit", - "Total number get object hit in db", - registry, - ) - .unwrap(), - indexing_module_resolver_in_mem_hit: register_int_counter_with_registry!( - "indexing_module_resolver_in_mem_hit", - "Total number module resolver hit in mem", - registry, - ) - .unwrap(), - indexing_package_resolver_in_mem_hit: register_int_counter_with_registry!( - "indexing_package_resolver_in_mem_hit", - "Total number package resolver hit in mem", - registry, - ) - .unwrap(), - checkpoint_objects_index_latency: register_histogram_with_registry!( - "checkpoint_object_index_latency", - "Time spent in indexing a checkpoint objects", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency: register_histogram_with_registry!( - "checkpoint_db_commit_latency", - "Time spent committing a checkpoint to the db", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - - checkpoint_db_commit_latency_step_1: register_histogram_with_registry!( - "checkpoint_db_commit_latency_step_1", - "Time spent committing a checkpoint to the db, step 1", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_transactions: register_histogram_with_registry!( - "checkpoint_db_commit_latency_transactions", - "Time spent committing transactions", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_transactions_chunks: register_histogram_with_registry!( - "checkpoint_db_commit_latency_transactions_chunks", - "Time spent committing transactions chunks", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_transactions_chunks_transformation: register_histogram_with_registry!( - "checkpoint_db_commit_latency_transactions_transaformation", - "Time spent in transactions chunks transformation prior to commit", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_objects: register_histogram_with_registry!( - "checkpoint_db_commit_latency_objects", - "Time spent committing objects", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_objects_snapshot: register_histogram_with_registry!( - "checkpoint_db_commit_latency_objects_snapshot", - "Time spent committing objects snapshots", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_objects_version: register_histogram_with_registry!( - "checkpoint_db_commit_latency_objects_version", - "Time spent committing objects version", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - checkpoint_db_commit_latency_objects_history: register_histogram_with_registry!( - "checkpoint_db_commit_latency_objects_history", - "Time spent committing objects history", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - checkpoint_db_commit_latency_full_objects_history: register_histogram_with_registry!( - "checkpoint_db_commit_latency_full_objects_history", - "Time spent committing full objects history", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - checkpoint_db_commit_latency_objects_chunks: register_histogram_with_registry!( - "checkpoint_db_commit_latency_objects_chunks", - "Time spent committing objects chunks", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_objects_snapshot_chunks: register_histogram_with_registry!( - "checkpoint_db_commit_latency_objects_snapshot_chunks", - "Time spent committing objects snapshot chunks", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_objects_version_chunks: register_histogram_with_registry!( - "checkpoint_db_commit_latency_objects_version_chunks", - "Time spent committing objects version chunks", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - checkpoint_db_commit_latency_objects_history_chunks: register_histogram_with_registry!( - "checkpoint_db_commit_latency_objects_history_chunks", - "Time spent committing objects history chunks", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - checkpoint_db_commit_latency_full_objects_history_chunks: register_histogram_with_registry!( - "checkpoint_db_commit_latency_full_objects_history_chunks", - "Time spent committing full objects history chunks", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_events: register_histogram_with_registry!( - "checkpoint_db_commit_latency_events", - "Time spent committing events", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_events_chunks: register_histogram_with_registry!( - "checkpoint_db_commit_latency_events_chunks", - "Time spent committing events chunks", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_event_indices: register_histogram_with_registry!( - "checkpoint_db_commit_latency_event_indices", - "Time spent committing event indices", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_event_indices_chunks: register_histogram_with_registry!( - "checkpoint_db_commit_latency_event_indices_chunks", - "Time spent committing event indices chunks", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_packages: register_histogram_with_registry!( - "checkpoint_db_commit_latency_packages", - "Time spent committing packages", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_tx_indices: register_histogram_with_registry!( - "checkpoint_db_commit_latency_tx_indices", - "Time spent committing tx indices", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_tx_indices_chunks: register_histogram_with_registry!( - "checkpoint_db_commit_latency_tx_indices_chunks", - "Time spent committing tx_indices chunks", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_checkpoints: register_histogram_with_registry!( - "checkpoint_db_commit_latency_checkpoints", - "Time spent committing checkpoints", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_epoch: register_histogram_with_registry!( - "checkpoint_db_commit_latency_epochs", - "Time spent committing epochs", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - checkpoint_db_commit_latency_watermarks: register_histogram_with_registry!( - "checkpoint_db_commit_latency_watermarks", - "Time spent committing watermarks", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - tokio_blocking_task_wait_latency: register_histogram_with_registry!( - "tokio_blocking_task_wait_latency", - "Time spent to wait for tokio blocking task pool", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - thousand_transaction_avg_db_commit_latency: register_histogram_with_registry!( - "transaction_db_commit_latency", - "Average time spent committing 1000 transactions to the db", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - object_db_commit_latency: register_histogram_with_registry!( - "object_db_commit_latency", - "Time spent committing a object to the db", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - object_mutation_db_commit_latency: register_histogram_with_registry!( - "object_mutation_db_commit_latency", - "Time spent committing a object mutation to the db", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - object_deletion_db_commit_latency: register_histogram_with_registry!( - "object_deletion_db_commit_latency", - "Time spent committing a object deletion to the db", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - epoch_db_commit_latency: register_histogram_with_registry!( - "epoch_db_commit_latency", - "Time spent committing a epoch to the db", - DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - advance_epoch_latency: register_histogram_with_registry!( - "advance_epoch_latency", - "Time spent in advancing epoch", - DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - subscription_process_latency: register_histogram_with_registry!( - "subscription_process_latency", - "Time spent in process Websocket subscription", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(), - transaction_per_checkpoint: register_histogram_with_registry!( - "transaction_per_checkpoint", - "Number of transactions per checkpoint", - vec![1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0, 5000.0], - registry, - ) - .unwrap(), - get_transaction_block_latency: register_histogram_with_registry!( - "get_transaction_block_latency", - "Time spent in get_transaction_block on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - multi_get_transaction_blocks_latency: register_histogram_with_registry!( - "multi_get_transaction_blocks_latency", - "Time spent in multi_get_transaction_blocks on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - get_object_latency: register_histogram_with_registry!( - "get_object_latency", - "Time spent in get_object on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - multi_get_objects_latency: register_histogram_with_registry!( - "multi_get_objects_latency", - "Time spent in multi_get_objects on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - try_get_past_object_latency: register_histogram_with_registry!( - "try_get_past_object_latency", - "Time spent in try_get_past_object on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - try_multi_get_past_objects_latency: register_histogram_with_registry!( - "try_multi_get_past_objects_latency", - "Time spent in try_multi_get_past_objects on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - get_checkpoint_latency: register_histogram_with_registry!( - "get_checkpoint_latency", - "Time spent in get_checkpoint on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - get_checkpoints_latency: register_histogram_with_registry!( - "get_checkpoints_latency", - "Time spent in get_checkpoints on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - get_events_latency: register_histogram_with_registry!( - "get_events_latency", - "Time spent in get_events on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - get_total_transaction_blocks_latency: register_histogram_with_registry!( - "get_total_transaction_blocks_latency", - "Time spent in get_total_transaction_blocks on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - get_latest_checkpoint_sequence_number_latency: register_histogram_with_registry!( - "get_latest_checkpoint_sequence_number_latency", - "Time spent in get_latest_checkpoint_sequence_number on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - get_owned_objects_latency: register_histogram_with_registry!( - "get_owned_objects_latency", - "Time spent in get_owned_objects on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - query_transaction_blocks_latency: register_histogram_with_registry!( - "query_transaction_blocks_latency", - "Time spent in query_transaction_blocks on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - query_events_latency: register_histogram_with_registry!( - "query_events_latency", - "Time spent in query_events on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - get_dynamic_fields_latency: register_histogram_with_registry!( - "get_dynamic_fields_latency", - "Time spent in get_dynamic_fields on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - get_dynamic_field_object_latency: register_histogram_with_registry!( - "get_dynamic_field_object_latency", - "Time spent in get_dynamic_field_object on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - get_loaded_child_objects_latency: register_histogram_with_registry!( - "get_loaded_child_objects_latency", - "Time spent in get_loaded_child_objects_latency on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - get_protocol_config_latency: register_histogram_with_registry!( - "get_protocol_config_latency", - "Time spent in get_protocol_config_latency on the fullnode behind.", - JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(), - registry - ) - .unwrap(), - db_conn_pool_size: register_int_gauge_with_registry!( - "db_conn_pool_size", - "Size of the database connection pool", - registry - ).unwrap(), - idle_db_conn: register_int_gauge_with_registry!( - "idle_db_conn", - "Number of idle database connections", - registry - ).unwrap(), - address_processor_failure: register_int_counter_with_registry!( - "address_processor_failure", - "Total number of address processor failure", - registry, - ) - .unwrap(), - checkpoint_metrics_processor_failure: register_int_counter_with_registry!( - "checkpoint_metrics_processor_failure", - "Total number of checkpoint metrics processor failure", - registry, - ) - .unwrap(), - last_pruned_epoch: register_int_gauge_with_registry!( - "last_pruned_epoch", - "Last pruned epoch number", - registry, - ) - .unwrap(), - last_pruned_checkpoint: register_int_gauge_with_registry!( - "last_pruned_checkpoint", - "Last pruned checkpoint sequence number", - registry, - ) - .unwrap(), - last_pruned_transaction: register_int_gauge_with_registry!( - "last_pruned_transaction", - "Last pruned transaction sequence number", - registry, - ).unwrap(), - epoch_pruning_latency: register_histogram_with_registry!( - "epoch_pruning_latency", - "Time spent in pruning one epoch", - DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS.to_vec(), - registry - ).unwrap(), - } - } -} - -pub fn spawn_connection_pool_metric_collector( - metrics: IndexerMetrics, - connection_pool: crate::database::ConnectionPool, -) { - tokio::spawn(async move { - loop { - let cp_state = connection_pool.state(); - tracing::debug!( - connection_pool_size =% cp_state.connections, - idle_connections =% cp_state.idle_connections, - ); - metrics.db_conn_pool_size.set(cp_state.connections as i64); - metrics.idle_db_conn.set(cp_state.idle_connections as i64); - tokio::time::sleep(tokio::time::Duration::from_secs(60)).await; - } - }); -} diff --git a/crates/sui-mvr-indexer/src/models/checkpoints.rs b/crates/sui-mvr-indexer/src/models/checkpoints.rs deleted file mode 100644 index d18c1a1a7ce9e..0000000000000 --- a/crates/sui-mvr-indexer/src/models/checkpoints.rs +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use diesel::prelude::*; - -use sui_json_rpc_types::Checkpoint as RpcCheckpoint; -use sui_types::base_types::TransactionDigest; -use sui_types::digests::CheckpointDigest; -use sui_types::gas::GasCostSummary; - -use crate::errors::IndexerError; -use crate::schema::{chain_identifier, checkpoints, pruner_cp_watermark}; -use crate::types::IndexedCheckpoint; - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = chain_identifier)] -pub struct StoredChainIdentifier { - pub checkpoint_digest: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = checkpoints)] -pub struct StoredCheckpoint { - pub sequence_number: i64, - pub checkpoint_digest: Vec, - pub epoch: i64, - pub network_total_transactions: i64, - pub previous_checkpoint_digest: Option>, - pub end_of_epoch: bool, - pub tx_digests: Vec>>, - pub timestamp_ms: i64, - pub total_gas_cost: i64, - pub computation_cost: i64, - pub storage_cost: i64, - pub storage_rebate: i64, - pub non_refundable_storage_fee: i64, - pub checkpoint_commitments: Vec, - pub validator_signature: Vec, - pub end_of_epoch_data: Option>, - pub min_tx_sequence_number: Option, - pub max_tx_sequence_number: Option, -} - -impl From<&IndexedCheckpoint> for StoredCheckpoint { - fn from(c: &IndexedCheckpoint) -> Self { - Self { - sequence_number: c.sequence_number as i64, - checkpoint_digest: c.checkpoint_digest.into_inner().to_vec(), - epoch: c.epoch as i64, - tx_digests: c - .tx_digests - .iter() - .map(|tx| Some(tx.into_inner().to_vec())) - .collect(), - network_total_transactions: c.network_total_transactions as i64, - previous_checkpoint_digest: c - .previous_checkpoint_digest - .as_ref() - .map(|d| (*d).into_inner().to_vec()), - timestamp_ms: c.timestamp_ms as i64, - total_gas_cost: c.total_gas_cost, - computation_cost: c.computation_cost as i64, - storage_cost: c.storage_cost as i64, - storage_rebate: c.storage_rebate as i64, - non_refundable_storage_fee: c.non_refundable_storage_fee as i64, - checkpoint_commitments: bcs::to_bytes(&c.checkpoint_commitments).unwrap(), - validator_signature: bcs::to_bytes(&c.validator_signature).unwrap(), - end_of_epoch_data: c - .end_of_epoch_data - .as_ref() - .map(|d| bcs::to_bytes(d).unwrap()), - end_of_epoch: c.end_of_epoch_data.is_some(), - min_tx_sequence_number: Some(c.min_tx_sequence_number as i64), - max_tx_sequence_number: Some(c.max_tx_sequence_number as i64), - } - } -} - -impl TryFrom for RpcCheckpoint { - type Error = IndexerError; - fn try_from(checkpoint: StoredCheckpoint) -> Result { - let parsed_digest = CheckpointDigest::try_from(checkpoint.checkpoint_digest.clone()) - .map_err(|e| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Failed to decode checkpoint digest: {:?} with err: {:?}", - checkpoint.checkpoint_digest, e - )) - })?; - - let parsed_previous_digest: Option = checkpoint - .previous_checkpoint_digest - .map(|digest| { - CheckpointDigest::try_from(digest.clone()).map_err(|e| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Failed to decode previous checkpoint digest: {:?} with err: {:?}", - digest, e - )) - }) - }) - .transpose()?; - - let transactions: Vec = { - checkpoint - .tx_digests - .into_iter() - .map(|tx_digest| match tx_digest { - None => Err(IndexerError::PersistentStorageDataCorruptionError( - "tx_digests should not contain null elements".to_string(), - )), - Some(tx_digest) => { - TransactionDigest::try_from(tx_digest.as_slice()).map_err(|e| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Failed to decode transaction digest: {:?} with err: {:?}", - tx_digest, e - )) - }) - } - }) - .collect::, IndexerError>>()? - }; - let validator_signature = - bcs::from_bytes(&checkpoint.validator_signature).map_err(|e| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Failed to decode validator signature: {:?} with err: {:?}", - checkpoint.validator_signature, e - )) - })?; - - let checkpoint_commitments = - bcs::from_bytes(&checkpoint.checkpoint_commitments).map_err(|e| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Failed to decode checkpoint commitments: {:?} with err: {:?}", - checkpoint.checkpoint_commitments, e - )) - })?; - - let end_of_epoch_data = checkpoint - .end_of_epoch_data - .map(|data| { - bcs::from_bytes(&data).map_err(|e| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Failed to decode end of epoch data: {:?} with err: {:?}", - data, e - )) - }) - }) - .transpose()?; - - Ok(RpcCheckpoint { - epoch: checkpoint.epoch as u64, - sequence_number: checkpoint.sequence_number as u64, - digest: parsed_digest, - previous_digest: parsed_previous_digest, - end_of_epoch_data, - epoch_rolling_gas_cost_summary: GasCostSummary { - computation_cost: checkpoint.computation_cost as u64, - storage_cost: checkpoint.storage_cost as u64, - storage_rebate: checkpoint.storage_rebate as u64, - non_refundable_storage_fee: checkpoint.non_refundable_storage_fee as u64, - }, - network_total_transactions: checkpoint.network_total_transactions as u64, - timestamp_ms: checkpoint.timestamp_ms as u64, - transactions, - validator_signature, - checkpoint_commitments, - }) - } -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = pruner_cp_watermark)] -pub struct StoredCpTx { - pub checkpoint_sequence_number: i64, - pub min_tx_sequence_number: i64, - pub max_tx_sequence_number: i64, -} - -impl From<&IndexedCheckpoint> for StoredCpTx { - fn from(c: &IndexedCheckpoint) -> Self { - Self { - checkpoint_sequence_number: c.sequence_number as i64, - min_tx_sequence_number: c.min_tx_sequence_number as i64, - max_tx_sequence_number: c.max_tx_sequence_number as i64, - } - } -} diff --git a/crates/sui-mvr-indexer/src/models/display.rs b/crates/sui-mvr-indexer/src/models/display.rs deleted file mode 100644 index 33a1c7c7cb0cb..0000000000000 --- a/crates/sui-mvr-indexer/src/models/display.rs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use diesel::prelude::*; -use serde::Deserialize; - -use sui_types::display::DisplayVersionUpdatedEvent; - -use crate::schema::display; - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Deserialize)] -#[diesel(table_name = display)] -pub struct StoredDisplay { - pub object_type: String, - pub id: Vec, - pub version: i16, - pub bcs: Vec, -} - -impl StoredDisplay { - pub fn try_from_event(event: &sui_types::event::Event) -> Option { - let (ty, display_event) = DisplayVersionUpdatedEvent::try_from_event(event)?; - - Some(Self { - object_type: ty.to_canonical_string(/* with_prefix */ true), - id: display_event.id.bytes.to_vec(), - version: display_event.version as i16, - bcs: event.contents.clone(), - }) - } - - pub fn to_display_update_event(&self) -> Result { - bcs::from_bytes(&self.bcs) - } -} diff --git a/crates/sui-mvr-indexer/src/models/epoch.rs b/crates/sui-mvr-indexer/src/models/epoch.rs deleted file mode 100644 index d8e943f4c245c..0000000000000 --- a/crates/sui-mvr-indexer/src/models/epoch.rs +++ /dev/null @@ -1,278 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::schema::epochs; -use crate::{errors::IndexerError, schema::feature_flags, schema::protocol_configs}; -use diesel::prelude::{AsChangeset, Identifiable}; -use diesel::{Insertable, Queryable, Selectable}; -use sui_json_rpc_types::{EndOfEpochInfo, EpochInfo}; -use sui_types::event::SystemEpochInfoEvent; -use sui_types::messages_checkpoint::CertifiedCheckpointSummary; -use sui_types::sui_system_state::sui_system_state_summary::SuiSystemStateSummary; - -#[derive(Queryable, Insertable, Debug, Clone, Default)] -#[diesel(table_name = epochs)] -pub struct StoredEpochInfo { - pub epoch: i64, - pub first_checkpoint_id: i64, - pub epoch_start_timestamp: i64, - pub reference_gas_price: i64, - pub protocol_version: i64, - pub total_stake: i64, - pub storage_fund_balance: i64, - pub system_state: Option>, - pub epoch_total_transactions: Option, - pub last_checkpoint_id: Option, - pub epoch_end_timestamp: Option, - pub storage_fund_reinvestment: Option, - pub storage_charge: Option, - pub storage_rebate: Option, - pub stake_subsidy_amount: Option, - pub total_gas_fees: Option, - pub total_stake_rewards_distributed: Option, - pub leftover_storage_fund_inflow: Option, - pub epoch_commitments: Option>, - /// This is the system state summary at the beginning of the epoch, serialized as JSON. - pub system_state_summary_json: Option, - /// First transaction sequence number of this epoch. - pub first_tx_sequence_number: Option, -} - -#[derive(Insertable, Identifiable, AsChangeset, Clone, Debug)] -#[diesel(primary_key(epoch))] -#[diesel(table_name = epochs)] -pub struct StartOfEpochUpdate { - pub epoch: i64, - pub first_checkpoint_id: i64, - pub first_tx_sequence_number: i64, - pub epoch_start_timestamp: i64, - pub reference_gas_price: i64, - pub protocol_version: i64, - pub total_stake: i64, - pub storage_fund_balance: i64, - pub system_state_summary_json: serde_json::Value, -} - -#[derive(Identifiable, AsChangeset, Clone, Debug)] -#[diesel(primary_key(epoch))] -#[diesel(table_name = epochs)] -pub struct EndOfEpochUpdate { - pub epoch: i64, - pub epoch_total_transactions: i64, - pub last_checkpoint_id: i64, - pub epoch_end_timestamp: i64, - pub storage_fund_reinvestment: i64, - pub storage_charge: i64, - pub storage_rebate: i64, - pub stake_subsidy_amount: i64, - pub total_gas_fees: i64, - pub total_stake_rewards_distributed: i64, - pub leftover_storage_fund_inflow: i64, - pub epoch_commitments: Vec, -} - -#[derive(Queryable, Insertable, Debug, Clone, Default)] -#[diesel(table_name = protocol_configs)] -pub struct StoredProtocolConfig { - pub protocol_version: i64, - pub config_name: String, - pub config_value: Option, -} - -#[derive(Queryable, Insertable, Debug, Clone, Default)] -#[diesel(table_name = feature_flags)] -pub struct StoredFeatureFlag { - pub protocol_version: i64, - pub flag_name: String, - pub flag_value: bool, -} - -#[derive(Queryable, Selectable, Clone)] -#[diesel(table_name = epochs)] -pub struct QueryableEpochInfo { - pub epoch: i64, - pub first_checkpoint_id: i64, - pub epoch_start_timestamp: i64, - pub reference_gas_price: i64, - pub protocol_version: i64, - pub total_stake: i64, - pub storage_fund_balance: i64, - pub epoch_total_transactions: Option, - pub first_tx_sequence_number: Option, - pub last_checkpoint_id: Option, - pub epoch_end_timestamp: Option, - pub storage_fund_reinvestment: Option, - pub storage_charge: Option, - pub storage_rebate: Option, - pub stake_subsidy_amount: Option, - pub total_gas_fees: Option, - pub total_stake_rewards_distributed: Option, - pub leftover_storage_fund_inflow: Option, - pub epoch_commitments: Option>, -} - -#[derive(Queryable)] -pub struct QueryableEpochSystemState { - pub epoch: i64, - pub system_state: Vec, -} - -#[derive(Default)] -pub struct EpochStartInfo { - pub first_checkpoint_id: u64, - pub first_tx_sequence_number: u64, - pub total_stake: u64, - pub storage_fund_balance: u64, -} - -impl EpochStartInfo { - pub fn new( - first_checkpoint_id: u64, - first_tx_sequence_number: u64, - epoch_event_opt: Option<&SystemEpochInfoEvent>, - ) -> Self { - Self { - first_checkpoint_id, - first_tx_sequence_number, - total_stake: epoch_event_opt.map(|e| e.total_stake).unwrap_or_default(), - storage_fund_balance: epoch_event_opt - .map(|e| e.storage_fund_balance) - .unwrap_or_default(), - } - } -} - -impl StartOfEpochUpdate { - pub fn new( - new_system_state_summary: SuiSystemStateSummary, - epoch_start_info: EpochStartInfo, - ) -> Self { - Self { - epoch: new_system_state_summary.epoch as i64, - system_state_summary_json: serde_json::to_value(new_system_state_summary.clone()) - .unwrap(), - first_checkpoint_id: epoch_start_info.first_checkpoint_id as i64, - first_tx_sequence_number: epoch_start_info.first_tx_sequence_number as i64, - epoch_start_timestamp: new_system_state_summary.epoch_start_timestamp_ms as i64, - reference_gas_price: new_system_state_summary.reference_gas_price as i64, - protocol_version: new_system_state_summary.protocol_version as i64, - total_stake: epoch_start_info.total_stake as i64, - storage_fund_balance: epoch_start_info.storage_fund_balance as i64, - } - } -} - -#[derive(Default)] -pub struct EpochEndInfo { - pub storage_fund_reinvestment: u64, - pub storage_charge: u64, - pub storage_rebate: u64, - pub leftover_storage_fund_inflow: u64, - pub stake_subsidy_amount: u64, - pub total_gas_fees: u64, - pub total_stake_rewards_distributed: u64, -} - -impl EpochEndInfo { - pub fn new(epoch_event_opt: Option<&SystemEpochInfoEvent>) -> Self { - epoch_event_opt.map_or_else(Self::default, |epoch_event| Self { - storage_fund_reinvestment: epoch_event.storage_fund_reinvestment, - storage_charge: epoch_event.storage_charge, - storage_rebate: epoch_event.storage_rebate, - leftover_storage_fund_inflow: epoch_event.leftover_storage_fund_inflow, - stake_subsidy_amount: epoch_event.stake_subsidy_amount, - total_gas_fees: epoch_event.total_gas_fees, - total_stake_rewards_distributed: epoch_event.total_stake_rewards_distributed, - }) - } -} - -impl EndOfEpochUpdate { - pub fn new( - last_checkpoint_summary: &CertifiedCheckpointSummary, - first_tx_sequence_number: u64, - epoch_end_info: EpochEndInfo, - ) -> Self { - Self { - epoch: last_checkpoint_summary.epoch as i64, - epoch_total_transactions: (last_checkpoint_summary.network_total_transactions - - first_tx_sequence_number) as i64, - last_checkpoint_id: *last_checkpoint_summary.sequence_number() as i64, - epoch_end_timestamp: last_checkpoint_summary.timestamp_ms as i64, - storage_fund_reinvestment: epoch_end_info.storage_fund_reinvestment as i64, - storage_charge: epoch_end_info.storage_charge as i64, - storage_rebate: epoch_end_info.storage_rebate as i64, - leftover_storage_fund_inflow: epoch_end_info.leftover_storage_fund_inflow as i64, - stake_subsidy_amount: epoch_end_info.stake_subsidy_amount as i64, - total_gas_fees: epoch_end_info.total_gas_fees as i64, - total_stake_rewards_distributed: epoch_end_info.total_stake_rewards_distributed as i64, - epoch_commitments: bcs::to_bytes( - &last_checkpoint_summary - .end_of_epoch_data - .clone() - .unwrap() - .epoch_commitments, - ) - .unwrap(), - } - } -} - -impl StoredEpochInfo { - pub fn get_json_system_state_summary(&self) -> Result { - let Some(system_state_summary_json) = self.system_state_summary_json.clone() else { - return Err(IndexerError::PersistentStorageDataCorruptionError( - "System state summary is null for the given epoch".into(), - )); - }; - let system_state_summary: SuiSystemStateSummary = - serde_json::from_value(system_state_summary_json).map_err(|_| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Failed to deserialize `system_state` for epoch {:?}", - self.epoch, - )) - })?; - debug_assert_eq!(system_state_summary.epoch, self.epoch as u64); - Ok(system_state_summary) - } -} - -impl From<&StoredEpochInfo> for Option { - fn from(info: &StoredEpochInfo) -> Option { - Some(EndOfEpochInfo { - reference_gas_price: (info.reference_gas_price as u64), - protocol_version: (info.protocol_version as u64), - last_checkpoint_id: info.last_checkpoint_id.map(|v| v as u64)?, - total_stake: info.total_stake as u64, - storage_fund_balance: info.storage_fund_balance as u64, - epoch_end_timestamp: info.epoch_end_timestamp.map(|v| v as u64)?, - storage_fund_reinvestment: info.storage_fund_reinvestment.map(|v| v as u64)?, - storage_charge: info.storage_charge.map(|v| v as u64)?, - storage_rebate: info.storage_rebate.map(|v| v as u64)?, - stake_subsidy_amount: info.stake_subsidy_amount.map(|v| v as u64)?, - total_gas_fees: info.total_gas_fees.map(|v| v as u64)?, - total_stake_rewards_distributed: info - .total_stake_rewards_distributed - .map(|v| v as u64)?, - leftover_storage_fund_inflow: info.leftover_storage_fund_inflow.map(|v| v as u64)?, - }) - } -} - -impl TryFrom for EpochInfo { - type Error = IndexerError; - - fn try_from(value: StoredEpochInfo) -> Result { - let end_of_epoch_info = (&value).into(); - let system_state_summary = value.get_json_system_state_summary()?; - Ok(EpochInfo { - epoch: value.epoch as u64, - validators: system_state_summary.active_validators, - epoch_total_transactions: value.epoch_total_transactions.unwrap_or(0) as u64, - first_checkpoint_id: value.first_checkpoint_id as u64, - epoch_start_timestamp: value.epoch_start_timestamp as u64, - end_of_epoch_info, - reference_gas_price: Some(value.reference_gas_price as u64), - }) - } -} diff --git a/crates/sui-mvr-indexer/src/models/event_indices.rs b/crates/sui-mvr-indexer/src/models/event_indices.rs deleted file mode 100644 index 08f17cce339d5..0000000000000 --- a/crates/sui-mvr-indexer/src/models/event_indices.rs +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::{ - schema::{ - event_emit_module, event_emit_package, event_senders, event_struct_instantiation, - event_struct_module, event_struct_name, event_struct_package, - }, - types::EventIndex, -}; -use diesel::prelude::*; - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = event_emit_package)] -pub struct StoredEventEmitPackage { - pub tx_sequence_number: i64, - pub event_sequence_number: i64, - pub package: Vec, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = event_emit_module)] -pub struct StoredEventEmitModule { - pub tx_sequence_number: i64, - pub event_sequence_number: i64, - pub package: Vec, - pub module: String, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = event_senders)] -pub struct StoredEventSenders { - pub tx_sequence_number: i64, - pub event_sequence_number: i64, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = event_struct_package)] -pub struct StoredEventStructPackage { - pub tx_sequence_number: i64, - pub event_sequence_number: i64, - pub package: Vec, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = event_struct_module)] -pub struct StoredEventStructModule { - pub tx_sequence_number: i64, - pub event_sequence_number: i64, - pub package: Vec, - pub module: String, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = event_struct_name)] -pub struct StoredEventStructName { - pub tx_sequence_number: i64, - pub event_sequence_number: i64, - pub package: Vec, - pub module: String, - pub type_name: String, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = event_struct_instantiation)] -pub struct StoredEventStructInstantiation { - pub tx_sequence_number: i64, - pub event_sequence_number: i64, - pub package: Vec, - pub module: String, - pub type_instantiation: String, - pub sender: Vec, -} - -impl EventIndex { - pub fn split( - self: EventIndex, - ) -> ( - StoredEventEmitPackage, - StoredEventEmitModule, - StoredEventSenders, - StoredEventStructPackage, - StoredEventStructModule, - StoredEventStructName, - StoredEventStructInstantiation, - ) { - let tx_sequence_number = self.tx_sequence_number as i64; - let event_sequence_number = self.event_sequence_number as i64; - ( - StoredEventEmitPackage { - tx_sequence_number, - event_sequence_number, - package: self.emit_package.to_vec(), - sender: self.sender.to_vec(), - }, - StoredEventEmitModule { - tx_sequence_number, - event_sequence_number, - package: self.emit_package.to_vec(), - module: self.emit_module.clone(), - sender: self.sender.to_vec(), - }, - StoredEventSenders { - tx_sequence_number, - event_sequence_number, - sender: self.sender.to_vec(), - }, - StoredEventStructPackage { - tx_sequence_number, - event_sequence_number, - package: self.type_package.to_vec(), - sender: self.sender.to_vec(), - }, - StoredEventStructModule { - tx_sequence_number, - event_sequence_number, - package: self.type_package.to_vec(), - module: self.type_module.clone(), - sender: self.sender.to_vec(), - }, - StoredEventStructName { - tx_sequence_number, - event_sequence_number, - package: self.type_package.to_vec(), - module: self.type_module.clone(), - type_name: self.type_name.clone(), - sender: self.sender.to_vec(), - }, - StoredEventStructInstantiation { - tx_sequence_number, - event_sequence_number, - package: self.type_package.to_vec(), - module: self.type_module.clone(), - type_instantiation: self.type_instantiation.clone(), - sender: self.sender.to_vec(), - }, - ) - } -} diff --git a/crates/sui-mvr-indexer/src/models/mod.rs b/crates/sui-mvr-indexer/src/models/mod.rs deleted file mode 100644 index 84e8b308bc0d5..0000000000000 --- a/crates/sui-mvr-indexer/src/models/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -pub mod checkpoints; -pub mod display; -pub mod epoch; -pub mod event_indices; -pub mod events; -pub mod obj_indices; -pub mod objects; -pub mod packages; -pub mod raw_checkpoints; -pub mod transactions; -pub mod tx_indices; -pub mod watermarks; diff --git a/crates/sui-mvr-indexer/src/models/obj_indices.rs b/crates/sui-mvr-indexer/src/models/obj_indices.rs deleted file mode 100644 index 4acc554565522..0000000000000 --- a/crates/sui-mvr-indexer/src/models/obj_indices.rs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use diesel::prelude::*; - -use crate::schema::objects_version; -/// Model types related to tables that support efficient execution of queries on the `objects`, -/// `objects_history` and `objects_snapshot` tables. - -#[derive(Queryable, Insertable, Debug, Identifiable, Clone, QueryableByName, Selectable)] -#[diesel(table_name = objects_version, primary_key(object_id, object_version))] -pub struct StoredObjectVersion { - pub object_id: Vec, - pub object_version: i64, - pub cp_sequence_number: i64, -} diff --git a/crates/sui-mvr-indexer/src/models/objects.rs b/crates/sui-mvr-indexer/src/models/objects.rs deleted file mode 100644 index 321aaebe2d4ed..0000000000000 --- a/crates/sui-mvr-indexer/src/models/objects.rs +++ /dev/null @@ -1,579 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::HashMap; -use std::sync::Arc; - -use diesel::prelude::*; -use serde::de::DeserializeOwned; - -use move_core_types::annotated_value::MoveTypeLayout; -use sui_json_rpc::coin_api::parse_to_struct_tag; -use sui_json_rpc_types::{Balance, Coin as SuiCoin}; -use sui_package_resolver::{PackageStore, Resolver}; -use sui_types::base_types::{ObjectID, ObjectRef}; -use sui_types::digests::ObjectDigest; -use sui_types::dynamic_field::{DynamicFieldType, Field}; -use sui_types::object::{Object, ObjectRead}; - -use crate::errors::IndexerError; -use crate::schema::{full_objects_history, objects, objects_history, objects_snapshot}; -use crate::types::{owner_to_owner_info, IndexedDeletedObject, IndexedObject, ObjectStatus}; - -#[derive(Queryable)] -pub struct DynamicFieldColumn { - pub object_id: Vec, - pub object_version: i64, - pub object_digest: Vec, - pub df_kind: Option, - pub df_name: Option>, - pub df_object_type: Option, - pub df_object_id: Option>, -} - -#[derive(Queryable)] -pub struct ObjectRefColumn { - pub object_id: Vec, - pub object_version: i64, - pub object_digest: Vec, -} - -// NOTE: please add updating statement like below in pg_indexer_store.rs, -// if new columns are added here: -// objects::epoch.eq(excluded(objects::epoch)) -#[derive(Queryable, Insertable, Debug, Identifiable, Clone, QueryableByName)] -#[diesel(table_name = objects, primary_key(object_id))] -pub struct StoredObject { - pub object_id: Vec, - pub object_version: i64, - pub object_digest: Vec, - pub owner_type: i16, - pub owner_id: Option>, - /// The full type of this object, including package id, module, name and type parameters. - /// This and following three fields will be None if the object is a Package - pub object_type: Option, - pub object_type_package: Option>, - pub object_type_module: Option, - /// Name of the object type, e.g., "Coin", without type parameters. - pub object_type_name: Option, - pub serialized_object: Vec, - pub coin_type: Option, - // TODO deal with overflow - pub coin_balance: Option, - pub df_kind: Option, -} - -impl From for StoredObject { - fn from(o: IndexedObject) -> Self { - let IndexedObject { - checkpoint_sequence_number: _, - object, - df_kind, - } = o; - let (owner_type, owner_id) = owner_to_owner_info(&object.owner); - let coin_type = object - .coin_type_maybe() - .map(|t| t.to_canonical_string(/* with_prefix */ true)); - let coin_balance = if coin_type.is_some() { - Some(object.get_coin_value_unsafe()) - } else { - None - }; - Self { - object_id: object.id().to_vec(), - object_version: object.version().value() as i64, - object_digest: object.digest().into_inner().to_vec(), - owner_type: owner_type as i16, - owner_id: owner_id.map(|id| id.to_vec()), - object_type: object - .type_() - .map(|t| t.to_canonical_string(/* with_prefix */ true)), - object_type_package: object.type_().map(|t| t.address().to_vec()), - object_type_module: object.type_().map(|t| t.module().to_string()), - object_type_name: object.type_().map(|t| t.name().to_string()), - serialized_object: bcs::to_bytes(&object).unwrap(), - coin_type, - coin_balance: coin_balance.map(|b| b as i64), - df_kind: df_kind.map(|k| match k { - DynamicFieldType::DynamicField => 0, - DynamicFieldType::DynamicObject => 1, - }), - } - } -} - -#[derive(Queryable, Insertable, Debug, Identifiable, Clone, QueryableByName)] -#[diesel(table_name = objects, primary_key(object_id))] -pub struct StoredDeletedObject { - pub object_id: Vec, - pub object_version: i64, -} - -impl From for StoredDeletedObject { - fn from(o: IndexedDeletedObject) -> Self { - Self { - object_id: o.object_id.to_vec(), - object_version: o.object_version as i64, - } - } -} - -#[derive(Queryable, Insertable, Selectable, Debug, Identifiable, Clone, QueryableByName)] -#[diesel(table_name = objects_snapshot, primary_key(object_id))] -pub struct StoredObjectSnapshot { - pub object_id: Vec, - pub object_version: i64, - pub object_status: i16, - pub object_digest: Option>, - pub checkpoint_sequence_number: i64, - pub owner_type: Option, - pub owner_id: Option>, - pub object_type: Option, - pub object_type_package: Option>, - pub object_type_module: Option, - pub object_type_name: Option, - pub serialized_object: Option>, - pub coin_type: Option, - pub coin_balance: Option, - pub df_kind: Option, -} - -impl From for StoredObjectSnapshot { - fn from(o: IndexedObject) -> Self { - let IndexedObject { - checkpoint_sequence_number, - object, - df_kind, - } = o; - let (owner_type, owner_id) = owner_to_owner_info(&object.owner); - let coin_type = object - .coin_type_maybe() - .map(|t| t.to_canonical_string(/* with_prefix */ true)); - let coin_balance = if coin_type.is_some() { - Some(object.get_coin_value_unsafe()) - } else { - None - }; - - Self { - object_id: object.id().to_vec(), - object_version: object.version().value() as i64, - object_status: ObjectStatus::Active as i16, - object_digest: Some(object.digest().into_inner().to_vec()), - checkpoint_sequence_number: checkpoint_sequence_number as i64, - owner_type: Some(owner_type as i16), - owner_id: owner_id.map(|id| id.to_vec()), - object_type: object - .type_() - .map(|t| t.to_canonical_string(/* with_prefix */ true)), - object_type_package: object.type_().map(|t| t.address().to_vec()), - object_type_module: object.type_().map(|t| t.module().to_string()), - object_type_name: object.type_().map(|t| t.name().to_string()), - serialized_object: Some(bcs::to_bytes(&object).unwrap()), - coin_type, - coin_balance: coin_balance.map(|b| b as i64), - df_kind: df_kind.map(|k| match k { - DynamicFieldType::DynamicField => 0, - DynamicFieldType::DynamicObject => 1, - }), - } - } -} - -impl From for StoredObjectSnapshot { - fn from(o: IndexedDeletedObject) -> Self { - Self { - object_id: o.object_id.to_vec(), - object_version: o.object_version as i64, - object_status: ObjectStatus::WrappedOrDeleted as i16, - object_digest: None, - checkpoint_sequence_number: o.checkpoint_sequence_number as i64, - owner_type: None, - owner_id: None, - object_type: None, - object_type_package: None, - object_type_module: None, - object_type_name: None, - serialized_object: None, - coin_type: None, - coin_balance: None, - df_kind: None, - } - } -} - -#[derive(Queryable, Insertable, Selectable, Debug, Identifiable, Clone, QueryableByName)] -#[diesel(table_name = objects_history, primary_key(object_id, object_version, checkpoint_sequence_number))] -pub struct StoredHistoryObject { - pub object_id: Vec, - pub object_version: i64, - pub object_status: i16, - pub object_digest: Option>, - pub checkpoint_sequence_number: i64, - pub owner_type: Option, - pub owner_id: Option>, - pub object_type: Option, - pub object_type_package: Option>, - pub object_type_module: Option, - pub object_type_name: Option, - pub serialized_object: Option>, - pub coin_type: Option, - pub coin_balance: Option, - pub df_kind: Option, -} - -impl From for StoredHistoryObject { - fn from(o: IndexedObject) -> Self { - let IndexedObject { - checkpoint_sequence_number, - object, - df_kind, - } = o; - let (owner_type, owner_id) = owner_to_owner_info(&object.owner); - let coin_type = object - .coin_type_maybe() - .map(|t| t.to_canonical_string(/* with_prefix */ true)); - let coin_balance = if coin_type.is_some() { - Some(object.get_coin_value_unsafe()) - } else { - None - }; - - Self { - object_id: object.id().to_vec(), - object_version: object.version().value() as i64, - object_status: ObjectStatus::Active as i16, - object_digest: Some(object.digest().into_inner().to_vec()), - checkpoint_sequence_number: checkpoint_sequence_number as i64, - owner_type: Some(owner_type as i16), - owner_id: owner_id.map(|id| id.to_vec()), - object_type: object - .type_() - .map(|t| t.to_canonical_string(/* with_prefix */ true)), - object_type_package: object.type_().map(|t| t.address().to_vec()), - object_type_module: object.type_().map(|t| t.module().to_string()), - object_type_name: object.type_().map(|t| t.name().to_string()), - serialized_object: Some(bcs::to_bytes(&object).unwrap()), - coin_type, - coin_balance: coin_balance.map(|b| b as i64), - df_kind: df_kind.map(|k| match k { - DynamicFieldType::DynamicField => 0, - DynamicFieldType::DynamicObject => 1, - }), - } - } -} - -impl From for StoredHistoryObject { - fn from(o: IndexedDeletedObject) -> Self { - Self { - object_id: o.object_id.to_vec(), - object_version: o.object_version as i64, - object_status: ObjectStatus::WrappedOrDeleted as i16, - object_digest: None, - checkpoint_sequence_number: o.checkpoint_sequence_number as i64, - owner_type: None, - owner_id: None, - object_type: None, - object_type_package: None, - object_type_module: None, - object_type_name: None, - serialized_object: None, - coin_type: None, - coin_balance: None, - df_kind: None, - } - } -} - -impl TryFrom for Object { - type Error = IndexerError; - - fn try_from(o: StoredObject) -> Result { - bcs::from_bytes(&o.serialized_object).map_err(|e| { - IndexerError::SerdeError(format!( - "Failed to deserialize object: {:?}, error: {}", - o.object_id, e - )) - }) - } -} - -impl StoredObject { - pub async fn try_into_object_read( - self, - package_resolver: Arc>, - ) -> Result { - let oref = self.get_object_ref()?; - let object: sui_types::object::Object = self.try_into()?; - let Some(move_object) = object.data.try_as_move().cloned() else { - return Err(IndexerError::PostgresReadError(format!( - "Object {:?} is not a Move object", - oref, - ))); - }; - - let move_type_layout = package_resolver - .type_layout(move_object.type_().clone().into()) - .await - .map_err(|e| { - IndexerError::ResolveMoveStructError(format!( - "Failed to convert into object read for obj {}:{}, type: {}. Error: {e}", - object.id(), - object.version(), - move_object.type_(), - )) - })?; - let move_struct_layout = match move_type_layout { - MoveTypeLayout::Struct(s) => Ok(s), - _ => Err(IndexerError::ResolveMoveStructError( - "MoveTypeLayout is not Struct".to_string(), - )), - }?; - - Ok(ObjectRead::Exists(oref, object, Some(*move_struct_layout))) - } - - pub fn get_object_ref(&self) -> Result { - let object_id = ObjectID::from_bytes(self.object_id.clone()).map_err(|_| { - IndexerError::SerdeError(format!("Can't convert {:?} to object_id", self.object_id)) - })?; - let object_digest = - ObjectDigest::try_from(self.object_digest.as_slice()).map_err(|_| { - IndexerError::SerdeError(format!( - "Can't convert {:?} to object_digest", - self.object_digest - )) - })?; - Ok(( - object_id, - (self.object_version as u64).into(), - object_digest, - )) - } - - pub fn to_dynamic_field(&self) -> Option> - where - K: DeserializeOwned, - V: DeserializeOwned, - { - let object: Object = bcs::from_bytes(&self.serialized_object).ok()?; - - let object = object.data.try_as_move()?; - let ty = object.type_(); - - if !ty.is_dynamic_field() { - return None; - } - - bcs::from_bytes(object.contents()).ok() - } -} - -impl TryFrom for SuiCoin { - type Error = IndexerError; - - fn try_from(o: StoredObject) -> Result { - let object: Object = o.clone().try_into()?; - let (coin_object_id, version, digest) = o.get_object_ref()?; - let coin_type_canonical = - o.coin_type - .ok_or(IndexerError::PersistentStorageDataCorruptionError(format!( - "Object {} is supposed to be a coin but has an empty coin_type column", - coin_object_id, - )))?; - let coin_type = parse_to_struct_tag(coin_type_canonical.as_str()) - .map_err(|_| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "The type of object {} cannot be parsed as a struct tag", - coin_object_id, - )) - })? - .to_string(); - let balance = o - .coin_balance - .ok_or(IndexerError::PersistentStorageDataCorruptionError(format!( - "Object {} is supposed to be a coin but has an empty coin_balance column", - coin_object_id, - )))?; - Ok(SuiCoin { - coin_type, - coin_object_id, - version, - digest, - balance: balance as u64, - previous_transaction: object.previous_transaction, - }) - } -} - -#[derive(QueryableByName)] -pub struct CoinBalance { - #[diesel(sql_type = diesel::sql_types::Text)] - pub coin_type: String, - #[diesel(sql_type = diesel::sql_types::BigInt)] - pub coin_num: i64, - #[diesel(sql_type = diesel::sql_types::BigInt)] - pub coin_balance: i64, -} - -impl TryFrom for Balance { - type Error = IndexerError; - - fn try_from(c: CoinBalance) -> Result { - let coin_type = parse_to_struct_tag(c.coin_type.as_str()) - .map_err(|_| { - IndexerError::PersistentStorageDataCorruptionError( - "The type of coin balance cannot be parsed as a struct tag".to_string(), - ) - })? - .to_string(); - Ok(Self { - coin_type, - coin_object_count: c.coin_num as usize, - // TODO: deal with overflow - total_balance: c.coin_balance as u128, - locked_balance: HashMap::default(), - }) - } -} - -#[derive(Queryable, Insertable, Debug, Identifiable, Clone, QueryableByName, Selectable)] -#[diesel(table_name = full_objects_history, primary_key(object_id, object_version))] -pub struct StoredFullHistoryObject { - pub object_id: Vec, - pub object_version: i64, - pub serialized_object: Option>, -} - -impl From for StoredFullHistoryObject { - fn from(o: IndexedObject) -> Self { - let object = o.object; - Self { - object_id: object.id().to_vec(), - object_version: object.version().value() as i64, - serialized_object: Some(bcs::to_bytes(&object).unwrap()), - } - } -} - -impl From for StoredFullHistoryObject { - fn from(o: IndexedDeletedObject) -> Self { - Self { - object_id: o.object_id.to_vec(), - object_version: o.object_version as i64, - serialized_object: None, - } - } -} - -#[cfg(test)] -mod tests { - use move_core_types::{account_address::AccountAddress, language_storage::StructTag}; - use sui_types::{ - coin::Coin, - digests::TransactionDigest, - gas_coin::{GasCoin, GAS}, - object::{Data, MoveObject, ObjectInner, Owner}, - Identifier, TypeTag, - }; - - use super::*; - - #[test] - fn test_canonical_string_of_object_type_for_coin() { - let test_obj = Object::new_gas_for_testing(); - let indexed_obj = IndexedObject::from_object(1, test_obj, None); - - let stored_obj = StoredObject::from(indexed_obj); - - match stored_obj.object_type { - Some(t) => { - assert_eq!(t, "0x0000000000000000000000000000000000000000000000000000000000000002::coin::Coin<0x0000000000000000000000000000000000000000000000000000000000000002::sui::SUI>"); - } - None => { - panic!("object_type should not be none"); - } - } - } - - #[test] - fn test_convert_stored_obj_to_sui_coin() { - let test_obj = Object::new_gas_for_testing(); - let indexed_obj = IndexedObject::from_object(1, test_obj, None); - - let stored_obj = StoredObject::from(indexed_obj); - - let sui_coin = SuiCoin::try_from(stored_obj).unwrap(); - assert_eq!(sui_coin.coin_type, "0x2::sui::SUI"); - } - - #[test] - fn test_output_format_coin_balance() { - let test_obj = Object::new_gas_for_testing(); - let indexed_obj = IndexedObject::from_object(1, test_obj, None); - - let stored_obj = StoredObject::from(indexed_obj); - let test_balance = CoinBalance { - coin_type: stored_obj.coin_type.unwrap(), - coin_num: 1, - coin_balance: 100, - }; - let balance = Balance::try_from(test_balance).unwrap(); - assert_eq!(balance.coin_type, "0x2::sui::SUI"); - } - - #[test] - fn test_vec_of_coin_sui_conversion() { - // 0xe7::vec_coin::VecCoin>> - let vec_coins_type = TypeTag::Vector(Box::new( - Coin::type_(TypeTag::Struct(Box::new(GAS::type_()))).into(), - )); - let object_type = StructTag { - address: AccountAddress::from_hex_literal("0xe7").unwrap(), - module: Identifier::new("vec_coin").unwrap(), - name: Identifier::new("VecCoin").unwrap(), - type_params: vec![vec_coins_type], - }; - - let id = ObjectID::ZERO; - let gas = 10; - - let contents = bcs::to_bytes(&vec![GasCoin::new(id, gas)]).unwrap(); - let data = Data::Move( - unsafe { - MoveObject::new_from_execution_with_limit( - object_type.into(), - true, - 1.into(), - contents, - 256, - ) - } - .unwrap(), - ); - - let owner = AccountAddress::from_hex_literal("0x1").unwrap(); - - let object = ObjectInner { - owner: Owner::AddressOwner(owner.into()), - data, - previous_transaction: TransactionDigest::genesis_marker(), - storage_rebate: 0, - } - .into(); - - let indexed_obj = IndexedObject::from_object(1, object, None); - - let stored_obj = StoredObject::from(indexed_obj); - - match stored_obj.object_type { - Some(t) => { - assert_eq!(t, "0x00000000000000000000000000000000000000000000000000000000000000e7::vec_coin::VecCoin>>"); - } - None => { - panic!("object_type should not be none"); - } - } - } -} diff --git a/crates/sui-mvr-indexer/src/models/packages.rs b/crates/sui-mvr-indexer/src/models/packages.rs deleted file mode 100644 index 97c8e8fc5b459..0000000000000 --- a/crates/sui-mvr-indexer/src/models/packages.rs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::schema::packages; -use crate::types::IndexedPackage; - -use diesel::prelude::*; - -#[derive(Queryable, Insertable, Selectable, Clone, Debug, Identifiable)] -#[diesel(table_name = packages, primary_key(package_id))] -pub struct StoredPackage { - pub package_id: Vec, - pub original_id: Vec, - pub package_version: i64, - pub move_package: Vec, - pub checkpoint_sequence_number: i64, -} - -impl From for StoredPackage { - fn from(p: IndexedPackage) -> Self { - Self { - package_id: p.package_id.to_vec(), - original_id: p.move_package.original_package_id().to_vec(), - package_version: p.move_package.version().value() as i64, - move_package: bcs::to_bytes(&p.move_package).unwrap(), - checkpoint_sequence_number: p.checkpoint_sequence_number as i64, - } - } -} diff --git a/crates/sui-mvr-indexer/src/models/raw_checkpoints.rs b/crates/sui-mvr-indexer/src/models/raw_checkpoints.rs deleted file mode 100644 index 98fafba928705..0000000000000 --- a/crates/sui-mvr-indexer/src/models/raw_checkpoints.rs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::schema::raw_checkpoints; -use crate::types::IndexedCheckpoint; -use diesel::prelude::*; - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = raw_checkpoints)] -pub struct StoredRawCheckpoint { - pub sequence_number: i64, - /// BCS serialized CertifiedCheckpointSummary - pub certified_checkpoint: Vec, - /// BCS serialized CheckpointContents - pub checkpoint_contents: Vec, -} - -impl From<&IndexedCheckpoint> for StoredRawCheckpoint { - fn from(c: &IndexedCheckpoint) -> Self { - Self { - sequence_number: c.sequence_number as i64, - certified_checkpoint: bcs::to_bytes(c.certified_checkpoint.as_ref().unwrap()).unwrap(), - checkpoint_contents: bcs::to_bytes(c.checkpoint_contents.as_ref().unwrap()).unwrap(), - } - } -} diff --git a/crates/sui-mvr-indexer/src/models/transactions.rs b/crates/sui-mvr-indexer/src/models/transactions.rs deleted file mode 100644 index 1856025c5be4d..0000000000000 --- a/crates/sui-mvr-indexer/src/models/transactions.rs +++ /dev/null @@ -1,353 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::Arc; - -use diesel::prelude::*; - -use move_core_types::annotated_value::{MoveDatatypeLayout, MoveTypeLayout}; -use move_core_types::language_storage::TypeTag; -use sui_json_rpc_types::{ - BalanceChange, ObjectChange, SuiEvent, SuiTransactionBlock, SuiTransactionBlockEffects, - SuiTransactionBlockEvents, SuiTransactionBlockResponse, SuiTransactionBlockResponseOptions, -}; -use sui_package_resolver::{PackageStore, Resolver}; -use sui_types::digests::TransactionDigest; -use sui_types::effects::TransactionEffects; -use sui_types::effects::TransactionEvents; -use sui_types::event::Event; -use sui_types::transaction::SenderSignedData; - -use crate::errors::IndexerError; -use crate::schema::transactions; -use crate::types::IndexedObjectChange; -use crate::types::IndexedTransaction; -use crate::types::IndexerResult; - -#[derive(Clone, Debug, Queryable, Insertable, QueryableByName, Selectable)] -#[diesel(table_name = transactions)] -pub struct StoredTransaction { - pub tx_sequence_number: i64, - pub transaction_digest: Vec, - pub raw_transaction: Vec, - pub raw_effects: Vec, - pub checkpoint_sequence_number: i64, - pub timestamp_ms: i64, - pub object_changes: Vec>>, - pub balance_changes: Vec>>, - pub events: Vec>>, - pub transaction_kind: i16, - pub success_command_count: i16, -} - -pub type StoredTransactionEvents = Vec>>; - -#[derive(Debug, Queryable)] -pub struct TxSeq { - pub seq: i64, -} - -impl Default for TxSeq { - fn default() -> Self { - Self { seq: -1 } - } -} - -#[derive(Clone, Debug, Queryable)] -pub struct StoredTransactionTimestamp { - pub tx_sequence_number: i64, - pub timestamp_ms: i64, -} - -#[derive(Clone, Debug, Queryable)] -pub struct StoredTransactionCheckpoint { - pub tx_sequence_number: i64, - pub checkpoint_sequence_number: i64, -} - -#[derive(Clone, Debug, Queryable)] -pub struct StoredTransactionSuccessCommandCount { - pub tx_sequence_number: i64, - pub checkpoint_sequence_number: i64, - pub success_command_count: i16, - pub timestamp_ms: i64, -} - -impl From<&IndexedTransaction> for StoredTransaction { - fn from(tx: &IndexedTransaction) -> Self { - StoredTransaction { - tx_sequence_number: tx.tx_sequence_number as i64, - transaction_digest: tx.tx_digest.into_inner().to_vec(), - raw_transaction: bcs::to_bytes(&tx.sender_signed_data).unwrap(), - raw_effects: bcs::to_bytes(&tx.effects).unwrap(), - checkpoint_sequence_number: tx.checkpoint_sequence_number as i64, - object_changes: tx - .object_changes - .iter() - .map(|oc| Some(bcs::to_bytes(&oc).unwrap())) - .collect(), - balance_changes: tx - .balance_change - .iter() - .map(|bc| Some(bcs::to_bytes(&bc).unwrap())) - .collect(), - events: tx - .events - .iter() - .map(|e| Some(bcs::to_bytes(&e).unwrap())) - .collect(), - timestamp_ms: tx.timestamp_ms as i64, - transaction_kind: tx.transaction_kind.clone() as i16, - success_command_count: tx.successful_tx_num as i16, - } - } -} - -impl StoredTransaction { - pub fn get_balance_len(&self) -> usize { - self.balance_changes.len() - } - - pub fn get_balance_at_idx(&self, idx: usize) -> Option> { - self.balance_changes.get(idx).cloned().flatten() - } - - pub fn get_object_len(&self) -> usize { - self.object_changes.len() - } - - pub fn get_object_at_idx(&self, idx: usize) -> Option> { - self.object_changes.get(idx).cloned().flatten() - } - - pub fn get_event_len(&self) -> usize { - self.events.len() - } - - pub fn get_event_at_idx(&self, idx: usize) -> Option> { - self.events.get(idx).cloned().flatten() - } - - pub async fn try_into_sui_transaction_block_response( - self, - options: SuiTransactionBlockResponseOptions, - package_resolver: Arc>, - ) -> IndexerResult { - let options = options.clone(); - let tx_digest = - TransactionDigest::try_from(self.transaction_digest.as_slice()).map_err(|e| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Can't convert {:?} as tx_digest. Error: {e}", - self.transaction_digest - )) - })?; - - let transaction = if options.show_input { - let sender_signed_data = self.try_into_sender_signed_data()?; - let tx_block = SuiTransactionBlock::try_from_with_package_resolver( - sender_signed_data, - package_resolver.clone(), - ) - .await?; - Some(tx_block) - } else { - None - }; - - let effects = if options.show_effects { - let effects = self.try_into_sui_transaction_effects()?; - Some(effects) - } else { - None - }; - - let raw_transaction = if options.show_raw_input { - self.raw_transaction - } else { - Vec::new() - }; - - let events = if options.show_events { - let events = { - self - .events - .into_iter() - .map(|event| match event { - Some(event) => { - let event: Event = bcs::from_bytes(&event).map_err(|e| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Can't convert event bytes into Event. tx_digest={:?} Error: {e}", - tx_digest - )) - })?; - Ok(event) - } - None => Err(IndexerError::PersistentStorageDataCorruptionError(format!( - "Event should not be null, tx_digest={:?}", - tx_digest - ))), - }) - .collect::, IndexerError>>()? - }; - let timestamp = self.timestamp_ms as u64; - let tx_events = TransactionEvents { data: events }; - - tx_events_to_sui_tx_events(tx_events, package_resolver, tx_digest, timestamp).await? - } else { - None - }; - - let object_changes = if options.show_object_changes { - let object_changes = { - self.object_changes.into_iter().map(|object_change| { - match object_change { - Some(object_change) => { - let object_change: IndexedObjectChange = bcs::from_bytes(&object_change) - .map_err(|e| IndexerError::PersistentStorageDataCorruptionError( - format!("Can't convert object_change bytes into IndexedObjectChange. tx_digest={:?} Error: {e}", tx_digest) - ))?; - Ok(ObjectChange::from(object_change)) - } - None => Err(IndexerError::PersistentStorageDataCorruptionError(format!("object_change should not be null, tx_digest={:?}", tx_digest))), - } - }).collect::, IndexerError>>()? - }; - Some(object_changes) - } else { - None - }; - - let balance_changes = if options.show_balance_changes { - let balance_changes = { - self.balance_changes.into_iter().map(|balance_change| { - match balance_change { - Some(balance_change) => { - let balance_change: BalanceChange = bcs::from_bytes(&balance_change) - .map_err(|e| IndexerError::PersistentStorageDataCorruptionError( - format!("Can't convert balance_change bytes into BalanceChange. tx_digest={:?} Error: {e}", tx_digest) - ))?; - Ok(balance_change) - } - None => Err(IndexerError::PersistentStorageDataCorruptionError(format!("object_change should not be null, tx_digest={:?}", tx_digest))), - } - }).collect::, IndexerError>>()? - }; - Some(balance_changes) - } else { - None - }; - - Ok(SuiTransactionBlockResponse { - digest: tx_digest, - transaction, - raw_transaction, - effects, - events, - object_changes, - balance_changes, - timestamp_ms: Some(self.timestamp_ms as u64), - checkpoint: Some(self.checkpoint_sequence_number as u64), - confirmed_local_execution: None, - errors: vec![], - raw_effects: self.raw_effects, - }) - } - fn try_into_sender_signed_data(&self) -> IndexerResult { - let sender_signed_data: SenderSignedData = - bcs::from_bytes(&self.raw_transaction).map_err(|e| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Can't convert raw_transaction of {} into SenderSignedData. Error: {e}", - self.tx_sequence_number - )) - })?; - Ok(sender_signed_data) - } - - pub fn try_into_sui_transaction_effects(&self) -> IndexerResult { - let effects: TransactionEffects = bcs::from_bytes(&self.raw_effects).map_err(|e| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Can't convert raw_effects of {} into TransactionEffects. Error: {e}", - self.tx_sequence_number - )) - })?; - let effects = SuiTransactionBlockEffects::try_from(effects)?; - Ok(effects) - } -} - -pub fn stored_events_to_events( - stored_events: StoredTransactionEvents, -) -> Result, IndexerError> { - stored_events - .into_iter() - .map(|event| match event { - Some(event) => { - let event: Event = bcs::from_bytes(&event).map_err(|e| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Can't convert event bytes into Event. Error: {e}", - )) - })?; - Ok(event) - } - None => Err(IndexerError::PersistentStorageDataCorruptionError( - "Event should not be null".to_string(), - )), - }) - .collect::, IndexerError>>() -} - -pub async fn tx_events_to_sui_tx_events( - tx_events: TransactionEvents, - package_resolver: Arc>, - tx_digest: TransactionDigest, - timestamp: u64, -) -> Result, IndexerError> { - let mut sui_event_futures = vec![]; - let tx_events_data_len = tx_events.data.len(); - for tx_event in tx_events.data.clone() { - let package_resolver_clone = package_resolver.clone(); - sui_event_futures.push(tokio::task::spawn(async move { - let resolver = package_resolver_clone; - resolver - .type_layout(TypeTag::Struct(Box::new(tx_event.type_.clone()))) - .await - })); - } - let event_move_type_layouts = futures::future::join_all(sui_event_futures) - .await - .into_iter() - .collect::, _>>()? - .into_iter() - .collect::, _>>() - .map_err(|e| { - IndexerError::ResolveMoveStructError(format!( - "Failed to convert to sui event with Error: {e}", - )) - })?; - let event_move_datatype_layouts = event_move_type_layouts - .into_iter() - .filter_map(|move_type_layout| match move_type_layout { - MoveTypeLayout::Struct(s) => Some(MoveDatatypeLayout::Struct(s)), - MoveTypeLayout::Enum(e) => Some(MoveDatatypeLayout::Enum(e)), - _ => None, - }) - .collect::>(); - assert!(tx_events_data_len == event_move_datatype_layouts.len()); - let sui_events = tx_events - .data - .into_iter() - .enumerate() - .zip(event_move_datatype_layouts) - .map(|((seq, tx_event), move_datatype_layout)| { - SuiEvent::try_from( - tx_event, - tx_digest, - seq as u64, - Some(timestamp), - move_datatype_layout, - ) - }) - .collect::, _>>()?; - let sui_tx_events = SuiTransactionBlockEvents { data: sui_events }; - Ok(Some(sui_tx_events)) -} diff --git a/crates/sui-mvr-indexer/src/models/tx_indices.rs b/crates/sui-mvr-indexer/src/models/tx_indices.rs deleted file mode 100644 index a00b715eedf98..0000000000000 --- a/crates/sui-mvr-indexer/src/models/tx_indices.rs +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::{ - schema::{ - tx_affected_addresses, tx_affected_objects, tx_calls_fun, tx_calls_mod, tx_calls_pkg, - tx_changed_objects, tx_digests, tx_input_objects, tx_kinds, - }, - types::TxIndex, -}; -use diesel::prelude::*; -use itertools::Itertools; - -#[derive(QueryableByName)] -pub struct TxSequenceNumber { - #[diesel(sql_type = diesel::sql_types::BigInt)] - pub tx_sequence_number: i64, -} - -#[derive(QueryableByName)] -pub struct TxDigest { - #[diesel(sql_type = diesel::sql_types::Binary)] - pub transaction_digest: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = tx_affected_addresses)] -pub struct StoredTxAffectedAddresses { - pub tx_sequence_number: i64, - pub affected: Vec, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = tx_affected_objects)] -pub struct StoredTxAffectedObjects { - pub tx_sequence_number: i64, - pub affected: Vec, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = tx_input_objects)] -pub struct StoredTxInputObject { - pub tx_sequence_number: i64, - pub object_id: Vec, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = tx_changed_objects)] -pub struct StoredTxChangedObject { - pub tx_sequence_number: i64, - pub object_id: Vec, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = tx_calls_pkg)] -pub struct StoredTxPkg { - pub tx_sequence_number: i64, - pub package: Vec, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = tx_calls_mod)] -pub struct StoredTxMod { - pub tx_sequence_number: i64, - pub package: Vec, - pub module: String, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = tx_calls_fun)] -pub struct StoredTxFun { - pub tx_sequence_number: i64, - pub package: Vec, - pub module: String, - pub func: String, - pub sender: Vec, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = tx_digests)] -pub struct StoredTxDigest { - pub tx_digest: Vec, - pub tx_sequence_number: i64, -} - -#[derive(Queryable, Insertable, Selectable, Debug, Clone, Default)] -#[diesel(table_name = tx_kinds)] -pub struct StoredTxKind { - pub tx_kind: i16, - pub tx_sequence_number: i64, -} - -#[allow(clippy::type_complexity)] -impl TxIndex { - pub fn split( - self: TxIndex, - ) -> ( - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - ) { - let tx_sequence_number = self.tx_sequence_number as i64; - - let tx_affected_addresses = self - .recipients - .iter() - .chain(self.payers.iter()) - .chain(std::iter::once(&self.sender)) - .unique() - .map(|a| StoredTxAffectedAddresses { - tx_sequence_number, - affected: a.to_vec(), - sender: self.sender.to_vec(), - }) - .collect(); - - let tx_affected_objects = self - .affected_objects - .iter() - .map(|o| StoredTxAffectedObjects { - tx_sequence_number, - affected: o.to_vec(), - sender: self.sender.to_vec(), - }) - .collect(); - - let tx_input_objects = self - .input_objects - .iter() - .map(|o| StoredTxInputObject { - tx_sequence_number, - object_id: bcs::to_bytes(&o).unwrap(), - sender: self.sender.to_vec(), - }) - .collect(); - - let tx_changed_objects = self - .changed_objects - .iter() - .map(|o| StoredTxChangedObject { - tx_sequence_number, - object_id: bcs::to_bytes(&o).unwrap(), - sender: self.sender.to_vec(), - }) - .collect(); - - let mut packages = Vec::new(); - let mut packages_modules = Vec::new(); - let mut packages_modules_funcs = Vec::new(); - - for (pkg, pkg_mod, pkg_mod_func) in self - .move_calls - .iter() - .map(|(p, m, f)| (*p, (*p, m.clone()), (*p, m.clone(), f.clone()))) - { - packages.push(pkg); - packages_modules.push(pkg_mod); - packages_modules_funcs.push(pkg_mod_func); - } - - let tx_pkgs = packages - .iter() - .map(|p| StoredTxPkg { - tx_sequence_number, - package: p.to_vec(), - sender: self.sender.to_vec(), - }) - .collect(); - - let tx_mods = packages_modules - .iter() - .map(|(p, m)| StoredTxMod { - tx_sequence_number, - package: p.to_vec(), - module: m.to_string(), - sender: self.sender.to_vec(), - }) - .collect(); - - let tx_calls = packages_modules_funcs - .iter() - .map(|(p, m, f)| StoredTxFun { - tx_sequence_number, - package: p.to_vec(), - module: m.to_string(), - func: f.to_string(), - sender: self.sender.to_vec(), - }) - .collect(); - - let stored_tx_digest = StoredTxDigest { - tx_digest: self.transaction_digest.into_inner().to_vec(), - tx_sequence_number, - }; - - let tx_kind = StoredTxKind { - tx_kind: self.tx_kind as i16, - tx_sequence_number, - }; - - ( - tx_affected_addresses, - tx_affected_objects, - tx_input_objects, - tx_changed_objects, - tx_pkgs, - tx_mods, - tx_calls, - vec![stored_tx_digest], - vec![tx_kind], - ) - } -} diff --git a/crates/sui-mvr-indexer/src/models/watermarks.rs b/crates/sui-mvr-indexer/src/models/watermarks.rs deleted file mode 100644 index 1ff3d3cfe52ac..0000000000000 --- a/crates/sui-mvr-indexer/src/models/watermarks.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::str::FromStr; - -use diesel::prelude::*; - -use crate::{ - handlers::{pruner::PrunableTable, CommitterWatermark}, - schema::watermarks::{self}, -}; - -/// Represents a row in the `watermarks` table. -#[derive(Queryable, Insertable, Default, QueryableByName, Clone)] -#[diesel(table_name = watermarks, primary_key(entity))] -pub struct StoredWatermark { - /// The table governed by this watermark, i.e `epochs`, `checkpoints`, `transactions`. - pub pipeline: String, - /// Inclusive upper epoch bound for this entity's data. Committer updates this field. Pruner uses - /// this to determine if pruning is necessary based on the retention policy. - pub epoch_hi_inclusive: i64, - /// Inclusive upper checkpoint bound for this entity's data. Committer updates this field. All - /// data of this entity in the checkpoint must be persisted before advancing this watermark. The - /// committer refers to this on disaster recovery to resume writing. - pub checkpoint_hi_inclusive: i64, - /// Exclusive upper transaction sequence number bound for this entity's data. Committer updates - /// this field. - pub tx_hi: i64, - /// Inclusive lower epoch bound for this entity's data. Pruner updates this field when the epoch range exceeds the retention policy. - pub epoch_lo: i64, - /// Inclusive low watermark that the pruner advances. Corresponds to the epoch id, checkpoint - /// sequence number, or tx sequence number depending on the entity. Data before this watermark is - /// considered pruned by a reader. The underlying data may still exist in the db instance. - pub reader_lo: i64, - /// Updated using the database's current timestamp when the pruner sees that some data needs to - /// be dropped. The pruner uses this column to determine whether to prune or wait long enough - /// that all in-flight reads complete or timeout before it acts on an updated watermark. - pub timestamp_ms: i64, - /// Column used by the pruner to track its true progress. Data below this watermark can be - /// immediately pruned. - pub pruner_hi: i64, -} - -impl StoredWatermark { - pub fn from_upper_bound_update(entity: &str, watermark: CommitterWatermark) -> Self { - StoredWatermark { - pipeline: entity.to_string(), - epoch_hi_inclusive: watermark.epoch_hi_inclusive as i64, - checkpoint_hi_inclusive: watermark.checkpoint_hi_inclusive as i64, - tx_hi: watermark.tx_hi as i64, - ..StoredWatermark::default() - } - } - - pub fn from_lower_bound_update(entity: &str, epoch_lo: u64, reader_lo: u64) -> Self { - StoredWatermark { - pipeline: entity.to_string(), - epoch_lo: epoch_lo as i64, - reader_lo: reader_lo as i64, - ..StoredWatermark::default() - } - } - - pub fn entity(&self) -> Option { - PrunableTable::from_str(&self.pipeline).ok() - } - - /// Determine whether to set a new epoch lower bound based on the retention policy. - pub fn new_epoch_lo(&self, retention: u64) -> Option { - if self.epoch_lo as u64 + retention <= self.epoch_hi_inclusive as u64 { - Some((self.epoch_hi_inclusive as u64).saturating_sub(retention - 1)) - } else { - None - } - } -} diff --git a/crates/sui-mvr-indexer/src/restorer/archives.rs b/crates/sui-mvr-indexer/src/restorer/archives.rs deleted file mode 100644 index f70336f76d3c5..0000000000000 --- a/crates/sui-mvr-indexer/src/restorer/archives.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::num::NonZeroUsize; - -use prometheus::Registry; -use sui_types::digests::CheckpointDigest; -use tracing::info; - -use sui_archival::reader::{ArchiveReader, ArchiveReaderMetrics}; -use sui_config::node::ArchiveReaderConfig; -use sui_config::object_storage_config::{ObjectStoreConfig, ObjectStoreType}; - -use crate::errors::IndexerError; -use crate::types::IndexerResult; - -#[derive(Clone, Debug)] -pub struct RestoreCheckpointInfo { - pub next_checkpoint_after_epoch: u64, - pub chain_identifier: CheckpointDigest, -} - -pub async fn read_restore_checkpoint_info( - archive_bucket: Option, - epoch: u64, -) -> IndexerResult { - let archive_store_config = ObjectStoreConfig { - object_store: Some(ObjectStoreType::GCS), - bucket: archive_bucket, - object_store_connection_limit: 50, - no_sign_request: false, - ..Default::default() - }; - let archive_reader_config = ArchiveReaderConfig { - remote_store_config: archive_store_config, - download_concurrency: NonZeroUsize::new(50).unwrap(), - use_for_pruning_watermark: false, - }; - let metrics = ArchiveReaderMetrics::new(&Registry::default()); - let archive_reader = ArchiveReader::new(archive_reader_config, &metrics)?; - archive_reader.sync_manifest_once().await?; - let manifest = archive_reader.get_manifest().await?; - let next_checkpoint_after_epoch = manifest.next_checkpoint_after_epoch(epoch); - info!( - "Read from archives: next checkpoint sequence after epoch {} is: {}", - epoch, next_checkpoint_after_epoch - ); - let cp_summaries = archive_reader - .get_summaries_for_list_no_verify(vec![0]) - .await - .map_err(|e| IndexerError::ArchiveReaderError(format!("Failed to get summaries: {}", e)))?; - let first_cp = cp_summaries - .first() - .ok_or_else(|| IndexerError::ArchiveReaderError("No checkpoint found".to_string()))?; - let chain_identifier = *first_cp.digest(); - Ok(RestoreCheckpointInfo { - next_checkpoint_after_epoch, - chain_identifier, - }) -} diff --git a/crates/sui-mvr-indexer/src/restorer/mod.rs b/crates/sui-mvr-indexer/src/restorer/mod.rs deleted file mode 100644 index 1899227725b62..0000000000000 --- a/crates/sui-mvr-indexer/src/restorer/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -pub mod archives; -pub mod formal_snapshot; diff --git a/crates/sui-mvr-indexer/src/schema.patch b/crates/sui-mvr-indexer/src/schema.patch deleted file mode 100644 index c935f4d862fe0..0000000000000 --- a/crates/sui-mvr-indexer/src/schema.patch +++ /dev/null @@ -1,7 +0,0 @@ -diff --git a/crates/sui-mvr-indexer/src/schema.rs b/crates/sui-mvr-indexer/src/schema.rs ---- a/crates/sui-mvr-indexer/src/schema.rs -+++ b/crates/sui-mvr-indexer/src/schema.rs -@@ -1 +1,3 @@ -+// Copyright (c) Mysten Labs, Inc. -+// SPDX-License-Identifier: Apache-2.0 - // @generated automatically by Diesel CLI. diff --git a/crates/sui-mvr-indexer/src/schema.rs b/crates/sui-mvr-indexer/src/schema.rs deleted file mode 100644 index 447b45557922c..0000000000000 --- a/crates/sui-mvr-indexer/src/schema.rs +++ /dev/null @@ -1,404 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 -// @generated automatically by Diesel CLI. - -diesel::table! { - chain_identifier (checkpoint_digest) { - checkpoint_digest -> Bytea, - } -} - -diesel::table! { - checkpoints (sequence_number) { - sequence_number -> Int8, - checkpoint_digest -> Bytea, - epoch -> Int8, - network_total_transactions -> Int8, - previous_checkpoint_digest -> Nullable, - end_of_epoch -> Bool, - tx_digests -> Array>, - timestamp_ms -> Int8, - total_gas_cost -> Int8, - computation_cost -> Int8, - storage_cost -> Int8, - storage_rebate -> Int8, - non_refundable_storage_fee -> Int8, - checkpoint_commitments -> Bytea, - validator_signature -> Bytea, - end_of_epoch_data -> Nullable, - min_tx_sequence_number -> Nullable, - max_tx_sequence_number -> Nullable, - } -} - -diesel::table! { - display (object_type) { - object_type -> Text, - id -> Bytea, - version -> Int2, - bcs -> Bytea, - } -} - -diesel::table! { - epochs (epoch) { - epoch -> Int8, - first_checkpoint_id -> Int8, - epoch_start_timestamp -> Int8, - reference_gas_price -> Int8, - protocol_version -> Int8, - total_stake -> Int8, - storage_fund_balance -> Int8, - system_state -> Nullable, - epoch_total_transactions -> Nullable, - last_checkpoint_id -> Nullable, - epoch_end_timestamp -> Nullable, - storage_fund_reinvestment -> Nullable, - storage_charge -> Nullable, - storage_rebate -> Nullable, - stake_subsidy_amount -> Nullable, - total_gas_fees -> Nullable, - total_stake_rewards_distributed -> Nullable, - leftover_storage_fund_inflow -> Nullable, - epoch_commitments -> Nullable, - system_state_summary_json -> Nullable, - first_tx_sequence_number -> Nullable, - } -} - -diesel::table! { - event_emit_module (package, module, tx_sequence_number, event_sequence_number) { - package -> Bytea, - module -> Text, - tx_sequence_number -> Int8, - event_sequence_number -> Int8, - sender -> Bytea, - } -} - -diesel::table! { - event_emit_package (package, tx_sequence_number, event_sequence_number) { - package -> Bytea, - tx_sequence_number -> Int8, - event_sequence_number -> Int8, - sender -> Bytea, - } -} - -diesel::table! { - event_senders (sender, tx_sequence_number, event_sequence_number) { - sender -> Bytea, - tx_sequence_number -> Int8, - event_sequence_number -> Int8, - } -} - -diesel::table! { - event_struct_instantiation (package, module, type_instantiation, tx_sequence_number, event_sequence_number) { - package -> Bytea, - module -> Text, - type_instantiation -> Text, - tx_sequence_number -> Int8, - event_sequence_number -> Int8, - sender -> Bytea, - } -} - -diesel::table! { - event_struct_module (package, module, tx_sequence_number, event_sequence_number) { - package -> Bytea, - module -> Text, - tx_sequence_number -> Int8, - event_sequence_number -> Int8, - sender -> Bytea, - } -} - -diesel::table! { - event_struct_name (package, module, type_name, tx_sequence_number, event_sequence_number) { - package -> Bytea, - module -> Text, - type_name -> Text, - tx_sequence_number -> Int8, - event_sequence_number -> Int8, - sender -> Bytea, - } -} - -diesel::table! { - event_struct_package (package, tx_sequence_number, event_sequence_number) { - package -> Bytea, - tx_sequence_number -> Int8, - event_sequence_number -> Int8, - sender -> Bytea, - } -} - -diesel::table! { - events (tx_sequence_number, event_sequence_number) { - tx_sequence_number -> Int8, - event_sequence_number -> Int8, - transaction_digest -> Bytea, - senders -> Array>, - package -> Bytea, - module -> Text, - event_type -> Text, - timestamp_ms -> Int8, - bcs -> Bytea, - sender -> Nullable, - } -} - -diesel::table! { - feature_flags (protocol_version, flag_name) { - protocol_version -> Int8, - flag_name -> Text, - flag_value -> Bool, - } -} - -diesel::table! { - full_objects_history (object_id, object_version) { - object_id -> Bytea, - object_version -> Int8, - serialized_object -> Nullable, - } -} - -diesel::table! { - objects (object_id) { - object_id -> Bytea, - object_version -> Int8, - object_digest -> Bytea, - owner_type -> Int2, - owner_id -> Nullable, - object_type -> Nullable, - object_type_package -> Nullable, - object_type_module -> Nullable, - object_type_name -> Nullable, - serialized_object -> Bytea, - coin_type -> Nullable, - coin_balance -> Nullable, - df_kind -> Nullable, - } -} - -diesel::table! { - objects_history (checkpoint_sequence_number, object_id, object_version) { - object_id -> Bytea, - object_version -> Int8, - object_status -> Int2, - object_digest -> Nullable, - checkpoint_sequence_number -> Int8, - owner_type -> Nullable, - owner_id -> Nullable, - object_type -> Nullable, - object_type_package -> Nullable, - object_type_module -> Nullable, - object_type_name -> Nullable, - serialized_object -> Nullable, - coin_type -> Nullable, - coin_balance -> Nullable, - df_kind -> Nullable, - } -} - -diesel::table! { - objects_snapshot (object_id) { - object_id -> Bytea, - object_version -> Int8, - object_status -> Int2, - object_digest -> Nullable, - checkpoint_sequence_number -> Int8, - owner_type -> Nullable, - owner_id -> Nullable, - object_type -> Nullable, - object_type_package -> Nullable, - object_type_module -> Nullable, - object_type_name -> Nullable, - serialized_object -> Nullable, - coin_type -> Nullable, - coin_balance -> Nullable, - df_kind -> Nullable, - } -} - -diesel::table! { - objects_version (object_id, object_version) { - object_id -> Bytea, - object_version -> Int8, - cp_sequence_number -> Int8, - } -} - -diesel::table! { - packages (package_id, original_id, package_version) { - package_id -> Bytea, - original_id -> Bytea, - package_version -> Int8, - move_package -> Bytea, - checkpoint_sequence_number -> Int8, - } -} - -diesel::table! { - protocol_configs (protocol_version, config_name) { - protocol_version -> Int8, - config_name -> Text, - config_value -> Nullable, - } -} - -diesel::table! { - pruner_cp_watermark (checkpoint_sequence_number) { - checkpoint_sequence_number -> Int8, - min_tx_sequence_number -> Int8, - max_tx_sequence_number -> Int8, - } -} - -diesel::table! { - raw_checkpoints (sequence_number) { - sequence_number -> Int8, - certified_checkpoint -> Bytea, - checkpoint_contents -> Bytea, - } -} - -diesel::table! { - transactions (tx_sequence_number) { - tx_sequence_number -> Int8, - transaction_digest -> Bytea, - raw_transaction -> Bytea, - raw_effects -> Bytea, - checkpoint_sequence_number -> Int8, - timestamp_ms -> Int8, - object_changes -> Array>, - balance_changes -> Array>, - events -> Array>, - transaction_kind -> Int2, - success_command_count -> Int2, - } -} - -diesel::table! { - tx_affected_addresses (affected, tx_sequence_number) { - tx_sequence_number -> Int8, - affected -> Bytea, - sender -> Bytea, - } -} - -diesel::table! { - tx_affected_objects (affected, tx_sequence_number) { - tx_sequence_number -> Int8, - affected -> Bytea, - sender -> Bytea, - } -} - -diesel::table! { - tx_calls_fun (package, module, func, tx_sequence_number) { - tx_sequence_number -> Int8, - package -> Bytea, - module -> Text, - func -> Text, - sender -> Bytea, - } -} - -diesel::table! { - tx_calls_mod (package, module, tx_sequence_number) { - tx_sequence_number -> Int8, - package -> Bytea, - module -> Text, - sender -> Bytea, - } -} - -diesel::table! { - tx_calls_pkg (package, tx_sequence_number) { - tx_sequence_number -> Int8, - package -> Bytea, - sender -> Bytea, - } -} - -diesel::table! { - tx_changed_objects (object_id, tx_sequence_number) { - tx_sequence_number -> Int8, - object_id -> Bytea, - sender -> Bytea, - } -} - -diesel::table! { - tx_digests (tx_digest) { - tx_digest -> Bytea, - tx_sequence_number -> Int8, - } -} - -diesel::table! { - tx_input_objects (object_id, tx_sequence_number) { - tx_sequence_number -> Int8, - object_id -> Bytea, - sender -> Bytea, - } -} - -diesel::table! { - tx_kinds (tx_kind, tx_sequence_number) { - tx_sequence_number -> Int8, - tx_kind -> Int2, - } -} - -diesel::table! { - watermarks (pipeline) { - pipeline -> Text, - epoch_hi_inclusive -> Int8, - checkpoint_hi_inclusive -> Int8, - tx_hi -> Int8, - epoch_lo -> Int8, - reader_lo -> Int8, - timestamp_ms -> Int8, - pruner_hi -> Int8, - } -} - -diesel::allow_tables_to_appear_in_same_query!( - chain_identifier, - checkpoints, - display, - epochs, - event_emit_module, - event_emit_package, - event_senders, - event_struct_instantiation, - event_struct_module, - event_struct_name, - event_struct_package, - events, - feature_flags, - full_objects_history, - objects, - objects_history, - objects_snapshot, - objects_version, - packages, - protocol_configs, - pruner_cp_watermark, - raw_checkpoints, - transactions, - tx_affected_addresses, - tx_affected_objects, - tx_calls_fun, - tx_calls_mod, - tx_calls_pkg, - tx_changed_objects, - tx_digests, - tx_input_objects, - tx_kinds, - watermarks, -); diff --git a/crates/sui-mvr-indexer/src/store/indexer_store.rs b/crates/sui-mvr-indexer/src/store/indexer_store.rs deleted file mode 100644 index 998b37f286b1e..0000000000000 --- a/crates/sui-mvr-indexer/src/store/indexer_store.rs +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::BTreeMap; - -use async_trait::async_trait; -use strum::IntoEnumIterator; - -use crate::errors::IndexerError; -use crate::handlers::pruner::PrunableTable; -use crate::handlers::{CommitterWatermark, EpochToCommit, TransactionObjectChangesToCommit}; -use crate::models::display::StoredDisplay; -use crate::models::obj_indices::StoredObjectVersion; -use crate::models::objects::{StoredDeletedObject, StoredObject}; -use crate::models::raw_checkpoints::StoredRawCheckpoint; -use crate::models::watermarks::StoredWatermark; -use crate::types::{ - EventIndex, IndexedCheckpoint, IndexedEvent, IndexedPackage, IndexedTransaction, TxIndex, -}; - -#[allow(clippy::large_enum_variant)] -pub enum ObjectsToCommit { - MutatedObject(StoredObject), - DeletedObject(StoredDeletedObject), -} - -#[async_trait] -pub trait IndexerStore: Clone + Sync + Send + 'static { - async fn get_latest_checkpoint_sequence_number(&self) -> Result, IndexerError>; - - async fn get_available_epoch_range(&self) -> Result<(u64, u64), IndexerError>; - - async fn get_available_checkpoint_range(&self) -> Result<(u64, u64), IndexerError>; - - async fn get_latest_object_snapshot_checkpoint_sequence_number( - &self, - ) -> Result, IndexerError>; - - async fn get_chain_identifier(&self) -> Result>, IndexerError>; - - async fn persist_protocol_configs_and_feature_flags( - &self, - chain_id: Vec, - ) -> Result<(), IndexerError>; - - async fn persist_objects( - &self, - object_changes: Vec, - ) -> Result<(), IndexerError>; - - async fn persist_object_history( - &self, - object_changes: Vec, - ) -> Result<(), IndexerError>; - - async fn persist_full_objects_history( - &self, - object_changes: Vec, - ) -> Result<(), IndexerError>; - - async fn persist_objects_version( - &self, - object_versions: Vec, - ) -> Result<(), IndexerError>; - - async fn persist_objects_snapshot( - &self, - object_changes: Vec, - ) -> Result<(), IndexerError>; - - async fn persist_checkpoints( - &self, - checkpoints: Vec, - ) -> Result<(), IndexerError>; - - async fn persist_chain_identifier( - &self, - checkpoint_digest: Vec, - ) -> Result<(), IndexerError>; - - async fn persist_transactions( - &self, - transactions: Vec, - ) -> Result<(), IndexerError>; - - async fn persist_tx_indices(&self, indices: Vec) -> Result<(), IndexerError>; - - async fn persist_events(&self, events: Vec) -> Result<(), IndexerError>; - async fn persist_event_indices( - &self, - event_indices: Vec, - ) -> Result<(), IndexerError>; - - async fn persist_displays( - &self, - display_updates: BTreeMap, - ) -> Result<(), IndexerError>; - - async fn persist_packages(&self, packages: Vec) -> Result<(), IndexerError>; - - /// Updates the current epoch with end-of-epoch data, and writes a new epoch to the database. - async fn persist_epoch(&self, epoch: EpochToCommit) -> Result<(), IndexerError>; - - /// Updates epoch-partitioned tables to accept data from the new epoch. - async fn advance_epoch(&self, epoch: EpochToCommit) -> Result<(), IndexerError>; - - async fn prune_epoch(&self, epoch: u64) -> Result<(), IndexerError>; - - async fn get_network_total_transactions_by_end_of_epoch( - &self, - epoch: u64, - ) -> Result, IndexerError>; - - async fn upload_display(&self, epoch: u64) -> Result<(), IndexerError>; - - async fn restore_display(&self, bytes: bytes::Bytes) -> Result<(), IndexerError>; - - async fn persist_raw_checkpoints( - &self, - checkpoints: Vec, - ) -> Result<(), IndexerError>; - - /// Update the upper bound of the watermarks for the given tables. - async fn update_watermarks_upper_bound( - &self, - watermark: CommitterWatermark, - ) -> Result<(), IndexerError> - where - E::Iterator: Iterator>; - - /// Updates each watermark entry's lower bounds per the list of tables and their new epoch lower - /// bounds. - async fn update_watermarks_lower_bound( - &self, - watermarks: Vec<(PrunableTable, u64)>, - ) -> Result<(), IndexerError>; - - /// Load all watermark entries from the store, and the latest timestamp from the db. - async fn get_watermarks(&self) -> Result<(Vec, i64), IndexerError>; -} diff --git a/crates/sui-mvr-indexer/src/store/mod.rs b/crates/sui-mvr-indexer/src/store/mod.rs deleted file mode 100644 index 9d6bf65cc26b4..0000000000000 --- a/crates/sui-mvr-indexer/src/store/mod.rs +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::time::Duration; - -use diesel_async::{scoped_futures::ScopedBoxFuture, AsyncPgConnection}; -pub(crate) use indexer_store::*; -pub use pg_indexer_store::PgIndexerStore; - -use crate::{database::ConnectionPool, errors::IndexerError}; - -pub mod indexer_store; -pub mod package_resolver; -mod pg_indexer_store; -pub mod pg_partition_manager; - -pub async fn transaction_with_retry<'a, Q, T>( - pool: &ConnectionPool, - timeout: Duration, - query: Q, -) -> Result -where - Q: for<'r> FnOnce( - &'r mut AsyncPgConnection, - ) -> ScopedBoxFuture<'a, 'r, Result> - + Send, - Q: Clone, - T: 'a, -{ - let backoff = backoff::ExponentialBackoff { - max_elapsed_time: Some(timeout), - ..Default::default() - }; - backoff::future::retry(backoff, || async { - let mut connection = pool.get().await.map_err(|e| backoff::Error::Transient { - err: IndexerError::PostgresWriteError(e.to_string()), - retry_after: None, - })?; - - connection - .build_transaction() - .read_write() - .run(query.clone()) - .await - .map_err(|e| { - tracing::error!("Error with persisting data into DB: {:?}, retrying...", e); - backoff::Error::Transient { - err: IndexerError::PostgresWriteError(e.to_string()), - retry_after: None, - } - }) - }) - .await -} - -pub async fn read_with_retry<'a, Q, T>( - pool: &ConnectionPool, - timeout: Duration, - query: Q, -) -> Result -where - Q: for<'r> FnOnce( - &'r mut AsyncPgConnection, - ) -> ScopedBoxFuture<'a, 'r, Result> - + Send, - Q: Clone, - T: 'a, -{ - let backoff = backoff::ExponentialBackoff { - max_elapsed_time: Some(timeout), - ..Default::default() - }; - backoff::future::retry(backoff, || async { - let mut connection = pool.get().await.map_err(|e| backoff::Error::Transient { - err: IndexerError::PostgresWriteError(e.to_string()), - retry_after: None, - })?; - - connection - .build_transaction() - .read_only() - .run(query.clone()) - .await - .map_err(|e| { - tracing::error!("Error with reading data from DB: {:?}, retrying...", e); - backoff::Error::Transient { - err: IndexerError::PostgresWriteError(e.to_string()), - retry_after: None, - } - }) - }) - .await -} diff --git a/crates/sui-mvr-indexer/src/store/package_resolver.rs b/crates/sui-mvr-indexer/src/store/package_resolver.rs deleted file mode 100644 index f4cedd6500871..0000000000000 --- a/crates/sui-mvr-indexer/src/store/package_resolver.rs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::Arc; - -use crate::database::ConnectionPool; -use crate::schema::objects; -use anyhow::anyhow; -use async_trait::async_trait; -use diesel::ExpressionMethods; -use diesel::QueryDsl; -use diesel_async::RunQueryDsl; -use move_core_types::account_address::AccountAddress; -use sui_package_resolver::{error::Error as PackageResolverError, Package, PackageStore}; -use sui_types::object::Object; - -/// A package resolver that reads packages from the database. -#[derive(Clone)] -pub struct IndexerStorePackageResolver { - pool: ConnectionPool, -} - -impl IndexerStorePackageResolver { - pub fn new(pool: ConnectionPool) -> Self { - Self { pool } - } -} - -#[async_trait] -impl PackageStore for IndexerStorePackageResolver { - async fn fetch(&self, id: AccountAddress) -> Result, PackageResolverError> { - let pkg = self - .get_package_from_db(id) - .await - .map_err(|e| PackageResolverError::Store { - store: "PostgresDB", - error: e.to_string(), - })?; - Ok(Arc::new(pkg)) - } -} - -impl IndexerStorePackageResolver { - async fn get_package_from_db(&self, id: AccountAddress) -> Result { - let mut connection = self.pool.get().await?; - - let bcs = objects::dsl::objects - .select(objects::dsl::serialized_object) - .filter(objects::dsl::object_id.eq(id.to_vec())) - .get_result::>(&mut connection) - .await - .map_err(|e| anyhow!("Package not found in DB: {e}"))?; - - let object = bcs::from_bytes::(&bcs)?; - Package::read_from_object(&object) - .map_err(|e| anyhow!("Failed parsing object to package: {e}")) - } -} diff --git a/crates/sui-mvr-indexer/src/store/pg_indexer_store.rs b/crates/sui-mvr-indexer/src/store/pg_indexer_store.rs deleted file mode 100644 index b1d1af7b31ed6..0000000000000 --- a/crates/sui-mvr-indexer/src/store/pg_indexer_store.rs +++ /dev/null @@ -1,2495 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::{BTreeMap, HashMap}; -use std::io::Cursor; -use std::time::Duration; - -use async_trait::async_trait; -use core::result::Result::Ok; -use csv::{ReaderBuilder, Writer}; -use diesel::dsl::{max, min}; -use diesel::ExpressionMethods; -use diesel::OptionalExtension; -use diesel::QueryDsl; -use diesel_async::scoped_futures::ScopedFutureExt; -use futures::future::Either; -use itertools::Itertools; -use object_store::path::Path; -use strum::IntoEnumIterator; -use sui_types::base_types::ObjectID; -use tap::TapFallible; -use tracing::{info, warn}; - -use sui_config::object_storage_config::{ObjectStoreConfig, ObjectStoreType}; -use sui_protocol_config::ProtocolConfig; -use sui_storage::object_store::util::put; - -use crate::config::UploadOptions; -use crate::database::ConnectionPool; -use crate::errors::{Context, IndexerError}; -use crate::handlers::pruner::PrunableTable; -use crate::handlers::TransactionObjectChangesToCommit; -use crate::handlers::{CommitterWatermark, EpochToCommit}; -use crate::metrics::IndexerMetrics; -use crate::models::checkpoints::StoredChainIdentifier; -use crate::models::checkpoints::StoredCheckpoint; -use crate::models::checkpoints::StoredCpTx; -use crate::models::display::StoredDisplay; -use crate::models::epoch::StoredEpochInfo; -use crate::models::epoch::{StoredFeatureFlag, StoredProtocolConfig}; -use crate::models::events::StoredEvent; -use crate::models::obj_indices::StoredObjectVersion; -use crate::models::objects::{ - StoredDeletedObject, StoredFullHistoryObject, StoredHistoryObject, StoredObject, - StoredObjectSnapshot, -}; -use crate::models::packages::StoredPackage; -use crate::models::transactions::StoredTransaction; -use crate::models::watermarks::StoredWatermark; -use crate::schema::{ - chain_identifier, checkpoints, display, epochs, event_emit_module, event_emit_package, - event_senders, event_struct_instantiation, event_struct_module, event_struct_name, - event_struct_package, events, feature_flags, full_objects_history, objects, objects_history, - objects_snapshot, objects_version, packages, protocol_configs, pruner_cp_watermark, - raw_checkpoints, transactions, tx_affected_addresses, tx_affected_objects, tx_calls_fun, - tx_calls_mod, tx_calls_pkg, tx_changed_objects, tx_digests, tx_input_objects, tx_kinds, - watermarks, -}; -use crate::store::{read_with_retry, transaction_with_retry}; -use crate::types::{EventIndex, IndexedDeletedObject, IndexedObject}; -use crate::types::{IndexedCheckpoint, IndexedEvent, IndexedPackage, IndexedTransaction, TxIndex}; - -use super::pg_partition_manager::{EpochPartitionData, PgPartitionManager}; -use super::IndexerStore; - -use crate::models::raw_checkpoints::StoredRawCheckpoint; -use diesel::upsert::excluded; -use sui_types::digests::{ChainIdentifier, CheckpointDigest}; - -#[macro_export] -macro_rules! chunk { - ($data: expr, $size: expr) => {{ - $data - .into_iter() - .chunks($size) - .into_iter() - .map(|c| c.collect()) - .collect::>>() - }}; -} - -// In one DB transaction, the update could be chunked into -// a few statements, this is the amount of rows to update in one statement -// TODO: I think with the `per_db_tx` params, `PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX` -// is now less relevant. We should do experiments and remove it if it's true. -const PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX: usize = 1000; -// The amount of rows to update in one DB transaction -const PG_COMMIT_PARALLEL_CHUNK_SIZE: usize = 100; -// The amount of rows to update in one DB transaction, for objects particularly -// Having this number too high may cause many db deadlocks because of -// optimistic locking. -const PG_COMMIT_OBJECTS_PARALLEL_CHUNK_SIZE: usize = 500; -const PG_DB_COMMIT_SLEEP_DURATION: Duration = Duration::from_secs(3600); - -#[derive(Clone)] -pub struct PgIndexerStoreConfig { - pub parallel_chunk_size: usize, - pub parallel_objects_chunk_size: usize, - pub gcs_cred_path: Option, - pub gcs_display_bucket: Option, -} - -#[derive(Clone)] -pub struct PgIndexerStore { - pool: ConnectionPool, - metrics: IndexerMetrics, - partition_manager: PgPartitionManager, - config: PgIndexerStoreConfig, -} - -impl PgIndexerStore { - pub fn new( - pool: ConnectionPool, - upload_options: UploadOptions, - metrics: IndexerMetrics, - ) -> Self { - let parallel_chunk_size = std::env::var("PG_COMMIT_PARALLEL_CHUNK_SIZE") - .unwrap_or_else(|_e| PG_COMMIT_PARALLEL_CHUNK_SIZE.to_string()) - .parse::() - .unwrap(); - let parallel_objects_chunk_size = std::env::var("PG_COMMIT_OBJECTS_PARALLEL_CHUNK_SIZE") - .unwrap_or_else(|_e| PG_COMMIT_OBJECTS_PARALLEL_CHUNK_SIZE.to_string()) - .parse::() - .unwrap(); - let partition_manager = - PgPartitionManager::new(pool.clone()).expect("Failed to initialize partition manager"); - let config = PgIndexerStoreConfig { - parallel_chunk_size, - parallel_objects_chunk_size, - gcs_cred_path: upload_options.gcs_cred_path, - gcs_display_bucket: upload_options.gcs_display_bucket, - }; - - Self { - pool, - metrics, - partition_manager, - config, - } - } - - pub fn pool(&self) -> ConnectionPool { - self.pool.clone() - } - - /// Get the range of the protocol versions that need to be indexed. - pub async fn get_protocol_version_index_range(&self) -> Result<(i64, i64), IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - // We start indexing from the next protocol version after the latest one stored in the db. - let start = protocol_configs::table - .select(max(protocol_configs::protocol_version)) - .first::>(&mut connection) - .await - .map_err(Into::into) - .context("Failed reading latest protocol version from PostgresDB")? - .map_or(1, |v| v + 1); - - // We end indexing at the protocol version of the latest epoch stored in the db. - let end = epochs::table - .select(max(epochs::protocol_version)) - .first::>(&mut connection) - .await - .map_err(Into::into) - .context("Failed reading latest epoch protocol version from PostgresDB")? - .unwrap_or(1); - Ok((start, end)) - } - - async fn get_chain_identifier(&self) -> Result>, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - chain_identifier::table - .select(chain_identifier::checkpoint_digest) - .first::>(&mut connection) - .await - .optional() - .map_err(Into::into) - .context("Failed reading chain id from PostgresDB") - } - - // `pub` is needed for wait_for_checkpoint in tests - pub async fn get_latest_checkpoint_sequence_number(&self) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - checkpoints::table - .select(max(checkpoints::sequence_number)) - .first::>(&mut connection) - .await - .map_err(Into::into) - .map(|v| v.map(|v| v as u64)) - .context("Failed reading latest checkpoint sequence number from PostgresDB") - } - - async fn get_available_checkpoint_range(&self) -> Result<(u64, u64), IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - checkpoints::table - .select(( - min(checkpoints::sequence_number), - max(checkpoints::sequence_number), - )) - .first::<(Option, Option)>(&mut connection) - .await - .map_err(Into::into) - .map(|(min, max)| { - ( - min.unwrap_or_default() as u64, - max.unwrap_or_default() as u64, - ) - }) - .context("Failed reading min and max checkpoint sequence numbers from PostgresDB") - } - - async fn get_prunable_epoch_range(&self) -> Result<(u64, u64), IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - epochs::table - .select((min(epochs::epoch), max(epochs::epoch))) - .first::<(Option, Option)>(&mut connection) - .await - .map_err(Into::into) - .map(|(min, max)| { - ( - min.unwrap_or_default() as u64, - max.unwrap_or_default() as u64, - ) - }) - .context("Failed reading min and max epoch numbers from PostgresDB") - } - - async fn get_min_prunable_checkpoint(&self) -> Result { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - pruner_cp_watermark::table - .select(min(pruner_cp_watermark::checkpoint_sequence_number)) - .first::>(&mut connection) - .await - .map_err(Into::into) - .map(|v| v.unwrap_or_default() as u64) - .context("Failed reading min prunable checkpoint sequence number from PostgresDB") - } - - pub async fn get_checkpoint_range_for_epoch( - &self, - epoch: u64, - ) -> Result<(u64, Option), IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - epochs::table - .select((epochs::first_checkpoint_id, epochs::last_checkpoint_id)) - .filter(epochs::epoch.eq(epoch as i64)) - .first::<(i64, Option)>(&mut connection) - .await - .map_err(Into::into) - .map(|(min, max)| (min as u64, max.map(|v| v as u64))) - .context("Failed reading checkpoint range from PostgresDB") - } - - pub async fn get_transaction_range_for_checkpoint( - &self, - checkpoint: u64, - ) -> Result<(u64, u64), IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - pruner_cp_watermark::table - .select(( - pruner_cp_watermark::min_tx_sequence_number, - pruner_cp_watermark::max_tx_sequence_number, - )) - .filter(pruner_cp_watermark::checkpoint_sequence_number.eq(checkpoint as i64)) - .first::<(i64, i64)>(&mut connection) - .await - .map_err(Into::into) - .map(|(min, max)| (min as u64, max as u64)) - .context("Failed reading transaction range from PostgresDB") - } - - pub async fn get_latest_object_snapshot_checkpoint_sequence_number( - &self, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - objects_snapshot::table - .select(max(objects_snapshot::checkpoint_sequence_number)) - .first::>(&mut connection) - .await - .map_err(Into::into) - .map(|v| v.map(|v| v as u64)) - .context( - "Failed reading latest object snapshot checkpoint sequence number from PostgresDB", - ) - } - - async fn persist_display_updates( - &self, - display_updates: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - diesel::insert_into(display::table) - .values(display_updates) - .on_conflict(display::object_type) - .do_update() - .set(( - display::id.eq(excluded(display::id)), - display::version.eq(excluded(display::version)), - display::bcs.eq(excluded(display::bcs)), - )) - .execute(conn) - .await?; - - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await?; - - Ok(()) - } - - async fn persist_object_mutation_chunk( - &self, - mutated_object_mutation_chunk: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - let guard = self - .metrics - .checkpoint_db_commit_latency_objects_chunks - .start_timer(); - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - diesel::insert_into(objects::table) - .values(mutated_object_mutation_chunk.clone()) - .on_conflict(objects::object_id) - .do_update() - .set(( - objects::object_id.eq(excluded(objects::object_id)), - objects::object_version.eq(excluded(objects::object_version)), - objects::object_digest.eq(excluded(objects::object_digest)), - objects::owner_type.eq(excluded(objects::owner_type)), - objects::owner_id.eq(excluded(objects::owner_id)), - objects::object_type.eq(excluded(objects::object_type)), - objects::serialized_object.eq(excluded(objects::serialized_object)), - objects::coin_type.eq(excluded(objects::coin_type)), - objects::coin_balance.eq(excluded(objects::coin_balance)), - objects::df_kind.eq(excluded(objects::df_kind)), - )) - .execute(conn) - .await?; - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - guard.stop_and_record(); - }) - .tap_err(|e| { - tracing::error!("Failed to persist object mutations with error: {}", e); - }) - } - - async fn persist_object_deletion_chunk( - &self, - deleted_objects_chunk: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - let guard = self - .metrics - .checkpoint_db_commit_latency_objects_chunks - .start_timer(); - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - diesel::delete( - objects::table.filter( - objects::object_id.eq_any( - deleted_objects_chunk - .iter() - .map(|o| o.object_id.clone()) - .collect::>(), - ), - ), - ) - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to write object deletion to PostgresDB")?; - - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - guard.stop_and_record(); - }) - .tap_err(|e| { - tracing::error!("Failed to persist object deletions with error: {}", e); - }) - } - - async fn persist_object_snapshot_mutation_chunk( - &self, - objects_snapshot_mutations: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - let guard = self - .metrics - .checkpoint_db_commit_latency_objects_snapshot_chunks - .start_timer(); - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for mutation_chunk in - objects_snapshot_mutations.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(objects_snapshot::table) - .values(mutation_chunk) - .on_conflict(objects_snapshot::object_id) - .do_update() - .set(( - objects_snapshot::object_version - .eq(excluded(objects_snapshot::object_version)), - objects_snapshot::object_status - .eq(excluded(objects_snapshot::object_status)), - objects_snapshot::object_digest - .eq(excluded(objects_snapshot::object_digest)), - objects_snapshot::owner_type.eq(excluded(objects_snapshot::owner_type)), - objects_snapshot::owner_id.eq(excluded(objects_snapshot::owner_id)), - objects_snapshot::object_type_package - .eq(excluded(objects_snapshot::object_type_package)), - objects_snapshot::object_type_module - .eq(excluded(objects_snapshot::object_type_module)), - objects_snapshot::object_type_name - .eq(excluded(objects_snapshot::object_type_name)), - objects_snapshot::object_type - .eq(excluded(objects_snapshot::object_type)), - objects_snapshot::serialized_object - .eq(excluded(objects_snapshot::serialized_object)), - objects_snapshot::coin_type.eq(excluded(objects_snapshot::coin_type)), - objects_snapshot::coin_balance - .eq(excluded(objects_snapshot::coin_balance)), - objects_snapshot::df_kind.eq(excluded(objects_snapshot::df_kind)), - objects_snapshot::checkpoint_sequence_number - .eq(excluded(objects_snapshot::checkpoint_sequence_number)), - )) - .execute(conn) - .await?; - } - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - guard.stop_and_record(); - }) - .tap_err(|e| { - tracing::error!("Failed to persist object snapshot with error: {}", e); - }) - } - - async fn persist_object_snapshot_deletion_chunk( - &self, - objects_snapshot_deletions: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - let guard = self - .metrics - .checkpoint_db_commit_latency_objects_snapshot_chunks - .start_timer(); - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for deletion_chunk in - objects_snapshot_deletions.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::delete( - objects_snapshot::table.filter( - objects_snapshot::object_id.eq_any( - deletion_chunk - .iter() - .map(|o| o.object_id.clone()) - .collect::>(), - ), - ), - ) - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to write object deletion to PostgresDB")?; - } - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!( - elapsed, - "Deleted {} chunked object snapshots", - objects_snapshot_deletions.len(), - ); - }) - .tap_err(|e| { - tracing::error!( - "Failed to persist object snapshot deletions with error: {}", - e - ); - }) - } - - async fn persist_objects_history_chunk( - &self, - stored_objects_history: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - let guard = self - .metrics - .checkpoint_db_commit_latency_objects_history_chunks - .start_timer(); - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for stored_objects_history_chunk in - stored_objects_history.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - let error_message = concat!( - "Failed to write to ", - stringify!((objects_history::table)), - " DB" - ); - diesel::insert_into(objects_history::table) - .values(stored_objects_history_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context(error_message)?; - } - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - guard.stop_and_record(); - }) - .tap_err(|e| { - tracing::error!("Failed to persist object history with error: {}", e); - }) - } - - async fn persist_full_objects_history_chunk( - &self, - objects: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - let guard = self - .metrics - .checkpoint_db_commit_latency_full_objects_history_chunks - .start_timer(); - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for objects_chunk in objects.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - diesel::insert_into(full_objects_history::table) - .values(objects_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to write to full_objects_history table")?; - } - - Ok(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!( - elapsed, - "Persisted {} chunked full objects history", - objects.len(), - ); - }) - .tap_err(|e| { - tracing::error!("Failed to persist full object history with error: {}", e); - }) - } - - async fn persist_objects_version_chunk( - &self, - object_versions: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - let guard = self - .metrics - .checkpoint_db_commit_latency_objects_version_chunks - .start_timer(); - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for object_version_chunk in object_versions.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(objects_version::table) - .values(object_version_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to write to objects_version table")?; - } - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!( - elapsed, - "Persisted {} chunked object versions", - object_versions.len(), - ); - }) - .tap_err(|e| { - tracing::error!("Failed to persist object versions with error: {}", e); - }) - } - - async fn persist_raw_checkpoints_impl( - &self, - raw_checkpoints: &[StoredRawCheckpoint], - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - diesel::insert_into(raw_checkpoints::table) - .values(raw_checkpoints) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to write to raw_checkpoints table")?; - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - } - - async fn persist_checkpoints( - &self, - checkpoints: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - let Some(first_checkpoint) = checkpoints.as_slice().first() else { - return Ok(()); - }; - - // If the first checkpoint has sequence number 0, we need to persist the digest as - // chain identifier. - if first_checkpoint.sequence_number == 0 { - let checkpoint_digest = first_checkpoint.checkpoint_digest.into_inner().to_vec(); - self.persist_protocol_configs_and_feature_flags(checkpoint_digest.clone()) - .await?; - self.persist_chain_identifier(checkpoint_digest).await?; - } - let guard = self - .metrics - .checkpoint_db_commit_latency_checkpoints - .start_timer(); - - let stored_cp_txs = checkpoints.iter().map(StoredCpTx::from).collect::>(); - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for stored_cp_tx_chunk in stored_cp_txs.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - diesel::insert_into(pruner_cp_watermark::table) - .values(stored_cp_tx_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to write to pruner_cp_watermark table")?; - } - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - info!( - "Persisted {} pruner_cp_watermark rows.", - stored_cp_txs.len(), - ); - }) - .tap_err(|e| { - tracing::error!("Failed to persist pruner_cp_watermark with error: {}", e); - })?; - - let stored_checkpoints = checkpoints - .iter() - .map(StoredCheckpoint::from) - .collect::>(); - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for stored_checkpoint_chunk in - stored_checkpoints.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(checkpoints::table) - .values(stored_checkpoint_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to write to checkpoints table")?; - let time_now_ms = chrono::Utc::now().timestamp_millis(); - for stored_checkpoint in stored_checkpoint_chunk { - self.metrics - .db_commit_lag_ms - .set(time_now_ms - stored_checkpoint.timestamp_ms); - self.metrics - .max_committed_checkpoint_sequence_number - .set(stored_checkpoint.sequence_number); - self.metrics - .committed_checkpoint_timestamp_ms - .set(stored_checkpoint.timestamp_ms); - } - - for stored_checkpoint in stored_checkpoint_chunk { - info!( - "Indexer lag: \ - persisted checkpoint {} with time now {} and checkpoint time {}", - stored_checkpoint.sequence_number, - time_now_ms, - stored_checkpoint.timestamp_ms - ); - } - } - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!( - elapsed, - "Persisted {} checkpoints", - stored_checkpoints.len() - ); - }) - .tap_err(|e| { - tracing::error!("Failed to persist checkpoints with error: {}", e); - }) - } - - async fn persist_transactions_chunk( - &self, - transactions: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - let guard = self - .metrics - .checkpoint_db_commit_latency_transactions_chunks - .start_timer(); - let transformation_guard = self - .metrics - .checkpoint_db_commit_latency_transactions_chunks_transformation - .start_timer(); - let transactions = transactions - .iter() - .map(StoredTransaction::from) - .collect::>(); - drop(transformation_guard); - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for transaction_chunk in transactions.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - let error_message = concat!( - "Failed to write to ", - stringify!((transactions::table)), - " DB" - ); - diesel::insert_into(transactions::table) - .values(transaction_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context(error_message)?; - } - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!( - elapsed, - "Persisted {} chunked transactions", - transactions.len() - ); - }) - .tap_err(|e| { - tracing::error!("Failed to persist transactions with error: {}", e); - }) - } - - async fn persist_events_chunk(&self, events: Vec) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - let guard = self - .metrics - .checkpoint_db_commit_latency_events_chunks - .start_timer(); - let len = events.len(); - let events = events - .into_iter() - .map(StoredEvent::from) - .collect::>(); - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for event_chunk in events.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - let error_message = - concat!("Failed to write to ", stringify!((events::table)), " DB"); - diesel::insert_into(events::table) - .values(event_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context(error_message)?; - } - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted {} chunked events", len); - }) - .tap_err(|e| { - tracing::error!("Failed to persist events with error: {}", e); - }) - } - - async fn persist_packages(&self, packages: Vec) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - if packages.is_empty() { - return Ok(()); - } - let guard = self - .metrics - .checkpoint_db_commit_latency_packages - .start_timer(); - let packages = packages - .into_iter() - .map(StoredPackage::from) - .collect::>(); - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for packages_chunk in packages.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - diesel::insert_into(packages::table) - .values(packages_chunk) - .on_conflict(packages::package_id) - .do_update() - .set(( - packages::package_id.eq(excluded(packages::package_id)), - packages::package_version.eq(excluded(packages::package_version)), - packages::move_package.eq(excluded(packages::move_package)), - packages::checkpoint_sequence_number - .eq(excluded(packages::checkpoint_sequence_number)), - )) - .execute(conn) - .await?; - } - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted {} packages", packages.len()); - }) - .tap_err(|e| { - tracing::error!("Failed to persist packages with error: {}", e); - }) - } - - async fn persist_event_indices_chunk( - &self, - indices: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - let guard = self - .metrics - .checkpoint_db_commit_latency_event_indices_chunks - .start_timer(); - let len = indices.len(); - let ( - event_emit_packages, - event_emit_modules, - event_senders, - event_struct_packages, - event_struct_modules, - event_struct_names, - event_struct_instantiations, - ) = indices.into_iter().map(|i| i.split()).fold( - ( - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - ), - |( - mut event_emit_packages, - mut event_emit_modules, - mut event_senders, - mut event_struct_packages, - mut event_struct_modules, - mut event_struct_names, - mut event_struct_instantiations, - ), - index| { - event_emit_packages.push(index.0); - event_emit_modules.push(index.1); - event_senders.push(index.2); - event_struct_packages.push(index.3); - event_struct_modules.push(index.4); - event_struct_names.push(index.5); - event_struct_instantiations.push(index.6); - ( - event_emit_packages, - event_emit_modules, - event_senders, - event_struct_packages, - event_struct_modules, - event_struct_names, - event_struct_instantiations, - ) - }, - ); - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for event_emit_packages_chunk in - event_emit_packages.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(event_emit_package::table) - .values(event_emit_packages_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for event_emit_modules_chunk in - event_emit_modules.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(event_emit_module::table) - .values(event_emit_modules_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for event_senders_chunk in event_senders.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - diesel::insert_into(event_senders::table) - .values(event_senders_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for event_struct_packages_chunk in - event_struct_packages.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(event_struct_package::table) - .values(event_struct_packages_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for event_struct_modules_chunk in - event_struct_modules.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(event_struct_module::table) - .values(event_struct_modules_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for event_struct_names_chunk in - event_struct_names.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(event_struct_name::table) - .values(event_struct_names_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for event_struct_instantiations_chunk in - event_struct_instantiations.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(event_struct_instantiation::table) - .values(event_struct_instantiations_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - Ok(()) - } - .scope_boxed() - }) - .await?; - - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted {} chunked event indices", len); - Ok(()) - } - - async fn persist_tx_indices_chunk(&self, indices: Vec) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - let guard = self - .metrics - .checkpoint_db_commit_latency_tx_indices_chunks - .start_timer(); - let len = indices.len(); - let ( - affected_addresses, - affected_objects, - input_objects, - changed_objects, - pkgs, - mods, - funs, - digests, - kinds, - ) = indices.into_iter().map(|i| i.split()).fold( - ( - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - ), - |( - mut tx_affected_addresses, - mut tx_affected_objects, - mut tx_input_objects, - mut tx_changed_objects, - mut tx_pkgs, - mut tx_mods, - mut tx_funs, - mut tx_digests, - mut tx_kinds, - ), - index| { - tx_affected_addresses.extend(index.0); - tx_affected_objects.extend(index.1); - tx_input_objects.extend(index.2); - tx_changed_objects.extend(index.3); - tx_pkgs.extend(index.4); - tx_mods.extend(index.5); - tx_funs.extend(index.6); - tx_digests.extend(index.7); - tx_kinds.extend(index.8); - ( - tx_affected_addresses, - tx_affected_objects, - tx_input_objects, - tx_changed_objects, - tx_pkgs, - tx_mods, - tx_funs, - tx_digests, - tx_kinds, - ) - }, - ); - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for affected_addresses_chunk in - affected_addresses.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(tx_affected_addresses::table) - .values(affected_addresses_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for affected_objects_chunk in - affected_objects.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(tx_affected_objects::table) - .values(affected_objects_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for input_objects_chunk in input_objects.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - diesel::insert_into(tx_input_objects::table) - .values(input_objects_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for changed_objects_chunk in - changed_objects.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) - { - diesel::insert_into(tx_changed_objects::table) - .values(changed_objects_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for pkgs_chunk in pkgs.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - diesel::insert_into(tx_calls_pkg::table) - .values(pkgs_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for mods_chunk in mods.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - diesel::insert_into(tx_calls_mod::table) - .values(mods_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for funs_chunk in funs.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - diesel::insert_into(tx_calls_fun::table) - .values(funs_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for digests_chunk in digests.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - diesel::insert_into(tx_digests::table) - .values(digests_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - for kinds_chunk in kinds.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - diesel::insert_into(tx_kinds::table) - .values(kinds_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await?; - } - - Ok(()) - } - .scope_boxed() - }) - .await?; - - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted {} chunked tx_indices", len); - Ok(()) - } - - async fn persist_epoch(&self, epoch: EpochToCommit) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - let guard = self - .metrics - .checkpoint_db_commit_latency_epoch - .start_timer(); - let epoch_id = epoch.new_epoch.epoch; - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - if let Some(last_epoch) = &epoch.last_epoch { - let last_epoch_id = last_epoch.epoch; - - info!(last_epoch_id, "Persisting epoch end data."); - diesel::update(epochs::table.filter(epochs::epoch.eq(last_epoch_id))) - .set(last_epoch) - .execute(conn) - .await?; - } - - let epoch_id = epoch.new_epoch.epoch; - info!(epoch_id, "Persisting epoch beginning info"); - let error_message = - concat!("Failed to write to ", stringify!((epochs::table)), " DB"); - diesel::insert_into(epochs::table) - .values(epoch.new_epoch) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context(error_message)?; - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!(elapsed, epoch_id, "Persisted epoch beginning info"); - }) - .tap_err(|e| { - tracing::error!("Failed to persist epoch with error: {}", e); - }) - } - - async fn advance_epoch(&self, epoch_to_commit: EpochToCommit) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let last_epoch_id = epoch_to_commit.last_epoch.as_ref().map(|e| e.epoch); - // partition_0 has been created, so no need to advance it. - if let Some(last_epoch_id) = last_epoch_id { - let last_db_epoch: Option = epochs::table - .filter(epochs::epoch.eq(last_epoch_id)) - .first::(&mut connection) - .await - .optional() - .map_err(Into::into) - .context("Failed to read last epoch from PostgresDB")?; - if let Some(last_epoch) = last_db_epoch { - let epoch_partition_data = - EpochPartitionData::compose_data(epoch_to_commit, last_epoch); - let table_partitions = self.partition_manager.get_table_partitions().await?; - for (table, (_, last_partition)) in table_partitions { - // Only advance epoch partition for epoch partitioned tables. - if !self - .partition_manager - .get_strategy(&table) - .is_epoch_partitioned() - { - continue; - } - let guard = self.metrics.advance_epoch_latency.start_timer(); - self.partition_manager - .advance_epoch(table.clone(), last_partition, &epoch_partition_data) - .await?; - let elapsed = guard.stop_and_record(); - info!( - elapsed, - "Advanced epoch partition {} for table {}", - last_partition, - table.clone() - ); - } - } else { - tracing::error!("Last epoch: {} from PostgresDB is None.", last_epoch_id); - } - } - - Ok(()) - } - - async fn prune_checkpoints_table(&self, cp: u64) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - diesel::delete( - checkpoints::table.filter(checkpoints::sequence_number.eq(cp as i64)), - ) - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to prune checkpoints table")?; - - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - } - - async fn prune_event_indices_table( - &self, - min_tx: u64, - max_tx: u64, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - let (min_tx, max_tx) = (min_tx as i64, max_tx as i64); - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - diesel::delete( - event_emit_module::table - .filter(event_emit_module::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete( - event_emit_package::table - .filter(event_emit_package::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete( - event_senders::table - .filter(event_senders::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete(event_struct_instantiation::table.filter( - event_struct_instantiation::tx_sequence_number.between(min_tx, max_tx), - )) - .execute(conn) - .await?; - - diesel::delete( - event_struct_module::table - .filter(event_struct_module::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete( - event_struct_name::table - .filter(event_struct_name::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete( - event_struct_package::table - .filter(event_struct_package::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - Ok(()) - } - .scope_boxed() - }) - .await - } - - async fn prune_tx_indices_table(&self, min_tx: u64, max_tx: u64) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - let (min_tx, max_tx) = (min_tx as i64, max_tx as i64); - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - diesel::delete( - tx_affected_addresses::table - .filter(tx_affected_addresses::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete( - tx_affected_objects::table - .filter(tx_affected_objects::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete( - tx_input_objects::table - .filter(tx_input_objects::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete( - tx_changed_objects::table - .filter(tx_changed_objects::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete( - tx_calls_pkg::table - .filter(tx_calls_pkg::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete( - tx_calls_mod::table - .filter(tx_calls_mod::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete( - tx_calls_fun::table - .filter(tx_calls_fun::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - diesel::delete( - tx_digests::table - .filter(tx_digests::tx_sequence_number.between(min_tx, max_tx)), - ) - .execute(conn) - .await?; - - Ok(()) - } - .scope_boxed() - }) - .await - } - - async fn prune_cp_tx_table(&self, cp: u64) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - diesel::delete( - pruner_cp_watermark::table - .filter(pruner_cp_watermark::checkpoint_sequence_number.eq(cp as i64)), - ) - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to prune pruner_cp_watermark table")?; - Ok(()) - } - .scope_boxed() - }) - .await - } - - async fn get_network_total_transactions_by_end_of_epoch( - &self, - epoch: u64, - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - // TODO: (wlmyng) update to read from epochs::network_total_transactions - - Ok(Some( - checkpoints::table - .filter(checkpoints::epoch.eq(epoch as i64)) - .select(checkpoints::network_total_transactions) - .order_by(checkpoints::sequence_number.desc()) - .first::(&mut connection) - .await - .map_err(Into::into) - .context("Failed to get network total transactions in epoch") - .map(|v| v as u64)?, - )) - } - - async fn update_watermarks_upper_bound( - &self, - watermark: CommitterWatermark, - ) -> Result<(), IndexerError> - where - E::Iterator: Iterator>, - { - use diesel_async::RunQueryDsl; - - let guard = self - .metrics - .checkpoint_db_commit_latency_watermarks - .start_timer(); - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - let upper_bound_updates = E::iter() - .map(|table| StoredWatermark::from_upper_bound_update(table.as_ref(), watermark)) - .collect::>(); - async { - diesel::insert_into(watermarks::table) - .values(upper_bound_updates) - .on_conflict(watermarks::pipeline) - .do_update() - .set(( - watermarks::epoch_hi_inclusive.eq(excluded(watermarks::epoch_hi_inclusive)), - watermarks::checkpoint_hi_inclusive - .eq(excluded(watermarks::checkpoint_hi_inclusive)), - watermarks::tx_hi.eq(excluded(watermarks::tx_hi)), - )) - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to update watermarks upper bound")?; - - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted watermarks"); - }) - .tap_err(|e| { - tracing::error!("Failed to persist watermarks with error: {}", e); - }) - } - - async fn map_epochs_to_cp_tx( - &self, - epochs: &[u64], - ) -> Result, IndexerError> { - use diesel_async::RunQueryDsl; - - let mut connection = self.pool.get().await?; - - let results: Vec<(i64, i64, Option)> = epochs::table - .filter(epochs::epoch.eq_any(epochs.iter().map(|&e| e as i64))) - .select(( - epochs::epoch, - epochs::first_checkpoint_id, - epochs::first_tx_sequence_number, - )) - .load::<(i64, i64, Option)>(&mut connection) - .await - .map_err(Into::into) - .context("Failed to fetch first checkpoint and tx seq num for epochs")?; - - Ok(results - .into_iter() - .map(|(epoch, checkpoint, tx)| { - ( - epoch as u64, - (checkpoint as u64, tx.unwrap_or_default() as u64), - ) - }) - .collect()) - } - - async fn update_watermarks_lower_bound( - &self, - watermarks: Vec<(PrunableTable, u64)>, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - let epochs: Vec = watermarks.iter().map(|(_table, epoch)| *epoch).collect(); - let epoch_mapping = self.map_epochs_to_cp_tx(&epochs).await?; - let lookups: Result, IndexerError> = watermarks - .into_iter() - .map(|(table, epoch)| { - let (checkpoint, tx) = epoch_mapping.get(&epoch).ok_or_else(|| { - IndexerError::PersistentStorageDataCorruptionError(format!( - "Epoch {} not found in epoch mapping", - epoch - )) - })?; - - Ok(StoredWatermark::from_lower_bound_update( - table.as_ref(), - epoch, - table.select_reader_lo(*checkpoint, *tx), - )) - }) - .collect(); - let lower_bound_updates = lookups?; - - let guard = self - .metrics - .checkpoint_db_commit_latency_watermarks - .start_timer(); - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - use diesel::dsl::sql; - use diesel::query_dsl::methods::FilterDsl; - - diesel::insert_into(watermarks::table) - .values(lower_bound_updates) - .on_conflict(watermarks::pipeline) - .do_update() - .set(( - watermarks::reader_lo.eq(excluded(watermarks::reader_lo)), - watermarks::epoch_lo.eq(excluded(watermarks::epoch_lo)), - watermarks::timestamp_ms.eq(sql::( - "(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) * 1000)::bigint", - )), - )) - .filter(excluded(watermarks::reader_lo).gt(watermarks::reader_lo)) - .filter(excluded(watermarks::epoch_lo).gt(watermarks::epoch_lo)) - .filter( - diesel::dsl::sql::( - "(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) * 1000)::bigint", - ) - .gt(watermarks::timestamp_ms), - ) - .execute(conn) - .await?; - - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted watermarks"); - }) - .tap_err(|e| { - tracing::error!("Failed to persist watermarks with error: {}", e); - }) - } - - async fn get_watermarks(&self) -> Result<(Vec, i64), IndexerError> { - use diesel_async::RunQueryDsl; - - // read_only transaction, otherwise this will block and get blocked by write transactions to - // the same table. - read_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - let stored = watermarks::table - .load::(conn) - .await - .map_err(Into::into) - .context("Failed reading watermarks from PostgresDB")?; - - let timestamp = diesel::select(diesel::dsl::sql::( - "(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) * 1000)::bigint", - )) - .get_result(conn) - .await - .map_err(Into::into) - .context("Failed reading current timestamp from PostgresDB")?; - - Ok((stored, timestamp)) - } - .scope_boxed() - }) - .await - } -} - -#[async_trait] -impl IndexerStore for PgIndexerStore { - async fn get_latest_checkpoint_sequence_number(&self) -> Result, IndexerError> { - self.get_latest_checkpoint_sequence_number().await - } - - async fn get_available_epoch_range(&self) -> Result<(u64, u64), IndexerError> { - self.get_prunable_epoch_range().await - } - - async fn get_available_checkpoint_range(&self) -> Result<(u64, u64), IndexerError> { - self.get_available_checkpoint_range().await - } - - async fn get_chain_identifier(&self) -> Result>, IndexerError> { - self.get_chain_identifier().await - } - - async fn get_latest_object_snapshot_checkpoint_sequence_number( - &self, - ) -> Result, IndexerError> { - self.get_latest_object_snapshot_checkpoint_sequence_number() - .await - } - - async fn persist_objects( - &self, - object_changes: Vec, - ) -> Result<(), IndexerError> { - if object_changes.is_empty() { - return Ok(()); - } - let guard = self - .metrics - .checkpoint_db_commit_latency_objects - .start_timer(); - let (indexed_mutations, indexed_deletions) = retain_latest_indexed_objects(object_changes); - let object_mutations = indexed_mutations - .into_iter() - .map(StoredObject::from) - .collect::>(); - let object_deletions = indexed_deletions - .into_iter() - .map(StoredDeletedObject::from) - .collect::>(); - let mutation_len = object_mutations.len(); - let deletion_len = object_deletions.len(); - - let object_mutation_chunks = - chunk!(object_mutations, self.config.parallel_objects_chunk_size); - let object_deletion_chunks = - chunk!(object_deletions, self.config.parallel_objects_chunk_size); - let mutation_futures = object_mutation_chunks - .into_iter() - .map(|c| self.persist_object_mutation_chunk(c)) - .map(Either::Left); - let deletion_futures = object_deletion_chunks - .into_iter() - .map(|c| self.persist_object_deletion_chunk(c)) - .map(Either::Right); - let all_futures = mutation_futures.chain(deletion_futures).collect::>(); - - futures::future::join_all(all_futures) - .await - .into_iter() - .collect::, _>>() - .map_err(|e| { - IndexerError::PostgresWriteError(format!( - "Failed to persist all object mutation or deletion chunks: {:?}", - e - )) - })?; - let elapsed = guard.stop_and_record(); - info!( - elapsed, - "Persisted {} objects mutations and {} deletions", mutation_len, deletion_len - ); - Ok(()) - } - - async fn persist_objects_snapshot( - &self, - object_changes: Vec, - ) -> Result<(), IndexerError> { - if object_changes.is_empty() { - return Ok(()); - } - let guard = self - .metrics - .checkpoint_db_commit_latency_objects_snapshot - .start_timer(); - let (indexed_mutations, indexed_deletions) = retain_latest_indexed_objects(object_changes); - let object_snapshot_mutations: Vec = indexed_mutations - .into_iter() - .map(StoredObjectSnapshot::from) - .collect(); - let object_snapshot_deletions: Vec = indexed_deletions - .into_iter() - .map(StoredObjectSnapshot::from) - .collect(); - let mutation_len = object_snapshot_mutations.len(); - let deletion_len = object_snapshot_deletions.len(); - let object_snapshot_mutation_chunks = chunk!( - object_snapshot_mutations, - self.config.parallel_objects_chunk_size - ); - let object_snapshot_deletion_chunks = chunk!( - object_snapshot_deletions, - self.config.parallel_objects_chunk_size - ); - let mutation_futures = object_snapshot_mutation_chunks - .into_iter() - .map(|c| self.persist_object_snapshot_mutation_chunk(c)) - .map(Either::Left) - .collect::>(); - let deletion_futures = object_snapshot_deletion_chunks - .into_iter() - .map(|c| self.persist_object_snapshot_deletion_chunk(c)) - .map(Either::Right) - .collect::>(); - let all_futures = mutation_futures - .into_iter() - .chain(deletion_futures) - .collect::>(); - futures::future::join_all(all_futures) - .await - .into_iter() - .collect::, _>>() - .map_err(|e| { - IndexerError::PostgresWriteError(format!( - "Failed to persist object snapshot mutation or deletion chunks: {:?}", - e - )) - }) - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!( - elapsed, - "Persisted {} objects snapshot mutations and {} deletions", - mutation_len, - deletion_len - ); - }) - .tap_err(|e| { - tracing::error!( - "Failed to persist object snapshot mutation or deletion chunks: {:?}", - e - ) - })?; - Ok(()) - } - - async fn persist_object_history( - &self, - object_changes: Vec, - ) -> Result<(), IndexerError> { - let skip_history = std::env::var("SKIP_OBJECT_HISTORY") - .map(|val| val.eq_ignore_ascii_case("true")) - .unwrap_or(false); - if skip_history { - info!("skipping object history"); - return Ok(()); - } - - if object_changes.is_empty() { - return Ok(()); - } - let objects = make_objects_history_to_commit(object_changes); - let guard = self - .metrics - .checkpoint_db_commit_latency_objects_history - .start_timer(); - - let len = objects.len(); - let chunks = chunk!(objects, self.config.parallel_objects_chunk_size); - let futures = chunks - .into_iter() - .map(|c| self.persist_objects_history_chunk(c)) - .collect::>(); - - futures::future::join_all(futures) - .await - .into_iter() - .collect::, _>>() - .map_err(|e| { - IndexerError::PostgresWriteError(format!( - "Failed to persist all objects history chunks: {:?}", - e - )) - })?; - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted {} objects history", len); - Ok(()) - } - - // TODO: There are quite some shared boiler-plate code in all functions. - // We should clean them up eventually. - async fn persist_full_objects_history( - &self, - object_changes: Vec, - ) -> Result<(), IndexerError> { - let skip_history = std::env::var("SKIP_OBJECT_HISTORY") - .map(|val| val.eq_ignore_ascii_case("true")) - .unwrap_or(false); - if skip_history { - info!("skipping object history"); - return Ok(()); - } - - if object_changes.is_empty() { - return Ok(()); - } - let objects: Vec = object_changes - .into_iter() - .flat_map(|c| { - let TransactionObjectChangesToCommit { - changed_objects, - deleted_objects, - } = c; - changed_objects - .into_iter() - .map(|o| o.into()) - .chain(deleted_objects.into_iter().map(|o| o.into())) - }) - .collect(); - let guard = self - .metrics - .checkpoint_db_commit_latency_full_objects_history - .start_timer(); - - let len = objects.len(); - let chunks = chunk!(objects, self.config.parallel_objects_chunk_size); - let futures = chunks - .into_iter() - .map(|c| self.persist_full_objects_history_chunk(c)) - .collect::>(); - - futures::future::join_all(futures) - .await - .into_iter() - .collect::, _>>() - .map_err(|e| { - IndexerError::PostgresWriteError(format!( - "Failed to persist all full objects history chunks: {:?}", - e - )) - })?; - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted {} full objects history", len); - Ok(()) - } - - async fn persist_objects_version( - &self, - object_versions: Vec, - ) -> Result<(), IndexerError> { - if object_versions.is_empty() { - return Ok(()); - } - - let guard = self - .metrics - .checkpoint_db_commit_latency_objects_version - .start_timer(); - - let len = object_versions.len(); - let chunks = chunk!(object_versions, self.config.parallel_objects_chunk_size); - let futures = chunks - .into_iter() - .map(|c| self.persist_objects_version_chunk(c)) - .collect::>(); - - futures::future::join_all(futures) - .await - .into_iter() - .collect::, _>>() - .map_err(|e| { - IndexerError::PostgresWriteError(format!( - "Failed to persist all objects version chunks: {:?}", - e - )) - })?; - - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted {} object versions", len); - Ok(()) - } - - async fn persist_checkpoints( - &self, - checkpoints: Vec, - ) -> Result<(), IndexerError> { - self.persist_checkpoints(checkpoints).await - } - - async fn persist_transactions( - &self, - transactions: Vec, - ) -> Result<(), IndexerError> { - let guard = self - .metrics - .checkpoint_db_commit_latency_transactions - .start_timer(); - let len = transactions.len(); - - let chunks = chunk!(transactions, self.config.parallel_chunk_size); - let futures = chunks - .into_iter() - .map(|c| self.persist_transactions_chunk(c)) - .collect::>(); - - futures::future::join_all(futures) - .await - .into_iter() - .collect::, _>>() - .map_err(|e| { - IndexerError::PostgresWriteError(format!( - "Failed to persist all transactions chunks: {:?}", - e - )) - })?; - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted {} transactions", len); - Ok(()) - } - - async fn persist_events(&self, events: Vec) -> Result<(), IndexerError> { - if events.is_empty() { - return Ok(()); - } - let len = events.len(); - let guard = self - .metrics - .checkpoint_db_commit_latency_events - .start_timer(); - let chunks = chunk!(events, self.config.parallel_chunk_size); - let futures = chunks - .into_iter() - .map(|c| self.persist_events_chunk(c)) - .collect::>(); - - futures::future::join_all(futures) - .await - .into_iter() - .collect::, _>>() - .map_err(|e| { - IndexerError::PostgresWriteError(format!( - "Failed to persist all events chunks: {:?}", - e - )) - })?; - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted {} events", len); - Ok(()) - } - - async fn persist_displays( - &self, - display_updates: BTreeMap, - ) -> Result<(), IndexerError> { - if display_updates.is_empty() { - return Ok(()); - } - self.persist_display_updates(display_updates.values().cloned().collect::>()) - .await - } - - async fn persist_packages(&self, packages: Vec) -> Result<(), IndexerError> { - if packages.is_empty() { - return Ok(()); - } - self.persist_packages(packages).await - } - - async fn persist_event_indices(&self, indices: Vec) -> Result<(), IndexerError> { - if indices.is_empty() { - return Ok(()); - } - let len = indices.len(); - let guard = self - .metrics - .checkpoint_db_commit_latency_event_indices - .start_timer(); - let chunks = chunk!(indices, self.config.parallel_chunk_size); - - let futures = chunks - .into_iter() - .map(|chunk| self.persist_event_indices_chunk(chunk)) - .collect::>(); - futures::future::join_all(futures) - .await - .into_iter() - .collect::, _>>() - .map_err(|e| { - IndexerError::PostgresWriteError(format!( - "Failed to persist all event_indices chunks: {:?}", - e - )) - }) - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted {} event_indices chunks", len); - }) - .tap_err(|e| tracing::error!("Failed to persist all event_indices chunks: {:?}", e))?; - Ok(()) - } - - async fn persist_tx_indices(&self, indices: Vec) -> Result<(), IndexerError> { - if indices.is_empty() { - return Ok(()); - } - let len = indices.len(); - let guard = self - .metrics - .checkpoint_db_commit_latency_tx_indices - .start_timer(); - let chunks = chunk!(indices, self.config.parallel_chunk_size); - - let futures = chunks - .into_iter() - .map(|chunk| self.persist_tx_indices_chunk(chunk)) - .collect::>(); - futures::future::join_all(futures) - .await - .into_iter() - .collect::, _>>() - .map_err(|e| { - IndexerError::PostgresWriteError(format!( - "Failed to persist all tx_indices chunks: {:?}", - e - )) - }) - .tap_ok(|_| { - let elapsed = guard.stop_and_record(); - info!(elapsed, "Persisted {} tx_indices chunks", len); - }) - .tap_err(|e| tracing::error!("Failed to persist all tx_indices chunks: {:?}", e))?; - Ok(()) - } - - async fn persist_epoch(&self, epoch: EpochToCommit) -> Result<(), IndexerError> { - self.persist_epoch(epoch).await - } - - async fn advance_epoch(&self, epoch: EpochToCommit) -> Result<(), IndexerError> { - self.advance_epoch(epoch).await - } - - async fn prune_epoch(&self, epoch: u64) -> Result<(), IndexerError> { - let (mut min_cp, max_cp) = match self.get_checkpoint_range_for_epoch(epoch).await? { - (min_cp, Some(max_cp)) => Ok((min_cp, max_cp)), - _ => Err(IndexerError::PostgresReadError(format!( - "Failed to get checkpoint range for epoch {}", - epoch - ))), - }?; - - // NOTE: for disaster recovery, min_cp is the min cp of the current epoch, which is likely - // partially pruned already. min_prunable_cp is the min cp to be pruned. - // By std::cmp::max, we will resume the pruning process from the next checkpoint, instead of - // the first cp of the current epoch. - let min_prunable_cp = self.get_min_prunable_checkpoint().await?; - min_cp = std::cmp::max(min_cp, min_prunable_cp); - for cp in min_cp..=max_cp { - // NOTE: the order of pruning tables is crucial: - // 1. prune checkpoints table, checkpoints table is the source table of available range, - // we prune it first to make sure that we always have full data for checkpoints within the available range; - // 2. then prune tx_* tables; - // 3. then prune pruner_cp_watermark table, which is the checkpoint pruning watermark table and also tx seq source - // of a checkpoint to prune tx_* tables; - // 4. lastly we prune epochs table when all checkpoints of the epoch have been pruned. - info!( - "Pruning checkpoint {} of epoch {} (min_prunable_cp: {})", - cp, epoch, min_prunable_cp - ); - self.prune_checkpoints_table(cp).await?; - - let (min_tx, max_tx) = self.get_transaction_range_for_checkpoint(cp).await?; - self.prune_tx_indices_table(min_tx, max_tx).await?; - info!( - "Pruned transactions for checkpoint {} from tx {} to tx {}", - cp, min_tx, max_tx - ); - self.prune_event_indices_table(min_tx, max_tx).await?; - info!( - "Pruned events of transactions for checkpoint {} from tx {} to tx {}", - cp, min_tx, max_tx - ); - self.metrics.last_pruned_transaction.set(max_tx as i64); - - self.prune_cp_tx_table(cp).await?; - info!("Pruned checkpoint {} of epoch {}", cp, epoch); - self.metrics.last_pruned_checkpoint.set(cp as i64); - } - - Ok(()) - } - - async fn upload_display(&self, epoch_number: u64) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - let mut connection = self.pool.get().await?; - let mut buffer = Cursor::new(Vec::new()); - { - let mut writer = Writer::from_writer(&mut buffer); - let displays = display::table - .load::(&mut connection) - .await - .map_err(Into::into) - .context("Failed to get display from database")?; - info!("Read {} displays", displays.len()); - writer - .write_record(["object_type", "id", "version", "bcs"]) - .map_err(|_| { - IndexerError::GcsError("Failed to write display to csv".to_string()) - })?; - for display in displays { - writer - .write_record(&[ - display.object_type, - hex::encode(display.id), - display.version.to_string(), - hex::encode(display.bcs), - ]) - .map_err(|_| IndexerError::GcsError("Failed to write to csv".to_string()))?; - } - writer - .flush() - .map_err(|_| IndexerError::GcsError("Failed to flush csv".to_string()))?; - } - - if let (Some(cred_path), Some(bucket)) = ( - self.config.gcs_cred_path.clone(), - self.config.gcs_display_bucket.clone(), - ) { - let remote_store_config = ObjectStoreConfig { - object_store: Some(ObjectStoreType::GCS), - bucket: Some(bucket), - google_service_account: Some(cred_path), - object_store_connection_limit: 200, - no_sign_request: false, - ..Default::default() - }; - let remote_store = remote_store_config.make().map_err(|e| { - IndexerError::GcsError(format!("Failed to make GCS remote store: {}", e)) - })?; - let path = Path::from(format!("display_{}.csv", epoch_number).as_str()); - put(&remote_store, &path, buffer.into_inner().into()) - .await - .map_err(|e| IndexerError::GcsError(format!("Failed to put to GCS: {}", e)))?; - } else { - warn!("Either GCS cred path or bucket is not set, skipping display upload."); - } - Ok(()) - } - - async fn restore_display(&self, bytes: bytes::Bytes) -> Result<(), IndexerError> { - let cursor = Cursor::new(bytes); - let mut csv_reader = ReaderBuilder::new().has_headers(true).from_reader(cursor); - let displays = csv_reader - .deserialize() - .collect::, csv::Error>>() - .map_err(|e| { - IndexerError::GcsError(format!("Failed to deserialize display records: {}", e)) - })?; - self.persist_display_updates(displays).await - } - - async fn get_network_total_transactions_by_end_of_epoch( - &self, - epoch: u64, - ) -> Result, IndexerError> { - self.get_network_total_transactions_by_end_of_epoch(epoch) - .await - } - - /// Persist protocol configs and feature flags until the protocol version for the latest epoch - /// we have stored in the db, inclusive. - async fn persist_protocol_configs_and_feature_flags( - &self, - chain_id: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - let chain_id = ChainIdentifier::from( - CheckpointDigest::try_from(chain_id).expect("Unable to convert chain id"), - ); - - let mut all_configs = vec![]; - let mut all_flags = vec![]; - - let (start_version, end_version) = self.get_protocol_version_index_range().await?; - info!( - "Persisting protocol configs with start_version: {}, end_version: {}", - start_version, end_version - ); - - // Gather all protocol configs and feature flags for all versions between start and end. - for version in start_version..=end_version { - let protocol_configs = ProtocolConfig::get_for_version_if_supported( - (version as u64).into(), - chain_id.chain(), - ) - .ok_or(IndexerError::GenericError(format!( - "Unable to fetch protocol version {} and chain {:?}", - version, - chain_id.chain() - )))?; - let configs_vec = protocol_configs - .attr_map() - .into_iter() - .map(|(k, v)| StoredProtocolConfig { - protocol_version: version, - config_name: k, - config_value: v.map(|v| v.to_string()), - }) - .collect::>(); - all_configs.extend(configs_vec); - - let feature_flags = protocol_configs - .feature_map() - .into_iter() - .map(|(k, v)| StoredFeatureFlag { - protocol_version: version, - flag_name: k, - flag_value: v, - }) - .collect::>(); - all_flags.extend(feature_flags); - } - - // Now insert all of them into the db. - // TODO: right now the size of these updates is manageable but later we may consider batching. - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - for config_chunk in all_configs.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { - diesel::insert_into(protocol_configs::table) - .values(config_chunk) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to write to protocol_configs table")?; - } - - diesel::insert_into(feature_flags::table) - .values(all_flags.clone()) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context("Failed to write to feature_flags table")?; - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await?; - Ok(()) - } - - async fn persist_chain_identifier( - &self, - checkpoint_digest: Vec, - ) -> Result<(), IndexerError> { - use diesel_async::RunQueryDsl; - - transaction_with_retry(&self.pool, PG_DB_COMMIT_SLEEP_DURATION, |conn| { - async { - diesel::insert_into(chain_identifier::table) - .values(StoredChainIdentifier { checkpoint_digest }) - .on_conflict_do_nothing() - .execute(conn) - .await - .map_err(IndexerError::from) - .context("failed to write to chain_identifier table")?; - Ok::<(), IndexerError>(()) - } - .scope_boxed() - }) - .await?; - Ok(()) - } - - async fn persist_raw_checkpoints( - &self, - checkpoints: Vec, - ) -> Result<(), IndexerError> { - self.persist_raw_checkpoints_impl(&checkpoints).await - } - - async fn update_watermarks_upper_bound( - &self, - watermark: CommitterWatermark, - ) -> Result<(), IndexerError> - where - E::Iterator: Iterator>, - { - self.update_watermarks_upper_bound::(watermark).await - } - - async fn update_watermarks_lower_bound( - &self, - watermarks: Vec<(PrunableTable, u64)>, - ) -> Result<(), IndexerError> { - self.update_watermarks_lower_bound(watermarks).await - } - - async fn get_watermarks(&self) -> Result<(Vec, i64), IndexerError> { - self.get_watermarks().await - } -} - -fn make_objects_history_to_commit( - tx_object_changes: Vec, -) -> Vec { - let deleted_objects: Vec = tx_object_changes - .clone() - .into_iter() - .flat_map(|changes| changes.deleted_objects) - .map(|o| o.into()) - .collect(); - let mutated_objects: Vec = tx_object_changes - .into_iter() - .flat_map(|changes| changes.changed_objects) - .map(|o| o.into()) - .collect(); - deleted_objects.into_iter().chain(mutated_objects).collect() -} - -// Partition object changes into deletions and mutations, -// within partition of mutations or deletions, retain the latest with highest version; -// For overlappings of mutations and deletions, only keep one with higher version. -// This is necessary b/c after this step, DB commit will be done in parallel and not in order. -fn retain_latest_indexed_objects( - tx_object_changes: Vec, -) -> (Vec, Vec) { - // Only the last deleted / mutated object will be in the map, - // b/c tx_object_changes are in order and versions always increment, - let (mutations, deletions) = tx_object_changes - .into_iter() - .flat_map(|change| { - change - .changed_objects - .into_iter() - .map(Either::Left) - .chain( - change - .deleted_objects - .into_iter() - .map(Either::Right), - ) - }) - .fold( - (HashMap::::new(), HashMap::::new()), - |(mut mutations, mut deletions), either_change| { - match either_change { - // Remove mutation / deletion with a following deletion / mutation, - // b/c following deletion / mutation always has a higher version. - // Technically, assertions below are not required, double check just in case. - Either::Left(mutation) => { - let id = mutation.object.id(); - let mutation_version = mutation.object.version(); - if let Some(existing) = deletions.remove(&id) { - assert!( - existing.object_version < mutation_version.value(), - "Mutation version ({:?}) should be greater than existing deletion version ({:?}) for object {:?}", - mutation_version, - existing.object_version, - id - ); - } - if let Some(existing) = mutations.insert(id, mutation) { - assert!( - existing.object.version() < mutation_version, - "Mutation version ({:?}) should be greater than existing mutation version ({:?}) for object {:?}", - mutation_version, - existing.object.version(), - id - ); - } - } - Either::Right(deletion) => { - let id = deletion.object_id; - let deletion_version = deletion.object_version; - if let Some(existing) = mutations.remove(&id) { - assert!( - existing.object.version().value() < deletion_version, - "Deletion version ({:?}) should be greater than existing mutation version ({:?}) for object {:?}", - deletion_version, - existing.object.version(), - id - ); - } - if let Some(existing) = deletions.insert(id, deletion) { - assert!( - existing.object_version < deletion_version, - "Deletion version ({:?}) should be greater than existing deletion version ({:?}) for object {:?}", - deletion_version, - existing.object_version, - id - ); - } - } - } - (mutations, deletions) - }, - ); - ( - mutations.into_values().collect(), - deletions.into_values().collect(), - ) -} diff --git a/crates/sui-mvr-indexer/src/store/pg_partition_manager.rs b/crates/sui-mvr-indexer/src/store/pg_partition_manager.rs deleted file mode 100644 index 876a1b9c56146..0000000000000 --- a/crates/sui-mvr-indexer/src/store/pg_partition_manager.rs +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use diesel::sql_types::{BigInt, VarChar}; -use diesel::QueryableByName; -use diesel_async::scoped_futures::ScopedFutureExt; -use std::collections::{BTreeMap, HashMap}; -use std::time::Duration; -use tracing::{error, info}; - -use crate::database::ConnectionPool; -use crate::errors::IndexerError; -use crate::handlers::EpochToCommit; -use crate::models::epoch::StoredEpochInfo; -use crate::store::transaction_with_retry; - -const GET_PARTITION_SQL: &str = r" -SELECT parent.relname AS table_name, - MIN(CAST(SUBSTRING(child.relname FROM '\d+$') AS BIGINT)) AS first_partition, - MAX(CAST(SUBSTRING(child.relname FROM '\d+$') AS BIGINT)) AS last_partition -FROM pg_inherits - JOIN pg_class parent ON pg_inherits.inhparent = parent.oid - JOIN pg_class child ON pg_inherits.inhrelid = child.oid - JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace - JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace -WHERE parent.relkind = 'p' -GROUP BY table_name; -"; - -#[derive(Clone)] -pub struct PgPartitionManager { - pool: ConnectionPool, - - partition_strategies: HashMap<&'static str, PgPartitionStrategy>, -} - -#[derive(Clone, Copy)] -pub enum PgPartitionStrategy { - CheckpointSequenceNumber, - TxSequenceNumber, - ObjectId, -} - -impl PgPartitionStrategy { - pub fn is_epoch_partitioned(&self) -> bool { - matches!( - self, - Self::CheckpointSequenceNumber | Self::TxSequenceNumber - ) - } -} - -#[derive(Clone, Debug)] -pub struct EpochPartitionData { - last_epoch: u64, - next_epoch: u64, - last_epoch_start_cp: u64, - next_epoch_start_cp: u64, - last_epoch_start_tx: u64, - next_epoch_start_tx: u64, -} - -impl EpochPartitionData { - pub fn compose_data(epoch: EpochToCommit, last_db_epoch: StoredEpochInfo) -> Self { - let last_epoch = last_db_epoch.epoch as u64; - let last_epoch_start_cp = last_db_epoch.first_checkpoint_id as u64; - let next_epoch = epoch.new_epoch_id(); - let next_epoch_start_cp = epoch.new_epoch_first_checkpoint_id(); - let next_epoch_start_tx = epoch.new_epoch_first_tx_sequence_number(); - let last_epoch_start_tx = - next_epoch_start_tx - epoch.last_epoch_total_transactions().unwrap(); - - Self { - last_epoch, - next_epoch, - last_epoch_start_cp, - next_epoch_start_cp, - last_epoch_start_tx, - next_epoch_start_tx, - } - } -} - -impl PgPartitionManager { - pub fn new(pool: ConnectionPool) -> Result { - let mut partition_strategies = HashMap::new(); - partition_strategies.insert("events", PgPartitionStrategy::TxSequenceNumber); - partition_strategies.insert("transactions", PgPartitionStrategy::TxSequenceNumber); - partition_strategies.insert("objects_version", PgPartitionStrategy::ObjectId); - let manager = Self { - pool, - partition_strategies, - }; - Ok(manager) - } - - pub async fn get_table_partitions(&self) -> Result, IndexerError> { - #[derive(QueryableByName, Debug, Clone)] - struct PartitionedTable { - #[diesel(sql_type = VarChar)] - table_name: String, - #[diesel(sql_type = BigInt)] - first_partition: i64, - #[diesel(sql_type = BigInt)] - last_partition: i64, - } - - let mut connection = self.pool.get().await?; - - Ok( - diesel_async::RunQueryDsl::load(diesel::sql_query(GET_PARTITION_SQL), &mut connection) - .await? - .into_iter() - .map(|table: PartitionedTable| { - ( - table.table_name, - (table.first_partition as u64, table.last_partition as u64), - ) - }) - .collect(), - ) - } - - /// Tries to fetch the partitioning strategy for the given partitioned table. Defaults to - /// `CheckpointSequenceNumber` as the majority of our tables are partitioned on an epoch's - /// checkpoints today. - pub fn get_strategy(&self, table_name: &str) -> PgPartitionStrategy { - self.partition_strategies - .get(table_name) - .copied() - .unwrap_or(PgPartitionStrategy::CheckpointSequenceNumber) - } - - pub fn determine_epoch_partition_range( - &self, - table_name: &str, - data: &EpochPartitionData, - ) -> Option<(u64, u64)> { - match self.get_strategy(table_name) { - PgPartitionStrategy::CheckpointSequenceNumber => { - Some((data.last_epoch_start_cp, data.next_epoch_start_cp)) - } - PgPartitionStrategy::TxSequenceNumber => { - Some((data.last_epoch_start_tx, data.next_epoch_start_tx)) - } - PgPartitionStrategy::ObjectId => None, - } - } - - pub async fn advance_epoch( - &self, - table: String, - last_partition: u64, - data: &EpochPartitionData, - ) -> Result<(), IndexerError> { - let Some(partition_range) = self.determine_epoch_partition_range(&table, data) else { - return Ok(()); - }; - if data.next_epoch == 0 { - tracing::info!("Epoch 0 partition has been created in the initial setup."); - return Ok(()); - } - if last_partition == data.last_epoch { - transaction_with_retry(&self.pool, Duration::from_secs(10), |conn| { - async { - diesel_async::RunQueryDsl::execute( - diesel::sql_query("CALL advance_partition($1, $2, $3, $4, $5)") - .bind::(table.clone()) - .bind::(data.last_epoch as i64) - .bind::(data.next_epoch as i64) - .bind::(partition_range.0 as i64) - .bind::(partition_range.1 as i64), - conn, - ) - .await?; - - Ok(()) - } - .scope_boxed() - }) - .await?; - - info!( - "Advanced epoch partition for table {} from {} to {}, prev partition upper bound {}", - table, last_partition, data.next_epoch, partition_range.0 - ); - } else if last_partition != data.next_epoch { - // skip when the partition is already advanced once, which is possible when indexer - // crashes and restarts; error otherwise. - error!( - "Epoch partition for table {} is not in sync with the last epoch {}.", - table, data.last_epoch - ); - } else { - info!( - "Epoch has been advanced to {} already, skipping.", - data.next_epoch - ); - } - Ok(()) - } - - pub async fn drop_table_partition( - &self, - table: String, - partition: u64, - ) -> Result<(), IndexerError> { - transaction_with_retry(&self.pool, Duration::from_secs(10), |conn| { - async { - diesel_async::RunQueryDsl::execute( - diesel::sql_query("CALL drop_partition($1, $2)") - .bind::(table.clone()) - .bind::(partition as i64), - conn, - ) - .await?; - Ok(()) - } - .scope_boxed() - }) - .await?; - Ok(()) - } -} diff --git a/crates/sui-mvr-indexer/src/store/query.rs b/crates/sui-mvr-indexer/src/store/query.rs deleted file mode 100644 index 93d57b298044d..0000000000000 --- a/crates/sui-mvr-indexer/src/store/query.rs +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use sui_json_rpc_types::SuiObjectDataFilter; -use sui_types::base_types::ObjectID; - -pub trait DBFilter { - fn to_objects_history_sql(&self, cursor: Option, limit: usize, columns: Vec<&str>) - -> String; - fn to_latest_objects_sql(&self, cursor: Option, limit: usize, columns: Vec<&str>) -> String; -} - -impl DBFilter for SuiObjectDataFilter { - fn to_objects_history_sql( - &self, - cursor: Option, - limit: usize, - columns: Vec<&str>, - ) -> String { - let inner_clauses = to_clauses(self); - let inner_clauses = if let Some(inner_clauses) = inner_clauses { - format!("\n AND {inner_clauses}") - } else { - "".to_string() - }; - let outer_clauses = to_outer_clauses(self); - let outer_clauses = if let Some(outer_clauses) = outer_clauses { - format!("\nAND {outer_clauses}") - } else { - "".to_string() - }; - let cursor = if let Some(cursor) = cursor { - format!("\n AND o.object_id > '{cursor}'") - } else { - "".to_string() - }; - - let columns = columns - .iter() - .map(|c| format!("t1.{c}")) - .collect::>() - .join(", "); - // NOTE: order by checkpoint DESC so that whenever a row from checkpoint is available, - // we will pick that over the one from fast-path, which has checkpoint of -1. - format!( - "SELECT {columns} -FROM (SELECT DISTINCT ON (o.object_id) * - FROM objects_history o - WHERE o.checkpoint <= $1{cursor}{inner_clauses} - ORDER BY o.object_id, version, o.checkpoint DESC) AS t1 -WHERE t1.object_status NOT IN ('deleted', 'wrapped', 'unwrapped_then_deleted'){outer_clauses} -LIMIT {limit};" - ) - } - - fn to_latest_objects_sql( - &self, - cursor: Option, - limit: usize, - columns: Vec<&str>, - ) -> String { - let columns = columns - .iter() - .map(|c| format!("o.{c}")) - .collect::>() - .join(", "); - - let cursor = if let Some(cursor) = cursor { - format!(" AND o.object_id > '{cursor}'") - } else { - "".to_string() - }; - - let inner_clauses = to_latest_objects_clauses(self); - let inner_clauses = if let Some(inner_clauses) = inner_clauses { - format!(" AND {inner_clauses}") - } else { - "".to_string() - }; - - format!( - "SELECT {columns} -FROM objects o WHERE o.object_status NOT IN ('deleted', 'wrapped', 'unwrapped_then_deleted'){cursor}{inner_clauses} -LIMIT {limit};" - ) - } -} - -fn to_latest_objects_clauses(filter: &SuiObjectDataFilter) -> Option { - match filter { - SuiObjectDataFilter::AddressOwner(a) => Some(format!( - "(o.owner_type = 'address_owner' AND o.owner_address = '{a}')" - )), - _ => None, - } -} - -fn to_clauses(filter: &SuiObjectDataFilter) -> Option { - match filter { - SuiObjectDataFilter::MatchAll(sub_filters) => { - let sub_filters = sub_filters.iter().flat_map(to_clauses).collect::>(); - if sub_filters.is_empty() { - None - } else if sub_filters.len() == 1 { - Some(sub_filters[0].to_string()) - } else { - Some(format!("({})", sub_filters.join(" AND "))) - } - } - SuiObjectDataFilter::MatchAny(sub_filters) => { - let sub_filters = sub_filters.iter().flat_map(to_clauses).collect::>(); - if sub_filters.is_empty() { - // Any default to false - Some("FALSE".to_string()) - } else if sub_filters.len() == 1 { - Some(sub_filters[0].to_string()) - } else { - Some(format!("({})", sub_filters.join(" OR "))) - } - } - SuiObjectDataFilter::MatchNone(sub_filters) => { - let sub_filters = sub_filters.iter().flat_map(to_clauses).collect::>(); - if sub_filters.is_empty() { - None - } else { - Some(format!("NOT ({})", sub_filters.join(" OR "))) - } - } - SuiObjectDataFilter::Package(p) => Some(format!("o.object_type LIKE '{}::%'", p.to_hex_literal())), - SuiObjectDataFilter::MoveModule { package, module } => Some(format!( - "o.object_type LIKE '{}::{}::%'", - package.to_hex_literal(), - module - )), - SuiObjectDataFilter::StructType(s) => { - // If people do not provide type_params, we will match all type_params - // e.g. `0x2::coin::Coin` can match `0x2::coin::Coin<0x2::sui::SUI>` - if s.type_params.is_empty() { - Some(format!("o.object_type LIKE '{s}%'")) - } else { - Some(format!("o.object_type = '{s}'")) - } - }, - SuiObjectDataFilter::AddressOwner(a) => { - Some(format!("((o.owner_type = 'address_owner' AND o.owner_address = '{a}') OR (o.old_owner_type = 'address_owner' AND o.old_owner_address = '{a}'))")) - } - SuiObjectDataFilter::ObjectOwner(o) => { - Some(format!("((o.owner_type = 'object_owner' AND o.owner_address = '{o}') OR (o.old_owner_type = 'object_owner' AND o.old_owner_address = '{o}'))")) - } - SuiObjectDataFilter::ObjectId(id) => { - Some(format!("o.object_id = '{id}'")) - } - SuiObjectDataFilter::ObjectIds(ids) => { - if ids.is_empty() { - None - } else { - let ids = ids - .iter() - .map(|o| o.to_string()) - .collect::>() - .join(", "); - Some(format!("o.object_id IN '{ids}'")) - } - } - SuiObjectDataFilter::Version(v) => Some(format!("o.version = {v}")), - } -} - -fn to_outer_clauses(filter: &SuiObjectDataFilter) -> Option { - match filter { - SuiObjectDataFilter::MatchNone(sub_filters) => { - let sub_filters = sub_filters - .iter() - .flat_map(to_outer_clauses) - .collect::>(); - if sub_filters.is_empty() { - None - } else { - Some(format!("NOT ({})", sub_filters.join(" OR "))) - } - } - SuiObjectDataFilter::MatchAll(sub_filters) => { - let sub_filters = sub_filters - .iter() - .flat_map(to_outer_clauses) - .collect::>(); - if sub_filters.is_empty() { - None - } else if sub_filters.len() == 1 { - Some(sub_filters[0].to_string()) - } else { - Some(format!("({})", sub_filters.join(" AND "))) - } - } - SuiObjectDataFilter::MatchAny(sub_filters) => { - let sub_filters = sub_filters - .iter() - .flat_map(to_outer_clauses) - .collect::>(); - if sub_filters.is_empty() { - None - } else if sub_filters.len() == 1 { - Some(sub_filters[0].to_string()) - } else { - Some(format!("({})", sub_filters.join(" OR "))) - } - } - SuiObjectDataFilter::AddressOwner(a) => Some(format!("t1.owner_address = '{a}'")), - _ => None, - } -} - -#[cfg(test)] -mod test { - use std::str::FromStr; - - use move_core_types::ident_str; - - use sui_json_rpc_types::SuiObjectDataFilter; - use sui_types::base_types::{ObjectID, SuiAddress}; - use sui_types::parse_sui_struct_tag; - - use crate::store::query::DBFilter; - - #[test] - fn test_address_filter() { - let address = SuiAddress::from_str( - "0x92dd4d9b0150c251661d821583ef078024ae9e9ee11063e216500861eec7f381", - ) - .unwrap(); - let filter = SuiObjectDataFilter::AddressOwner(address); - - let expected_sql = "SELECT t1.* -FROM (SELECT DISTINCT ON (o.object_id) * - FROM objects_history o - WHERE o.checkpoint <= $1 - AND ((o.owner_type = 'address_owner' AND o.owner_address = '0x92dd4d9b0150c251661d821583ef078024ae9e9ee11063e216500861eec7f381') OR (o.old_owner_type = 'address_owner' AND o.old_owner_address = '0x92dd4d9b0150c251661d821583ef078024ae9e9ee11063e216500861eec7f381')) - ORDER BY o.object_id, version, o.checkpoint DESC) AS t1 -WHERE t1.object_status NOT IN ('deleted', 'wrapped', 'unwrapped_then_deleted') -AND t1.owner_address = '0x92dd4d9b0150c251661d821583ef078024ae9e9ee11063e216500861eec7f381' -LIMIT 100;"; - assert_eq!( - expected_sql, - filter.to_objects_history_sql(None, 100, vec!["*"]) - ); - } - - #[test] - fn test_move_module_filter() { - let filter = SuiObjectDataFilter::MoveModule { - package: ObjectID::from_str( - "0x485d947e293f07e659127dc5196146b49cdf2efbe4b233f4d293fc56aff2aa17", - ) - .unwrap(), - module: ident_str!("test_module").into(), - }; - let expected_sql = "SELECT t1.* -FROM (SELECT DISTINCT ON (o.object_id) * - FROM objects_history o - WHERE o.checkpoint <= $1 - AND o.object_type LIKE '0x485d947e293f07e659127dc5196146b49cdf2efbe4b233f4d293fc56aff2aa17::test_module::%' - ORDER BY o.object_id, version, o.checkpoint DESC) AS t1 -WHERE t1.object_status NOT IN ('deleted', 'wrapped', 'unwrapped_then_deleted') -LIMIT 100;"; - assert_eq!( - expected_sql, - filter.to_objects_history_sql(None, 100, vec!["*"]) - ); - } - - #[test] - fn test_empty_all_filter() { - let filter = SuiObjectDataFilter::MatchAll(vec![]); - let expected_sql = "SELECT t1.* -FROM (SELECT DISTINCT ON (o.object_id) * - FROM objects_history o - WHERE o.checkpoint <= $1 - ORDER BY o.object_id, version, o.checkpoint DESC) AS t1 -WHERE t1.object_status NOT IN ('deleted', 'wrapped', 'unwrapped_then_deleted') -LIMIT 100;"; - assert_eq!( - expected_sql, - filter.to_objects_history_sql(None, 100, vec!["*"]) - ); - } - - #[test] - fn test_empty_any_filter() { - let filter = SuiObjectDataFilter::MatchAny(vec![]); - let expected_sql = "SELECT t1.* -FROM (SELECT DISTINCT ON (o.object_id) * - FROM objects_history o - WHERE o.checkpoint <= $1 - AND FALSE - ORDER BY o.object_id, version, o.checkpoint DESC) AS t1 -WHERE t1.object_status NOT IN ('deleted', 'wrapped', 'unwrapped_then_deleted') -LIMIT 100;"; - assert_eq!( - expected_sql, - filter.to_objects_history_sql(None, 100, vec!["*"]) - ); - } - - #[test] - fn test_all_filter() { - let filter = SuiObjectDataFilter::MatchAll(vec![ - SuiObjectDataFilter::ObjectId( - ObjectID::from_str( - "0xef9fb75a7b3d4cb5551ef0b08c83528b94d5f5cd8be28b1d08a87dbbf3731738", - ) - .unwrap(), - ), - SuiObjectDataFilter::StructType(parse_sui_struct_tag("0x2::test::Test").unwrap()), - ]); - - let expected_sql = "SELECT t1.* -FROM (SELECT DISTINCT ON (o.object_id) * - FROM objects_history o - WHERE o.checkpoint <= $1 - AND (o.object_id = '0xef9fb75a7b3d4cb5551ef0b08c83528b94d5f5cd8be28b1d08a87dbbf3731738' AND o.object_type LIKE '0x2::test::Test%') - ORDER BY o.object_id, version, o.checkpoint DESC) AS t1 -WHERE t1.object_status NOT IN ('deleted', 'wrapped', 'unwrapped_then_deleted') -LIMIT 100;"; - assert_eq!( - expected_sql, - filter.to_objects_history_sql(None, 100, vec!["*"]) - ); - } -} diff --git a/crates/sui-mvr-indexer/src/system_package_task.rs b/crates/sui-mvr-indexer/src/system_package_task.rs deleted file mode 100644 index 8c2d6586f72d5..0000000000000 --- a/crates/sui-mvr-indexer/src/system_package_task.rs +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::indexer_reader::IndexerReader; -use std::time::Duration; -use sui_types::SYSTEM_PACKAGE_ADDRESSES; -use tokio_util::sync::CancellationToken; - -/// Background task responsible for evicting system packages from the package resolver's cache after -/// detecting an epoch boundary. -pub(crate) struct SystemPackageTask { - /// Holds the DB connection and also the package resolver to evict packages from. - reader: IndexerReader, - /// Signal to cancel the task. - cancel: CancellationToken, - /// Interval to sleep for between checks. - interval: Duration, -} - -impl SystemPackageTask { - pub(crate) fn new( - reader: IndexerReader, - cancel: CancellationToken, - interval: Duration, - ) -> Self { - Self { - reader, - cancel, - interval, - } - } - - pub(crate) async fn run(&self) { - let mut last_epoch: i64 = 0; - loop { - tokio::select! { - _ = self.cancel.cancelled() => { - tracing::info!( - "Shutdown signal received, terminating system package eviction task" - ); - return; - } - _ = tokio::time::sleep(self.interval) => { - let next_epoch = match self.reader.get_latest_epoch_info_from_db().await { - Ok(epoch) => epoch.epoch, - Err(e) => { - tracing::error!("Failed to fetch latest epoch: {:?}", e); - continue; - } - }; - - if next_epoch > last_epoch { - last_epoch = next_epoch; - tracing::info!( - "Detected epoch boundary, evicting system packages from cache" - ); - self.reader - .package_resolver() - .package_store() - .evict(SYSTEM_PACKAGE_ADDRESSES.iter().copied()); - } - } - } - } - } -} diff --git a/crates/sui-mvr-indexer/src/tempdb.rs b/crates/sui-mvr-indexer/src/tempdb.rs deleted file mode 100644 index d63f34a02a3de..0000000000000 --- a/crates/sui-mvr-indexer/src/tempdb.rs +++ /dev/null @@ -1,343 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use anyhow::anyhow; -use anyhow::Context; -use anyhow::Result; -use std::fs::OpenOptions; -use std::{ - path::{Path, PathBuf}, - process::{Child, Command}, - time::{Duration, Instant}, -}; -use tracing::trace; -use url::Url; - -/// A temporary, local postgres database -pub struct TempDb { - database: LocalDatabase, - - // Directory used for the ephemeral database. - // - // On drop the directory will be cleaned an its contents deleted. - // - // NOTE: This needs to be the last entry in this struct so that the database is dropped before - // and has a chance to gracefully shutdown before the directory is deleted. - dir: tempfile::TempDir, -} - -impl TempDb { - /// Create and start a new temporary postgres database. - /// - /// A fresh database will be initialized in a temporary directory that will be cleandup on drop. - /// The running `postgres` service will be serving traffic on an available, os-assigned port. - pub fn new() -> Result { - let dir = tempfile::TempDir::new()?; - let port = get_available_port(); - - let database = LocalDatabase::new_initdb(dir.path().to_owned(), port)?; - - Ok(Self { dir, database }) - } - - pub fn database(&self) -> &LocalDatabase { - &self.database - } - - pub fn database_mut(&mut self) -> &mut LocalDatabase { - &mut self.database - } - - pub fn dir(&self) -> &Path { - self.dir.path() - } -} - -#[derive(Debug)] -struct PostgresProcess { - dir: PathBuf, - inner: Child, -} - -impl PostgresProcess { - fn start(dir: PathBuf, port: u16) -> Result { - let child = Command::new("postgres") - // Set the data directory to use - .arg("-D") - .arg(&dir) - // Set the port to listen for incoming connections - .args(["-p", &port.to_string()]) - // Disable creating and listening on a UDS - .args(["-c", "unix_socket_directories="]) - // pipe stdout and stderr to files located in the data directory - .stdout( - OpenOptions::new() - .create(true) - .append(true) - .open(dir.join("stdout"))?, - ) - .stderr( - OpenOptions::new() - .create(true) - .append(true) - .open(dir.join("stderr"))?, - ) - .spawn() - .context("command not found: postgres")?; - - Ok(Self { dir, inner: child }) - } - - // https://www.postgresql.org/docs/16/app-pg-ctl.html - fn pg_ctl_stop(&mut self) -> Result<()> { - let output = Command::new("pg_ctl") - .arg("stop") - .arg("-D") - .arg(&self.dir) - .arg("-mfast") - .output() - .context("command not found: pg_ctl")?; - - if output.status.success() { - Ok(()) - } else { - Err(anyhow!("couldn't shut down postgres")) - } - } - - fn dump_stdout_stderr(&self) -> Result<(String, String)> { - let stdout = std::fs::read_to_string(self.dir.join("stdout"))?; - let stderr = std::fs::read_to_string(self.dir.join("stderr"))?; - - Ok((stdout, stderr)) - } -} - -impl Drop for PostgresProcess { - // When the Process struct goes out of scope we need to kill the child process - fn drop(&mut self) { - tracing::error!("dropping postgres"); - // check if the process has already been terminated - match self.inner.try_wait() { - // The child process has already terminated, perhaps due to a crash - Ok(Some(_)) => {} - - // The process is still running so we need to attempt to kill it - _ => { - if self.pg_ctl_stop().is_err() { - // Couldn't gracefully stop server so we'll just kill it - self.inner.kill().expect("postgres couldn't be killed"); - } - self.inner.wait().unwrap(); - } - } - - // Dump the contents of stdout/stderr if TRACE is enabled - if tracing::event_enabled!(tracing::Level::TRACE) { - if let Ok((stdout, stderr)) = self.dump_stdout_stderr() { - trace!("stdout: {stdout}"); - trace!("stderr: {stderr}"); - } - } - } -} - -/// Local instance of a `postgres` server. -/// -/// See for more info. -pub struct LocalDatabase { - dir: PathBuf, - port: u16, - url: Url, - process: Option, -} - -impl LocalDatabase { - /// Start a local `postgres` database service. - /// - /// `dir`: The location of the on-disk postgres database. The database must already exist at - /// the provided path. If you instead want to create a new database see `Self::new_initdb`. - /// - /// `port`: The port to listen for incoming connection on. - pub fn new(dir: PathBuf, port: u16) -> Result { - let url = format!( - "postgres://postgres:postgrespw@localhost:{port}/{db_name}", - db_name = "postgres" - ) - .parse() - .unwrap(); - let mut db = Self { - dir, - port, - url, - process: None, - }; - db.start()?; - Ok(db) - } - - /// Initialize and start a local `postgres` database service. - /// - /// Unlike `Self::new`, this will initialize a clean database at the provided path. - pub fn new_initdb(dir: PathBuf, port: u16) -> Result { - initdb(&dir)?; - Self::new(dir, port) - } - - /// Return the url used to connect to the database - pub fn url(&self) -> &Url { - &self.url - } - - fn start(&mut self) -> Result<()> { - if self.process.is_none() { - self.process = Some(PostgresProcess::start(self.dir.clone(), self.port)?); - self.wait_till_ready() - .map_err(|e| anyhow!("unable to start postgres: {e:?}"))?; - } - - Ok(()) - } - - fn health_check(&mut self) -> Result<(), HealthCheckError> { - if let Some(p) = &mut self.process { - match p.inner.try_wait() { - // This would mean the child process has crashed - Ok(Some(_)) => Err(HealthCheckError::NotRunning), - - // This is the case where the process is still running - Ok(None) => pg_isready(self.port), - - // Some other unknown error - Err(e) => Err(HealthCheckError::Unknown(e.to_string())), - } - } else { - Err(HealthCheckError::NotRunning) - } - } - - fn wait_till_ready(&mut self) -> Result<(), HealthCheckError> { - let start = Instant::now(); - - while start.elapsed() < Duration::from_secs(10) { - match self.health_check() { - Ok(()) => return Ok(()), - Err(HealthCheckError::NotReady) => {} - Err(HealthCheckError::NotRunning | HealthCheckError::Unknown(_)) => break, - } - - std::thread::sleep(Duration::from_millis(50)); - } - - Err(HealthCheckError::Unknown( - "timeout reached when waiting for service to be ready".to_owned(), - )) - } -} - -#[derive(Debug)] -enum HealthCheckError { - NotRunning, - NotReady, - #[allow(unused)] - Unknown(String), -} - -/// Run the postgres `pg_isready` command to get the status of database -/// -/// See for more info -fn pg_isready(port: u16) -> Result<(), HealthCheckError> { - let output = Command::new("pg_isready") - .arg("--host=localhost") - .arg("-p") - .arg(port.to_string()) - .arg("--username=postgres") - .output() - .map_err(|e| HealthCheckError::Unknown(format!("command not found: pg_ctl: {e}")))?; - - trace!("pg_isready code: {:?}", output.status.code()); - trace!("pg_isready output: {}", output.stderr.escape_ascii()); - trace!("pg_isready output: {}", output.stdout.escape_ascii()); - if output.status.success() { - Ok(()) - } else { - Err(HealthCheckError::NotReady) - } -} - -/// Run the postgres `initdb` command to initialize a database at the provided path -/// -/// See for more info -fn initdb(dir: &Path) -> Result<()> { - let output = Command::new("initdb") - .arg("-D") - .arg(dir) - .arg("--no-instructions") - .arg("--username=postgres") - .output() - .context("command not found: initdb")?; - - if output.status.success() { - Ok(()) - } else { - Err(anyhow!( - "unable to initialize database: {:?}", - String::from_utf8(output.stderr) - )) - } -} - -/// Return an ephemeral, available port. On unix systems, the port returned will be in the -/// TIME_WAIT state ensuring that the OS won't hand out this port for some grace period. -/// Callers should be able to bind to this port given they use SO_REUSEADDR. -pub fn get_available_port() -> u16 { - const MAX_PORT_RETRIES: u32 = 1000; - - for _ in 0..MAX_PORT_RETRIES { - if let Ok(port) = get_ephemeral_port() { - return port; - } - } - - panic!("Error: could not find an available port"); -} - -fn get_ephemeral_port() -> std::io::Result { - // Request a random available port from the OS - let listener = std::net::TcpListener::bind(("127.0.0.1", 0))?; - let addr = listener.local_addr()?; - - // Create and accept a connection (which we'll promptly drop) in order to force the port - // into the TIME_WAIT state, ensuring that the port will be reserved from some limited - // amount of time (roughly 60s on some Linux systems) - let _sender = std::net::TcpStream::connect(addr)?; - let _incoming = listener.accept()?; - - Ok(addr.port()) -} - -#[cfg(test)] -mod test { - #[tokio::test] - async fn smoketest() { - use crate::database::Connection; - use crate::tempdb::TempDb; - use diesel_async::RunQueryDsl; - - telemetry_subscribers::init_for_testing(); - - let db = TempDb::new().unwrap(); - println!("dir: {:?}", db.dir.path()); - - let url = db.database.url(); - println!("url: {}", url.as_str()); - let mut connection = Connection::dedicated(url).await.unwrap(); - - // Run a simple query to verify the db can properly be queried - let resp = diesel::sql_query("SELECT datname FROM pg_database") - .execute(&mut connection) - .await - .unwrap(); - println!("resp: {:?}", resp); - } -} diff --git a/crates/sui-mvr-indexer/src/test_utils.rs b/crates/sui-mvr-indexer/src/test_utils.rs deleted file mode 100644 index 3faea10eeeec8..0000000000000 --- a/crates/sui-mvr-indexer/src/test_utils.rs +++ /dev/null @@ -1,340 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use mysten_metrics::init_metrics; -use tokio::task::JoinHandle; -use tokio_util::sync::CancellationToken; - -use simulacrum::Simulacrum; -use std::net::SocketAddr; -use std::path::PathBuf; -use std::sync::Arc; -use std::time::Duration; -use sui_json_rpc_types::SuiTransactionBlockResponse; - -use crate::config::{IngestionConfig, RetentionConfig, SnapshotLagConfig, UploadOptions}; -use crate::database::Connection; -use crate::database::ConnectionPool; -use crate::db::ConnectionPoolConfig; -use crate::errors::IndexerError; -use crate::indexer::Indexer; -use crate::store::PgIndexerStore; -use crate::tempdb::get_available_port; -use crate::tempdb::TempDb; -use crate::IndexerMetrics; - -/// Wrapper over `Indexer::start_reader` to make it easier to configure an indexer jsonrpc reader -/// for testing. -pub async fn start_indexer_jsonrpc_for_testing( - db_url: String, - fullnode_url: String, - json_rpc_url: String, - cancel: Option, -) -> (JoinHandle>, CancellationToken) { - let token = cancel.unwrap_or_default(); - - // Reduce the connection pool size to 10 for testing - // to prevent maxing out - let pool_config = ConnectionPoolConfig { - pool_size: 5, - connection_timeout: Duration::from_secs(10), - statement_timeout: Duration::from_secs(30), - }; - - println!("db_url: {db_url}"); - println!("pool_config: {pool_config:?}"); - - let registry = prometheus::Registry::default(); - init_metrics(®istry); - - let pool = ConnectionPool::new(db_url.parse().unwrap(), pool_config) - .await - .unwrap(); - - let handle = { - let config = crate::config::JsonRpcConfig { - name_service_options: crate::config::NameServiceOptions::default(), - rpc_address: json_rpc_url.parse().unwrap(), - rpc_client_url: fullnode_url, - }; - let token_clone = token.clone(); - tokio::spawn( - async move { Indexer::start_reader(&config, ®istry, pool, token_clone).await }, - ) - }; - - (handle, token) -} - -/// Wrapper over `Indexer::start_writer_with_config` to make it easier to configure an indexer -/// writer for testing. If the config options are null, default values that have historically worked -/// for testing will be used. -pub async fn start_indexer_writer_for_testing( - db_url: String, - snapshot_config: Option, - retention_config: Option, - data_ingestion_path: Option, - cancel: Option, - start_checkpoint: Option, - end_checkpoint: Option, -) -> ( - PgIndexerStore, - JoinHandle>, - CancellationToken, -) { - let token = cancel.unwrap_or_default(); - let snapshot_config = snapshot_config.unwrap_or(SnapshotLagConfig { - snapshot_min_lag: 5, - sleep_duration: 0, - }); - - // Reduce the connection pool size to 10 for testing to prevent maxing out - let pool_config = ConnectionPoolConfig { - pool_size: 5, - connection_timeout: Duration::from_secs(10), - statement_timeout: Duration::from_secs(30), - }; - - println!("db_url: {db_url}"); - println!("pool_config: {pool_config:?}"); - println!("{data_ingestion_path:?}"); - - let registry = prometheus::Registry::default(); - init_metrics(®istry); - let indexer_metrics = IndexerMetrics::new(®istry); - - let pool = ConnectionPool::new(db_url.parse().unwrap(), pool_config) - .await - .unwrap(); - let store = PgIndexerStore::new( - pool.clone(), - UploadOptions::default(), - indexer_metrics.clone(), - ); - - let handle = { - let connection = Connection::dedicated(&db_url.parse().unwrap()) - .await - .unwrap(); - crate::db::reset_database(connection).await.unwrap(); - - let store_clone = store.clone(); - let mut ingestion_config = IngestionConfig { - start_checkpoint, - end_checkpoint, - ..Default::default() - }; - ingestion_config.sources.data_ingestion_path = data_ingestion_path; - let token_clone = token.clone(); - - tokio::spawn(async move { - Indexer::start_writer( - ingestion_config, - store_clone, - indexer_metrics, - snapshot_config, - retention_config, - token_clone, - ) - .await - }) - }; - - (store, handle, token) -} - -#[derive(Clone)] -pub struct SuiTransactionBlockResponseBuilder<'a> { - response: SuiTransactionBlockResponse, - full_response: &'a SuiTransactionBlockResponse, -} - -impl<'a> SuiTransactionBlockResponseBuilder<'a> { - pub fn new(full_response: &'a SuiTransactionBlockResponse) -> Self { - Self { - response: SuiTransactionBlockResponse::default(), - full_response, - } - } - - pub fn with_input(mut self) -> Self { - self.response = SuiTransactionBlockResponse { - transaction: self.full_response.transaction.clone(), - ..self.response - }; - self - } - - pub fn with_raw_input(mut self) -> Self { - self.response = SuiTransactionBlockResponse { - raw_transaction: self.full_response.raw_transaction.clone(), - ..self.response - }; - self - } - - pub fn with_effects(mut self) -> Self { - self.response = SuiTransactionBlockResponse { - effects: self.full_response.effects.clone(), - ..self.response - }; - self - } - - pub fn with_events(mut self) -> Self { - self.response = SuiTransactionBlockResponse { - events: self.full_response.events.clone(), - ..self.response - }; - self - } - - pub fn with_balance_changes(mut self) -> Self { - self.response = SuiTransactionBlockResponse { - balance_changes: self.full_response.balance_changes.clone(), - ..self.response - }; - self - } - - pub fn with_object_changes(mut self) -> Self { - self.response = SuiTransactionBlockResponse { - object_changes: self.full_response.object_changes.clone(), - ..self.response - }; - self - } - - pub fn with_input_and_changes(mut self) -> Self { - self.response = SuiTransactionBlockResponse { - transaction: self.full_response.transaction.clone(), - balance_changes: self.full_response.balance_changes.clone(), - object_changes: self.full_response.object_changes.clone(), - ..self.response - }; - self - } - - pub fn build(self) -> SuiTransactionBlockResponse { - SuiTransactionBlockResponse { - transaction: self.response.transaction, - raw_transaction: self.response.raw_transaction, - effects: self.response.effects, - events: self.response.events, - balance_changes: self.response.balance_changes, - object_changes: self.response.object_changes, - // Use full response for any fields that aren't showable - ..self.full_response.clone() - } - } -} - -/// Set up a test indexer fetching from a REST endpoint served by the given Simulacrum. -pub async fn set_up( - sim: Arc, - data_ingestion_path: PathBuf, -) -> ( - JoinHandle<()>, - PgIndexerStore, - JoinHandle>, - TempDb, -) { - let database = TempDb::new().unwrap(); - let server_url: SocketAddr = format!("127.0.0.1:{}", get_available_port()) - .parse() - .unwrap(); - - let server_handle = tokio::spawn(async move { - sui_rpc_api::RpcService::new_without_version(sim) - .start_service(server_url) - .await; - }); - // Starts indexer - let (pg_store, pg_handle, _) = start_indexer_writer_for_testing( - database.database().url().as_str().to_owned(), - None, - None, - Some(data_ingestion_path), - None, /* cancel */ - None, /* start_checkpoint */ - None, /* end_checkpoint */ - ) - .await; - (server_handle, pg_store, pg_handle, database) -} - -pub async fn set_up_with_start_and_end_checkpoints( - sim: Arc, - data_ingestion_path: PathBuf, - start_checkpoint: u64, - end_checkpoint: u64, -) -> ( - JoinHandle<()>, - PgIndexerStore, - JoinHandle>, - TempDb, -) { - let database = TempDb::new().unwrap(); - let server_url: SocketAddr = format!("127.0.0.1:{}", get_available_port()) - .parse() - .unwrap(); - let server_handle = tokio::spawn(async move { - sui_rpc_api::RpcService::new_without_version(sim) - .start_service(server_url) - .await; - }); - // Starts indexer - let (pg_store, pg_handle, _) = start_indexer_writer_for_testing( - database.database().url().as_str().to_owned(), - None, - None, - Some(data_ingestion_path), - None, /* cancel */ - Some(start_checkpoint), - Some(end_checkpoint), - ) - .await; - (server_handle, pg_store, pg_handle, database) -} - -/// Wait for the indexer to catch up to the given checkpoint sequence number. -pub async fn wait_for_checkpoint( - pg_store: &PgIndexerStore, - checkpoint_sequence_number: u64, -) -> Result<(), IndexerError> { - tokio::time::timeout(Duration::from_secs(30), async { - while { - let cp_opt = pg_store - .get_latest_checkpoint_sequence_number() - .await - .unwrap(); - cp_opt.is_none() || (cp_opt.unwrap() < checkpoint_sequence_number) - } { - tokio::time::sleep(Duration::from_millis(100)).await; - } - }) - .await - .expect("Timeout waiting for indexer to catchup to checkpoint"); - Ok(()) -} - -/// Wait for the indexer to catch up to the given checkpoint sequence number for objects snapshot. -pub async fn wait_for_objects_snapshot( - pg_store: &PgIndexerStore, - checkpoint_sequence_number: u64, -) -> Result<(), IndexerError> { - tokio::time::timeout(Duration::from_secs(30), async { - while { - let cp_opt = pg_store - .get_latest_object_snapshot_checkpoint_sequence_number() - .await - .unwrap(); - cp_opt.is_none() || (cp_opt.unwrap() < checkpoint_sequence_number) - } { - tokio::time::sleep(Duration::from_millis(100)).await; - } - }) - .await - .expect("Timeout waiting for indexer to catchup to checkpoint for objects snapshot"); - Ok(()) -} diff --git a/crates/sui-mvr-indexer/src/types.rs b/crates/sui-mvr-indexer/src/types.rs deleted file mode 100644 index a69272bb44149..0000000000000 --- a/crates/sui-mvr-indexer/src/types.rs +++ /dev/null @@ -1,676 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use move_core_types::language_storage::StructTag; -use rand::Rng; -use serde::{Deserialize, Serialize}; -use serde_with::serde_as; -use sui_json_rpc_types::{ - ObjectChange, SuiTransactionBlockResponse, SuiTransactionBlockResponseOptions, -}; -use sui_types::base_types::{ObjectDigest, SequenceNumber}; -use sui_types::base_types::{ObjectID, SuiAddress}; -use sui_types::crypto::AggregateAuthoritySignature; -use sui_types::digests::TransactionDigest; -use sui_types::dynamic_field::DynamicFieldType; -use sui_types::effects::TransactionEffects; -use sui_types::messages_checkpoint::{ - CertifiedCheckpointSummary, CheckpointCommitment, CheckpointContents, CheckpointDigest, - CheckpointSequenceNumber, EndOfEpochData, -}; -use sui_types::move_package::MovePackage; -use sui_types::object::{Object, Owner}; -use sui_types::sui_serde::SuiStructTag; -use sui_types::transaction::SenderSignedData; - -use crate::errors::IndexerError; - -pub type IndexerResult = Result; - -#[derive(Debug, Default)] -pub struct IndexedCheckpoint { - // TODO: A lot of fields are now redundant with certified_checkpoint and checkpoint_contents. - pub sequence_number: u64, - pub checkpoint_digest: CheckpointDigest, - pub epoch: u64, - pub tx_digests: Vec, - pub network_total_transactions: u64, - pub previous_checkpoint_digest: Option, - pub timestamp_ms: u64, - pub total_gas_cost: i64, // total gas cost could be negative - pub computation_cost: u64, - pub storage_cost: u64, - pub storage_rebate: u64, - pub non_refundable_storage_fee: u64, - pub checkpoint_commitments: Vec, - pub validator_signature: AggregateAuthoritySignature, - pub successful_tx_num: usize, - pub end_of_epoch_data: Option, - pub end_of_epoch: bool, - pub min_tx_sequence_number: u64, - pub max_tx_sequence_number: u64, - // FIXME: Remove the Default derive and make these fields mandatory. - pub certified_checkpoint: Option, - pub checkpoint_contents: Option, -} - -impl IndexedCheckpoint { - pub fn from_sui_checkpoint( - checkpoint: &CertifiedCheckpointSummary, - contents: &CheckpointContents, - successful_tx_num: usize, - ) -> Self { - let total_gas_cost = checkpoint.epoch_rolling_gas_cost_summary.computation_cost as i64 - + checkpoint.epoch_rolling_gas_cost_summary.storage_cost as i64 - - checkpoint.epoch_rolling_gas_cost_summary.storage_rebate as i64; - let tx_digests = contents.iter().map(|t| t.transaction).collect::>(); - let max_tx_sequence_number = checkpoint.network_total_transactions - 1; - // NOTE: + 1u64 first to avoid subtraction with overflow - let min_tx_sequence_number = max_tx_sequence_number + 1u64 - tx_digests.len() as u64; - let auth_sig = &checkpoint.auth_sig().signature; - Self { - sequence_number: checkpoint.sequence_number, - checkpoint_digest: *checkpoint.digest(), - epoch: checkpoint.epoch, - tx_digests, - previous_checkpoint_digest: checkpoint.previous_digest, - end_of_epoch_data: checkpoint.end_of_epoch_data.clone(), - end_of_epoch: checkpoint.end_of_epoch_data.clone().is_some(), - total_gas_cost, - computation_cost: checkpoint.epoch_rolling_gas_cost_summary.computation_cost, - storage_cost: checkpoint.epoch_rolling_gas_cost_summary.storage_cost, - storage_rebate: checkpoint.epoch_rolling_gas_cost_summary.storage_rebate, - non_refundable_storage_fee: checkpoint - .epoch_rolling_gas_cost_summary - .non_refundable_storage_fee, - successful_tx_num, - network_total_transactions: checkpoint.network_total_transactions, - timestamp_ms: checkpoint.timestamp_ms, - validator_signature: auth_sig.clone(), - checkpoint_commitments: checkpoint.checkpoint_commitments.clone(), - min_tx_sequence_number, - max_tx_sequence_number, - certified_checkpoint: Some(checkpoint.clone()), - checkpoint_contents: Some(contents.clone()), - } - } -} - -#[derive(Debug, Clone)] -pub struct IndexedEvent { - pub tx_sequence_number: u64, - pub event_sequence_number: u64, - pub checkpoint_sequence_number: u64, - pub transaction_digest: TransactionDigest, - pub sender: SuiAddress, - pub package: ObjectID, - pub module: String, - pub event_type: String, - pub event_type_package: ObjectID, - pub event_type_module: String, - /// Struct name of the event, without type parameters. - pub event_type_name: String, - pub bcs: Vec, - pub timestamp_ms: u64, -} - -impl IndexedEvent { - pub fn from_event( - tx_sequence_number: u64, - event_sequence_number: u64, - checkpoint_sequence_number: u64, - transaction_digest: TransactionDigest, - event: &sui_types::event::Event, - timestamp_ms: u64, - ) -> Self { - Self { - tx_sequence_number, - event_sequence_number, - checkpoint_sequence_number, - transaction_digest, - sender: event.sender, - package: event.package_id, - module: event.transaction_module.to_string(), - event_type: event.type_.to_canonical_string(/* with_prefix */ true), - event_type_package: event.type_.address.into(), - event_type_module: event.type_.module.to_string(), - event_type_name: event.type_.name.to_string(), - bcs: event.contents.clone(), - timestamp_ms, - } - } -} - -#[derive(Debug, Clone)] -pub struct EventIndex { - pub tx_sequence_number: u64, - pub event_sequence_number: u64, - pub sender: SuiAddress, - pub emit_package: ObjectID, - pub emit_module: String, - pub type_package: ObjectID, - pub type_module: String, - /// Struct name of the event, without type parameters. - pub type_name: String, - /// Type instantiation of the event, with type name and type parameters, if any. - pub type_instantiation: String, -} - -// for ingestion test -impl EventIndex { - pub fn random() -> Self { - let mut rng = rand::thread_rng(); - EventIndex { - tx_sequence_number: rng.gen(), - event_sequence_number: rng.gen(), - sender: SuiAddress::random_for_testing_only(), - emit_package: ObjectID::random(), - emit_module: rng.gen::().to_string(), - type_package: ObjectID::random(), - type_module: rng.gen::().to_string(), - type_name: rng.gen::().to_string(), - type_instantiation: rng.gen::().to_string(), - } - } -} - -impl EventIndex { - pub fn from_event( - tx_sequence_number: u64, - event_sequence_number: u64, - event: &sui_types::event::Event, - ) -> Self { - let type_instantiation = event - .type_ - .to_canonical_string(/* with_prefix */ true) - .splitn(3, "::") - .collect::>()[2] - .to_string(); - Self { - tx_sequence_number, - event_sequence_number, - sender: event.sender, - emit_package: event.package_id, - emit_module: event.transaction_module.to_string(), - type_package: event.type_.address.into(), - type_module: event.type_.module.to_string(), - type_name: event.type_.name.to_string(), - type_instantiation, - } - } -} - -#[derive(Debug, Copy, Clone)] -pub enum OwnerType { - Immutable = 0, - Address = 1, - Object = 2, - Shared = 3, -} - -pub enum ObjectStatus { - Active = 0, - WrappedOrDeleted = 1, -} - -impl TryFrom for ObjectStatus { - type Error = IndexerError; - - fn try_from(value: i16) -> Result { - Ok(match value { - 0 => ObjectStatus::Active, - 1 => ObjectStatus::WrappedOrDeleted, - value => { - return Err(IndexerError::PersistentStorageDataCorruptionError(format!( - "{value} as ObjectStatus" - ))) - } - }) - } -} - -impl TryFrom for OwnerType { - type Error = IndexerError; - - fn try_from(value: i16) -> Result { - Ok(match value { - 0 => OwnerType::Immutable, - 1 => OwnerType::Address, - 2 => OwnerType::Object, - 3 => OwnerType::Shared, - value => { - return Err(IndexerError::PersistentStorageDataCorruptionError(format!( - "{value} as OwnerType" - ))) - } - }) - } -} - -// Returns owner_type, owner_address -pub fn owner_to_owner_info(owner: &Owner) -> (OwnerType, Option) { - match owner { - Owner::AddressOwner(address) => (OwnerType::Address, Some(*address)), - Owner::ObjectOwner(address) => (OwnerType::Object, Some(*address)), - Owner::Shared { .. } => (OwnerType::Shared, None), - Owner::Immutable => (OwnerType::Immutable, None), - // ConsensusV2 objects are treated as singly-owned for now in indexers. - // This will need to be updated if additional Authenticators are added. - Owner::ConsensusV2 { authenticator, .. } => { - (OwnerType::Address, Some(*authenticator.as_single_owner())) - } - } -} - -#[derive(Debug, Copy, Clone)] -pub enum DynamicFieldKind { - DynamicField = 0, - DynamicObject = 1, -} - -#[derive(Clone, Debug)] -pub struct IndexedObject { - pub checkpoint_sequence_number: CheckpointSequenceNumber, - pub object: Object, - pub df_kind: Option, -} - -impl IndexedObject { - pub fn random() -> Self { - let mut rng = rand::thread_rng(); - let random_address = SuiAddress::random_for_testing_only(); - IndexedObject { - checkpoint_sequence_number: rng.gen(), - object: Object::with_owner_for_testing(random_address), - df_kind: { - let random_value = rng.gen_range(0..3); - match random_value { - 0 => Some(DynamicFieldType::DynamicField), - 1 => Some(DynamicFieldType::DynamicObject), - _ => None, - } - }, - } - } -} - -impl IndexedObject { - pub fn from_object( - checkpoint_sequence_number: CheckpointSequenceNumber, - object: Object, - df_kind: Option, - ) -> Self { - Self { - checkpoint_sequence_number, - object, - df_kind, - } - } -} - -#[derive(Clone, Debug)] -pub struct IndexedDeletedObject { - pub object_id: ObjectID, - pub object_version: u64, - pub checkpoint_sequence_number: u64, -} - -impl IndexedDeletedObject { - pub fn random() -> Self { - let mut rng = rand::thread_rng(); - IndexedDeletedObject { - object_id: ObjectID::random(), - object_version: rng.gen(), - checkpoint_sequence_number: rng.gen(), - } - } -} - -#[derive(Debug)] -pub struct IndexedPackage { - pub package_id: ObjectID, - pub move_package: MovePackage, - pub checkpoint_sequence_number: u64, -} - -#[derive(Debug, Clone)] -pub enum TransactionKind { - SystemTransaction = 0, - ProgrammableTransaction = 1, -} - -#[derive(Debug, Clone)] -pub struct IndexedTransaction { - pub tx_sequence_number: u64, - pub tx_digest: TransactionDigest, - pub sender_signed_data: SenderSignedData, - pub effects: TransactionEffects, - pub checkpoint_sequence_number: u64, - pub timestamp_ms: u64, - pub object_changes: Vec, - pub balance_change: Vec, - pub events: Vec, - pub transaction_kind: TransactionKind, - pub successful_tx_num: u64, -} - -#[derive(Debug, Clone)] -pub struct TxIndex { - pub tx_sequence_number: u64, - pub tx_kind: TransactionKind, - pub transaction_digest: TransactionDigest, - pub checkpoint_sequence_number: u64, - pub input_objects: Vec, - pub changed_objects: Vec, - pub affected_objects: Vec, - pub payers: Vec, - pub sender: SuiAddress, - pub recipients: Vec, - pub move_calls: Vec<(ObjectID, String, String)>, -} - -impl TxIndex { - pub fn random() -> Self { - let mut rng = rand::thread_rng(); - TxIndex { - tx_sequence_number: rng.gen(), - tx_kind: if rng.gen_bool(0.5) { - TransactionKind::SystemTransaction - } else { - TransactionKind::ProgrammableTransaction - }, - transaction_digest: TransactionDigest::random(), - checkpoint_sequence_number: rng.gen(), - input_objects: (0..1000).map(|_| ObjectID::random()).collect(), - changed_objects: (0..1000).map(|_| ObjectID::random()).collect(), - affected_objects: (0..1000).map(|_| ObjectID::random()).collect(), - payers: (0..rng.gen_range(0..100)) - .map(|_| SuiAddress::random_for_testing_only()) - .collect(), - sender: SuiAddress::random_for_testing_only(), - recipients: (0..rng.gen_range(0..1000)) - .map(|_| SuiAddress::random_for_testing_only()) - .collect(), - move_calls: (0..rng.gen_range(0..1000)) - .map(|_| { - ( - ObjectID::random(), - rng.gen::().to_string(), - rng.gen::().to_string(), - ) - }) - .collect(), - } - } -} - -// ObjectChange is not bcs deserializable, IndexedObjectChange is. -#[serde_as] -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] -pub enum IndexedObjectChange { - Published { - package_id: ObjectID, - version: SequenceNumber, - digest: ObjectDigest, - modules: Vec, - }, - Transferred { - sender: SuiAddress, - recipient: Owner, - #[serde_as(as = "SuiStructTag")] - object_type: StructTag, - object_id: ObjectID, - version: SequenceNumber, - digest: ObjectDigest, - }, - /// Object mutated. - Mutated { - sender: SuiAddress, - owner: Owner, - #[serde_as(as = "SuiStructTag")] - object_type: StructTag, - object_id: ObjectID, - version: SequenceNumber, - previous_version: SequenceNumber, - digest: ObjectDigest, - }, - /// Delete object - Deleted { - sender: SuiAddress, - #[serde_as(as = "SuiStructTag")] - object_type: StructTag, - object_id: ObjectID, - version: SequenceNumber, - }, - /// Wrapped object - Wrapped { - sender: SuiAddress, - #[serde_as(as = "SuiStructTag")] - object_type: StructTag, - object_id: ObjectID, - version: SequenceNumber, - }, - /// New object creation - Created { - sender: SuiAddress, - owner: Owner, - #[serde_as(as = "SuiStructTag")] - object_type: StructTag, - object_id: ObjectID, - version: SequenceNumber, - digest: ObjectDigest, - }, -} - -impl From for IndexedObjectChange { - fn from(oc: ObjectChange) -> Self { - match oc { - ObjectChange::Published { - package_id, - version, - digest, - modules, - } => Self::Published { - package_id, - version, - digest, - modules, - }, - ObjectChange::Transferred { - sender, - recipient, - object_type, - object_id, - version, - digest, - } => Self::Transferred { - sender, - recipient, - object_type, - object_id, - version, - digest, - }, - ObjectChange::Mutated { - sender, - owner, - object_type, - object_id, - version, - previous_version, - digest, - } => Self::Mutated { - sender, - owner, - object_type, - object_id, - version, - previous_version, - digest, - }, - ObjectChange::Deleted { - sender, - object_type, - object_id, - version, - } => Self::Deleted { - sender, - object_type, - object_id, - version, - }, - ObjectChange::Wrapped { - sender, - object_type, - object_id, - version, - } => Self::Wrapped { - sender, - object_type, - object_id, - version, - }, - ObjectChange::Created { - sender, - owner, - object_type, - object_id, - version, - digest, - } => Self::Created { - sender, - owner, - object_type, - object_id, - version, - digest, - }, - } - } -} - -impl From for ObjectChange { - fn from(val: IndexedObjectChange) -> Self { - match val { - IndexedObjectChange::Published { - package_id, - version, - digest, - modules, - } => ObjectChange::Published { - package_id, - version, - digest, - modules, - }, - IndexedObjectChange::Transferred { - sender, - recipient, - object_type, - object_id, - version, - digest, - } => ObjectChange::Transferred { - sender, - recipient, - object_type, - object_id, - version, - digest, - }, - IndexedObjectChange::Mutated { - sender, - owner, - object_type, - object_id, - version, - previous_version, - digest, - } => ObjectChange::Mutated { - sender, - owner, - object_type, - object_id, - version, - previous_version, - digest, - }, - IndexedObjectChange::Deleted { - sender, - object_type, - object_id, - version, - } => ObjectChange::Deleted { - sender, - object_type, - object_id, - version, - }, - IndexedObjectChange::Wrapped { - sender, - object_type, - object_id, - version, - } => ObjectChange::Wrapped { - sender, - object_type, - object_id, - version, - }, - IndexedObjectChange::Created { - sender, - owner, - object_type, - object_id, - version, - digest, - } => ObjectChange::Created { - sender, - owner, - object_type, - object_id, - version, - digest, - }, - } - } -} - -// SuiTransactionBlockResponseWithOptions is only used on the reading path -pub struct SuiTransactionBlockResponseWithOptions { - pub response: SuiTransactionBlockResponse, - pub options: SuiTransactionBlockResponseOptions, -} - -impl From for SuiTransactionBlockResponse { - fn from(value: SuiTransactionBlockResponseWithOptions) -> Self { - let SuiTransactionBlockResponseWithOptions { response, options } = value; - - SuiTransactionBlockResponse { - digest: response.digest, - transaction: options.show_input.then_some(response.transaction).flatten(), - raw_transaction: options - .show_raw_input - .then_some(response.raw_transaction) - .unwrap_or_default(), - effects: options.show_effects.then_some(response.effects).flatten(), - events: options.show_events.then_some(response.events).flatten(), - object_changes: options - .show_object_changes - .then_some(response.object_changes) - .flatten(), - balance_changes: options - .show_balance_changes - .then_some(response.balance_changes) - .flatten(), - timestamp_ms: response.timestamp_ms, - confirmed_local_execution: response.confirmed_local_execution, - checkpoint: response.checkpoint, - errors: vec![], - raw_effects: options - .show_raw_effects - .then_some(response.raw_effects) - .unwrap_or_default(), - } - } -} diff --git a/crates/sui-mvr-indexer/tests/ingestion_tests.rs b/crates/sui-mvr-indexer/tests/ingestion_tests.rs deleted file mode 100644 index 351b243594e4b..0000000000000 --- a/crates/sui-mvr-indexer/tests/ingestion_tests.rs +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 -use std::sync::Arc; - -use diesel::ExpressionMethods; -use diesel::QueryDsl; -use diesel_async::RunQueryDsl; -use simulacrum::Simulacrum; -use sui_mvr_indexer::errors::IndexerError; -use sui_mvr_indexer::handlers::TransactionObjectChangesToCommit; -use sui_mvr_indexer::models::{checkpoints::StoredCheckpoint, objects::StoredObjectSnapshot}; -use sui_mvr_indexer::schema::{checkpoints, objects_snapshot}; -use sui_mvr_indexer::store::indexer_store::IndexerStore; -use sui_mvr_indexer::test_utils::{ - set_up, set_up_with_start_and_end_checkpoints, wait_for_checkpoint, wait_for_objects_snapshot, -}; -use sui_mvr_indexer::types::EventIndex; -use sui_mvr_indexer::types::IndexedDeletedObject; -use sui_mvr_indexer::types::IndexedObject; -use sui_mvr_indexer::types::TxIndex; -use sui_types::base_types::SuiAddress; -use tempfile::tempdir; - -#[tokio::test] -pub async fn test_checkpoint_range_ingestion() -> Result<(), IndexerError> { - let tempdir = tempdir().unwrap(); - let mut sim = Simulacrum::new(); - let data_ingestion_path = tempdir.path().to_path_buf(); - sim.set_data_ingestion_path(data_ingestion_path.clone()); - - // Create multiple checkpoints - for _ in 0..10 { - let transfer_recipient = SuiAddress::random_for_testing_only(); - let (transaction, _) = sim.transfer_txn(transfer_recipient); - let (_, err) = sim.execute_transaction(transaction).unwrap(); - assert!(err.is_none()); - sim.create_checkpoint(); - } - - // Set up indexer with specific start and end checkpoints - let start_checkpoint = 2; - let end_checkpoint = 4; - let (_, pg_store, _, _database) = set_up_with_start_and_end_checkpoints( - Arc::new(sim), - data_ingestion_path, - start_checkpoint, - end_checkpoint, - ) - .await; - - // Wait for the indexer to catch up to the end checkpoint - wait_for_checkpoint(&pg_store, end_checkpoint).await?; - - // Verify that only checkpoints within the specified range were ingested - let mut connection = pg_store.pool().dedicated_connection().await.unwrap(); - let checkpoint_count: i64 = checkpoints::table - .count() - .get_result(&mut connection) - .await - .expect("Failed to count checkpoints"); - assert_eq!(checkpoint_count, 3, "Expected 3 checkpoints to be ingested"); - - // Verify the range of ingested checkpoints - let min_checkpoint = checkpoints::table - .select(diesel::dsl::min(checkpoints::sequence_number)) - .first::>(&mut connection) - .await - .expect("Failed to get min checkpoint") - .expect("Min checkpoint should be Some"); - let max_checkpoint = checkpoints::table - .select(diesel::dsl::max(checkpoints::sequence_number)) - .first::>(&mut connection) - .await - .expect("Failed to get max checkpoint") - .expect("Max checkpoint should be Some"); - assert_eq!( - min_checkpoint, start_checkpoint as i64, - "Minimum ingested checkpoint should be {}", - start_checkpoint - ); - assert_eq!( - max_checkpoint, end_checkpoint as i64, - "Maximum ingested checkpoint should be {}", - end_checkpoint - ); - - Ok(()) -} - -#[tokio::test] -pub async fn test_objects_snapshot() -> Result<(), IndexerError> { - let tempdir = tempdir().unwrap(); - let mut sim = Simulacrum::new(); - let data_ingestion_path = tempdir.path().to_path_buf(); - sim.set_data_ingestion_path(data_ingestion_path.clone()); - - // Run 10 transfer transactions and create 10 checkpoints - let mut last_transaction = None; - let total_checkpoint_sequence_number = 7usize; - for _ in 0..total_checkpoint_sequence_number { - let transfer_recipient = SuiAddress::random_for_testing_only(); - let (transaction, _) = sim.transfer_txn(transfer_recipient); - let (_, err) = sim.execute_transaction(transaction.clone()).unwrap(); - assert!(err.is_none()); - last_transaction = Some(transaction); - let _ = sim.create_checkpoint(); - } - - let (_, pg_store, _, _database) = set_up(Arc::new(sim), data_ingestion_path).await; - - // Wait for objects snapshot at checkpoint max_expected_checkpoint_sequence_number - let max_expected_checkpoint_sequence_number = total_checkpoint_sequence_number - 5; - wait_for_objects_snapshot(&pg_store, max_expected_checkpoint_sequence_number as u64).await?; - - let mut connection = pg_store.pool().dedicated_connection().await.unwrap(); - // Get max checkpoint_sequence_number from objects_snapshot table and assert it's expected - let max_checkpoint_sequence_number = objects_snapshot::table - .select(objects_snapshot::checkpoint_sequence_number) - .order(objects_snapshot::checkpoint_sequence_number.desc()) - .limit(1) - .first::(&mut connection) - .await - .expect("Failed to read max checkpoint_sequence_number from objects_snapshot"); - assert_eq!( - max_checkpoint_sequence_number, - max_expected_checkpoint_sequence_number as i64 - ); - - // Get the object state at max_expected_checkpoint_sequence_number and assert. - let last_tx = last_transaction.unwrap(); - let obj_id = last_tx.gas()[0].0; - let gas_owner_id = last_tx.sender_address(); - - let snapshot_object = objects_snapshot::table - .filter(objects_snapshot::object_id.eq(obj_id.to_vec())) - .filter( - objects_snapshot::checkpoint_sequence_number - .eq(max_expected_checkpoint_sequence_number as i64), - ) - .first::(&mut connection) - .await - .expect("Failed reading object from objects_snapshot"); - // Assert that the object state is as expected at checkpoint max_expected_checkpoint_sequence_number - assert_eq!(snapshot_object.object_id, obj_id.to_vec()); - assert_eq!( - snapshot_object.checkpoint_sequence_number, - max_expected_checkpoint_sequence_number as i64 - ); - assert_eq!(snapshot_object.owner_type, Some(1)); - assert_eq!(snapshot_object.owner_id, Some(gas_owner_id.to_vec())); - Ok(()) -} - -#[tokio::test] -pub async fn test_objects_ingestion() -> Result<(), IndexerError> { - let tempdir = tempdir().unwrap(); - let mut sim = Simulacrum::new(); - let data_ingestion_path = tempdir.path().to_path_buf(); - sim.set_data_ingestion_path(data_ingestion_path.clone()); - - let (_, pg_store, _, _database) = set_up(Arc::new(sim), data_ingestion_path).await; - - let mut objects = Vec::new(); - for _ in 0..1000 { - objects.push(TransactionObjectChangesToCommit { - changed_objects: vec![IndexedObject::random()], - deleted_objects: vec![IndexedDeletedObject::random()], - }); - } - pg_store.persist_objects(objects).await?; - Ok(()) -} - -// test insert large batch of tx_indices -#[tokio::test] -pub async fn test_insert_large_batch_tx_indices() -> Result<(), IndexerError> { - let tempdir = tempdir().unwrap(); - let mut sim = Simulacrum::new(); - let data_ingestion_path = tempdir.path().to_path_buf(); - sim.set_data_ingestion_path(data_ingestion_path.clone()); - - let (_, pg_store, _, _database) = set_up(Arc::new(sim), data_ingestion_path).await; - - let mut v = Vec::new(); - for _ in 0..1000 { - v.push(TxIndex::random()); - } - pg_store.persist_tx_indices(v).await?; - Ok(()) -} - -// test insert large batch of event_indices -#[tokio::test] -pub async fn test_insert_large_batch_event_indices() -> Result<(), IndexerError> { - let tempdir = tempdir().unwrap(); - let mut sim = Simulacrum::new(); - let data_ingestion_path = tempdir.path().to_path_buf(); - sim.set_data_ingestion_path(data_ingestion_path.clone()); - - let (_, pg_store, _, _database) = set_up(Arc::new(sim), data_ingestion_path).await; - - let mut v = Vec::new(); - for _ in 0..1000 { - v.push(EventIndex::random()); - } - pg_store.persist_event_indices(v).await?; - Ok(()) -} - -#[tokio::test] -pub async fn test_epoch_boundary() -> Result<(), IndexerError> { - println!("test_epoch_boundary"); - let tempdir = tempdir().unwrap(); - let mut sim = Simulacrum::new(); - let data_ingestion_path = tempdir.path().to_path_buf(); - sim.set_data_ingestion_path(data_ingestion_path.clone()); - - let transfer_recipient = SuiAddress::random_for_testing_only(); - let (transaction, _) = sim.transfer_txn(transfer_recipient); - let (_, err) = sim.execute_transaction(transaction.clone()).unwrap(); - assert!(err.is_none()); - - sim.create_checkpoint(); // checkpoint 1 - sim.advance_epoch(true); // checkpoint 2 and epoch 1 - - let (transaction, _) = sim.transfer_txn(transfer_recipient); - let (_, err) = sim.execute_transaction(transaction.clone()).unwrap(); - sim.create_checkpoint(); // checkpoint 3 - assert!(err.is_none()); - - let (_, pg_store, _, _database) = set_up(Arc::new(sim), data_ingestion_path).await; - wait_for_checkpoint(&pg_store, 3).await?; - let mut connection = pg_store.pool().dedicated_connection().await.unwrap(); - let db_checkpoint: StoredCheckpoint = checkpoints::table - .order(checkpoints::sequence_number.desc()) - .first::(&mut connection) - .await - .expect("Failed reading checkpoint from PostgresDB"); - assert_eq!(db_checkpoint.sequence_number, 3); - assert_eq!(db_checkpoint.epoch, 1); - Ok(()) -} diff --git a/crates/sui-mvr-indexer/tests/json_rpc_tests.rs b/crates/sui-mvr-indexer/tests/json_rpc_tests.rs deleted file mode 100644 index 4124e5837c746..0000000000000 --- a/crates/sui-mvr-indexer/tests/json_rpc_tests.rs +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::path::PathBuf; - -use sui_json_rpc_api::{CoinReadApiClient, IndexerApiClient, ReadApiClient}; -use sui_json_rpc_types::{ - CoinPage, EventFilter, SuiObjectDataOptions, SuiObjectResponse, SuiObjectResponseQuery, -}; -use sui_swarm_config::genesis_config::DEFAULT_GAS_AMOUNT; -use sui_test_transaction_builder::publish_package; -use sui_types::{event::EventID, transaction::CallArg}; -use test_cluster::TestClusterBuilder; - -#[tokio::test] -async fn test_get_owned_objects() -> Result<(), anyhow::Error> { - let cluster = TestClusterBuilder::new() - .with_indexer_backed_rpc() - .build() - .await; - - let http_client = cluster.rpc_client(); - let address = cluster.get_address_0(); - - let data_option = SuiObjectDataOptions::new().with_owner(); - let objects = http_client - .get_owned_objects( - address, - Some(SuiObjectResponseQuery::new_with_options( - data_option.clone(), - )), - None, - None, - ) - .await? - .data; - let fullnode_objects = cluster - .fullnode_handle - .rpc_client - .get_owned_objects( - address, - Some(SuiObjectResponseQuery::new_with_options( - data_option.clone(), - )), - None, - None, - ) - .await? - .data; - assert_eq!(5, objects.len()); - // TODO: right now we compare the results from indexer and fullnode, but as we deprecate fullnode rpc, - // we should change this to compare the results with the object id/digest from genesis potentially. - assert_eq!(objects, fullnode_objects); - - for obj in &objects { - let oref = obj.clone().into_object().unwrap(); - let result = http_client - .get_object(oref.object_id, Some(data_option.clone())) - .await?; - assert!( - matches!(result, SuiObjectResponse { data: Some(object), .. } if oref.object_id == object.object_id && object.owner.clone().unwrap().get_owner_address()? == address) - ); - } - - // Multiget objectIDs test - let object_ids: Vec<_> = objects - .iter() - .map(|o| o.object().unwrap().object_id) - .collect(); - - let object_resp = http_client - .multi_get_objects(object_ids.clone(), None) - .await?; - let fullnode_object_resp = cluster - .fullnode_handle - .rpc_client - .multi_get_objects(object_ids, None) - .await?; - assert_eq!(5, object_resp.len()); - // TODO: right now we compare the results from indexer and fullnode, but as we deprecate fullnode rpc, - // we should change this to compare the results with the object id/digest from genesis potentially. - assert_eq!(object_resp, fullnode_object_resp); - Ok(()) -} - -#[tokio::test] -async fn test_get_coins() -> Result<(), anyhow::Error> { - let cluster = TestClusterBuilder::new() - .with_indexer_backed_rpc() - .build() - .await; - let http_client = cluster.rpc_client(); - let address = cluster.get_address_0(); - - let result: CoinPage = http_client.get_coins(address, None, None, None).await?; - assert_eq!(5, result.data.len()); - assert!(!result.has_next_page); - - // We should get 0 coins for a non-existent coin type. - let result: CoinPage = http_client - .get_coins(address, Some("0x2::sui::TestCoin".into()), None, None) - .await?; - assert_eq!(0, result.data.len()); - - // We should get all the 5 coins for SUI with the right balance. - let result: CoinPage = http_client - .get_coins(address, Some("0x2::sui::SUI".into()), None, None) - .await?; - assert_eq!(5, result.data.len()); - assert_eq!(result.data[0].balance, DEFAULT_GAS_AMOUNT); - assert!(!result.has_next_page); - - // When we have more than 3 coins, we should get a next page. - let result: CoinPage = http_client - .get_coins(address, Some("0x2::sui::SUI".into()), None, Some(3)) - .await?; - assert_eq!(3, result.data.len()); - assert!(result.has_next_page); - - // We should get the remaining 2 coins with the next page. - let result: CoinPage = http_client - .get_coins( - address, - Some("0x2::sui::SUI".into()), - result.next_cursor, - Some(3), - ) - .await?; - assert_eq!(2, result.data.len(), "{:?}", result); - assert!(!result.has_next_page); - - // No more coins after the last page. - let result: CoinPage = http_client - .get_coins( - address, - Some("0x2::sui::SUI".into()), - result.next_cursor, - None, - ) - .await?; - assert_eq!(0, result.data.len(), "{:?}", result); - assert!(!result.has_next_page); - - Ok(()) -} - -#[tokio::test] -async fn test_events() -> Result<(), anyhow::Error> { - let cluster = TestClusterBuilder::new() - .with_indexer_backed_rpc() - .build() - .await; - - // publish package - let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.push("tests/move_test_code"); - let move_package = publish_package(&cluster.wallet, path).await.0; - - // execute a transaction to generate events - let function = "emit_3"; - let arguments = vec![CallArg::Pure(bcs::to_bytes(&5u64).unwrap())]; - let transaction = cluster - .test_transaction_builder() - .await - .move_call(move_package, "events_queries", function, arguments) - .build(); - let signed_transaction = cluster.wallet.sign_transaction(&transaction); - cluster.execute_transaction(signed_transaction).await; - - // query for events - let http_client = cluster.rpc_client(); - - // start with ascending order - let event_filter = EventFilter::All([]); - let mut cursor: Option = None; - let mut limit = None; - let mut descending_order = Some(false); - let result = http_client - .query_events(event_filter.clone(), cursor, limit, descending_order) - .await?; - assert_eq!(3, result.data.len()); - assert!(!result.has_next_page); - let forward_paginated_events = result.data; - - // Fetch the initial event - limit = Some(1); - let result = http_client - .query_events(event_filter.clone(), cursor, limit, descending_order) - .await?; - assert_eq!(1, result.data.len()); - assert!(result.has_next_page); - assert_eq!(forward_paginated_events[0], result.data[0]); - - // Fetch remaining events - cursor = result.next_cursor; - limit = None; - let result = http_client - .query_events(event_filter.clone(), cursor, limit, descending_order) - .await?; - assert_eq!(2, result.data.len()); - assert_eq!(forward_paginated_events[1..], result.data[..]); - - // now descending order - make sure to reset parameters - cursor = None; - descending_order = Some(true); - limit = None; - let result = http_client - .query_events(event_filter.clone(), cursor, limit, descending_order) - .await?; - assert_eq!(3, result.data.len()); - assert!(!result.has_next_page); - let backward_paginated_events = result.data; - - // Fetch the initial event - limit = Some(1); - let result = http_client - .query_events(event_filter.clone(), cursor, limit, descending_order) - .await?; - assert_eq!(1, result.data.len()); - assert!(result.has_next_page); - assert_eq!(backward_paginated_events[0], result.data[0]); - assert_eq!(forward_paginated_events[2], result.data[0]); - - // Fetch remaining events - cursor = result.next_cursor; - limit = None; - let result = http_client - .query_events(event_filter.clone(), cursor, limit, descending_order) - .await?; - assert_eq!(2, result.data.len()); - assert_eq!(backward_paginated_events[1..], result.data[..]); - - // check that the forward and backward paginated events are in reverse order - assert_eq!( - forward_paginated_events - .into_iter() - .rev() - .collect::>(), - backward_paginated_events - ); - - Ok(()) -} diff --git a/crates/sui-mvr-indexer/tests/move_test_code/Move.toml b/crates/sui-mvr-indexer/tests/move_test_code/Move.toml deleted file mode 100644 index 09e9e50f000f0..0000000000000 --- a/crates/sui-mvr-indexer/tests/move_test_code/Move.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "move_test_code" -version = "0.0.1" -edition = "2024.beta" - -[dependencies] -Sui = { local = "../../../sui-framework/packages/sui-framework" } - -[addresses] -move_test_code = "0x0" diff --git a/crates/sui-mvr-indexer/tests/move_test_code/sources/events.move b/crates/sui-mvr-indexer/tests/move_test_code/sources/events.move deleted file mode 100644 index f32cc7fe109f3..0000000000000 --- a/crates/sui-mvr-indexer/tests/move_test_code/sources/events.move +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - - -module move_test_code::events_queries { - use sui::event; - - public struct EventA has copy, drop { - new_value: u64 - } - - public entry fun emit_1(value: u64) { - event::emit(EventA { new_value: value }) - } - - public entry fun emit_2(value: u64) { - event::emit(EventA { new_value: value }); - event::emit(EventA { new_value: value + 1}) - } - - public entry fun emit_3(value: u64) { - event::emit(EventA { new_value: value }); - event::emit(EventA { new_value: value + 1}); - event::emit(EventA { new_value: value + 2}); - } -} diff --git a/crates/sui-mvr-indexer/tests/read_api_tests.rs b/crates/sui-mvr-indexer/tests/read_api_tests.rs deleted file mode 100644 index d17b431888f01..0000000000000 --- a/crates/sui-mvr-indexer/tests/read_api_tests.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use jsonrpsee::core::RpcResult; -use simulacrum::Simulacrum; -use std::sync::Arc; -use sui_json_rpc_api::ReadApiServer; -use sui_mvr_indexer::apis::read_api::ReadApi; -use sui_mvr_indexer::indexer_reader::IndexerReader; -use sui_mvr_indexer::test_utils::{set_up, wait_for_checkpoint}; -use tempfile::tempdir; - -#[tokio::test] -async fn test_checkpoint_apis() -> RpcResult<()> { - let tempdir = tempdir().unwrap(); - let mut sim = Simulacrum::new(); - let data_ingestion_path = tempdir.path().to_path_buf(); - sim.set_data_ingestion_path(data_ingestion_path.clone()); - sim.create_checkpoint(); - sim.create_checkpoint(); - - let (_, pg_store, _, _database) = set_up(Arc::new(sim), data_ingestion_path).await; - wait_for_checkpoint(&pg_store, 2).await.unwrap(); - - // Test get_latest_checkpoint_sequence_number - let read_api = ReadApi::new(IndexerReader::new(pg_store.pool())); - let latest_checkpoint = read_api.get_latest_checkpoint_sequence_number().await?; - assert_eq!(latest_checkpoint.into_inner(), 2); - - // Test get_checkpoint - let checkpoint_id = sui_json_rpc_types::CheckpointId::SequenceNumber(1); - let checkpoint = read_api.get_checkpoint(checkpoint_id).await?; - assert_eq!(checkpoint.sequence_number, 1); - - // Test get_checkpoints - let checkpoints = read_api.get_checkpoints(None, Some(10), false).await?; - assert_eq!(checkpoints.data.len(), 3); // 0, 1, 2 - assert!(!checkpoints.has_next_page); - assert_eq!(checkpoints.next_cursor, Some(2.into())); - - let checkpoints = read_api - .get_checkpoints(Some(2.into()), Some(2), true) - .await?; - assert_eq!(checkpoints.data.len(), 2); - assert!(!checkpoints.has_next_page); - assert_eq!(checkpoints.next_cursor, Some(0.into())); - assert_eq!(checkpoints.data[0].sequence_number, 1); - assert_eq!(checkpoints.data[1].sequence_number, 0); - Ok(()) -} diff --git a/docker/sui-mvr-indexer/build.sh b/docker/sui-mvr-indexer/build.sh deleted file mode 100755 index 5e1c8c1623fe7..0000000000000 --- a/docker/sui-mvr-indexer/build.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh -# Copyright (c) Mysten Labs, Inc. -# SPDX-License-Identifier: Apache-2.0 - -# fast fail. -set -e - -DIR="$( cd "$( dirname "$0" )" && pwd )" -REPO_ROOT="$(git rev-parse --show-toplevel)" -DOCKERFILE="$DIR/Dockerfile" -GIT_REVISION="$(git describe --always --abbrev=12 --dirty --exclude '*')" -BUILD_DATE="$(date -u +'%Y-%m-%d')" - -echo -echo "Building sui-mvr-indexer docker image" -echo "Dockerfile: \t$DOCKERFILE" -echo "docker context: $REPO_ROOT" -echo "build date: \t$BUILD_DATE" -echo "git revision: \t$GIT_REVISION" -echo - -docker build -f "$DOCKERFILE" "$REPO_ROOT" \ - --build-arg GIT_REVISION="$GIT_REVISION" \ - --build-arg BUILD_DATE="$BUILD_DATE" \ - "$@"