From 3282dada757b80bd9bf5d0da94d6f2af50934bfc Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Fri, 8 Nov 2024 12:18:02 +0100 Subject: [PATCH] feat: implement `/v1/blocks/{hash_or_height}` endpoint --- package.json | 1 + src/api/errors.ts | 9 +++ src/api/routes/blocks.ts | 89 +++++++++++++++++++++- src/api/schemas.ts | 56 +++++++++++++- src/helpers.ts | 5 ++ src/pg/pg-store.ts | 146 ++++++++++++++++++++++++++++++++++++- tests/db/endpoints.test.ts | 69 ++++++++++++++++++ 7 files changed, 368 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index a90aabf..3bcaff4 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "testenv:stop": "docker compose -f docker/docker-compose.dev.postgres.yml down -v -t 0", "testenv:logs": "docker compose -f docker/docker-compose.dev.postgres.yml logs -t -f", "migrate": "ts-node node_modules/.bin/node-pg-migrate -j ts", + "lint": "npm run lint:eslint && npm run lint:prettier", "lint:eslint": "eslint . --ext .js,.jsx,.ts,.tsx -f unix", "lint:prettier": "prettier --check src/**/*.ts tests/**/*.ts migrations/**/*.ts", "generate:openapi": "rimraf ./tmp && node -r ts-node/register ./util/openapi-generator.ts", diff --git a/src/api/errors.ts b/src/api/errors.ts index 46d6144..79c5519 100644 --- a/src/api/errors.ts +++ b/src/api/errors.ts @@ -6,3 +6,12 @@ export class InvalidRequestError extends Error { this.status = status; } } + +export class NotFoundError extends Error { + status: number; + constructor(msg: string, status: number = 404) { + super(msg); + this.name = this.constructor.name; + this.status = status; + } +} diff --git a/src/api/routes/blocks.ts b/src/api/routes/blocks.ts index 0db8ca1..1e5d942 100644 --- a/src/api/routes/blocks.ts +++ b/src/api/routes/blocks.ts @@ -1,7 +1,16 @@ import { Type, TypeBoxTypeProvider } from '@fastify/type-provider-typebox'; import { FastifyPluginCallback } from 'fastify'; import { Server } from 'http'; -import { BlocksEntry, BlocksEntrySignerData, BlocksResponseSchema } from '../schemas'; +import { + BlockEntrySchema, + BlockParamsSchema, + BlocksEntry, + BlocksEntrySignerData, + BlocksResponseSchema, + cleanBlockHeightOrHashParam, + parseBlockParam, +} from '../schemas'; +import { NotFoundError } from '../errors'; export const BlockRoutes: FastifyPluginCallback< Record, @@ -33,7 +42,11 @@ export const BlockRoutes: FastifyPluginCallback< }, async (request, reply) => { const result = await fastify.db.sqlTransaction(async sql => { - const results = await fastify.db.getRecentBlocks(request.query.limit, request.query.offset); + const results = await fastify.db.getSignerDataForRecentBlocks({ + sql, + limit: request.query.limit, + offset: request.query.offset, + }); const formatted: BlocksEntry[] = results.map(result => { const entry: BlocksEntry = { @@ -88,5 +101,77 @@ export const BlockRoutes: FastifyPluginCallback< await reply.send(result); } ); + + fastify.get( + '/v1/blocks/:height_or_hash', + { + preValidation: (req, _reply, done) => { + cleanBlockHeightOrHashParam(req.params); + done(); + }, + schema: { + operationId: 'getBlock', + summary: 'Aggregated signer information for a block', + description: 'Aggregated signer information for a block', + tags: ['Blocks'], + params: BlockParamsSchema, + response: { + 200: BlockEntrySchema, + }, + }, + }, + async (request, reply) => { + const blockId = parseBlockParam(request.params.height_or_hash); + const result = await fastify.db.sqlTransaction(async sql => { + const result = await fastify.db.getSignerDataForBlock({ sql, blockId }); + if (!result) { + throw new NotFoundError(`Block not found`); + } + + const entry: BlocksEntry = { + block_height: result.block_height, + block_hash: result.block_hash, + index_block_hash: result.index_block_hash, + burn_block_height: result.burn_block_height, + tenure_height: result.tenure_height, + block_time: result.block_time, + }; + + if (!result.block_proposal_time_ms || !result.cycle_number) { + // no signer data available for this, only return the block header data + return entry; + } + + const entrySignerData: BlocksEntrySignerData = { + cycle_number: result.cycle_number, + total_signer_count: result.total_signer_count, + accepted_count: + result.signer_accepted_mined_count + result.signer_accepted_excluded_count, + rejected_count: result.signer_rejected_count, + missing_count: result.signer_missing_count, + + accepted_excluded_count: result.signer_accepted_excluded_count, + + average_response_time_ms: result.average_response_time_ms, + block_proposal_time_ms: Number.parseInt(result.block_proposal_time_ms), + + accepted_stacked_amount: ( + BigInt(result.accepted_mined_stacked_amount) + + BigInt(result.accepted_excluded_stacked_amount) + ).toString(), + rejected_stacked_amount: result.rejected_stacked_amount, + missing_stacked_amount: result.missing_stacked_amount, + + accepted_weight: result.accepted_mined_weight + result.accepted_excluded_weight, + rejected_weight: result.rejected_weight, + missing_weight: result.missing_weight, + }; + entry.signer_data = entrySignerData; + return entry; + }); + await reply.send(result); + } + ); + done(); }; diff --git a/src/api/schemas.ts b/src/api/schemas.ts index d9b8810..bb5ef04 100644 --- a/src/api/schemas.ts +++ b/src/api/schemas.ts @@ -1,6 +1,7 @@ import { SwaggerOptions } from '@fastify/swagger'; -import { SERVER_VERSION } from '@hirosystems/api-toolkit'; +import { has0xPrefix, SERVER_VERSION } from '@hirosystems/api-toolkit'; import { Static, TSchema, Type } from '@sinclair/typebox'; +import { BlockIdParam } from '../helpers'; export const OpenApiSchemaOptions: SwaggerOptions = { openapi: { @@ -30,7 +31,6 @@ export const OpenApiSchemaOptions: SwaggerOptions = { }, ], }, - exposeRoute: true, }; export const ApiStatusResponse = Type.Object( @@ -197,3 +197,55 @@ export type CycleSignersResponse = Static; export const CycleSignerResponseSchema = Type.Composite([CycleSignerSchema]); export type CycleSignerResponse = Static; + +const BlockHashParamSchema = Type.String({ + pattern: '^(0x)?[a-fA-F0-9]{64}$', + title: 'Block hash', + description: 'Block hash', + examples: ['0xdaf79950c5e8bb0c620751333967cdd62297137cdaf79950c5e8bb0c62075133'], +}); + +const BlockHeightParamSchema = Type.Integer({ + title: 'Block height', + description: 'Block height', + examples: [777678], +}); + +export const BlockParamsSchema = Type.Object( + { + height_or_hash: Type.Union([ + Type.Literal('latest'), + BlockHashParamSchema, + BlockHeightParamSchema, + ]), + }, + { additionalProperties: false } +); +export type BlockParams = Static; + +/** + * If a param can accept a block hash or height, then ensure that the hash is prefixed with '0x' so + * that hashes with only digits are not accidentally parsed as a number. + */ +export function cleanBlockHeightOrHashParam(params: { height_or_hash: string | number }) { + if ( + typeof params.height_or_hash === 'string' && + /^[a-fA-F0-9]{64}$/i.test(params.height_or_hash) + ) { + params.height_or_hash = '0x' + params.height_or_hash; + } +} + +export function parseBlockParam(value: BlockParams['height_or_hash']): BlockIdParam { + if (value === 'latest') { + return { type: 'latest', latest: true }; + } + value = typeof value === 'string' ? value : value.toString(); + if (/^(0x)?[a-fA-F0-9]{64}$/i.test(value)) { + return { type: 'hash', hash: has0xPrefix(value) ? value : `0x${value}` }; + } + if (/^[0-9]+$/.test(value)) { + return { type: 'height', height: parseInt(value) }; + } + throw new Error('Invalid block height or hash'); +} diff --git a/src/helpers.ts b/src/helpers.ts index 8bd4e73..84a2cdc 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -76,3 +76,8 @@ export function parseTime(timeStr: string): Date | null { // Return null if parsing failed return null; } + +export type BlockIdParam = + | { type: 'height'; height: number } + | { type: 'hash'; hash: string } + | { type: 'latest'; latest: true }; diff --git a/src/pg/pg-store.ts b/src/pg/pg-store.ts index a6fe1f4..dc4da9d 100644 --- a/src/pg/pg-store.ts +++ b/src/pg/pg-store.ts @@ -9,7 +9,8 @@ import { } from '@hirosystems/api-toolkit'; import * as path from 'path'; import { ChainhookPgStore } from './chainhook/chainhook-pg-store'; -import { normalizeHexString, sleep } from '../helpers'; +import { BlockIdParam, normalizeHexString, sleep } from '../helpers'; +import { Fragment } from 'postgres'; export const MIGRATIONS_DIR = path.join(__dirname, '../../migrations'); @@ -100,7 +101,15 @@ export class PgStore extends BasePgStore { return { rowUpdated: updateResult.count > 0 }; } - async getRecentBlocks(limit: number, offset: number) { + async getSignerDataForRecentBlocks({ + sql, + limit, + offset, + }: { + sql: PgSqlClient; + limit: number; + offset: number; + }) { // The `blocks` table (and its associated block_signer_signatures table) is the source of truth that is // never missing blocks and does not contain duplicate rows per block. // @@ -142,7 +151,7 @@ export class PgStore extends BasePgStore { // * rejected_weight: the total signer_weight of each signer in the rejected state // * missing_weight: the total signer_weight of each signer in the missing state - const result = await this.sql< + const result = await sql< { block_height: number; block_hash: string; @@ -255,6 +264,137 @@ export class PgStore extends BasePgStore { return result; } + async getSignerDataForBlock({ sql, blockId }: { sql: PgSqlClient; blockId: BlockIdParam }) { + let blockFilter: Fragment; + switch (blockId.type) { + case 'height': + blockFilter = sql`block_height = ${blockId.height}`; + break; + case 'hash': + blockFilter = sql`block_hash = ${normalizeHexString(blockId.hash)}`; + break; + case 'latest': + blockFilter = sql`block_height = (SELECT block_height FROM chain_tip)`; + break; + default: + throw new Error(`Invalid blockId type: ${blockId}`); + } + + const result = await sql< + { + block_height: number; + block_hash: string; + index_block_hash: string; + burn_block_height: number; + tenure_height: number; + block_time: number; + cycle_number: number | null; + block_proposal_time_ms: string | null; + total_signer_count: number; + signer_accepted_mined_count: number; + signer_accepted_excluded_count: number; + signer_rejected_count: number; + signer_missing_count: number; + average_response_time_ms: number; + accepted_mined_stacked_amount: string; + accepted_excluded_stacked_amount: string; + rejected_stacked_amount: string; + missing_stacked_amount: string; + accepted_mined_weight: number; + accepted_excluded_weight: number; + rejected_weight: number; + missing_weight: number; + chain_tip_block_height: number; + }[] + >` + WITH latest_blocks AS ( + SELECT * FROM blocks + WHERE ${blockFilter} + LIMIT 1 + ), + block_signers AS ( + SELECT + lb.id AS block_id, + lb.block_height, + lb.block_time, + lb.block_hash, + lb.index_block_hash, + lb.burn_block_height, + bp.reward_cycle AS cycle_number, + bp.received_at AS block_proposal_time_ms, + rs.signer_key, + COALESCE(rs.signer_weight, 0) AS signer_weight, + COALESCE(rs.signer_stacked_amount, 0) AS signer_stacked_amount, + CASE + WHEN bss.id IS NOT NULL THEN 'accepted_mined' + WHEN bss.id IS NULL AND fbr.accepted = TRUE THEN 'accepted_excluded' + WHEN bss.id IS NULL AND fbr.accepted = FALSE THEN 'rejected' + WHEN bss.id IS NULL AND fbr.id IS NULL THEN 'missing' + END AS signer_status, + EXTRACT(MILLISECOND FROM (fbr.received_at - bp.received_at)) AS response_time_ms + FROM latest_blocks lb + LEFT JOIN block_proposals bp ON lb.block_hash = bp.block_hash + LEFT JOIN reward_set_signers rs ON bp.reward_cycle = rs.cycle_number + LEFT JOIN block_signer_signatures bss ON lb.block_height = bss.block_height AND rs.signer_key = bss.signer_key + LEFT JOIN block_responses fbr ON fbr.signer_key = rs.signer_key AND fbr.signer_sighash = lb.block_hash + ), + signer_state_aggregation AS ( + SELECT + block_id, + MAX(cycle_number) AS cycle_number, + MAX(block_proposal_time_ms) AS block_proposal_time_ms, + COUNT(signer_key) AS total_signer_count, + COALESCE(COUNT(CASE WHEN signer_status = 'accepted_mined' THEN 1 END), 0) AS signer_accepted_mined_count, + COALESCE(COUNT(CASE WHEN signer_status = 'accepted_excluded' THEN 1 END), 0) AS signer_accepted_excluded_count, + COALESCE(COUNT(CASE WHEN signer_status = 'rejected' THEN 1 END), 0) AS signer_rejected_count, + COALESCE(COUNT(CASE WHEN signer_status = 'missing' THEN 1 END), 0) AS signer_missing_count, + COALESCE(AVG(response_time_ms) FILTER (WHERE signer_status IN ('accepted_mined', 'accepted_excluded', 'rejected')), 0) AS average_response_time_ms, + COALESCE(SUM(CASE WHEN signer_status = 'accepted_mined' THEN signer_stacked_amount END), 0) AS accepted_mined_stacked_amount, + COALESCE(SUM(CASE WHEN signer_status = 'accepted_excluded' THEN signer_stacked_amount END), 0) AS accepted_excluded_stacked_amount, + COALESCE(SUM(CASE WHEN signer_status = 'rejected' THEN signer_stacked_amount END), 0) AS rejected_stacked_amount, + COALESCE(SUM(CASE WHEN signer_status = 'missing' THEN signer_stacked_amount END), 0) AS missing_stacked_amount, + COALESCE(SUM(CASE WHEN signer_status = 'accepted_mined' THEN signer_weight END), 0) AS accepted_mined_weight, + COALESCE(SUM(CASE WHEN signer_status = 'accepted_excluded' THEN signer_weight END), 0) AS accepted_excluded_weight, + COALESCE(SUM(CASE WHEN signer_status = 'rejected' THEN signer_weight END), 0) AS rejected_weight, + COALESCE(SUM(CASE WHEN signer_status = 'missing' THEN signer_weight END), 0) AS missing_weight + FROM block_signers + GROUP BY block_id + ) + SELECT + lb.block_height, + lb.block_hash, + lb.index_block_hash, + lb.burn_block_height, + lb.tenure_height, + EXTRACT(EPOCH FROM lb.block_time)::integer AS block_time, + bsa.cycle_number, + (EXTRACT(EPOCH FROM bsa.block_proposal_time_ms) * 1000)::bigint AS block_proposal_time_ms, + bsa.total_signer_count::integer, + bsa.signer_accepted_mined_count::integer, + bsa.signer_accepted_excluded_count::integer, + bsa.signer_rejected_count::integer, + bsa.signer_missing_count::integer, + ROUND(bsa.average_response_time_ms, 3)::float8 AS average_response_time_ms, + bsa.accepted_mined_stacked_amount, + bsa.accepted_excluded_stacked_amount, + bsa.rejected_stacked_amount, + bsa.missing_stacked_amount, + bsa.accepted_mined_weight::integer, + bsa.accepted_excluded_weight::integer, + bsa.rejected_weight::integer, + bsa.missing_weight::integer, + ct.block_height AS chain_tip_block_height + FROM latest_blocks lb + JOIN signer_state_aggregation bsa ON lb.id = bsa.block_id + CROSS JOIN chain_tip ct + `; + if (result.length === 0) { + return null; + } else { + return result[0]; + } + } + async getSignersForCycle({ sql, cycleNumber, diff --git a/tests/db/endpoints.test.ts b/tests/db/endpoints.test.ts index 91e37ca..a03b0cf 100644 --- a/tests/db/endpoints.test.ts +++ b/tests/db/endpoints.test.ts @@ -119,6 +119,75 @@ describe('Endpoint tests', () => { expect(testBlock).toEqual(expectedBlockData); }); + test('get block by hash or height', async () => { + // block 112274 has all signer states (missing, rejected, accepted, accepted_excluded) + const blockHeight = 112274; + const blockHash = '0x782d69b5955a91b110859ee6fc6454cc19814d6434cdde61d5bc91697dee50fc'; + + const expectedBlockData: BlocksEntry = { + block_height: blockHeight, + block_hash: blockHash, + block_time: 1730554291, + index_block_hash: '0xb5c47a7c0e444b6a96331e0435d940a528dfde98966bb079b1b0b7d706b3016f', + burn_block_height: 65203, + tenure_height: 53405, + signer_data: { + cycle_number: 72, + total_signer_count: 11, + accepted_count: 7, + rejected_count: 1, + missing_count: 3, + accepted_excluded_count: 1, + average_response_time_ms: 11727.75, + block_proposal_time_ms: 1730554295560, + accepted_stacked_amount: '306370000003000', + rejected_stacked_amount: '9690000000000', + missing_stacked_amount: '21200000000000', + accepted_weight: 46, + rejected_weight: 1, + missing_weight: 3, + }, + }; + + const { body: testBlockHeight } = (await supertest(apiServer.server) + .get(`/signer-metrics/v1/blocks/${blockHeight}`) + .expect(200)) as { body: BlocksEntry }; + expect(testBlockHeight).toEqual(expectedBlockData); + + const { body: testBlockHash } = (await supertest(apiServer.server) + .get(`/signer-metrics/v1/blocks/${blockHash}`) + .expect(200)) as { body: BlocksEntry }; + expect(testBlockHash).toEqual(expectedBlockData); + + // Test 404 using a non-existent block hash + const nonExistentHash = '0x000069b5955a91b110859ee6fc6454cc19814d6434cdde61d5bc91697dee50f0'; + const notFoundResp = await supertest(apiServer.server) + .get(`/signer-metrics/v1/blocks/${nonExistentHash}`) + .expect(404); + expect(notFoundResp.body).toMatchObject({ + error: 'Not Found', + message: 'Block not found', + statusCode: 404, + }); + }); + + test('get block by latest', async () => { + // Note: the current block payload test data does not have signer data for the latest block + const expectedBlockData: BlocksEntry = { + block_height: 112291, + block_hash: '0x82ac0b52a4dde86ac05d04f59d81081d047125d0c7eaf424683191fc3fd839f2', + block_time: 1730554958, + index_block_hash: '0x7183de5c4ae700248283fede9264d31a37ab3ca1b54b4fd24adc449fbbd4c2b7', + burn_block_height: 65206, + tenure_height: 53408, + }; + + const { body: testBlockLatest } = (await supertest(apiServer.server) + .get(`/signer-metrics/v1/blocks/latest`) + .expect(200)) as { body: BlocksEntry }; + expect(testBlockLatest).toEqual(expectedBlockData); + }); + test('get signers for cycle', async () => { const responseTest = await supertest(apiServer.server) .get('/signer-metrics/v1/cycles/72/signers')