diff --git a/docs/pages/packages/providers/Compass.mdx b/docs/pages/packages/providers/Compass.mdx index fa43952b..04636d4c 100644 --- a/docs/pages/packages/providers/Compass.mdx +++ b/docs/pages/packages/providers/Compass.mdx @@ -86,3 +86,63 @@ unless there is a migration to be made. When the API key given is incorrect or e All errors derived from a malformed structure on the query will throw a `CompassRequestError` which has a public property called `errors` of the type `CompassError` with more information about what went wrong. + +## Batch Requests + +It is possible to do many requests that only differs on the variables given to the same +query. Such requests will be done in batch not to overload the server. + +For example, if what we want is to retrieve collectors of a drop: + +```typescript +const total = 800; // Number of collectors we want to retrieve +const limit = 100; // Number of collectors we retrieve per page + +const MY_DROP_COLLECTORS_QUERY = ` + query MyDropCollectors( + $offset: Int + $limit: Int + ) { + poaps( + where: { + drop_id: { _eq: 151249 } + } + offset: $offset + limit: $limit + ) { + collector_address + } + } +`; + +type CollectorsQueryResponse = { + poaps: Array<{ + collector_address: string; + }>; +}; + +const responses = await compass.batch< + CollectorsQueryResponse, + PaginatedVariables +>( + MY_DROP_COLLECTORS_QUERY, + Array.from( + { length: Math.ceil(total / limit) }, + (_, page): PaginatedVariables => ({ + offset: page * limit, + limit: limit, + }), + ), +); + +const collectors = responses.reduce( + (addresses, response) => [ + ...addresses, + ...response.data.poaps.map((poap) => poap.collector_address), + ], + [], +); +``` + +In this case, we are doing eight requests but in two groups requested in parallel, one of +five and one of three requests. diff --git a/packages/drops/package.json b/packages/drops/package.json index 6d45b7fc..a4fba4af 100644 --- a/packages/drops/package.json +++ b/packages/drops/package.json @@ -1,6 +1,6 @@ { "name": "@poap-xyz/drops", - "version": "0.2.4", + "version": "0.2.5", "description": "Drops module for the poap.js library", "main": "dist/cjs/index.cjs", "module": "dist/esm/index.mjs", @@ -29,7 +29,7 @@ "node": ">=18" }, "dependencies": { - "@poap-xyz/providers": "0.2.4", - "@poap-xyz/utils": "0.2.4" + "@poap-xyz/providers": "0.2.5", + "@poap-xyz/utils": "0.2.5" } } diff --git a/packages/moments/package.json b/packages/moments/package.json index c20513cb..0b4ff9d2 100644 --- a/packages/moments/package.json +++ b/packages/moments/package.json @@ -1,6 +1,6 @@ { "name": "@poap-xyz/moments", - "version": "0.2.4", + "version": "0.2.5", "description": "Moments module for the poap.js library", "main": "dist/cjs/index.cjs", "module": "dist/esm/index.mjs", @@ -26,8 +26,8 @@ "build": "rollup -c --bundleConfigAsCjs" }, "dependencies": { - "@poap-xyz/providers": "0.2.4", - "@poap-xyz/utils": "0.2.4", + "@poap-xyz/providers": "0.2.5", + "@poap-xyz/utils": "0.2.5", "uuid": "^9.0.0" }, "engines": { diff --git a/packages/poaps/package.json b/packages/poaps/package.json index ee36d197..3645181d 100644 --- a/packages/poaps/package.json +++ b/packages/poaps/package.json @@ -1,6 +1,6 @@ { "name": "@poap-xyz/poaps", - "version": "0.2.4", + "version": "0.2.5", "description": "Poaps module for the poap.js library", "main": "dist/cjs/index.cjs", "module": "dist/esm/index.mjs", @@ -26,8 +26,8 @@ "build": "rollup -c --bundleConfigAsCjs" }, "dependencies": { - "@poap-xyz/providers": "0.2.4", - "@poap-xyz/utils": "0.2.4" + "@poap-xyz/providers": "0.2.5", + "@poap-xyz/utils": "0.2.5" }, "engines": { "node": ">=18" diff --git a/packages/providers/package.json b/packages/providers/package.json index 40e2c510..1d7351c9 100644 --- a/packages/providers/package.json +++ b/packages/providers/package.json @@ -1,6 +1,6 @@ { "name": "@poap-xyz/providers", - "version": "0.2.4", + "version": "0.2.5", "description": "Providers module for the poap.js library", "main": "dist/cjs/index.cjs", "module": "dist/esm/index.mjs", @@ -26,11 +26,13 @@ "build": "rollup -c --bundleConfigAsCjs" }, "dependencies": { - "@poap-xyz/utils": "0.2.4", - "axios": "^1.3.5" + "@poap-xyz/utils": "0.2.5", + "axios": "^1.3.5", + "lodash.chunk": "^4.2.0" }, "devDependencies": { - "axios-mock-adapter": "^1.21.4" + "axios-mock-adapter": "^1.21.4", + "jest-fetch-mock": "^3.0.3" }, "engines": { "node": ">=18" diff --git a/packages/providers/src/core/PoapCompass/PoapCompass.ts b/packages/providers/src/core/PoapCompass/PoapCompass.ts index 4901192c..b8e8404c 100644 --- a/packages/providers/src/core/PoapCompass/PoapCompass.ts +++ b/packages/providers/src/core/PoapCompass/PoapCompass.ts @@ -1,3 +1,4 @@ +import chunk from 'lodash.chunk'; import { CompassProvider } from '../../ports/CompassProvider/CompassProvider'; import { CompassErrors } from '../../ports/CompassProvider/types/CompassErrors'; import { CompassError } from '../../ports/CompassProvider/types/CompassError'; @@ -14,6 +15,11 @@ const DEFAULT_COMPASS_BASE_URL = 'https://public.compass.poap.tech/v1/graphql'; * @implements {CompassProvider} */ export class PoapCompass implements CompassProvider { + /** + * Maximum number of request to do at the time in a batch. + */ + protected batchSize = 5; + private apiKey: string; private baseUrl: string; @@ -172,6 +178,24 @@ export class PoapCompass implements CompassProvider { ): Promise<{ data: D }> { return await this.fetchGraphQL<{ data: D }>(query, variables ?? {}, signal); } + + async batch( + query: string, + variables: V[], + signal?: AbortSignal, + ): Promise<{ data: D }[]> { + const results: { data: D }[] = []; + const chunks: V[][] = chunk(variables, this.batchSize); + + for (const chunk of chunks) { + const responses = await Promise.all( + chunk.map((variables) => this.request(query, variables, signal)), + ); + results.push(...responses); + } + + return results; + } } /** diff --git a/packages/providers/src/ports/CompassProvider/CompassProvider.ts b/packages/providers/src/ports/CompassProvider/CompassProvider.ts index c851724e..5f67cd5b 100644 --- a/packages/providers/src/ports/CompassProvider/CompassProvider.ts +++ b/packages/providers/src/ports/CompassProvider/CompassProvider.ts @@ -14,10 +14,31 @@ export interface CompassProvider { * @param {AbortSignal} signal - When given, the request can be aborted with its controller. * @returns {Promise<{ data: D }>} A Promise that resolves with the result of the query. * @template D - The type of the result's data. + * @template V - The type of the query's variables. */ request( query: string, variables?: null | undefined | V, signal?: AbortSignal, ): Promise<{ data: D }>; + + /** + * Batch fetch data from compass and return the deserialized responses. This + * is used to make many fetch calls at once, but in batches to avoid + * overloading the server. + * + * @function + * @name CompassProvider#batch + * @param {string} query - The query string to execute. + * @param {{ readonly [variable: string]: unknown }[]} variables - The variables to pass with the each query. + * @param {AbortSignal} [signal] - When given, the requests can be aborted with its controller. + * @returns {Promise<{ data: D }[]>} A Promise that resolves with the results of the queries. + * @template D - The type of the result's data. + * @template V - The type of the query's variables. + */ + batch( + query: string, + variables: V[], + signal?: AbortSignal, + ): Promise<{ data: D }[]>; } diff --git a/packages/providers/test/PoapCompass.spec.ts b/packages/providers/test/PoapCompass.spec.ts index 4a6da726..7975d913 100644 --- a/packages/providers/test/PoapCompass.spec.ts +++ b/packages/providers/test/PoapCompass.spec.ts @@ -1,86 +1,205 @@ -import { mock } from 'node:test'; +import { enableFetchMocks } from 'jest-fetch-mock'; import { PoapCompass } from '../src/core/PoapCompass/PoapCompass'; import { CompassRequestError } from '../src/ports/CompassProvider/errors/CompassRequestError'; import { CompassMissingDataError } from '../src/ports/CompassProvider/errors/CompassMissingDataError'; +import { CompassBadRequestError } from '../src/ports/CompassProvider/errors/CompassBadRequestError'; +import { CompassUnauthorizedError } from '../src/ports/CompassProvider/errors/CompassUnauthorizedError'; + +enableFetchMocks(); describe('PoapCompass', () => { - let apiKey: string; + let poapCompass: PoapCompass; beforeEach(() => { - apiKey = 'test-api-key'; + poapCompass = new PoapCompass({ + apiKey: 'test', + }); }); - it('should execute a GraphQL query successfully', async () => { - const query = 'query { test }'; - const variables = { key: 'value' }; - const responseData = { data: { test: 'result' } }; + describe('request', () => { + it('should execute a GraphQL query successfully', async () => { + const query = 'query { test }'; + const variables = { key: 'value' }; + const responseData = { data: { test: 'result' } }; + + fetchMock.mockOnce(() => + Promise.resolve({ + ok: true, + status: 200, + body: JSON.stringify(responseData), + }), + ); + + const result = await poapCompass.request(query, variables); - mock.method(global, 'fetch', () => { - return Promise.resolve({ - ok: true, - status: 200, - json: () => Promise.resolve(responseData), + expect(result).toEqual(responseData); + }); + + it('should allow changing the base url', async () => { + const query = 'query { test }'; + const variables = { key: 'value' }; + const responseData = { data: { test: 'result' } }; + + fetchMock.mockOnce(() => + Promise.resolve({ + ok: true, + status: 200, + body: JSON.stringify(responseData), + }), + ); + + poapCompass = new PoapCompass({ + baseUrl: 'another-compass.poap.tech', + apiKey: 'test', }); + + const result = await poapCompass.request(query, variables); + + expect(result).toEqual(responseData); }); - const poapCompass = new PoapCompass({ apiKey }); - const result = await poapCompass.request(query, variables); + it('should allow passing no variables', async () => { + const query = 'query { test }'; + const responseData = { data: { test: 'result' } }; - expect(result).toEqual(responseData); - }); + fetchMock.mockOnce(() => + Promise.resolve({ + ok: true, + status: 200, + body: JSON.stringify(responseData), + }), + ); - it('should throw an error when the API returns no data', async () => { - const query = 'query { test }'; - const variables = { key: 'value' }; - const responseData = {}; + const result = await poapCompass.request(query); - mock.method(global, 'fetch', () => { - return Promise.resolve({ - ok: true, - status: 200, - json: () => Promise.resolve(responseData), - }); + expect(result).toEqual(responseData); }); - const poapCompass = new PoapCompass({ apiKey }); + it('should throw an error when the API returns no data', async () => { + const query = 'query { test }'; + const variables = { key: 'value' }; + const responseData = {}; + + fetchMock.mockOnce(() => + Promise.resolve({ + ok: true, + status: 200, + body: JSON.stringify(responseData), + }), + ); + + await expect(poapCompass.request(query, variables)).rejects.toThrow( + CompassMissingDataError, + ); + }); - await expect(poapCompass.request(query, variables)).rejects.toThrow( - CompassMissingDataError, - ); - }); + it('should throw an error when the API returns an error', async () => { + const query = 'query { test }'; + const variables = { key: 'value' }; + const responseData = { errors: [{ message: 'Error message' }] }; + + fetchMock.mockOnce(() => + Promise.resolve({ + ok: true, + status: 200, + body: JSON.stringify(responseData), + }), + ); + + await expect(poapCompass.request(query, variables)).rejects.toThrow( + CompassRequestError, + ); + }); - it('should throw an error when the API returns an error', async () => { - const query = 'query { test }'; - const variables = { key: 'value' }; - const responseData = { errors: [{ message: 'Error message' }] }; + it('should throw a bad request error when the query is invalid', async () => { + const query = 'query { invalid_query }'; + const variables = { key: 'value' }; - mock.method(global, 'fetch', () => { - return Promise.resolve({ - ok: true, - status: 200, - json: () => Promise.resolve(responseData), - }); + fetchMock.mockOnce(() => + Promise.resolve({ + ok: true, + status: 400, + }), + ); + + await expect(poapCompass.request(query, variables)).rejects.toThrow( + CompassBadRequestError, + ); }); - const poapCompass = new PoapCompass({ apiKey }); + it('should throw an unauthorized error when the api key is invalid', async () => { + const query = 'query { test }'; + const variables = { key: 'value' }; - await expect(poapCompass.request(query, variables)).rejects.toThrow( - CompassRequestError, - ); - }); + fetchMock.mockOnce(() => + Promise.resolve({ + ok: true, + status: 401, + }), + ); - it('should throw a network error when the request fails', async () => { - const query = 'query { test }'; - const variables = { key: 'value' }; + poapCompass = new PoapCompass({ + apiKey: 'invalid', + }); - mock.method(global, 'fetch', () => { - return Promise.reject(new Error('Network error')); + await expect(poapCompass.request(query, variables)).rejects.toThrow( + CompassUnauthorizedError, + ); }); - const poapCompass = new PoapCompass({ apiKey }); + it('should throw an unknown error when the response is from an unknonw status code', async () => { + const query = 'query { test }'; + const variables = { key: 'value' }; + + fetchMock.mockOnce(() => + Promise.resolve({ + ok: true, + status: 100, + }), + ); + + await expect(poapCompass.request(query, variables)).rejects.toThrow( + /Response error/, + ); + }); - await expect(poapCompass.request(query, variables)).rejects.toThrow( - /Network error/, - ); + it('should throw a network error when the request fails', async () => { + const query = 'query { test }'; + const variables = { key: 'value' }; + + fetchMock.mockOnce(() => Promise.reject(new Error('Network error'))); + + await expect(poapCompass.request(query, variables)).rejects.toThrow( + /Network error/, + ); + }); + }); + + describe('batch', () => { + it('should execute many GraphQL requests', async () => { + const query = 'query { test }'; + const variables = [{ key: 'value-0' }, { key: 'value-1' }]; + const responseData0 = { data: { test: 'result-0' } }; + const responseData1 = { data: { test: 'result-1' } }; + + fetchMock.mockOnce(() => + Promise.resolve({ + ok: true, + status: 200, + body: JSON.stringify(responseData0), + }), + ); + fetchMock.mockOnce(() => + Promise.resolve({ + ok: true, + status: 200, + body: JSON.stringify(responseData1), + }), + ); + + const result = await poapCompass.batch(query, variables); + + expect(result).toEqual([responseData0, responseData1]); + }); }); }); diff --git a/packages/utils/package.json b/packages/utils/package.json index fc9aa7d2..0f0050b8 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,6 +1,6 @@ { "name": "@poap-xyz/utils", - "version": "0.2.4", + "version": "0.2.5", "description": "Utils module for the poap.js library", "main": "dist/cjs/index.cjs", "module": "dist/esm/index.mjs", diff --git a/yarn.lock b/yarn.lock index 58aceae7..e060ab4c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -884,8 +884,8 @@ __metadata: version: 0.0.0-use.local resolution: "@poap-xyz/drops@workspace:packages/drops" dependencies: - "@poap-xyz/providers": 0.2.4 - "@poap-xyz/utils": 0.2.4 + "@poap-xyz/providers": 0.2.5 + "@poap-xyz/utils": 0.2.5 languageName: unknown linkType: soft @@ -901,8 +901,8 @@ __metadata: version: 0.0.0-use.local resolution: "@poap-xyz/moments@workspace:packages/moments" dependencies: - "@poap-xyz/providers": 0.2.4 - "@poap-xyz/utils": 0.2.4 + "@poap-xyz/providers": 0.2.5 + "@poap-xyz/utils": 0.2.5 "@types/uuid": ^9.0.2 uuid: ^9.0.0 languageName: unknown @@ -912,22 +912,24 @@ __metadata: version: 0.0.0-use.local resolution: "@poap-xyz/poaps@workspace:packages/poaps" dependencies: - "@poap-xyz/providers": 0.2.4 - "@poap-xyz/utils": 0.2.4 + "@poap-xyz/providers": 0.2.5 + "@poap-xyz/utils": 0.2.5 languageName: unknown linkType: soft -"@poap-xyz/providers@*, @poap-xyz/providers@0.2.4, @poap-xyz/providers@workspace:packages/providers": +"@poap-xyz/providers@*, @poap-xyz/providers@0.2.5, @poap-xyz/providers@workspace:packages/providers": version: 0.0.0-use.local resolution: "@poap-xyz/providers@workspace:packages/providers" dependencies: - "@poap-xyz/utils": 0.2.4 + "@poap-xyz/utils": 0.2.5 axios: ^1.3.5 axios-mock-adapter: ^1.21.4 + jest-fetch-mock: ^3.0.3 + lodash.chunk: ^4.2.0 languageName: unknown linkType: soft -"@poap-xyz/utils@*, @poap-xyz/utils@0.2.4, @poap-xyz/utils@workspace:packages/utils": +"@poap-xyz/utils@*, @poap-xyz/utils@0.2.5, @poap-xyz/utils@workspace:packages/utils": version: 0.0.0-use.local resolution: "@poap-xyz/utils@workspace:packages/utils" languageName: unknown @@ -4962,6 +4964,13 @@ __metadata: languageName: node linkType: hard +"lodash.chunk@npm:^4.2.0": + version: 4.2.0 + resolution: "lodash.chunk@npm:4.2.0" + checksum: 6286c6d06814fbeda502164015c42ef53a9194e6ebaac52ec2b41e83344aefe7bc3d94fdfec525adcd2c66cefdf05dc333b6a1128e4de739797342315c17cbc7 + languageName: node + linkType: hard + "lodash.memoize@npm:4.x": version: 4.1.2 resolution: "lodash.memoize@npm:4.1.2"