From 3e1cfa77356fd773581a2d351168c0333d4ffec3 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Sun, 3 Nov 2024 01:39:24 +0000 Subject: [PATCH] feat(api): api update (#6) --- .gitignore | 2 +- .stats.yml | 4 +- README.md | 49 +++--- api.md | 12 -- scripts/build | 2 +- src/core.ts | 10 +- src/error.ts | 2 +- src/index.ts | 73 ++------- src/resources/index.ts | 8 - src/resources/sessions.ts | 8 +- src/resources/top-level.ts | 213 +------------------------- tests/api-resources/sessions.test.ts | 2 +- tests/api-resources/top-level.test.ts | 68 -------- tsconfig.deno.json | 11 +- tsconfig.json | 1 + 15 files changed, 53 insertions(+), 412 deletions(-) delete mode 100644 tests/api-resources/top-level.test.ts diff --git a/.gitignore b/.gitignore index 3eed6dd..d98d51a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ yarn-error.log codegen.log Brewfile.lock.json dist -/deno +dist-deno /*.tgz .idea/ diff --git a/.stats.yml b/.stats.yml index efcbe8e..ee11233 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,2 +1,2 @@ -configured_endpoints: 9 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/nen-labs%2Fsteel-17f11d5403c9a4e9b70c9695f5289d851a195e9d33f182156dde77087c10d21f.yml +configured_endpoints: 6 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/nen-labs%2Fsteel-e2f35e13822cd958a10f588d91b72e19c31c4c6ad0f5c1fe7c8d13b8f03e6843.yml diff --git a/README.md b/README.md index aaf8304..40d7fd0 100644 --- a/README.md +++ b/README.md @@ -48,11 +48,7 @@ const client = new Steel({ }); async function main() { - const params: Steel.ScrapeParams = { - url: 'https://www.eff.org/cyberspace-independence', - format: ['markdown'], - }; - const response: Steel.ScrapeResponse = await client.scrape(params); + const session: Steel.Session = await client.sessions.create(); } main(); @@ -69,17 +65,15 @@ a subclass of `APIError` will be thrown: ```ts async function main() { - const response = await client - .scrape({ url: 'https://www.eff.org/cyberspace-independence', format: ['markdown'] }) - .catch(async (err) => { - if (err instanceof Steel.APIError) { - console.log(err.status); // 400 - console.log(err.name); // BadRequestError - console.log(err.headers); // {server: 'nginx', ...} - } else { - throw err; - } - }); + const session = await client.sessions.create().catch(async (err) => { + if (err instanceof Steel.APIError) { + console.log(err.status); // 400 + console.log(err.name); // BadRequestError + console.log(err.headers); // {server: 'nginx', ...} + } else { + throw err; + } + }); } main(); @@ -114,7 +108,7 @@ const client = new Steel({ }); // Or, configure per-request: -await client.scrape({ url: 'https://www.eff.org/cyberspace-independence', format: ['markdown'] }, { +await client.sessions.create({ maxRetries: 5, }); ``` @@ -131,7 +125,7 @@ const client = new Steel({ }); // Override per-request: -await client.scrape({ url: 'https://www.eff.org/cyberspace-independence', format: ['markdown'] }, { +await client.sessions.create({ timeout: 5 * 1000, }); ``` @@ -183,17 +177,13 @@ You can also use the `.withResponse()` method to get the raw `Response` along wi ```ts const client = new Steel(); -const response = await client - .scrape({ url: 'https://www.eff.org/cyberspace-independence', format: ['markdown'] }) - .asResponse(); +const response = await client.sessions.create().asResponse(); console.log(response.headers.get('X-My-Header')); console.log(response.statusText); // access the underlying Response object -const { data: response, response: raw } = await client - .scrape({ url: 'https://www.eff.org/cyberspace-independence', format: ['markdown'] }) - .withResponse(); +const { data: session, response: raw } = await client.sessions.create().withResponse(); console.log(raw.headers.get('X-My-Header')); -console.log(response.content); +console.log(session.id); ``` ### Making custom/undocumented requests @@ -297,12 +287,9 @@ const client = new Steel({ }); // Override per-request: -await client.scrape( - { url: 'https://www.eff.org/cyberspace-independence', format: ['markdown'] }, - { - httpAgent: new http.Agent({ keepAlive: false }), - }, -); +await client.sessions.create({ + httpAgent: new http.Agent({ keepAlive: false }), +}); ``` ## Semantic versioning diff --git a/api.md b/api.md index 328188a..abc1e7a 100644 --- a/api.md +++ b/api.md @@ -1,17 +1,5 @@ # Steel -Types: - -- PdfResponse -- ScrapeResponse -- ScreenshotResponse - -Methods: - -- client.pdf({ ...params }) -> PdfResponse -- client.scrape({ ...params }) -> ScrapeResponse -- client.screenshot({ ...params }) -> ScreenshotResponse - # Sessions Types: diff --git a/scripts/build b/scripts/build index 6ad0468..1a24a46 100755 --- a/scripts/build +++ b/scripts/build @@ -50,7 +50,7 @@ node scripts/utils/postprocess-files.cjs (cd dist && node -e 'require("steel-sdk")') (cd dist && node -e 'import("steel-sdk")' --input-type=module) -if command -v deno &> /dev/null && [ -e ./scripts/build-deno ] +if [ -e ./scripts/build-deno ] then ./scripts/build-deno fi diff --git a/src/core.ts b/src/core.ts index 58272ea..0e8cd8b 100644 --- a/src/core.ts +++ b/src/core.ts @@ -396,7 +396,7 @@ export abstract class APIClient { error: Object | undefined, message: string | undefined, headers: Headers | undefined, - ) { + ): APIError { return APIError.generate(status, error, message, headers); } @@ -668,9 +668,9 @@ export abstract class AbstractPage implements AsyncIterable { return await this.#client.requestAPIList(this.constructor as any, nextOptions); } - async *iterPages() { + async *iterPages(): AsyncGenerator { // eslint-disable-next-line @typescript-eslint/no-this-alias - let page: AbstractPage = this; + let page: this = this; yield page; while (page.hasNextPage()) { page = await page.getNextPage(); @@ -678,7 +678,7 @@ export abstract class AbstractPage implements AsyncIterable { } } - async *[Symbol.asyncIterator]() { + async *[Symbol.asyncIterator](): AsyncGenerator { for await (const page of this.iterPages()) { for (const item of page.getPaginatedItems()) { yield item; @@ -721,7 +721,7 @@ export class PagePromise< * console.log(item) * } */ - async *[Symbol.asyncIterator]() { + async *[Symbol.asyncIterator](): AsyncGenerator { const page = await this; for await (const item of page) { yield item; diff --git a/src/error.ts b/src/error.ts index c8138b0..82ad07b 100644 --- a/src/error.ts +++ b/src/error.ts @@ -47,7 +47,7 @@ export class APIError extends SteelError { errorResponse: Object | undefined, message: string | undefined, headers: Headers | undefined, - ) { + ): APIError { if (!status) { return new APIConnectionError({ message, cause: castToError(errorResponse) }); } diff --git a/src/index.ts b/src/index.ts index 2df5089..4266ff5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,15 +7,6 @@ import * as Pagination from './pagination'; import { type SessionsCursorParams, SessionsCursorResponse } from './pagination'; import * as Uploads from './uploads'; import * as API from './resources/index'; -import * as TopLevelAPI from './resources/top-level'; -import { - PdfParams, - PdfResponse, - ScrapeParams, - ScrapeResponse, - ScreenshotParams, - ScreenshotResponse, -} from './resources/top-level'; import { Session, SessionContext, @@ -144,33 +135,6 @@ export class Steel extends Core.APIClient { sessions: API.Sessions = new API.Sessions(this); - /** - * Generates a PDF from a specified webpage. - */ - pdf(body: TopLevelAPI.PdfParams, options?: Core.RequestOptions): Core.APIPromise { - return this.post('/v1/pdf', { body, ...options }); - } - - /** - * Extracts content from a specified URL. - */ - scrape( - body: TopLevelAPI.ScrapeParams, - options?: Core.RequestOptions, - ): Core.APIPromise { - return this.post('/v1/scrape', { body, ...options }); - } - - /** - * Captures a screenshot of a specified webpage. - */ - screenshot( - body: TopLevelAPI.ScreenshotParams, - options?: Core.RequestOptions, - ): Core.APIPromise { - return this.post('/v1/screenshot', { body, ...options }); - } - protected override defaultQuery(): Core.DefaultQuery | undefined { return this._options.defaultQuery; } @@ -207,19 +171,21 @@ export class Steel extends Core.APIClient { static fileFromPath = Uploads.fileFromPath; } -export const SteelError = Errors.SteelError; -export const APIError = Errors.APIError; -export const APIConnectionError = Errors.APIConnectionError; -export const APIConnectionTimeoutError = Errors.APIConnectionTimeoutError; -export const APIUserAbortError = Errors.APIUserAbortError; -export const NotFoundError = Errors.NotFoundError; -export const ConflictError = Errors.ConflictError; -export const RateLimitError = Errors.RateLimitError; -export const BadRequestError = Errors.BadRequestError; -export const AuthenticationError = Errors.AuthenticationError; -export const InternalServerError = Errors.InternalServerError; -export const PermissionDeniedError = Errors.PermissionDeniedError; -export const UnprocessableEntityError = Errors.UnprocessableEntityError; +export { + SteelError, + APIError, + APIConnectionError, + APIConnectionTimeoutError, + APIUserAbortError, + NotFoundError, + ConflictError, + RateLimitError, + BadRequestError, + AuthenticationError, + InternalServerError, + PermissionDeniedError, + UnprocessableEntityError, +} from './error'; export import toFile = Uploads.toFile; export import fileFromPath = Uploads.fileFromPath; @@ -236,15 +202,6 @@ export declare namespace Steel { type SessionsCursorResponse as SessionsCursorResponse, }; - export { - type PdfResponse as PdfResponse, - type ScrapeResponse as ScrapeResponse, - type ScreenshotResponse as ScreenshotResponse, - type PdfParams as PdfParams, - type ScrapeParams as ScrapeParams, - type ScreenshotParams as ScreenshotParams, - }; - export { Sessions as Sessions, type Session as Session, diff --git a/src/resources/index.ts b/src/resources/index.ts index 37c4c10..165f2d9 100644 --- a/src/resources/index.ts +++ b/src/resources/index.ts @@ -12,11 +12,3 @@ export { type SessionCreateParams, type SessionListParams, } from './sessions'; -export { - type PdfResponse, - type ScrapeResponse, - type ScreenshotResponse, - type PdfParams, - type ScrapeParams, - type ScreenshotParams, -} from './top-level'; diff --git a/src/resources/sessions.ts b/src/resources/sessions.ts index 7d6b751..a37e851 100644 --- a/src/resources/sessions.ts +++ b/src/resources/sessions.ts @@ -403,14 +403,14 @@ export interface SessionCreateParams { sessionId?: string; /** - * Session timeout duration in milliseconds. Default is 900000 (15 minutes). + * Enable automatic captcha solving. Default is false. */ - sessionTimeout?: number; + solveCaptcha?: boolean; /** - * Enable automatic captcha solving. Default is false. + * Session timeout duration in milliseconds. Default is 900000 (15 minutes). */ - solveCaptcha?: boolean; + timeout?: number; /** * Enable Steel-provided residential proxy usage for the browser session. Default diff --git a/src/resources/top-level.ts b/src/resources/top-level.ts index 469536c..b7426a6 100644 --- a/src/resources/top-level.ts +++ b/src/resources/top-level.ts @@ -1,214 +1,3 @@ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -export interface PdfResponse { - /** - * URL where the PDF is hosted - */ - url: string; -} - -/** - * Response from a successful scrape request - */ -export interface ScrapeResponse { - content: ScrapeResponse.Content; - - links: Array; - - metadata: ScrapeResponse.Metadata; - - pdf?: ScrapeResponse.Pdf; - - screenshot?: ScrapeResponse.Screenshot; -} - -export namespace ScrapeResponse { - export interface Content { - /** - * Cleaned HTML content of the webpage - */ - cleaned_html?: string; - - /** - * Raw HTML content of the webpage - */ - html?: string; - - /** - * Webpage content converted to Markdown - */ - markdown?: string; - - /** - * Webpage content in Readability format - */ - readability?: Record; - } - - export interface Link { - /** - * Text content of the link - */ - text: string; - - /** - * URL of the link - */ - url: string; - } - - export interface Metadata { - /** - * HTTP status code of the response - */ - statusCode: number; - - /** - * Description of the webpage - */ - description?: string; - - /** - * Detected language of the webpage - */ - language?: string; - - /** - * Open Graph description - */ - ogDescription?: string; - - /** - * Open Graph image URL - */ - ogImage?: string; - - /** - * Open Graph title - */ - ogTitle?: string; - - /** - * Publication timestamp of the content (if available) - */ - published_timestamp?: string; - - /** - * Timestamp when the scrape was performed - */ - timestamp?: string; - - /** - * Title of the webpage - */ - title?: string; - - /** - * Source URL of the scraped page - */ - urlSource?: string; - } - - export interface Pdf { - /** - * URL of the generated PDF - */ - url: string; - } - - export interface Screenshot { - /** - * URL of the screenshot image - */ - url: string; - } -} - -export interface ScreenshotResponse { - /** - * URL where the screenshot is hosted - */ - url: string; -} - -export interface PdfParams { - /** - * URL of the webpage to convert to PDF - */ - url: string; - - /** - * Delay before generating the PDF (in milliseconds) - */ - delay?: number; - - /** - * Use a Steel-provided residential proxy for generating the PDF - */ - useProxy?: boolean; -} - -export interface ScrapeParams { - /** - * URL of the webpage to scrape - */ - url: string; - - /** - * Delay before scraping (in milliseconds) - */ - delay?: number; - - /** - * Desired format(s) for the scraped content. Default is `html`. - */ - format?: Array<'html' | 'readability' | 'cleaned_html' | 'markdown'>; - - /** - * Include a PDF in the response - */ - pdf?: boolean; - - /** - * Include a screenshot in the response - */ - screenshot?: boolean; - - /** - * Use a Steel-provided residential proxy for the scrape - */ - useProxy?: boolean; -} - -export interface ScreenshotParams { - /** - * URL of the webpage to capture - */ - url: string; - - /** - * Delay before capturing the screenshot (in milliseconds) - */ - delay?: number; - - /** - * Capture the full page screenshot. Default is `false`. - */ - fullPage?: boolean; - - /** - * Use a Steel-provided residential proxy for capturing the screenshot - */ - useProxy?: boolean; -} - -export declare namespace TopLevel { - export { - type PdfResponse as PdfResponse, - type ScrapeResponse as ScrapeResponse, - type ScreenshotResponse as ScreenshotResponse, - type PdfParams as PdfParams, - type ScrapeParams as ScrapeParams, - type ScreenshotParams as ScreenshotParams, - }; -} +export {}; diff --git a/tests/api-resources/sessions.test.ts b/tests/api-resources/sessions.test.ts index bbd7420..18fb76f 100644 --- a/tests/api-resources/sessions.test.ts +++ b/tests/api-resources/sessions.test.ts @@ -40,8 +40,8 @@ describe('resource sessions', () => { localStorage: [{ foo: 'bar' }, { foo: 'bar' }, { foo: 'bar' }], }, sessionId: '182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e', - sessionTimeout: 0, solveCaptcha: true, + timeout: 0, useProxy: true, userAgent: 'userAgent', }, diff --git a/tests/api-resources/top-level.test.ts b/tests/api-resources/top-level.test.ts deleted file mode 100644 index 6380ffd..0000000 --- a/tests/api-resources/top-level.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -import Steel from 'steel-sdk'; -import { Response } from 'node-fetch'; - -const client = new Steel({ - steelAPIKey: 'My Steel API Key', - baseURL: process.env['TEST_API_BASE_URL'] ?? 'http://127.0.0.1:4010', -}); - -describe('top level methods', () => { - test('pdf: only required params', async () => { - const responsePromise = client.pdf({ url: 'https://example.com' }); - const rawResponse = await responsePromise.asResponse(); - expect(rawResponse).toBeInstanceOf(Response); - const response = await responsePromise; - expect(response).not.toBeInstanceOf(Response); - const dataAndResponse = await responsePromise.withResponse(); - expect(dataAndResponse.data).toBe(response); - expect(dataAndResponse.response).toBe(rawResponse); - }); - - test('pdf: required and optional params', async () => { - const response = await client.pdf({ url: 'https://example.com', delay: 0, useProxy: true }); - }); - - test('scrape: only required params', async () => { - const responsePromise = client.scrape({ url: 'https://example.com' }); - const rawResponse = await responsePromise.asResponse(); - expect(rawResponse).toBeInstanceOf(Response); - const response = await responsePromise; - expect(response).not.toBeInstanceOf(Response); - const dataAndResponse = await responsePromise.withResponse(); - expect(dataAndResponse.data).toBe(response); - expect(dataAndResponse.response).toBe(rawResponse); - }); - - test('scrape: required and optional params', async () => { - const response = await client.scrape({ - url: 'https://example.com', - delay: 0, - format: ['html', 'readability', 'cleaned_html'], - pdf: true, - screenshot: true, - useProxy: true, - }); - }); - - test('screenshot: only required params', async () => { - const responsePromise = client.screenshot({ url: 'https://example.com' }); - const rawResponse = await responsePromise.asResponse(); - expect(rawResponse).toBeInstanceOf(Response); - const response = await responsePromise; - expect(response).not.toBeInstanceOf(Response); - const dataAndResponse = await responsePromise.withResponse(); - expect(dataAndResponse.data).toBe(response); - expect(dataAndResponse.response).toBe(rawResponse); - }); - - test('screenshot: required and optional params', async () => { - const response = await client.screenshot({ - url: 'https://example.com', - delay: 0, - fullPage: true, - useProxy: true, - }); - }); -}); diff --git a/tsconfig.deno.json b/tsconfig.deno.json index 5ead520..849e070 100644 --- a/tsconfig.deno.json +++ b/tsconfig.deno.json @@ -1,19 +1,14 @@ { "extends": "./tsconfig.json", - "include": ["deno"], + "include": ["dist-deno"], "exclude": [], "compilerOptions": { - "rootDir": "./deno", + "rootDir": "./dist-deno", "lib": ["es2020", "DOM"], - "paths": { - "steel-sdk/_shims/auto/*": ["deno/_shims/auto/*-deno"], - "steel-sdk/*": ["deno/*"], - "steel-sdk": ["deno/index.ts"], - }, "noEmit": true, "declaration": true, "declarationMap": true, - "outDir": "deno", + "outDir": "dist-deno", "pretty": true, "sourceMap": true } diff --git a/tsconfig.json b/tsconfig.json index 9e8de53..44ac620 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -32,6 +32,7 @@ "noUncheckedIndexedAccess": true, "noImplicitOverride": true, "noPropertyAccessFromIndexSignature": true, + "isolatedModules": false, "skipLibCheck": true }