From 916d489b7ca78156fa7ce5e40ba8bc9ff427bc47 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:02:06 +0000 Subject: [PATCH 1/7] chore(internal): bump cross-spawn to v7.0.6 (#28) Note: it is a dev transitive dependency. --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index bfd47d8..bb17942 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1370,9 +1370,9 @@ create-require@^1.1.0: integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== cross-spawn@^7.0.2, cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" From c35bfa6f9afaad73143e18ccd4b91c013a081f7f Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:15:37 +0000 Subject: [PATCH 2/7] chore(types): nicer error class types + jsdocs (#30) --- src/error.ts | 64 ++++++++++++++++++++-------------------------------- 1 file changed, 24 insertions(+), 40 deletions(-) diff --git a/src/error.ts b/src/error.ts index 82ad07b..b091361 100644 --- a/src/error.ts +++ b/src/error.ts @@ -4,17 +4,19 @@ import { castToError, Headers } from './core'; export class SteelError extends Error {} -export class APIError extends SteelError { - readonly status: number | undefined; - readonly headers: Headers | undefined; - readonly error: Object | undefined; - - constructor( - status: number | undefined, - error: Object | undefined, - message: string | undefined, - headers: Headers | undefined, - ) { +export class APIError< + TStatus extends number | undefined = number | undefined, + THeaders extends Headers | undefined = Headers | undefined, + TError extends Object | undefined = Object | undefined, +> extends SteelError { + /** HTTP status for the response that caused the error */ + readonly status: TStatus; + /** HTTP headers for the response that caused the error */ + readonly headers: THeaders; + /** JSON body of the response that caused the error */ + readonly error: TError; + + constructor(status: TStatus, error: TError, message: string | undefined, headers: THeaders) { super(`${APIError.makeMessage(status, error, message)}`); this.status = status; this.headers = headers; @@ -48,7 +50,7 @@ export class APIError extends SteelError { message: string | undefined, headers: Headers | undefined, ): APIError { - if (!status) { + if (!status || !headers) { return new APIConnectionError({ message, cause: castToError(errorResponse) }); } @@ -90,17 +92,13 @@ export class APIError extends SteelError { } } -export class APIUserAbortError extends APIError { - override readonly status: undefined = undefined; - +export class APIUserAbortError extends APIError { constructor({ message }: { message?: string } = {}) { super(undefined, undefined, message || 'Request was aborted.', undefined); } } -export class APIConnectionError extends APIError { - override readonly status: undefined = undefined; - +export class APIConnectionError extends APIError { constructor({ message, cause }: { message?: string | undefined; cause?: Error | undefined }) { super(undefined, undefined, message || 'Connection error.', undefined); // in some environments the 'cause' property is already declared @@ -115,32 +113,18 @@ export class APIConnectionTimeoutError extends APIConnectionError { } } -export class BadRequestError extends APIError { - override readonly status: 400 = 400; -} +export class BadRequestError extends APIError<400, Headers> {} -export class AuthenticationError extends APIError { - override readonly status: 401 = 401; -} +export class AuthenticationError extends APIError<401, Headers> {} -export class PermissionDeniedError extends APIError { - override readonly status: 403 = 403; -} +export class PermissionDeniedError extends APIError<403, Headers> {} -export class NotFoundError extends APIError { - override readonly status: 404 = 404; -} +export class NotFoundError extends APIError<404, Headers> {} -export class ConflictError extends APIError { - override readonly status: 409 = 409; -} +export class ConflictError extends APIError<409, Headers> {} -export class UnprocessableEntityError extends APIError { - override readonly status: 422 = 422; -} +export class UnprocessableEntityError extends APIError<422, Headers> {} -export class RateLimitError extends APIError { - override readonly status: 429 = 429; -} +export class RateLimitError extends APIError<429, Headers> {} -export class InternalServerError extends APIError {} +export class InternalServerError extends APIError {} From d417931a5ac720d8131e8955b050a7473daee5e6 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:06:29 +0000 Subject: [PATCH 3/7] chore(internal): codegen related update (#32) --- src/core.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core.ts b/src/core.ts index 0e8cd8b..22dbd5e 100644 --- a/src/core.ts +++ b/src/core.ts @@ -976,8 +976,8 @@ export const safeJSON = (text: string) => { } }; -// https://stackoverflow.com/a/19709846 -const startsWithSchemeRegexp = new RegExp('^(?:[a-z]+:)?//', 'i'); +// https://url.spec.whatwg.org/#url-scheme-string +const startsWithSchemeRegexp = /^[a-z][a-z0-9+.-]*:/i; const isAbsoluteURL = (url: string): boolean => { return startsWithSchemeRegexp.test(url); }; From fc64e4a7163c2ecfd5469c4480832a768f1c02cb Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:33:26 +0000 Subject: [PATCH 4/7] chore(internal): fix some typos (#33) --- src/core.ts | 4 ++-- tests/index.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core.ts b/src/core.ts index 22dbd5e..8f06b13 100644 --- a/src/core.ts +++ b/src/core.ts @@ -163,7 +163,7 @@ export abstract class APIClient { maxRetries = 2, timeout = 60000, // 1 minute httpAgent, - fetch: overridenFetch, + fetch: overriddenFetch, }: { baseURL: string; maxRetries?: number | undefined; @@ -176,7 +176,7 @@ export abstract class APIClient { this.timeout = validatePositiveInteger('timeout', timeout); this.httpAgent = httpAgent; - this.fetch = overridenFetch ?? fetch; + this.fetch = overriddenFetch ?? fetch; } protected authHeaders(opts: FinalRequestOptions): Headers { diff --git a/tests/index.test.ts b/tests/index.test.ts index 4482b0b..26d5f92 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -183,7 +183,7 @@ describe('instantiate client', () => { expect(client.steelAPIKey).toBe('My Steel API Key'); }); - test('with overriden environment variable arguments', () => { + test('with overridden environment variable arguments', () => { // set options via env var process.env['STEEL_API_KEY'] = 'another My Steel API Key'; const client = new Steel({ steelAPIKey: 'My Steel API Key' }); From b06e0b885e1cc4f112dc2b91e0bba80aaf31444b Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:32:50 +0000 Subject: [PATCH 5/7] codegen metadata --- .stats.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.stats.yml b/.stats.yml index e425cd4..6e991c2 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,2 +1,2 @@ configured_endpoints: 6 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/nen-labs%2Fsteel-514e0d3ea8fff8f5a712a5a8f780dfef6caf0c8ae99f472fd895dbd7c1eab6a4.yml +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/nen-labs%2Fsteel-73f12c3021fc56466b869b17a82fec982131d51e383151c897908d874bc188da.yml From 51a7bbe4d8be15b77f7831f5da74962212e9b29e Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:38:40 +0000 Subject: [PATCH 6/7] feat(api): api update (#34) --- .stats.yml | 2 +- api.md | 14 ++ src/index.ts | 45 ++++++ src/resources/index.ts | 8 + src/resources/top-level.ts | 214 ++++++++++++++++++++++++++ tests/api-resources/top-level.test.ts | 68 ++++++++ 6 files changed, 350 insertions(+), 1 deletion(-) create mode 100644 src/resources/top-level.ts create mode 100644 tests/api-resources/top-level.test.ts diff --git a/.stats.yml b/.stats.yml index 6e991c2..ff3783d 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,2 +1,2 @@ -configured_endpoints: 6 +configured_endpoints: 9 openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/nen-labs%2Fsteel-73f12c3021fc56466b869b17a82fec982131d51e383151c897908d874bc188da.yml diff --git a/api.md b/api.md index ab2d301..328188a 100644 --- a/api.md +++ b/api.md @@ -1,3 +1,17 @@ +# Steel + +Types: + +- PdfResponse +- ScrapeResponse +- ScreenshotResponse + +Methods: + +- client.pdf({ ...params }) -> PdfResponse +- client.scrape({ ...params }) -> ScrapeResponse +- client.screenshot({ ...params }) -> ScreenshotResponse + # Sessions Types: diff --git a/src/index.ts b/src/index.ts index 9e7645c..b02f68a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,15 @@ import * as Pagination from './pagination'; import { type SessionsCursorParams, SessionsCursorResponse } from './pagination'; import * as Uploads from './uploads'; import * as API from './resources/index'; +import * as TopLevelAPI from './resources/top-level'; +import { + PdfParams, + PdfResponse, + ScrapeParams, + ScrapeResponse, + ScreenshotParams, + ScreenshotResponse, +} from './resources/top-level'; import { Session, SessionContext, @@ -137,6 +146,33 @@ export class Steel extends Core.APIClient { sessions: API.Sessions = new API.Sessions(this); + /** + * Generates a PDF from a specified webpage. + */ + pdf(body: TopLevelAPI.PdfParams, options?: Core.RequestOptions): Core.APIPromise { + return this.post('/v1/pdf', { body, ...options }); + } + + /** + * Extracts content from a specified URL. + */ + scrape( + body: TopLevelAPI.ScrapeParams, + options?: Core.RequestOptions, + ): Core.APIPromise { + return this.post('/v1/scrape', { body, ...options }); + } + + /** + * Captures a screenshot of a specified webpage. + */ + screenshot( + body: TopLevelAPI.ScreenshotParams, + options?: Core.RequestOptions, + ): Core.APIPromise { + return this.post('/v1/screenshot', { body, ...options }); + } + protected override defaultQuery(): Core.DefaultQuery | undefined { return this._options.defaultQuery; } @@ -184,6 +220,15 @@ export declare namespace Steel { type SessionsCursorResponse as SessionsCursorResponse, }; + export { + type PdfResponse as PdfResponse, + type ScrapeResponse as ScrapeResponse, + type ScreenshotResponse as ScreenshotResponse, + type PdfParams as PdfParams, + type ScrapeParams as ScrapeParams, + type ScreenshotParams as ScreenshotParams, + }; + export { Sessions as Sessions, type Session as Session, diff --git a/src/resources/index.ts b/src/resources/index.ts index 5c4c83f..0586b0d 100644 --- a/src/resources/index.ts +++ b/src/resources/index.ts @@ -14,3 +14,11 @@ export { type SessionReleaseParams, type SessionReleaseAllParams, } from './sessions'; +export { + type PdfResponse, + type ScrapeResponse, + type ScreenshotResponse, + type PdfParams, + type ScrapeParams, + type ScreenshotParams, +} from './top-level'; diff --git a/src/resources/top-level.ts b/src/resources/top-level.ts new file mode 100644 index 0000000..469536c --- /dev/null +++ b/src/resources/top-level.ts @@ -0,0 +1,214 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +export interface PdfResponse { + /** + * URL where the PDF is hosted + */ + url: string; +} + +/** + * Response from a successful scrape request + */ +export interface ScrapeResponse { + content: ScrapeResponse.Content; + + links: Array; + + metadata: ScrapeResponse.Metadata; + + pdf?: ScrapeResponse.Pdf; + + screenshot?: ScrapeResponse.Screenshot; +} + +export namespace ScrapeResponse { + export interface Content { + /** + * Cleaned HTML content of the webpage + */ + cleaned_html?: string; + + /** + * Raw HTML content of the webpage + */ + html?: string; + + /** + * Webpage content converted to Markdown + */ + markdown?: string; + + /** + * Webpage content in Readability format + */ + readability?: Record; + } + + export interface Link { + /** + * Text content of the link + */ + text: string; + + /** + * URL of the link + */ + url: string; + } + + export interface Metadata { + /** + * HTTP status code of the response + */ + statusCode: number; + + /** + * Description of the webpage + */ + description?: string; + + /** + * Detected language of the webpage + */ + language?: string; + + /** + * Open Graph description + */ + ogDescription?: string; + + /** + * Open Graph image URL + */ + ogImage?: string; + + /** + * Open Graph title + */ + ogTitle?: string; + + /** + * Publication timestamp of the content (if available) + */ + published_timestamp?: string; + + /** + * Timestamp when the scrape was performed + */ + timestamp?: string; + + /** + * Title of the webpage + */ + title?: string; + + /** + * Source URL of the scraped page + */ + urlSource?: string; + } + + export interface Pdf { + /** + * URL of the generated PDF + */ + url: string; + } + + export interface Screenshot { + /** + * URL of the screenshot image + */ + url: string; + } +} + +export interface ScreenshotResponse { + /** + * URL where the screenshot is hosted + */ + url: string; +} + +export interface PdfParams { + /** + * URL of the webpage to convert to PDF + */ + url: string; + + /** + * Delay before generating the PDF (in milliseconds) + */ + delay?: number; + + /** + * Use a Steel-provided residential proxy for generating the PDF + */ + useProxy?: boolean; +} + +export interface ScrapeParams { + /** + * URL of the webpage to scrape + */ + url: string; + + /** + * Delay before scraping (in milliseconds) + */ + delay?: number; + + /** + * Desired format(s) for the scraped content. Default is `html`. + */ + format?: Array<'html' | 'readability' | 'cleaned_html' | 'markdown'>; + + /** + * Include a PDF in the response + */ + pdf?: boolean; + + /** + * Include a screenshot in the response + */ + screenshot?: boolean; + + /** + * Use a Steel-provided residential proxy for the scrape + */ + useProxy?: boolean; +} + +export interface ScreenshotParams { + /** + * URL of the webpage to capture + */ + url: string; + + /** + * Delay before capturing the screenshot (in milliseconds) + */ + delay?: number; + + /** + * Capture the full page screenshot. Default is `false`. + */ + fullPage?: boolean; + + /** + * Use a Steel-provided residential proxy for capturing the screenshot + */ + useProxy?: boolean; +} + +export declare namespace TopLevel { + export { + type PdfResponse as PdfResponse, + type ScrapeResponse as ScrapeResponse, + type ScreenshotResponse as ScreenshotResponse, + type PdfParams as PdfParams, + type ScrapeParams as ScrapeParams, + type ScreenshotParams as ScreenshotParams, + }; +} diff --git a/tests/api-resources/top-level.test.ts b/tests/api-resources/top-level.test.ts new file mode 100644 index 0000000..fa65ff7 --- /dev/null +++ b/tests/api-resources/top-level.test.ts @@ -0,0 +1,68 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +import Steel from 'steel-sdk'; +import { Response } from 'node-fetch'; + +const client = new Steel({ + steelAPIKey: 'My Steel API Key', + baseURL: process.env['TEST_API_BASE_URL'] ?? 'http://127.0.0.1:4010', +}); + +describe('top level methods', () => { + test('pdf: only required params', async () => { + const responsePromise = client.pdf({ url: 'https://example.com' }); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('pdf: required and optional params', async () => { + const response = await client.pdf({ url: 'https://example.com', delay: 0, useProxy: true }); + }); + + test('scrape: only required params', async () => { + const responsePromise = client.scrape({ url: 'https://example.com' }); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('scrape: required and optional params', async () => { + const response = await client.scrape({ + url: 'https://example.com', + delay: 0, + format: ['html'], + pdf: true, + screenshot: true, + useProxy: true, + }); + }); + + test('screenshot: only required params', async () => { + const responsePromise = client.screenshot({ url: 'https://example.com' }); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('screenshot: required and optional params', async () => { + const response = await client.screenshot({ + url: 'https://example.com', + delay: 0, + fullPage: true, + useProxy: true, + }); + }); +}); From 6651c08efbaf24e8bed61298021e9f4b209564fc Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:38:59 +0000 Subject: [PATCH 7/7] release: 0.1.0-beta.6 --- .release-please-manifest.json | 2 +- CHANGELOG.md | 16 ++++++++++++++++ package.json | 2 +- src/version.ts | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 4f5d6fd..5c9e1b3 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.1.0-beta.5" + ".": "0.1.0-beta.6" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 95fff45..c4e35bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 0.1.0-beta.6 (2024-12-19) + +Full Changelog: [v0.1.0-beta.5...v0.1.0-beta.6](https://github.com/steel-dev/steel-node/compare/v0.1.0-beta.5...v0.1.0-beta.6) + +### Features + +* **api:** api update ([#34](https://github.com/steel-dev/steel-node/issues/34)) ([51a7bbe](https://github.com/steel-dev/steel-node/commit/51a7bbe4d8be15b77f7831f5da74962212e9b29e)) + + +### Chores + +* **internal:** bump cross-spawn to v7.0.6 ([#28](https://github.com/steel-dev/steel-node/issues/28)) ([916d489](https://github.com/steel-dev/steel-node/commit/916d489b7ca78156fa7ce5e40ba8bc9ff427bc47)) +* **internal:** codegen related update ([#32](https://github.com/steel-dev/steel-node/issues/32)) ([d417931](https://github.com/steel-dev/steel-node/commit/d417931a5ac720d8131e8955b050a7473daee5e6)) +* **internal:** fix some typos ([#33](https://github.com/steel-dev/steel-node/issues/33)) ([fc64e4a](https://github.com/steel-dev/steel-node/commit/fc64e4a7163c2ecfd5469c4480832a768f1c02cb)) +* **types:** nicer error class types + jsdocs ([#30](https://github.com/steel-dev/steel-node/issues/30)) ([c35bfa6](https://github.com/steel-dev/steel-node/commit/c35bfa6f9afaad73143e18ccd4b91c013a081f7f)) + ## 0.1.0-beta.5 (2024-12-03) Full Changelog: [v0.1.0-beta.4...v0.1.0-beta.5](https://github.com/steel-dev/steel-node/compare/v0.1.0-beta.4...v0.1.0-beta.5) diff --git a/package.json b/package.json index 9771102..f4b848c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "steel-sdk", - "version": "0.1.0-beta.5", + "version": "0.1.0-beta.6", "description": "The official TypeScript library for the Steel API", "author": "Steel ", "types": "dist/index.d.ts", diff --git a/src/version.ts b/src/version.ts index 509a910..3ce249a 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const VERSION = '0.1.0-beta.5'; // x-release-please-version +export const VERSION = '0.1.0-beta.6'; // x-release-please-version