From 0d34161111f770140e9ec250ddaeb6199bd0ddfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Dr=C3=A4ger?= Date: Tue, 29 Jun 2021 23:12:23 +0200 Subject: [PATCH] #153: introduce 'response' function and add deprecation marker on 'expect' and 'extrqct' function --- README.md | 22 ++++---- examples/android/README.md | 2 +- .../main/kotlin/it/skrape/fetcher/Scraper.kt | 18 +++++-- .../it/skrape/fetcher/KtorAdapterTest.kt | 2 +- integrationtests/src/test/kotlin/DslTest.kt | 52 +++++++++---------- .../src/test/kotlin/ExperimentalDslTest.kt | 6 +-- 6 files changed, 55 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 4c41592d..6cd89cb6 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ class HtmlExtractionService { url = "http://localhost:8080" } - extract { + response { MySimpleDataClass( httpStatusCode = status { code }, httpStatusMessage = status { message }, @@ -253,7 +253,7 @@ fun `dsl can skrape by url`() { request { url = "http://localhost:8080/example" } - expect { + response { htmlDocument { // all official html and html5 elements are supported by the DSL div { @@ -309,7 +309,7 @@ fun `dsl can skrape by url`() { ```kotlin fun getDocumentByUrl(urlToScrape: String) = skrape(BrowserFetcher) { // <--- pass BrowserFetcher to include rendered JS request { url = urlToScrape } - extract { htmlDocument { this } } + response { htmlDocument { this } } } @@ -326,7 +326,7 @@ suspend fun getAllLinks(): Map = skrape(AsyncFetcher) { request { url = "https://my-fancy.website" } - extract { + response { htmlDocument { eachLink } } } @@ -367,7 +367,7 @@ class ExampleTest { @Test fun `can use preconfigured client`() { - myPreConfiguredClient.expect { + myPreConfiguredClient.response { status { code toBe 200 } // do more stuff } @@ -377,7 +377,7 @@ class ExampleTest { request { followRedirects = false } - }.expect { + }.response { status { code toBe 301 } // do more stuff } @@ -396,7 +396,7 @@ skrape(HttpFetcher) { headers = mapOf("Content-Type" to "application/json") body = """{"foo":"bar"}""" } - extract { + response { htmlDocument { ... ``` @@ -412,7 +412,7 @@ skrape(HttpFetcher) { contentType = "your-custom/content" // can optionally override content-type } } - extract { + response { htmlDocument { ... ``` @@ -432,7 +432,7 @@ skrape(HttpFetcher) { form("foo=bar") // will automatically set content-type header to "application/x-www-form-urlencoded" } } - extract { + response { htmlDocument { ... ``` @@ -456,7 +456,7 @@ skrape(HttpFetcher) { } } } - extract { + response { htmlDocument { ... ``` @@ -476,7 +476,7 @@ skrape(HttpFetcher) { } } } - extract { + response { htmlDocument { ... ``` diff --git a/examples/android/README.md b/examples/android/README.md index b6938bb9..f2dde34d 100644 --- a/examples/android/README.md +++ b/examples/android/README.md @@ -80,7 +80,7 @@ private suspend fun fetch(): List = request { url = "https://some.fancy/url" } - extract { + response { htmlDocument { ... ``` diff --git a/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Scraper.kt b/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Scraper.kt index 6f7be567..4a54dd9f 100644 --- a/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Scraper.kt +++ b/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Scraper.kt @@ -55,8 +55,9 @@ public suspend fun skrape(fetcher: NonBlockingFetcher, init: suspend S * Execute http call with a given Fetcher implementation and invoke the fetching result. */ @SkrapeItDsl +@Deprecated(message = "Please use 'response' instead", replaceWith = ReplaceWith("response(result)")) public suspend fun Scraper<*>.expect(result: Result.() -> Unit) { - extract(result) + response(result) } /** @@ -65,7 +66,7 @@ public suspend fun Scraper<*>.expect(result: Result.() -> Unit) { */ @SkrapeItDsl public fun Scraper<*>.expectBlocking(result: Result.() -> Unit) { - runBlocking { extract(result) } + runBlocking { response(result) } } /** @@ -73,7 +74,16 @@ public fun Scraper<*>.expectBlocking(result: Result.() -> Unit) { * @return T */ @SkrapeItDsl +@Deprecated(message = "Please use 'response' instead", replaceWith = ReplaceWith("response(result)")) public suspend fun Scraper<*>.extract(result: Result.() -> T): T = + response(result) + +/** + * Execute http call with a given Fetcher implementation and invoke the fetching result. + * @return T + */ +@SkrapeItDsl +public suspend fun Scraper<*>.response(result: Result.() -> T): T = scrape().result() /** @@ -82,7 +92,7 @@ public suspend fun Scraper<*>.extract(result: Result.() -> T): T = */ @SkrapeItDsl public fun Scraper<*>.extractBlocking(result: Result.() -> T): T = - runBlocking { extract(result) } + runBlocking { response(result) } /** * Execute http call with a given Fetcher implementation and invoke the fetching result as this and any given generic as it. @@ -92,7 +102,7 @@ public fun Scraper<*>.extractBlocking(result: Result.() -> T): T = @SkrapeItDsl public suspend inline fun Scraper<*>.extractIt(crossinline result: Result.(T) -> Unit): T = with(T::class) { - extract { createInstance().also { result(it) } } + response { createInstance().also { result(it) } } } /** diff --git a/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/KtorAdapterTest.kt b/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/KtorAdapterTest.kt index 44fa1fc7..2f198b6d 100644 --- a/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/KtorAdapterTest.kt +++ b/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/KtorAdapterTest.kt @@ -50,7 +50,7 @@ class KtorAdapterTest { url("${wiremock.httpUrl}/example") } - extract { this } + response { this } } expectThat(result.responseStatus.code).isEqualTo(200) diff --git a/integrationtests/src/test/kotlin/DslTest.kt b/integrationtests/src/test/kotlin/DslTest.kt index 6061d865..1df8d902 100644 --- a/integrationtests/src/test/kotlin/DslTest.kt +++ b/integrationtests/src/test/kotlin/DslTest.kt @@ -37,7 +37,7 @@ class DslTest { url = "${wiremock.httpUrl}/example" } - expect { + response { status { code toBe 200 @@ -83,9 +83,7 @@ class DslTest { sslRelaxed = true } - expect { - status { code toBe 200 } - } + response { status { code toBe 200 } } } } @@ -99,7 +97,7 @@ class DslTest { url = "${wiremock.httpUrl}/example" } - expect { + response { contentType toContain TEXT_HTML contentType toBe TEXT_HTML_UTF8 contentType toBeNot APPLICATION_XHTML @@ -125,7 +123,7 @@ class DslTest { url = "${wiremock.httpUrl}/" } - expect { + response { status { code toBe 302 message toBe "Found" @@ -142,7 +140,7 @@ class DslTest { request { url = "${wiremock.httpUrl}/" } - expect { + response { val header = httpHeader("Content-Type") { expectThat(this).isEqualTo("text/html;charset=utf-8") } @@ -164,7 +162,7 @@ class DslTest { request { url = "${wiremock.httpUrl}/" } - expect { + response { val headers = httpHeaders { expectThat(this).hasEntry("Content-Type", "text/html;charset=utf-8") } @@ -181,7 +179,7 @@ class DslTest { request { url = "${wiremock.httpUrl}/" } - expect { + response { htmlDocument { body { findFirst { @@ -201,7 +199,7 @@ class DslTest { request { url = "${wiremock.httpUrl}/" } - expect { + response { status { code toBe 404 message toBe "Not Found" @@ -219,7 +217,7 @@ class DslTest { url = "${wiremock.httpUrl}/" method = Method.POST } - expect { + response { //expectThat(request.method).isEqualTo(Method.POST) @@ -251,7 +249,7 @@ class DslTest { url = "${wiremock.httpUrl}/" timeout = 2000 } - expect {} + response {} } } } @@ -272,7 +270,7 @@ class DslTest { url = "${wiremock.httpUrl}/" } - val extracted = extract { + val extracted = response { status { MyObject(message, "", emptyList()) } @@ -387,7 +385,7 @@ class DslTest { url = "${wiremock.httpUrl}/" } - extract { + response { MySimpleDataClass( httpStatusCode = status { code }, httpStatusMessage = status { message }, @@ -415,7 +413,7 @@ class DslTest { request { url = "${wiremock.httpUrl}/" } - expect { + response { htmlDocument { findFirst(".nonExistent") {} } @@ -433,7 +431,7 @@ class DslTest { request { url = "${wiremock.httpUrl}/" } - expect { + response { htmlDocument { relaxed = true findAll(".nonExistent") { @@ -456,7 +454,7 @@ class DslTest { request { url = "${wiremock.httpUrl}/" } - expect { + response { htmlDocument { div { withId = "non-existend" @@ -482,7 +480,7 @@ class DslTest { url = "${wiremock.httpUrl}/" } - extract { + response { htmlDocument { MyObject( message = "", @@ -729,7 +727,7 @@ class DslTest { request { url = "${wiremock.httpUrl}/" } - extract { + response { htmlDocument { a { findAll { first { it.ownText == "relative link" } }.attribute("href") @@ -742,7 +740,7 @@ class DslTest { request { url = "${wiremock.httpUrl}$interestingLink" } - expect { + response { htmlDocument { title { findFirst { text toBe "i'm the title" } @@ -774,7 +772,7 @@ class DslTest { request { url = "${wiremock.httpUrl}/" } - expect { + response { htmlDocument { div(".dynamic") { findFirst { @@ -799,7 +797,7 @@ class DslTest { url = "${wiremock.httpUrl}/delayed" timeout = 15_000 } - expect { + response { status { code toBe 200 message toBe "OK" @@ -825,7 +823,7 @@ class DslTest { url = "${wiremock.httpUrl}/delayed" timeout = 15_000 } - expect { + response { status { code toBe 200 message toBe "OK" @@ -945,7 +943,7 @@ class DslTest { url = "https://docs.skrape.it/docs/" } - extract { + response { htmlDocument { toString() toContain "A Story of Deserializing HTML / XML." } @@ -959,7 +957,7 @@ class DslTest { request { url = "https://docs.skrape.it/docs/" } - extract { + response { htmlDocument { toString() toContain "A Story of Deserializing HTML / XML." } @@ -976,7 +974,7 @@ class DslTest { url = "$httpBin/basic-auth/cr1z/secure" } - expect { + response { status { code toBe 401 message toBe "UNAUTHORIZED" @@ -999,7 +997,7 @@ class DslTest { } } - expect { + response { status { code toBe 200 } diff --git a/integrationtests/src/test/kotlin/ExperimentalDslTest.kt b/integrationtests/src/test/kotlin/ExperimentalDslTest.kt index 8e5da350..1c07204b 100644 --- a/integrationtests/src/test/kotlin/ExperimentalDslTest.kt +++ b/integrationtests/src/test/kotlin/ExperimentalDslTest.kt @@ -1,7 +1,7 @@ import it.skrape.core.htmlDocument import it.skrape.fetcher.HttpFetcher -import it.skrape.fetcher.extract +import it.skrape.fetcher.response import it.skrape.fetcher.skrape import it.skrape.matchers.toBe import it.skrape.matchers.toBePresentExactlyOnce @@ -28,7 +28,7 @@ class ExperimentalDslTest { url = "${wiremock.httpUrl}/example" } - extract { + response { htmlDocument { div { withClass = "foo" and "bar" and "fizz" and "buzz" @@ -89,7 +89,7 @@ class ExperimentalDslTest { port = 12345 } } - extract { + response { // do something with the result }