Skip to content

Commit

Permalink
#153: introduce 'response' function and add deprecation marker on 'ex…
Browse files Browse the repository at this point in the history 
…pect' and 'extrqct' function
  • Loading branch information
@christian-draeger
christian-draeger committed Jun 29, 2021
1 parent 133fa5f commit 0d34161
Show file tree
Hide file tree
Showing 6 changed files with 55 additions and 47 deletions.
22 changes: 11 additions & 11 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ class HtmlExtractionService {
url = "http://localhost:8080"
}

extract {
response {
MySimpleDataClass(
httpStatusCode = status { code },
httpStatusMessage = status { message },
Expand Down Expand Up @@ -253,7 +253,7 @@ fun `dsl can skrape by url`() {
request {
url = "http://localhost:8080/example"
}
expect {
response {
htmlDocument {
// all official html and html5 elements are supported by the DSL
div {
Expand Down Expand Up @@ -309,7 +309,7 @@ fun `dsl can skrape by url`() {
```kotlin
fun getDocumentByUrl(urlToScrape: String) = skrape(BrowserFetcher) { // <--- pass BrowserFetcher to include rendered JS
request { url = urlToScrape }
extract { htmlDocument { this } }
response { htmlDocument { this } }
}


Expand All @@ -326,7 +326,7 @@ suspend fun getAllLinks(): Map<String, String> = skrape(AsyncFetcher) {
request {
url = "https://my-fancy.website"
}
extract {
response {
htmlDocument { eachLink }
}
}
Expand Down Expand Up @@ -367,7 +367,7 @@ class ExampleTest {
@Test
fun `can use preconfigured client`() {

myPreConfiguredClient.expect {
myPreConfiguredClient.response {
status { code toBe 200 }
// do more stuff
}
Expand All @@ -377,7 +377,7 @@ class ExampleTest {
request {
followRedirects = false
}
}.expect {
}.response {
status { code toBe 301 }
// do more stuff
}
Expand All @@ -396,7 +396,7 @@ skrape(HttpFetcher) {
headers = mapOf("Content-Type" to "application/json")
body = """{"foo":"bar"}"""
}
extract {
response {
htmlDocument {
...
```
Expand All @@ -412,7 +412,7 @@ skrape(HttpFetcher) {
contentType = "your-custom/content" // can optionally override content-type
}
}
extract {
response {
htmlDocument {
...
```
Expand All @@ -432,7 +432,7 @@ skrape(HttpFetcher) {
form("foo=bar") // will automatically set content-type header to "application/x-www-form-urlencoded"
}
}
extract {
response {
htmlDocument {
...
```
Expand All @@ -456,7 +456,7 @@ skrape(HttpFetcher) {
}
}
}
extract {
response {
htmlDocument {
...
```
Expand All @@ -476,7 +476,7 @@ skrape(HttpFetcher) {
}
}
}
extract {
response {
htmlDocument {
...
```
Expand Down
2 changes: 1 addition & 1 deletion examples/android/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ private suspend fun fetch(): List<User> =
request {
url = "https://some.fancy/url"
}
extract {
response {
htmlDocument {
...
```
18 changes: 14 additions & 4 deletions fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Scraper.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,9 @@ public suspend fun <R, T> skrape(fetcher: NonBlockingFetcher<R>, init: suspend S
* Execute http call with a given Fetcher implementation and invoke the fetching result.
*/
@SkrapeItDsl
@Deprecated(message = "Please use 'response' instead", replaceWith = ReplaceWith("response(result)"))
public suspend fun Scraper<*>.expect(result: Result.() -> Unit) {
extract(result)
response(result)
}

/**
Expand All @@ -65,15 +66,24 @@ public suspend fun Scraper<*>.expect(result: Result.() -> Unit) {
*/
@SkrapeItDsl
public fun Scraper<*>.expectBlocking(result: Result.() -> Unit) {
runBlocking { extract(result) }
runBlocking { response(result) }
}

/**
* Execute http call with a given Fetcher implementation and invoke the fetching result.
* @return T
*/
@SkrapeItDsl
@Deprecated(message = "Please use 'response' instead", replaceWith = ReplaceWith("response(result)"))
public suspend fun <T> Scraper<*>.extract(result: Result.() -> T): T =
response(result)

/**
* Execute http call with a given Fetcher implementation and invoke the fetching result.
* @return T
*/
@SkrapeItDsl
public suspend fun <T> Scraper<*>.response(result: Result.() -> T): T =
scrape().result()

/**
Expand All @@ -82,7 +92,7 @@ public suspend fun <T> Scraper<*>.extract(result: Result.() -> T): T =
*/
@SkrapeItDsl
public fun <T> Scraper<*>.extractBlocking(result: Result.() -> T): T =
runBlocking { extract(result) }
runBlocking { response(result) }

/**
* Execute http call with a given Fetcher implementation and invoke the fetching result as this and any given generic as it.
Expand All @@ -92,7 +102,7 @@ public fun <T> Scraper<*>.extractBlocking(result: Result.() -> T): T =
@SkrapeItDsl
public suspend inline fun <reified T : Any> Scraper<*>.extractIt(crossinline result: Result.(T) -> Unit): T =
with(T::class) {
extract { createInstance().also { result(it) } }
response { createInstance().also { result(it) } }
}

/**
Expand Down
2 changes: 1 addition & 1 deletion fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/KtorAdapterTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ class KtorAdapterTest {
url("${wiremock.httpUrl}/example")
}

extract { this }
response { this }
}

expectThat(result.responseStatus.code).isEqualTo(200)
Expand Down
52 changes: 25 additions & 27 deletions integrationtests/src/test/kotlin/DslTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class DslTest {
url = "${wiremock.httpUrl}/example"
}

expect {
response {

status {
code toBe 200
Expand Down Expand Up @@ -83,9 +83,7 @@ class DslTest {
sslRelaxed = true
}

expect {
status { code toBe 200 }
}
response { status { code toBe 200 } }
}

}
Expand All @@ -99,7 +97,7 @@ class DslTest {
url = "${wiremock.httpUrl}/example"
}

expect {
response {
contentType toContain TEXT_HTML
contentType toBe TEXT_HTML_UTF8
contentType toBeNot APPLICATION_XHTML
Expand All @@ -125,7 +123,7 @@ class DslTest {
url = "${wiremock.httpUrl}/"
}

expect {
response {
status {
code toBe 302
message toBe "Found"
Expand All @@ -142,7 +140,7 @@ class DslTest {
request {
url = "${wiremock.httpUrl}/"
}
expect {
response {
val header = httpHeader("Content-Type") {
expectThat(this).isEqualTo("text/html;charset=utf-8")
}
Expand All @@ -164,7 +162,7 @@ class DslTest {
request {
url = "${wiremock.httpUrl}/"
}
expect {
response {
val headers = httpHeaders {
expectThat(this).hasEntry("Content-Type", "text/html;charset=utf-8")
}
Expand All @@ -181,7 +179,7 @@ class DslTest {
request {
url = "${wiremock.httpUrl}/"
}
expect {
response {
htmlDocument {
body {
findFirst {
Expand All @@ -201,7 +199,7 @@ class DslTest {
request {
url = "${wiremock.httpUrl}/"
}
expect {
response {
status {
code toBe 404
message toBe "Not Found"
Expand All @@ -219,7 +217,7 @@ class DslTest {
url = "${wiremock.httpUrl}/"
method = Method.POST
}
expect {
response {

//expectThat(request.method).isEqualTo(Method.POST)

Expand Down Expand Up @@ -251,7 +249,7 @@ class DslTest {
url = "${wiremock.httpUrl}/"
timeout = 2000
}
expect {}
response {}
}
}
}
Expand All @@ -272,7 +270,7 @@ class DslTest {
url = "${wiremock.httpUrl}/"
}

val extracted = extract {
val extracted = response {
status {
MyObject(message, "", emptyList())
}
Expand Down Expand Up @@ -387,7 +385,7 @@ class DslTest {
url = "${wiremock.httpUrl}/"
}

extract {
response {
MySimpleDataClass(
httpStatusCode = status { code },
httpStatusMessage = status { message },
Expand Down Expand Up @@ -415,7 +413,7 @@ class DslTest {
request {
url = "${wiremock.httpUrl}/"
}
expect {
response {
htmlDocument {
findFirst(".nonExistent") {}
}
Expand All @@ -433,7 +431,7 @@ class DslTest {
request {
url = "${wiremock.httpUrl}/"
}
expect {
response {
htmlDocument {
relaxed = true
findAll(".nonExistent") {
Expand All @@ -456,7 +454,7 @@ class DslTest {
request {
url = "${wiremock.httpUrl}/"
}
expect {
response {
htmlDocument {
div {
withId = "non-existend"
Expand All @@ -482,7 +480,7 @@ class DslTest {
url = "${wiremock.httpUrl}/"
}

extract {
response {
htmlDocument {
MyObject(
message = "",
Expand Down Expand Up @@ -729,7 +727,7 @@ class DslTest {
request {
url = "${wiremock.httpUrl}/"
}
extract {
response {
htmlDocument {
a {
findAll { first { it.ownText == "relative link" } }.attribute("href")
Expand All @@ -742,7 +740,7 @@ class DslTest {
request {
url = "${wiremock.httpUrl}$interestingLink"
}
expect {
response {
htmlDocument {
title {
findFirst { text toBe "i'm the title" }
Expand Down Expand Up @@ -774,7 +772,7 @@ class DslTest {
request {
url = "${wiremock.httpUrl}/"
}
expect {
response {
htmlDocument {
div(".dynamic") {
findFirst {
Expand All @@ -799,7 +797,7 @@ class DslTest {
url = "${wiremock.httpUrl}/delayed"
timeout = 15_000
}
expect {
response {
status {
code toBe 200
message toBe "OK"
Expand All @@ -825,7 +823,7 @@ class DslTest {
url = "${wiremock.httpUrl}/delayed"
timeout = 15_000
}
expect {
response {
status {
code toBe 200
message toBe "OK"
Expand Down Expand Up @@ -945,7 +943,7 @@ class DslTest {
url = "https://docs.skrape.it/docs/"
}

extract {
response {
htmlDocument {
toString() toContain "A Story of Deserializing HTML / XML."
}
Expand All @@ -959,7 +957,7 @@ class DslTest {
request {
url = "https://docs.skrape.it/docs/"
}
extract {
response {
htmlDocument {
toString() toContain "A Story of Deserializing HTML / XML."
}
Expand All @@ -976,7 +974,7 @@ class DslTest {
url = "$httpBin/basic-auth/cr1z/secure"
}

expect {
response {
status {
code toBe 401
message toBe "UNAUTHORIZED"
Expand All @@ -999,7 +997,7 @@ class DslTest {
}
}

expect {
response {
status {
code toBe 200
}
Expand Down
Loading

0 comments on commit 0d34161

Please sign in to comment.