Skip to content

Commit

Permalink
Pagination help
Browse files Browse the repository at this point in the history
  • Loading branch information
@jserranohidalgo
jserranohidalgo committed Sep 16, 2022
1 parent 1a6cdd7 commit 6edd3c0
Showing 1 changed file with 83 additions and 214 deletions.
297 changes: 83 additions & 214 deletions README.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,28 +23,9 @@
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[32mimport \u001b[39m\u001b[36m$ivy.$ \n",
"\u001b[39m\r\n",
"\u001b[32mimport \u001b[39m\u001b[36m$ivy.$ \n",
"\u001b[39m\r\n",
"\u001b[32mimport \u001b[39m\u001b[36m$ivy.$ \n",
"\u001b[39m\r\n",
"\u001b[32mimport \u001b[39m\u001b[36m$ivy.$ \n",
"\u001b[39m\r\n",
"\u001b[32mimport \u001b[39m\u001b[36m$ivy.$ \u001b[39m"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import $ivy.`com.typesafe.akka::akka-http-spray-json:10.2.4`\n",
"import $ivy.`com.typesafe.akka::akka-slf4j:2.6.8`\n",
Expand All @@ -64,20 +45,9 @@
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[32mimport \u001b[39m\u001b[36m$cp.$ \u001b[39m"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import $cp.target.`scala-2.13`.`twitterapiv2_2.13-0.1.jar`"
]
Expand All @@ -91,30 +61,9 @@
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[32mimport \u001b[39m\u001b[36mscala.concurrent.{Future, Await, ExecutionContext, duration}, duration._\n",
"\u001b[39m\r\n",
"\u001b[32mimport \u001b[39m\u001b[36m_root_.akka.actor.typed.ActorSystem\n",
"\u001b[39m\r\n",
"\u001b[32mimport \u001b[39m\u001b[36m_root_.akka.actor.typed.scaladsl.Behaviors\n",
"\u001b[39m\r\n",
"\u001b[32mimport \u001b[39m\u001b[36mscala.util.Success\n",
"\u001b[39m\r\n",
"\u001b[32mimport \u001b[39m\u001b[36mscala.util.Failure\n",
"\u001b[39m\r\n",
"\u001b[32mimport \u001b[39m\u001b[36mdev.habla.twitter.{v2_akka, v2}, v2_akka._\u001b[39m"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import scala.concurrent.{Future, Await, ExecutionContext, duration}, duration._\n",
"import _root_.akka.actor.typed.ActorSystem\n",
Expand All @@ -133,30 +82,9 @@
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"SLF4J: Failed to load class \"org.slf4j.impl.StaticLoggerBinder\".\n",
"SLF4J: Defaulting to no-operation (NOP) logger implementation\n",
"SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.\n"
]
},
{
"data": {
"text/plain": [
"\u001b[36msystem\u001b[39m: \u001b[32mActorSystem\u001b[39m[\u001b[32mAny\u001b[39m] = akka://TwitterV2\r\n",
"\u001b[36mec\u001b[39m: \u001b[32mconcurrent\u001b[39m.\u001b[32mExecutionContextExecutor\u001b[39m = Dispatcher[akka.actor.default-dispatcher]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"implicit val system = ActorSystem(Behaviors.empty, \"TwitterV2\")\n",
"implicit val ec = system.executionContext"
Expand All @@ -171,20 +99,9 @@
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"defined \u001b[32mfunction\u001b[39m \u001b[36mbearerToken\u001b[39m"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def bearerToken = scala.util.Properties.envOrElse(\"BEARER_TOKEN\", \"undefined\")"
]
Expand All @@ -209,10 +126,16 @@
"metadata": {},
"outputs": [],
"source": [
"val request = v2.recents.SingleRequest(\"scala3\", bearerToken, max_results=Some(10))\n",
"\n",
"val response: Future[v2.recents.SingleResponse] = \n",
" v2.recents.SingleRequest(\"scala3\", bearerToken, max_results=Some(10))\n",
" .single"
"val response: Future[v2.recents.SingleResponse] = request.single"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"By default, we obtain 10 tweets per response:"
]
},
{
Expand All @@ -221,13 +144,15 @@
"metadata": {},
"outputs": [],
"source": [
"response.map{ case v2.recents.Tweets(_, r, s) => (r,s) }"
"response.map{ case v2.recents.Tweets(v2.recents.Tweets.Body(Some(tweets), _, _), _, _) => tweets.map(_.text) }"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"show(Await.result(response, 1.second))"
Expand Down Expand Up @@ -259,20 +184,9 @@
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[32mimport \u001b[39m\u001b[36makka.stream.scaladsl._\u001b[39m"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import akka.stream.scaladsl._"
]
Expand All @@ -281,25 +195,57 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"But the library can also do pagination and throttle management for us automatically:"
"In the last search for recent tweets, the `next_token` field of the response was not empty: "
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"\u001b[36mresponse\u001b[39m: \u001b[32mSource\u001b[39m[\u001b[32mv2\u001b[39m.\u001b[32mrecents\u001b[39m.\u001b[32mSingleResponse\u001b[39m, \u001b[32makka\u001b[39m.\u001b[32mNotUsed\u001b[39m] = Source(SourceShape(flatten.out(4893099)))"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"response.map{ case v2.recents.Tweets(v2.recents.Tweets.Body(_, _, meta), r, s) => (meta, r, s) }"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This means that we can issue a new request to get the next page of results (with another 10 tweets): "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"val next_request = request.copy(next_token = Some(\"b26v89c19zqg8o3fpzbjo0w1eqyrda77g2ag21zxdspkt\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"next_request.single"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This process can go on until we retrieve all the tweets from the response. To do this, we must take into account that there is a maximum number of requests that we can issue to the twitter API in a 15 minutes window. This library can help us with all this pagination and throttle management, by creating a _stream_ of responses instead of a single one:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"val response: Source[v2.recents.SingleResponse, akka.NotUsed] = \n",
" v2.recents.SingleRequest(\"scala3 -is:retweet\", bearerToken)\n",
Expand All @@ -308,86 +254,9 @@
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div class=\"jp-RenderedText\">\n",
"<pre><code><span style=\"color: rgb(0, 187, 187)\"><span class=\"ansi-cyan-fg\">res12</span></span>: <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">Future</span></span>[<span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">Seq</span></span>[<span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">v2</span></span>.<span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">recents</span></span>.<span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">SingleResponse</span></span>]] = <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\"><style>@keyframes fadein { from { opacity: 0; } to { opacity: 1; } }</style><span style=\"animation: fadein 2s;\"><span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">Success</span></span>(\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">Vector</span></span>(\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">Tweets</span></span>(\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">Body</span></span>(\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">Some</span></span>(\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">List</span></span>(\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">Tweet</span></span>(\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">&quot;1397895129571024896&quot;</span></span>,\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">&quot;scala3\\u306e\\u30a4\\u30f3\\u30c7\\u30f3\\u30c8\\u69cb\\u6587\\u3001\\u305f\\u3057\\u304b\\u306b\\u3056\\u308f\\u3064\\u304f\\u3051\\u3069\\u3001\\u3053\\u308c\\u3082\\u30b9\\u30b1\\u30fc\\u30e9\\u30d6\\u30eb\\u3055\\u3060\\u3063\\u305f\\u308a\\u3059\\u308b\\u3093\\u304b\\u306a\\u3001\\u3068\\u304b\\u601d\\u3063\\u305f\\u3002\\u3066\\u304b\\u3001\\u30d6\\u30ed\\u30c3\\u30af\\u3068\\u30a4\\u30f3\\u30c7\\u30f3\\u30c8\\u3063\\u3066\\u540c\\u5c45\\u3067\\u304d\\u308b\\u3093\\u3060\\u30fc\\u3063\\u3066\\u3044\\u3046\\u306e\\u3001\\u9762\\u767d\\u3044\\u3002\\u305d\\u306e\\u30a4\\u30f3\\u30c7\\u30f3\\u30c8\\u69cb\\u6587\\u306e\\u62e1\\u5f35\\u3067ruby\\u3063\\u307d\\u304f\\u3082\\u66f8\\u3051\\u308b\\u306e\\u3082\\u9762\\u767d\\u3044\\u3002&quot;</span></span>,\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">None</span></span>,\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">Some</span></span>(<span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">JsString</span></span>(<span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">&quot;139978757&quot;</span></span>)),\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">None</span></span>,\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">Some</span></span>(<span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">JsString</span></span>(<span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">&quot;1397895129571024896&quot;</span></span>)),\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">Some</span></span>(<span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">JsString</span></span>(<span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">&quot;2021-05-27T12:39:05.000Z&quot;</span></span>)),\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">None</span></span>,\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">None</span></span>,\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">None</span></span>,\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">Some</span></span>(<span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">JsString</span></span>(<span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">&quot;ja&quot;</span></span>)),\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">None</span></span>,\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">None</span></span>,\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">None</span></span>,\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">None</span></span>,\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">Some</span></span>(\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">JsObject</span></span>(\n",
" <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">TreeMap</span></span>(\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">&quot;like_count&quot;</span></span> -&gt; <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">JsNumber</span></span>(1),\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">&quot;quote_count&quot;</span></span> -&gt; <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">JsNumber</span></span>(0),\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">&quot;reply_count&quot;</span></span> -&gt; <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">JsNumber</span></span>(0),\n",
" <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">&quot;retweet_count&quot;</span></span> -&gt; <span style=\"color: yellow\"><span class=\"ansi-yellow-fg\">JsNumber</span></span>(0)\n",
" )\n",
" )\n",
"...</span></span></span></code></pre>\n",
"</div>"
],
"text/plain": [
"\u001b[36mres12\u001b[39m: \u001b[32mFuture\u001b[39m[\u001b[32mSeq\u001b[39m[\u001b[32mv2\u001b[39m.\u001b[32mrecents\u001b[39m.\u001b[32mSingleResponse\u001b[39m]] = \u001b[32m\u001b[33mSuccess\u001b[39m(\n",
" \u001b[33mVector\u001b[39m(\n",
" \u001b[33mTweets\u001b[39m(\n",
" \u001b[33mBody\u001b[39m(\n",
" \u001b[33mSome\u001b[39m(\n",
" \u001b[33mList\u001b[39m(\n",
" \u001b[33mTweet\u001b[39m(\n",
" \u001b[32m\"1397895129571024896\"\u001b[39m,\n",
" \u001b[32m\"scala3\\u306e\\u30a4\\u30f3\\u30c7\\u30f3\\u30c8\\u69cb\\u6587\\u3001\\u305f\\u3057\\u304b\\u306b\\u3056\\u308f\\u3064\\u304f\\u3051\\u3069\\u3001\\u3053\\u308c\\u3082\\u30b9\\u30b1\\u30fc\\u30e9\\u30d6\\u30eb\\u3055\\u3060\\u3063\\u305f\\u308a\\u3059\\u308b\\u3093\\u304b\\u306a\\u3001\\u3068\\u304b\\u601d\\u3063\\u305f\\u3002\\u3066\\u304b\\u3001\\u30d6\\u30ed\\u30c3\\u30af\\u3068\\u30a4\\u30f3\\u30c7\\u30f3\\u30c8\\u3063\\u3066\\u540c\\u5c45\\u3067\\u304d\\u308b\\u3093\\u3060\\u30fc\\u3063\\u3066\\u3044\\u3046\\u306e\\u3001\\u9762\\u767d\\u3044\\u3002\\u305d\\u306e\\u30a4\\u30f3\\u30c7\\u30f3\\u30c8\\u69cb\\u6587\\u306e\\u62e1\\u5f35\\u3067ruby\\u3063\\u307d\\u304f\\u3082\\u66f8\\u3051\\u308b\\u306e\\u3082\\u9762\\u767d\\u3044\\u3002\"\u001b[39m,\n",
" \u001b[32mNone\u001b[39m,\n",
" \u001b[33mSome\u001b[39m(\u001b[33mJsString\u001b[39m(\u001b[32m\"139978757\"\u001b[39m)),\n",
" \u001b[32mNone\u001b[39m,\n",
" \u001b[33mSome\u001b[39m(\u001b[33mJsString\u001b[39m(\u001b[32m\"1397895129571024896\"\u001b[39m)),\n",
" \u001b[33mSome\u001b[39m(\u001b[33mJsString\u001b[39m(\u001b[32m\"2021-05-27T12:39:05.000Z\"\u001b[39m)),\n",
" \u001b[32mNone\u001b[39m,\n",
" \u001b[32mNone\u001b[39m,\n",
" \u001b[32mNone\u001b[39m,\n",
" \u001b[33mSome\u001b[39m(\u001b[33mJsString\u001b[39m(\u001b[32m\"ja\"\u001b[39m)),\n",
" \u001b[32mNone\u001b[39m,\n",
" \u001b[32mNone\u001b[39m,\n",
" \u001b[32mNone\u001b[39m,\n",
" \u001b[32mNone\u001b[39m,\n",
" \u001b[33mSome\u001b[39m(\n",
" \u001b[33mJsObject\u001b[39m(\n",
" \u001b[33mTreeMap\u001b[39m(\n",
" \u001b[32m\"like_count\"\u001b[39m -> \u001b[33mJsNumber\u001b[39m(1),\n",
" \u001b[32m\"quote_count\"\u001b[39m -> \u001b[33mJsNumber\u001b[39m(0),\n",
" \u001b[32m\"reply_count\"\u001b[39m -> \u001b[33mJsNumber\u001b[39m(0),\n",
" \u001b[32m\"retweet_count\"\u001b[39m -> \u001b[33mJsNumber\u001b[39m(0)\n",
" )\n",
" )\n",
"...\u001b[39m"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"response.take(2)\n",
" .toMat(Sink.seq)(Keep.right) // RunnableGraph\n",
Expand Down Expand Up @@ -424,7 +293,7 @@
"metadata": {},
"outputs": [],
"source": [
"takeNTweets(v2.recents.SingleRequest(\"scala3\", bearerToken, tweet_fields = Some(\"id,text,created_at\")))(1)"
"takeNTweets(v2.recents.SingleRequest(\"scala3\", bearerToken, tweet_fields = Some(\"id,text,created_at\")))(5).map(_.map(_.text))"
]
},
{
Expand All @@ -447,7 +316,7 @@
"mimetype": "text/x-scala",
"name": "scala",
"nbconvert_exporter": "script",
"version": "2.13.1"
"version": "2.13.3"
}
},
"nbformat": 4,
Expand Down

0 comments on commit 6edd3c0

Please sign in to comment.