diff --git a/.gitignore b/.gitignore index 9d2b904..bcd15cb 100644 --- a/.gitignore +++ b/.gitignore @@ -23,10 +23,12 @@ src_managed/ project/boot/ project/plugins/project/ +desktop.ini # Scala-IDE specific .scala_dependencies .worksheet +*.sc ## ## ECLIPSE-Specific. @@ -125,8 +127,7 @@ node_modules *.py[co] __pycache__ *.egg-info -*~ -*.bak + .ipynb_checkpoints .tox .DS_Store @@ -135,9 +136,7 @@ __pycache__ .coverage .pytest_cache -*.swp *.map -.idea/ Read the Docs config.rst diff --git a/build.sbt b/build.sbt index 14aaa37..2ce2992 100644 --- a/build.sbt +++ b/build.sbt @@ -6,6 +6,7 @@ val AkkaVersion = "2.6.8" val AkkaHttpVersion = "10.2.4" libraryDependencies ++= Seq( + "com.github.alexarchambault" %% "case-app" % "2.0.1", "com.github.pureconfig" %% "pureconfig" % "0.14.0", "ch.qos.logback" % "logback-classic" % "1.2.3", @@ -17,7 +18,8 @@ libraryDependencies ++= Seq( "com.typesafe.akka" %% "akka-slf4j" % AkkaVersion, "com.typesafe.akka" %% "akka-actor-typed" % AkkaVersion, "com.typesafe.akka" %% "akka-stream" % AkkaVersion, - "com.typesafe.akka" %% "akka-http" % AkkaHttpVersion) + "com.typesafe.akka" %% "akka-http" % AkkaHttpVersion, + "com.lihaoyi" %% "requests" % "0.7.0") testFrameworks += new TestFramework("utest.runner.Framework") diff --git a/src/main/scala/dev/habla/twitter/main/Command.scala b/src/main/scala/dev/habla/twitter/main/Command.scala index 8802c4c..e39003f 100644 --- a/src/main/scala/dev/habla/twitter/main/Command.scala +++ b/src/main/scala/dev/habla/twitter/main/Command.scala @@ -1,7 +1,8 @@ package dev.habla.twitter -package v2 package main +import v2._ + sealed abstract class Command case class LookupTweet( diff --git a/src/main/scala/dev/habla/twitter/main/Main.scala b/src/main/scala/dev/habla/twitter/main/Main.scala index 89f8edb..5735f6c 100644 --- a/src/main/scala/dev/habla/twitter/main/Main.scala +++ b/src/main/scala/dev/habla/twitter/main/Main.scala @@ -1,18 +1,16 @@ package dev.habla.twitter -package v2 package main +import v2._ + import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} import _root_.akka.actor.typed.ActorSystem import _root_.akka.actor.typed.scaladsl.Behaviors import caseapp._ -import scala.util.Success -import scala.util.Failure - -object Main extends CommandApp[Command]{ - +import scala.util.{Failure, Success} +object Main extends CommandApp[Command] { def run(command: Command, rargs: RemainingArgs): Unit = @@ -23,20 +21,23 @@ object Main extends CommandApp[Command]{ case cmd: LookupUsers => runLookupUsers(cmd) } - def runLookupTweet(cmd: LookupTweet): Unit = withExecutionContext{ - implicit system => implicit ec => - v2_akka.lookupt.Run(cmd.toLookupTweetRequest) + def runLookupTweet(cmd: LookupTweet): Unit = withExecutionContext { + implicit system => + implicit ec => + v2_akka.lookupt.Run(cmd.toLookupTweetRequest) }(println, _.printStackTrace) - def runSearchRecent(search: SearchRecent): Unit = withExecutionContext{ - implicit system => implicit ec => search.toSearchRecentCommand match { - case Right(singleRequest: recents.SingleRequest) => - v2_akka.recents.Run(singleRequest) - case Right(pagination: recents.Pagination) => - v2_akka.recents.RunPagination(pagination) - case Left(error) => - Future.failed(new Exception(error)) - } + def runSearchRecent(search: SearchRecent): Unit = withExecutionContext { + implicit system => + implicit ec => + search.toSearchRecentCommand match { + case Right(singleRequest: recents.SingleRequest) => + v2_akka.recents.Run(singleRequest) + case Right(pagination: recents.Pagination) => + v2_akka.recents.RunPagination(pagination) + case Left(error) => + Future.failed(new Exception(error)) + } }(println, _.printStackTrace) def runLookupUser(cmd: LookupUser): Unit = withExecutionContext { @@ -52,9 +53,9 @@ object Main extends CommandApp[Command]{ }(println, _.printStackTrace) def withExecutionContext[A]( - run: ActorSystem[_] => ExecutionContext => Future[A])( - onSuccess: A => Unit, - onFailure: Throwable => Unit + run: ActorSystem[_] => ExecutionContext => Future[A])( + onSuccess: A => Unit, + onFailure: Throwable => Unit ): Unit = { val system = ActorSystem(Behaviors.empty, "TwitterV2") implicit val ec: ExecutionContextExecutor = system.executionContext @@ -64,4 +65,3 @@ object Main extends CommandApp[Command]{ } } } - diff --git a/src/main/scala/dev/habla/twitter/main/MainLi.scala b/src/main/scala/dev/habla/twitter/main/MainLi.scala new file mode 100644 index 0000000..5ca94e2 --- /dev/null +++ b/src/main/scala/dev/habla/twitter/main/MainLi.scala @@ -0,0 +1,26 @@ +package dev.habla.twitter +package main + +import caseapp._ + +import scala.util.Try + +object MainLi extends CommandApp[Command] { + + + def run(command: Command, rargs: RemainingArgs): Unit = + command match { + case cmd: LookupTweet => runLookupTweet(cmd) + case cmd: LookupUser => runLookupUser(cmd) + } + + def runLookupTweet(cmd: LookupTweet): Unit = + Try(v2_requests.lookupt.Run(cmd.toLookupTweetRequest)) + .fold(_.getMessage, println) + + def runLookupUser(cmd: LookupUser): Unit = { + cmd.toLookupUserRequest.left.map(new Exception(_)).toTry + .flatMap(req => Try(v2_requests.lookupuser.Run(req)) + .map(identity)).fold(_.getMessage, println) + } +} diff --git a/src/main/scala/dev/habla/twitter/v2_akka/HttpEndpoint.scala b/src/main/scala/dev/habla/twitter/v2_akka/HttpEndpoint.scala index 2fab4fd..3648e26 100644 --- a/src/main/scala/dev/habla/twitter/v2_akka/HttpEndpoint.scala +++ b/src/main/scala/dev/habla/twitter/v2_akka/HttpEndpoint.scala @@ -40,5 +40,8 @@ trait HttpEndpointSyntax{ trait HttpEndpointInstances{ implicit val lookuptEndpoint: HttpEndpoint.Aux[v2.lookupt.Request, v2.lookupt.Response] = v2_akka.lookupt.Run + + implicit val lookupuserEndpoint: HttpEndpoint.Aux[v2.lookupuser.Request, v2.lookupuser.Response] = + v2_akka.lookupuser.Run } diff --git a/src/main/scala/dev/habla/twitter/v2_requests/HttpBody.scala b/src/main/scala/dev/habla/twitter/v2_requests/HttpBody.scala new file mode 100644 index 0000000..3d09a3d --- /dev/null +++ b/src/main/scala/dev/habla/twitter/v2_requests/HttpBody.scala @@ -0,0 +1,15 @@ +package dev.habla.twitter +package v2_requests + +import spray.json._ +import scala.util.Try + +trait HttpBody{ + + def parseBody(response: requests.Response): Either[String, JsValue] = { + parseJson(response.text()) + } + + def parseJson(body: String): Either[String, JsValue] = + Try(body.parseJson).toEither.left.map(_ => body) +} diff --git a/src/main/scala/dev/habla/twitter/v2_requests/HttpEndpoint.scala b/src/main/scala/dev/habla/twitter/v2_requests/HttpEndpoint.scala new file mode 100644 index 0000000..5c30f64 --- /dev/null +++ b/src/main/scala/dev/habla/twitter/v2_requests/HttpEndpoint.scala @@ -0,0 +1,45 @@ +package dev.habla.twitter +package v2_requests + + +import requests.RequestBlob +import v2._ + +trait HttpEndpoint[Request]{ + /* abstract interface */ + + type Response + + def from(response: requests.Response): Response + + def to(request: Request): requests.Request + + /* concrete interface */ + + def apply(request: Request): Response = + from{ + requests.get + .apply(to(request), RequestBlob.EmptyRequestBlob, requests.chunkedUpload) + } +} + +object HttpEndpoint{ + type Aux[Req, Res] = HttpEndpoint[Req]{type Response = Res } +} + +trait HttpEndpointSyntax{ + + implicit class HttpEndpointRequestOps[Req, Res](request: Req)(implicit ep: HttpEndpoint.Aux[Req, Res]){ + def single: Res = + ep.apply(request) + } +} + +trait HttpEndpointInstances{ + implicit val lookuptEndpoint: HttpEndpoint.Aux[lookupt.Request, lookupt.Response] = + v2_requests.lookupt.Run + + implicit val lookupuserEndpoint: HttpEndpoint.Aux[lookupuser.Request, lookupuser.Response] = + v2_requests.lookupuser.Run +} + diff --git a/src/main/scala/dev/habla/twitter/v2_requests/QueryParams.scala b/src/main/scala/dev/habla/twitter/v2_requests/QueryParams.scala new file mode 100644 index 0000000..8d67a18 --- /dev/null +++ b/src/main/scala/dev/habla/twitter/v2_requests/QueryParams.scala @@ -0,0 +1,12 @@ +package dev.habla.twitter +package v2_requests + +trait QueryParams{ + + implicit class Params(params: Map[String, String]){ + def add(name: String, value: Option[String]): Map[String, String] = + value.fold(params){ v => params + ((name, v)) } + def add(name: String, value: String): Map[String, String] = + add(name, Some(value)) + } +} \ No newline at end of file diff --git a/src/main/scala/dev/habla/twitter/v2_requests/RateLimitHeaders.scala b/src/main/scala/dev/habla/twitter/v2_requests/RateLimitHeaders.scala new file mode 100644 index 0000000..2029a77 --- /dev/null +++ b/src/main/scala/dev/habla/twitter/v2_requests/RateLimitHeaders.scala @@ -0,0 +1,15 @@ +package dev.habla.twitter +package v2_requests + +import scala.util.Try + +trait RateLimitHeaders { + def parseRateLimitHeaders(response: requests.Response): Option[(Int, Long)] = { + for { + rateResetH <- response.headers("x-rate-limit-reset").headOption + rateReset <- Try(rateResetH.toLong).toOption + rateRemainingH <- response.headers("x-rate-limit-remaining").headOption + rateRemaining <- Try(rateRemainingH.toInt).toOption + } yield (rateRemaining, rateReset) + } +} diff --git a/src/main/scala/dev/habla/twitter/v2_requests/lookupt/From.scala b/src/main/scala/dev/habla/twitter/v2_requests/lookupt/From.scala new file mode 100644 index 0000000..441813a --- /dev/null +++ b/src/main/scala/dev/habla/twitter/v2_requests/lookupt/From.scala @@ -0,0 +1,38 @@ +package dev.habla.twitter +package v2_requests.lookupt + +import v2_requests._ +import v2.lookupt._ +import spray.json._ + +import scala.util.Try + + +trait From extends HttpBody with RateLimitHeaders{ + + def from(response: requests.Response): Response = { + val bodyE = parseBody(response) + parseTweetInfo(response, bodyE) + .orElse(parseRateLimitExceeded(response)) + .orElse(parseErroneousTextResponse(bodyE)) + .orElse(parseErroneousJsonResponse(bodyE)) + .getOrElse(ErroneousTextResponse("Not a lookup response")) + } + + def parseTweetInfo(response: requests.Response, bodyE: Either[String,JsValue]): Option[TweetInfo] = + for { + body <- bodyE.toOption if response.statusCode == 200 + tweets <- Try(body.convertTo[TweetInfo.Body]).toOption + (rateRemaining, rateReset) <- parseRateLimitHeaders(response) + } yield TweetInfo(tweets, rateRemaining, rateReset) + + def parseRateLimitExceeded(response: requests.Response): Option[RateLimitExceeded] = + if (response.statusCode != 429) None + else parseRateLimitHeaders(response).map{ case (_, l) => RateLimitExceeded(l) } + + def parseErroneousTextResponse(body: Either[String, JsValue]): Option[ErroneousJsonResponse] = + body.toOption.map(ErroneousJsonResponse) + + def parseErroneousJsonResponse(body: Either[String, JsValue]): Option[ErroneousTextResponse] = + body.swap.toOption.map(ErroneousTextResponse) +} diff --git a/src/main/scala/dev/habla/twitter/v2_requests/lookupt/Run.scala b/src/main/scala/dev/habla/twitter/v2_requests/lookupt/Run.scala new file mode 100644 index 0000000..a2e52bf --- /dev/null +++ b/src/main/scala/dev/habla/twitter/v2_requests/lookupt/Run.scala @@ -0,0 +1,9 @@ +package dev.habla.twitter +package v2_requests +package lookupt + +object Run extends HttpEndpoint[v2.lookupt.Request] + with From + with To{ + type Response = v2.lookupt.Response + } \ No newline at end of file diff --git a/src/main/scala/dev/habla/twitter/v2_requests/lookupt/To.scala b/src/main/scala/dev/habla/twitter/v2_requests/lookupt/To.scala new file mode 100644 index 0000000..4d47058 --- /dev/null +++ b/src/main/scala/dev/habla/twitter/v2_requests/lookupt/To.scala @@ -0,0 +1,19 @@ +package dev.habla.twitter +package v2_requests.lookupt + +import v2.lookupt.Request +import v2_requests.QueryParams + +trait To extends QueryParams{ + + def to(request: Request): requests.Request = { + requests.Request( + url = s"https://api.twitter.com/2/tweets/${request.id}", + params = Map[String, String]() + .add("expansions", request.expansions) + .add("tweet.fields", request.tweetFields) + .add("place.fields", request.placeFields), + headers = Map("Authorization" -> request.bearerToken) + ) + } +} diff --git a/src/main/scala/dev/habla/twitter/v2_requests/lookupuser/From.scala b/src/main/scala/dev/habla/twitter/v2_requests/lookupuser/From.scala new file mode 100644 index 0000000..d58a7f0 --- /dev/null +++ b/src/main/scala/dev/habla/twitter/v2_requests/lookupuser/From.scala @@ -0,0 +1,39 @@ +package dev.habla.twitter +package v2_requests +package lookupuser + +import dev.habla.twitter.v2 +import dev.habla.twitter.v2.lookupuser._ +import spray.json.JsValue + +import scala.util.Try + + +trait From extends HttpBody with RateLimitHeaders{ + + def from(response: requests.Response): Response = { + val bodyE = parseBody(response) + parseTweetInfo(response, bodyE) + .orElse(parseRateLimitExceeded(response)) + .orElse(parseErroneousTextResponse(bodyE)) + .orElse(parseErroneousJsonResponse(bodyE)) + .getOrElse(ErroneousTextResponse("Not a lookup response")) + } + + def parseTweetInfo(response: requests.Response, bodyE: Either[String,JsValue]): Option[UserInfo] = + for { + body <- bodyE.toOption if response.statusCode == 200 + users <- Try(body.convertTo[UserInfo.Body]).toOption + (rateRemaining, rateReset) <- parseRateLimitHeaders(response) + } yield UserInfo(users, rateRemaining, rateReset) + + def parseRateLimitExceeded(response: requests.Response): Option[RateLimitExceeded] = + if (response.statusCode != 429) None + else parseRateLimitHeaders(response).map{ case (_, l) => RateLimitExceeded(l) } + + def parseErroneousTextResponse(body: Either[String, JsValue]): Option[ErroneousJsonResponse] = + body.toOption.map(ErroneousJsonResponse) + + def parseErroneousJsonResponse(body: Either[String, JsValue]): Option[ErroneousTextResponse] = + body.swap.toOption.map(ErroneousTextResponse) +} diff --git a/src/main/scala/dev/habla/twitter/v2_requests/lookupuser/Run.scala b/src/main/scala/dev/habla/twitter/v2_requests/lookupuser/Run.scala new file mode 100644 index 0000000..38d89e8 --- /dev/null +++ b/src/main/scala/dev/habla/twitter/v2_requests/lookupuser/Run.scala @@ -0,0 +1,9 @@ +package dev.habla.twitter +package v2_requests +package lookupuser + +object Run extends HttpEndpoint[v2.lookupuser.Request] + with From + with To{ + type Response = v2.lookupuser.Response + } diff --git a/src/main/scala/dev/habla/twitter/v2_requests/lookupuser/To.scala b/src/main/scala/dev/habla/twitter/v2_requests/lookupuser/To.scala new file mode 100644 index 0000000..b33ea59 --- /dev/null +++ b/src/main/scala/dev/habla/twitter/v2_requests/lookupuser/To.scala @@ -0,0 +1,25 @@ +package dev.habla.twitter +package v2_requests +package lookupuser + +import v2.lookupuser.Request + +trait To extends QueryParams{ + + def to(request: Request): requests.Request = { + requests.Request( + url = s"https://api.twitter.com/2/users/${request + .idOrName + .fold( + identity, + username => s"by/username/$username" + ) + }", + params = Map[String, String]() + .add("expansions", request.expansions) + .add("tweet.fields", request.tweetFields) + .add("user.fields", request.userFields), + headers = Map("Authorization" -> request.bearerToken) + ) + } +} diff --git a/src/main/scala/dev/habla/twitter/v2_requests/package.scala b/src/main/scala/dev/habla/twitter/v2_requests/package.scala new file mode 100644 index 0000000..bcf96ea --- /dev/null +++ b/src/main/scala/dev/habla/twitter/v2_requests/package.scala @@ -0,0 +1,4 @@ +package dev.habla.twitter + +package object v2_requests extends HttpEndpointSyntax + with HttpEndpointInstances \ No newline at end of file