Parse using Circe

Switches using Alpakka and mdedetrich/akka-streams-json to parse `compare` API responses from GitHub.
2m · May 15, 2021 · f881449 · f881449
1 parent d358ff2
commit f881449
Show file tree

Hide file tree

Showing 5 changed files with 60 additions and 118 deletions.
diff --git a/build.sbt b/build.sbt
@@ -1,4 +1,6 @@
 val ScalaVersion = "2.12.13"
+val AkkaStreamsJson = "0.7.0"
+val Circe = "0.13.0"
 
 lazy val authors = project
   .in(file("."))
@@ -12,27 +14,24 @@ lazy val core = project
       val Akka = "2.6.14"
       val AkkaHttp = "10.2.4"
       libraryDependencies ++= Seq(
-        "com.typesafe.akka"    %% "akka-actor"               % Akka,
-        "com.typesafe.akka"    %% "akka-stream"              % Akka,
-        "com.typesafe.akka"    %% "akka-slf4j"               % Akka,
-        "com.typesafe.akka"    %% "akka-http"                % AkkaHttp,
-        "com.tradeshift"       %% "ts-reaktive-marshal-akka" % "0.16.3" exclude ("org.slf4j", "slf4j-log4j12"),
-        "com.madgag.scala-git" %% "scala-git"                % "4.2",
-        "ch.qos.logback"        % "logback-classic"          % "1.2.3",
-        "org.scalatest"        %% "scalatest"                % "3.2.8" % "test",
-        "com.typesafe.akka"    %% "akka-testkit"             % Akka    % "test",
-        // these come from ts-reaktive-marshal-akka
-        "com.typesafe.akka" %% "akka-persistence"       % Akka,
-        "com.typesafe.akka" %% "akka-remote"            % Akka,
-        "com.typesafe.akka" %% "akka-cluster"           % Akka,
-        "com.typesafe.akka" %% "akka-cluster-tools"     % Akka,
-        "com.typesafe.akka" %% "akka-distributed-data"  % Akka,
-        "com.typesafe.akka" %% "akka-persistence-query" % Akka,
-        "com.typesafe.akka" %% "akka-cluster-sharding"  % Akka,
-        "com.typesafe.akka" %% "akka-protobuf"          % Akka,
-        "com.typesafe.akka" %% "akka-http-jackson"      % AkkaHttp
+        "com.typesafe.akka"    %% "akka-actor"                         % Akka,
+        "com.typesafe.akka"    %% "akka-stream"                        % Akka,
+        "com.typesafe.akka"    %% "akka-slf4j"                         % Akka,
+        "com.typesafe.akka"    %% "akka-http"                          % AkkaHttp,
+        "com.madgag.scala-git" %% "scala-git"                          % "4.2",
+        "ch.qos.logback"        % "logback-classic"                    % "1.2.3",
+        "org.mdedetrich"       %% "akka-stream-circe"                  % AkkaStreamsJson,
+        "org.mdedetrich"       %% "akka-http-circe"                    % AkkaStreamsJson,
+        "com.lightbend.akka"   %% "akka-stream-alpakka-json-streaming" % "3.0.0-M1",
+        "io.circe"             %% "circe-generic"                      % Circe,
+        "io.circe"             %% "circe-generic-extras"               % Circe,
+        "org.scalatest"        %% "scalatest"                          % "3.2.8" % "test",
+        "com.typesafe.akka"    %% "akka-testkit"                       % Akka    % "test"
       )
-    }
+    },
+    addCompilerPlugin(
+      "org.scalamacros" % "paradise" % "2.1.1" cross CrossVersion.full
+    )
   )
   .enablePlugins(AutomateHeaderPlugin)
 

diff --git a/core/src/main/scala/Authors.scala b/core/src/main/scala/Authors.scala
@@ -28,22 +28,27 @@ import akka.http.scaladsl.Http
 import akka.http.scaladsl.marshalling.Marshal
 import akka.http.scaladsl.marshalling.PredefinedToRequestMarshallers._
 import akka.http.scaladsl.model.{HttpRequest, Uri}
+import akka.stream.alpakka.json.scaladsl.JsonReader
 import akka.stream.scaladsl.{Flow, Source}
 import akka.util.ByteString
 
 import com.madgag.git._
-import com.tradeshift.reaktive.marshal.stream.{ActsonReader, ProtocolReader}
 import org.eclipse.jgit.diff.DiffFormatter
 import org.eclipse.jgit.internal.storage.file.FileRepository
 import org.eclipse.jgit.storage.file.FileRepositoryBuilder
 import org.eclipse.jgit.util.io.DisabledOutputStream
+import org.mdedetrich.akka.stream.support.CirceStreamSupport
 
-import lt.dvim.authors.GithubProtocol.{AuthorStats, Commit, Stats}
+import lt.dvim.authors.Authors.{AuthorStats, Stats}
+import lt.dvim.authors.GithubProtocol.{Commit, GitAuthor, GithubAuthor}
 
 object Authors {
   final val MaxAuthors = 1024
   final val GithubApiUrl = "api.github.com"
 
+  final case class Stats(additions: Int, deletions: Int, commits: Int)
+  final case class AuthorStats(gitAuthor: GitAuthor, githubAuthor: Option[GithubAuthor], stats: Stats)
+
   def summary(repo: Option[String], from: String, to: String, path: String): Future[String] = {
     val cld = classOf[ActorSystem].getClassLoader
     implicit val sys = ActorSystem("Authors", classLoader = Some(cld))
@@ -55,8 +60,8 @@ object Authors {
       parseRepo(gitRepository.getConfig().getString("remote", "origin", "url"))
 
     DiffSource(repo.getOrElse(parsedRepo), from, to)
-      .via(ActsonReader.instance)
-      .via(ProtocolReader.of(GithubProtocol.compareProto))
+      .via(JsonReader.select("$.commits[*]"))
+      .via(CirceStreamSupport.decode[Commit])
       .via(StatsAggregator())
       .via(SortingMachine())
       .via(MarkdownConverter())
@@ -120,10 +125,10 @@ object SortingMachine {
 object StatsAggregator {
   def apply()(implicit repo: FileRepository, log: LoggingAdapter): Flow[Commit, AuthorStats, NotUsed] =
     Flow[Commit]
-      .filterNot(_.message.startsWith("Merge pull request"))
-      .groupBy(Authors.MaxAuthors, commit => commit.githubAuthor.map(_.login).getOrElse(commit.gitAuthor.email))
+      .filterNot(_.commit.message.startsWith("Merge pull request"))
+      .groupBy(Authors.MaxAuthors, commit => commit.author.map(_.login).getOrElse(commit.commit.author.email))
       .log("Commit")
-      .map(commit => AuthorStats(commit.gitAuthor, commit.githubAuthor, Authors.shaToStats(commit.sha)))
+      .map(commit => AuthorStats(commit.commit.author, commit.author, Authors.shaToStats(commit.sha)))
       .reduce((aggr, elem) =>
         aggr.copy(
           stats = Stats(
@@ -143,7 +148,7 @@ object MarkdownConverter {
         val authorId = author.githubAuthor.map { gh =>
           // using html instead of markdown, because default
           // avatars come from github not resized
-          s"""[<img width="20" alt="${gh.login}" src="${gh.avatar}&amp;s=40"/> **${gh.login}**](${gh.url})"""
+          s"""[<img width="20" alt="${gh.login}" src="${gh.avatarUrl}&amp;s=40"/> **${gh.login}**](${gh.htmlUrl})"""
         } getOrElse {
           author.gitAuthor.name
         }

diff --git a/core/src/main/scala/GithubProtocol.scala b/core/src/main/scala/GithubProtocol.scala
@@ -16,56 +16,13 @@
 
 package lt.dvim.authors
 
-import com.tradeshift.reaktive.json.JSONProtocol._
-import com.tradeshift.reaktive.marshal.Protocol._
-import io.vavr.control.{Option => VavrOption}
-import lt.dvim.scala.compat.vavr.OptionConverters._
+import io.circe.generic.extras.{Configuration, ConfiguredJsonCodec}
 
 object GithubProtocol {
-  final case class GithubAuthor(login: String, url: String, avatar: String)
-  final case class GitAuthor(name: String, email: String)
-  final case class Stats(additions: Int, deletions: Int, commits: Int)
-  final case class Commit(sha: String, message: String, gitAuthor: GitAuthor, githubAuthor: Option[GithubAuthor])
-  final case class AuthorStats(gitAuthor: GitAuthor, githubAuthor: Option[GithubAuthor], stats: Stats)
+  implicit val config: Configuration = Configuration.default.withSnakeCaseMemberNames.withSnakeCaseConstructorNames
 
-  val compareProto =
-    `object`(
-      field(
-        "commits",
-        array(commitProto)
-      )
-    )
-
-  lazy val commitProto =
-    `object`(
-      option(
-        field("author", githubAuthorProto)
-      ),
-      field(
-        "commit",
-        `object`(
-          field("author", gitAuthorProto),
-          field("message", stringValue),
-          (gitAuthor: GitAuthor, message: String) => (gitAuthor, message)
-        )
-      ),
-      field("sha", stringValue),
-      (githubAuthor: VavrOption[GithubAuthor], gitAuthorAndMessage: (GitAuthor, String), sha: String) =>
-        Commit(sha, gitAuthorAndMessage._2, gitAuthorAndMessage._1, githubAuthor)
-    )
-
-  lazy val gitAuthorProto =
-    `object`(
-      field("name", stringValue),
-      field("email", stringValue),
-      GitAuthor.apply(_, _)
-    )
-
-  lazy val githubAuthorProto =
-    `object`(
-      field("login", stringValue),
-      field("html_url", stringValue),
-      field("avatar_url", stringValue),
-      GithubAuthor.apply(_, _, _)
-    )
+  @ConfiguredJsonCodec final case class GithubAuthor(login: String, htmlUrl: String, avatarUrl: String)
+  @ConfiguredJsonCodec final case class GitAuthor(name: String, email: String)
+  @ConfiguredJsonCodec final case class GitCommit(message: String, author: GitAuthor)
+  @ConfiguredJsonCodec final case class Commit(sha: String, commit: GitCommit, author: Option[GithubAuthor])
 }
diff --git a/core/src/main/scala/ScalaVavrCompat.scala b/core/src/main/scala/ScalaVavrCompat.scala
diff --git a/core/src/test/scala/AuthorsSpec.scala b/core/src/test/scala/AuthorsSpec.scala
@@ -22,16 +22,18 @@ import scala.concurrent.duration._
 
 import akka.actor.ActorSystem
 import akka.event.Logging
+import akka.stream.alpakka.json.scaladsl.JsonReader
 import akka.stream.scaladsl.{FileIO, Sink, Source}
 import akka.testkit.TestKit
 
-import com.tradeshift.reaktive.marshal.stream.{ActsonReader, ProtocolReader}
 import com.typesafe.config.ConfigFactory
+import org.mdedetrich.akka.stream.support.CirceStreamSupport
 import org.scalatest.{BeforeAndAfterAll, Inside}
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpecLike
 
+import lt.dvim.authors.Authors._
 import lt.dvim.authors.GithubProtocol._
 
 object AuthorsSpec {
@@ -54,11 +56,22 @@ class AuthorsSpec
     "parse compare json" in {
       val res = FileIO
         .fromPath(Paths.get(getClass.getResource("/compare.json").toURI), 64)
-        .via(ActsonReader.instance)
-        .via(ProtocolReader.of(GithubProtocol.compareProto))
+        .via(JsonReader.select("$.commits[*]"))
+        .via(CirceStreamSupport.decode[Commit])
         .runFold(Seq.empty[Commit])(_ :+ _)
 
-      res.futureValue should have length 2
+      res.futureValue should contain theSameElementsInOrderAs Seq(
+        Commit(
+          "03ac2f41efff9cdfac9419ba7e6e34b30f9111e0",
+          GitCommit("Add some contents", GitAuthor("Martynas Mickevičius", "[email protected]")),
+          Some(GithubAuthor("2m", "https://github.com/2m", "https://avatars0.githubusercontent.com/u/422086?v=3"))
+        ),
+        Commit(
+          "e5fee6fbc982cea605a820c82a8ae8f14ead26e0",
+          GitCommit("Add some other contents", GitAuthor("Test User", "[email protected]")),
+          None
+        )
+      )
     }
 
     "get commit stats from sha" in {
@@ -72,14 +85,12 @@ class AuthorsSpec
         List(
           Commit(
             "f576a45",
-            "message",
-            GitAuthor("test", "[email protected]"),
+            GitCommit("message", GitAuthor("test", "[email protected]")),
             Some(GithubAuthor("test", "http://users/test", "http://avatars/test"))
           ),
           Commit(
             "bce0e63",
-            "message",
-            GitAuthor("test", "[email protected]"),
+            GitCommit("message", GitAuthor("test", "[email protected]")),
             Some(GithubAuthor("test", "http://users/test", "http://avatars/test"))
           )
         )
@@ -98,14 +109,12 @@ class AuthorsSpec
         List(
           Commit(
             "f576a45",
-            "message",
-            GitAuthor("test", "[email protected]"),
+            GitCommit("message", GitAuthor("test", "[email protected]")),
             Some(GithubAuthor("test", "http://users/test", "http://avatars/test"))
           ),
           Commit(
             "bce0e63",
-            "message",
-            GitAuthor("test", "[email protected]"),
+            GitCommit("message", GitAuthor("test", "[email protected]")),
             Some(GithubAuthor("test", "http://users/test", "http://avatars/test"))
           )
         )
@@ -124,8 +133,7 @@ class AuthorsSpec
         List(
           Commit(
             "901392a",
-            "message",
-            GitAuthor("test", "[email protected]"),
+            GitCommit("message", GitAuthor("test", "[email protected]")),
             Some(GithubAuthor("test", "http://users/test", "http://avatars/test"))
           )
         )