From 1a73ff641cf741c83ebdb1886fda8ae877e7df1d Mon Sep 17 00:00:00 2001
From: Leo Benkel <4960573+leobenkel@users.noreply.github.com>
Date: Mon, 19 Dec 2022 15:18:00 +0100
Subject: [PATCH] Upgrade to latest versions (#219)

* Upgrade to latest versions

Address https://github.com/leobenkel/ZparkIO/issues/217 and https://github.com/leobenkel/ZparkIO/issues/213

* Remove bad options for some scala version

* Simplify ci-cd

* Remove other conflicting scoptions

* Better system for the scalaCoptions

* More fixes

* More fixes

* Try again

* Add sbt cache
---
 .github/workflows/ci.yml                      | 82 ++++++++++++++--
 .github/workflows/release.yml                 | 47 ----------
 .../zparkio/Services/SparkModule.scala        |  2 +-
 .../com/leobenkel/zparkio/implicits.scala     |  8 +-
 build.sbt                                     | 94 +++++++++++++++----
 .../scallop/CommandLineArgumentScallop.scala  |  3 +-
 .../CommandLineArgumentScallopTest.scala      |  2 +-
 .../scala/com/leobenkel/example1/FileIO.scala |  2 +-
 .../leobenkel/example1/ApplicationTest.scala  |  7 +-
 .../example2/Services/Database.scala          |  2 +-
 .../leobenkel/example2/Services/FileIO.scala  |  2 +-
 .../Transformations/UserTransformations.scala |  2 +-
 .../leobenkel/example2/ApplicationTest.scala  |  9 +-
 project/build.properties                      |  2 +-
 project/plugins.sbt                           |  4 +
 project/soteria.sbt                           |  2 +-
 soteria.json                                  | 29 +-----
 sparkVersions                                 |  8 +-
 18 files changed, 183 insertions(+), 124 deletions(-)
 delete mode 100644 .github/workflows/release.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 96dd08b..463518c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,6 +1,8 @@
-name: CI
+name: CI-CD
 on:
   push:
+    branches: [ main ]
+    tags: [ "*" ]
     paths-ignore:
       - ".gitignore"
       - ".bettercodehub.yml"
@@ -26,15 +28,25 @@ on:
 jobs:
   build:
     strategy:
-      fail-fast: false
+      fail-fast: true
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2.3.4
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
-      - name: Coursier cache
-        uses: coursier/cache-action@v5
+        uses: actions/checkout@v2
+
+      - name: Cache SBT
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.ivy2/cache
+            ~/.sbt
+          key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt') }}
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'adopt'
+          java-version: '8'
+          cache: 'sbt'
 
       - name: clean
         run: make deep_clean
@@ -42,6 +54,7 @@ jobs:
       - name: build
         run: |
           ./scripts/spark-cross-compile.sh
+
       - name: test
         run: |
           ./scripts/spark-cross-fmt.sh
@@ -50,7 +63,58 @@ jobs:
       - name: unit-test-coverage
         run: make test_coverage
         env:
-          COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+
+      - name: clean-up
+        run: |
+          rm -rf "$HOME/.ivy2/local" || true
+          find $HOME/Library/Caches/Coursier/v1        -name "ivydata-*.properties" -delete || true
+          find $HOME/.ivy2/cache                       -name "ivydata-*.properties" -delete || true
+          find $HOME/.cache/coursier/v1                -name "ivydata-*.properties" -delete || true
+          find $HOME/.sbt
+
+  release:
+    needs: build
+    strategy:
+      fail-fast: true
+    runs-on: ubuntu-latest
+    env:
+      PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
+      PGP_SECRET: ${{ secrets.PGP_SECRET }}
+      SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
+      SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Cache SBT
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.ivy2/cache
+            ~/.sbt
+          key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt') }}
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'adopt'
+          java-version: '8'
+          cache: 'sbt'
+
+      - name: release-staging
+        run: |
+          ./scripts/spark-cross-publish.sh library true
+          ./scripts/spark-cross-publish.sh testHelper true
+          ./scripts/spark-cross-publish.sh libraryConfigsScallop true
+        if: startsWith(github.ref, 'refs/tags/') != true && github.ref == 'refs/heads/main'
+
+      - name: release-production
+        run: |
+          ./scripts/spark-cross-publish.sh library
+          ./scripts/spark-cross-publish.sh testHelper
+          ./scripts/spark-cross-publish.sh libraryConfigsScallop
+        if: startsWith(github.ref, 'refs/tags/v')
 
       - name: clean-up
         run: |
@@ -58,4 +122,4 @@ jobs:
           find $HOME/Library/Caches/Coursier/v1        -name "ivydata-*.properties" -delete || true
           find $HOME/.ivy2/cache                       -name "ivydata-*.properties" -delete || true
           find $HOME/.cache/coursier/v1                -name "ivydata-*.properties" -delete || true
-          find $HOME/.sbt                              -name "*.lock"               -delete || true
+          find $HOME/.sbt
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
deleted file mode 100644
index a7fda33..0000000
--- a/.github/workflows/release.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-name: Release
-on:
-  push:
-    branches: [main]
-    tags: ["*"]
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    env:
-      PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
-      PGP_SECRET: ${{ secrets.PGP_SECRET }}
-      SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
-      SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2.3.4
-
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v13
-
-      - name: Coursier cache
-        uses: coursier/cache-action@v5
-
-      - name: clean
-        run: sbt clean
-
-      - name: release-staging
-        run: |
-          ./scripts/spark-cross-publish.sh library true
-          ./scripts/spark-cross-publish.sh testHelper true
-          ./scripts/spark-cross-publish.sh libraryConfigsScallop true
-        if: startsWith(github.ref, 'refs/tags/') != true
-
-      - name: release-production
-        run: |
-          ./scripts/spark-cross-publish.sh library
-          ./scripts/spark-cross-publish.sh testHelper
-          ./scripts/spark-cross-publish.sh libraryConfigsScallop
-        if: startsWith(github.ref, 'refs/tags/v')
-
-      - name: clean-up
-        run: |
-          rm -rf "$HOME/.ivy2/local" || true
-          find $HOME/Library/Caches/Coursier/v1        -name "ivydata-*.properties" -delete || true
-          find $HOME/.ivy2/cache                       -name "ivydata-*.properties" -delete || true
-          find $HOME/.cache/coursier/v1                -name "ivydata-*.properties" -delete || true
-          find $HOME/.sbt
diff --git a/Library/src/main/scala/com/leobenkel/zparkio/Services/SparkModule.scala b/Library/src/main/scala/com/leobenkel/zparkio/Services/SparkModule.scala
index 355d90b..3130ccb 100644
--- a/Library/src/main/scala/com/leobenkel/zparkio/Services/SparkModule.scala
+++ b/Library/src/main/scala/com/leobenkel/zparkio/Services/SparkModule.scala
@@ -18,7 +18,7 @@ object SparkModule {
   }
 
   trait Factory[C <: CommandLineArguments.Service[C]] {
-    lazy private val sparkBuilder:         SparkSession.Builder = SparkSession.builder
+    lazy private val sparkBuilder:         SparkSession.Builder = SparkSession.builder()
     lazy private val sparkBuilderWithName: SparkSession.Builder = sparkBuilder.appName(appName)
 
     protected def appName: String
diff --git a/Library/src/main/scala/com/leobenkel/zparkio/implicits.scala b/Library/src/main/scala/com/leobenkel/zparkio/implicits.scala
index b032c24..55283a4 100644
--- a/Library/src/main/scala/com/leobenkel/zparkio/implicits.scala
+++ b/Library/src/main/scala/com/leobenkel/zparkio/implicits.scala
@@ -30,13 +30,13 @@ object implicits {
 
     def apply[A](f: SparkSession => Dataset[A]): ZDS[A] = ZDS.map(f)
 
-    def make[A <: Product : TypeTag : ClassTag, B <: Product : TypeTag : ClassTag](
+    def make[A <: Product, B <: Product : TypeTag](
         input: Dataset[A]
     )(
         f:     Dataset[A] => Encoder[B] => Dataset[B]
     ): ZDS[B] = ZDS(spark => f(input)(spark.implicits.newProductEncoder[B]))
 
-    def apply[A <: Product : TypeTag : ClassTag](data: A*): ZDS[A] =
+    def apply[A <: Product : TypeTag](data: A*): ZDS[A] =
       apply { spark =>
         import spark.implicits._
         data.toDS()
@@ -53,10 +53,10 @@ object implicits {
   }
 
   implicit class DatasetZ[R, A](zds: => ZIO[R, Throwable, Dataset[A]]) extends Serializable {
-    def mapDS[B <: Product : TypeTag : ClassTag](f: A => B): ZDS_R[R, B] =
+    def mapDS[B <: Product : TypeTag](f: A => B): ZDS_R[R, B] =
       SparkModule().flatMap(spark => zds.map(_.map(f)(spark.implicits.newProductEncoder[B])))
 
-    def zMap[B <: Product : TypeTag : ClassTag](
+    def zMap[B <: Product : TypeTag](
         f: A => ZIO[Any, Throwable, B]
     ): ZDS_R[R, B] =
       ZDS.flatMapR[R, B] { spark =>
diff --git a/build.sbt b/build.sbt
index 2b42573..d606a66 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,12 +2,15 @@ val projectName = IO.readLines(new File("PROJECT_NAME")).head
 val v           = IO.readLines(new File("VERSION")).head
 val sparkVersions: List[String] = IO.readLines(new File("sparkVersions")).map(_.trim)
 
-val scala11 = "2.11.12"
-val scala12 = "2.12.16"
+val Scala11 = "2.11.12"
+val Scala12 = "2.12.17"
+val Scala13 = "2.13.10"
 
-val Spark233 = "2.3.3"
-val Spark245 = "2.4.5"
-val Spark312 = "3.1.2"
+val Spark23 = "2.3.4"
+val Spark24 = "2.4.8"
+val Spark31 = "3.1.3"
+val Spark32 = "3.2.3"
+val Spark33 = "3.3.1"
 
 val sparkVersionSystem = System.getProperty("sparkVersion", sparkVersions.head)
 val sparkVersion       = settingKey[String]("Spark version")
@@ -29,13 +32,17 @@ lazy val rootSettings =
     sparkVersion       := sparkVersionSystem,
     crossScalaVersions := {
       sparkVersion.value match {
-        case Spark233 => Seq(scala11)
-        case Spark245 => Seq(scala11, scala12)
-        case Spark312 => Seq(scala12)
+        case Spark23 => Seq(Scala11)
+        case Spark24 => Seq(Scala12, Scala11)
+        case Spark31 => Seq(Scala12)
+        case Spark32 => Seq(Scala13, Scala12)
+        case Spark33 => Seq(Scala13, Scala12)
+        case s       =>
+          throw new Exception(s"crossScalaVersions: Do not know what to do with spark version $s")
       }
     },
     scalaVersion       := crossScalaVersions.value.head,
-    resolvers += Resolver.sonatypeRepo("releases"),
+    resolvers ++= Resolver.sonatypeOssRepos("releases"),
     soteriaAddSemantic := false,
     version ~= (v => s"${sparkVersionSystem}_$v"),
     dynver ~= (v => s"${sparkVersionSystem}_$v")
@@ -49,16 +56,57 @@ lazy val commonSettings =
       libraryDependencies ++=
         Seq(
           // https://zio.dev/docs/getting_started.html
-          "dev.zio"          %% "zio"        % zioVersion,
-          // https://mvnrepository.com/artifact/org.apache.spark/spark-core
-          "org.apache.spark" %% "spark-core" % sparkVersion.value % Provided,
-          // https://mvnrepository.com/artifact/org.apache.spark/spark-sql
-          "org.apache.spark" %% "spark-sql"  % sparkVersion.value % Provided,
-          "org.scalatest"    %% "scalatest"  % "3.2.14"           % Test
+          "dev.zio"          %% "zio"             % zioVersion,
+
+          // SPARK
+          "org.apache.spark" %% "spark-core"      % sparkVersion.value,
+          "org.apache.spark" %% "spark-streaming" % sparkVersion.value,
+          "org.apache.spark" %% "spark-sql"       % sparkVersion.value,
+          "org.apache.spark" %% "spark-hive"      % sparkVersion.value,
+          "org.apache.spark" %% "spark-catalyst"  % sparkVersion.value,
+          "org.apache.spark" %% "spark-yarn"      % sparkVersion.value,
+          "org.apache.spark" %% "spark-mllib"     % sparkVersion.value,
+
+          // TEST
+          "org.scalatest"    %% "scalatest"       % "3.2.14" % Test
         ),
+      libraryDependencies ++= {
+        sparkVersion.value match {
+          case Spark23 | Spark24           => Seq(
+              "org.apache.xbean" % "xbean-asm6-shaded" % "4.10"
+            )
+          case Spark31 | Spark32 | Spark33 => Seq(
+              "io.netty" % "netty-all"              % "4.1.77.Final",
+              "io.netty" % "netty-buffer"           % "4.1.77.Final",
+              "io.netty" % "netty-tcnative-classes" % "2.0.52.Final"
+            )
+          case _                           => Seq.empty
+        }
+      },
       updateOptions          := updateOptions.value.withGigahorse(false),
       Test / publishArtifact := false,
-      pomIncludeRepository   := (_ => false)
+      pomIncludeRepository   := (_ => false),
+      scalacOptions ++= {
+        scalaVersion.value match {
+          case Scala11 | Scala12 => Seq(
+              "-Ywarn-inaccessible",
+              "-Ywarn-unused-import"
+            )
+          case Scala13           => Seq.empty
+          case s                 => throw new Exception(s"scalacOptions: Unknown mapping for scala version $s")
+        }
+      }
+      // Can be needed in the future
+//      Compile / unmanagedSourceDirectories ++= {
+//        val pathWith: String => File = (p: String) => baseDirectory.value / "src" / "main" / p
+//        scalaVersion.value match {
+//          case Scala13           => Seq(pathWith("scala2.13"))
+//          case Scala11 | Scala12 => Seq(pathWith("scala2"))
+//          case s                 => throw new Exception(
+//              s"unmanagedSourceDirectories: Unknown mapping for scala version $s"
+//            )
+//        }
+//      }
     )
 
 lazy val root = (project in file("."))
@@ -73,11 +121,15 @@ lazy val library = (project in file("Library")).settings(
   name := projectName
 )
 
-lazy val sparkTestingBaseVersion =
+lazy val sparkTestingBaseVersion: String =
   sparkVersionSystem match {
     // https://mvnrepository.com/artifact/com.holdenkarau/spark-testing-base
-    case Spark312 => "3.1.2_1.1.0"
-    case _        => s"${sparkVersionSystem}_0.14.0"
+    case Spark23 => "2.3.3_0.14.0"
+    case Spark24 => "2.4.8_1.3.0"
+    case Spark31 => "3.1.2_1.3.0"
+    case Spark32 => "3.2.2_1.3.0"
+    case Spark33 => "3.3.1_1.3.0"
+    case s       => throw new Exception(s"sparkTestingBaseVersion: Unknown mapping for spark version $s")
   }
 
 lazy val testHelper = (project in file("testModules/TestHelper"))
@@ -142,3 +194,7 @@ lazy val example2Small = (project in file("examples/Example2_small"))
     libraryConfigsScallop,
     testHelper % Test
   )
+
+// https://github.com/sbt/sbt/issues/6997#issuecomment-1310637232
+ThisBuild / libraryDependencySchemes +=
+  "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always
diff --git a/configLibs/Scallop/src/main/scala/com/leobenkel/zparkio/config/scallop/CommandLineArgumentScallop.scala b/configLibs/Scallop/src/main/scala/com/leobenkel/zparkio/config/scallop/CommandLineArgumentScallop.scala
index 978544f..b0ea3eb 100644
--- a/configLibs/Scallop/src/main/scala/com/leobenkel/zparkio/config/scallop/CommandLineArgumentScallop.scala
+++ b/configLibs/Scallop/src/main/scala/com/leobenkel/zparkio/config/scallop/CommandLineArgumentScallop.scala
@@ -30,7 +30,8 @@ object CommandLineArgumentScallop {
       val (active, inactive) =
         filteredSummary(Set.empty).split('\n').partition(_.trim.startsWith("*"))
 
-      (active.sorted :+ "") ++ inactive.sorted
+      val a: Seq[String] = active.sorted.toSeq :+ ""
+      a ++ inactive.sorted
     }
 
     final val env: ScallopOption[Environment] =
diff --git a/configLibs/Scallop/src/test/scala/com/leobenkel/zparkio/config/scallop/CommandLineArgumentScallopTest.scala b/configLibs/Scallop/src/test/scala/com/leobenkel/zparkio/config/scallop/CommandLineArgumentScallopTest.scala
index 2c3f7b1..09ae860 100644
--- a/configLibs/Scallop/src/test/scala/com/leobenkel/zparkio/config/scallop/CommandLineArgumentScallopTest.scala
+++ b/configLibs/Scallop/src/test/scala/com/leobenkel/zparkio/config/scallop/CommandLineArgumentScallopTest.scala
@@ -114,7 +114,7 @@ class CommandLineArgumentScallopTest extends AnyFreeSpec {
           .run(
             arg.checkValidity().tapError { case h: HelpHandlerException => h.printHelpMessage }
           ) match {
-          case Success(a)  => assert(true)
+          case Success(_)  => assert(true)
           case Failure(ex) => assertThrows[HelpHandlerException](throw ex.squash)
         }
       }
diff --git a/examples/Example1_mini/src/main/scala/com/leobenkel/example1/FileIO.scala b/examples/Example1_mini/src/main/scala/com/leobenkel/example1/FileIO.scala
index 1a7b754..80da02d 100644
--- a/examples/Example1_mini/src/main/scala/com/leobenkel/example1/FileIO.scala
+++ b/examples/Example1_mini/src/main/scala/com/leobenkel/example1/FileIO.scala
@@ -18,7 +18,7 @@ object FileIO {
   private trait LiveService extends FileIO.Service {
     override protected def readFileContent(path: String): Seq[String] = {
       val file    = Source.fromFile(path)
-      val content = file.getLines().toArray
+      val content = file.getLines().toSeq
       file.close()
       content
     }
diff --git a/examples/Example1_mini/src/test/scala/com/leobenkel/example1/ApplicationTest.scala b/examples/Example1_mini/src/test/scala/com/leobenkel/example1/ApplicationTest.scala
index 00d6694..3fc614e 100644
--- a/examples/Example1_mini/src/test/scala/com/leobenkel/example1/ApplicationTest.scala
+++ b/examples/Example1_mini/src/test/scala/com/leobenkel/example1/ApplicationTest.scala
@@ -2,6 +2,7 @@ package com.leobenkel.example1
 
 import com.leobenkel.zparkio.ZparkioApp.ZIOEnv
 import com.leobenkel.zparkiotest.TestWithSpark
+import org.scalatest.Assertions
 import org.scalatest.freespec.AnyFreeSpec
 import zio.{Runtime, Unsafe, ZIO}
 import zio.Exit.{Failure, Success}
@@ -19,7 +20,7 @@ class ApplicationTest extends AnyFreeSpec with TestWithSpark {
           case Success(value) =>
             println(s"Read: $value")
             assertResult(0)(value)
-          case Failure(cause) => fail(cause.prettyPrint)
+          case Failure(cause) => Assertions.fail(cause.prettyPrint)
         }
       }
 
@@ -34,7 +35,7 @@ class ApplicationTest extends AnyFreeSpec with TestWithSpark {
           case Success(value) =>
             println(s"Read: $value")
             assertResult(1)(value)
-          case Failure(cause) => fail(cause.prettyPrint)
+          case Failure(cause) => Assertions.fail(cause.prettyPrint)
         }
       }
 
@@ -44,7 +45,7 @@ class ApplicationTest extends AnyFreeSpec with TestWithSpark {
           case Success(value) =>
             println(s"Read: $value")
             assertResult(0)(value)
-          case Failure(cause) => fail(cause.prettyPrint)
+          case Failure(cause) => Assertions.fail(cause.prettyPrint)
         }
       }
   }
diff --git a/examples/Example2_small/src/main/scala/com/leobenkel/example2/Services/Database.scala b/examples/Example2_small/src/main/scala/com/leobenkel/example2/Services/Database.scala
index b426c6b..f329068 100644
--- a/examples/Example2_small/src/main/scala/com/leobenkel/example2/Services/Database.scala
+++ b/examples/Example2_small/src/main/scala/com/leobenkel/example2/Services/Database.scala
@@ -43,7 +43,7 @@ object Database {
        * }}}
        * would go.
        */
-      Seq[A]().toDS
+      Seq[A]().toDS()
     }
   }
 
diff --git a/examples/Example2_small/src/main/scala/com/leobenkel/example2/Services/FileIO.scala b/examples/Example2_small/src/main/scala/com/leobenkel/example2/Services/FileIO.scala
index 322d223..72e5da3 100644
--- a/examples/Example2_small/src/main/scala/com/leobenkel/example2/Services/FileIO.scala
+++ b/examples/Example2_small/src/main/scala/com/leobenkel/example2/Services/FileIO.scala
@@ -16,7 +16,7 @@ object FileIO {
   trait LiveService extends FileIO.Service {
     override protected def readFileContent(path: String): Seq[String] = {
       val file    = Source.fromFile(path)
-      val content = file.getLines().toArray
+      val content = file.getLines().toSeq
       file.close()
       content
     }
diff --git a/examples/Example2_small/src/main/scala/com/leobenkel/example2/Transformations/UserTransformations.scala b/examples/Example2_small/src/main/scala/com/leobenkel/example2/Transformations/UserTransformations.scala
index 15f3525..0c444ad 100644
--- a/examples/Example2_small/src/main/scala/com/leobenkel/example2/Transformations/UserTransformations.scala
+++ b/examples/Example2_small/src/main/scala/com/leobenkel/example2/Transformations/UserTransformations.scala
@@ -20,7 +20,7 @@ object UserTransformations {
       authorIds <-
         ZDS.broadcast { spark =>
           import spark.implicits._
-          posts.map(_.authorId).distinct.collect
+          posts.map(_.authorId).distinct().collect()
         }
       authors   <- ZDS(_ => users.filter(u => authorIds.value.contains(u.userId)))
     } yield authors
diff --git a/examples/Example2_small/src/test/scala/com/leobenkel/example2/ApplicationTest.scala b/examples/Example2_small/src/test/scala/com/leobenkel/example2/ApplicationTest.scala
index c8fc3a3..c45aeb8 100644
--- a/examples/Example2_small/src/test/scala/com/leobenkel/example2/ApplicationTest.scala
+++ b/examples/Example2_small/src/test/scala/com/leobenkel/example2/ApplicationTest.scala
@@ -7,6 +7,7 @@ import com.leobenkel.zparkio.Services.Logger
 import com.leobenkel.zparkio.ZparkioApp.ZIOEnv
 import com.leobenkel.zparkiotest.{LoggerService, TestWithSpark}
 import org.apache.spark.sql._
+import org.scalatest.Assertions
 import org.scalatest.freespec.AnyFreeSpec
 import zio.{Console, Runtime, Unsafe, ZIO, ZLayer}
 import zio.Exit.{Failure, Success}
@@ -20,7 +21,7 @@ class ApplicationTest extends AnyFreeSpec with TestWithSpark {
           case Success(value) =>
             println(s"Read exit code: $value")
             assertResult(0)(value)
-          case Failure(cause) => fail(cause.prettyPrint)
+          case Failure(cause) => Assertions.fail(cause.prettyPrint)
         }
       }
     }
@@ -37,7 +38,7 @@ class ApplicationTest extends AnyFreeSpec with TestWithSpark {
           case Success(value) =>
             println(s"Read: $value")
             assertResult(1)(value)
-          case Failure(cause) => fail(cause.prettyPrint)
+          case Failure(cause) => Assertions.fail(cause.prettyPrint)
         }
       }
     }
@@ -49,7 +50,7 @@ class ApplicationTest extends AnyFreeSpec with TestWithSpark {
           case Success(value) =>
             println(s"Read exit code: $value")
             assertResult(0)(value)
-          case Failure(cause) => fail(cause.prettyPrint)
+          case Failure(cause) => Assertions.fail(cause.prettyPrint)
         }
       }
     }
@@ -107,7 +108,7 @@ case class TestApp(s: SparkSession) extends Application {
               }
 
             import spark.implicits._
-            rawSeq.map(_.asInstanceOf[A]).toDS
+            rawSeq.map(_.asInstanceOf[A]).toDS()
           }
         }
       }
diff --git a/project/build.properties b/project/build.properties
index d738b85..9a19778 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -1 +1 @@
-sbt.version = 1.7.1
+sbt.version = 1.8.0
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 11538d0..88a4f08 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,2 +1,6 @@
 // To publish
 addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.11")
+
+// https://github.com/sbt/sbt/issues/6997#issuecomment-1310637232
+ThisBuild / libraryDependencySchemes +=
+  "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always
diff --git a/project/soteria.sbt b/project/soteria.sbt
index 7451de4..362415f 100644
--- a/project/soteria.sbt
+++ b/project/soteria.sbt
@@ -1,2 +1,2 @@
 // https://github.com/leobenkel/safety_plugin
-addSbtPlugin("com.leobenkel" % "soteria" % "0.4.9")
+addSbtPlugin("com.leobenkel" % "soteria" % "0.5.1")
diff --git a/soteria.json b/soteria.json
index b225656..233491e 100644
--- a/soteria.json
+++ b/soteria.json
@@ -56,24 +56,6 @@
         "version": "2.5"
       }
     },
-    "io.netty": {
-      "netty": {
-        "version": "3.10.6.Final"
-      },
-      "netty-all": {
-        "overrideIsEnough": false,
-        "version": "4.1.48.Final"
-      },
-      "netty-codec": {
-        "version": "4.1.48.Final"
-      },
-      "netty-codec-http": {
-        "version": "4.1.48.Final"
-      },
-      "netty-handler": {
-        "version": "4.1.48.Final"
-      }
-    },
     "jline": {
       "jline": {
         "overrideIsEnough": false,
@@ -106,7 +88,6 @@
       },
       "hadoop-hdfs": {
         "dependenciesToRemove": [
-          "io.netty | netty-all",
           "org.codehaus.jackson | jackson-mapper-asl"
         ],
         "version": "2.7.5"
@@ -151,7 +132,6 @@
           "commons-beanutils | commons-beanutils-core",
           "commons-collections | commons-collections",
           "commons-io | commons-io",
-          "io.netty | netty-all",
           "org.apache.xbean | xbean-asm6-shaded",
           "org.apache.commons | commons-compress",
           "org.apache.hadoop | hadoop-hdfs",
@@ -162,7 +142,6 @@
           "org.codehaus.jackson | jackson-mapper-asl"
         ],
         "exactName": false,
-        "shouldBeProvided": true,
         "version": "none"
       }
     },
@@ -179,14 +158,12 @@
       }
     }
   },
-  "sbtVersion": "1.7.1",
+  "sbtVersion": "1.8.0",
   "scalaCFlags": [
     "-feature",
     "-Yrangepos",
     "-Ywarn-dead-code",
-    "-Ywarn-inaccessible",
     "-Ywarn-unused",
-    "-Ywarn-unused-import",
     "-Ywarn-value-discard",
     "-deprecation",
     "-encoding",
@@ -194,7 +171,7 @@
   ],
   "scalaVersions": [
     "2.11.12",
-    "2.12.16",
-    "2.13.3"
+    "2.12.17",
+    "2.13.10"
   ]
 }
diff --git a/sparkVersions b/sparkVersions
index 6b2881b..f891589 100644
--- a/sparkVersions
+++ b/sparkVersions
@@ -1,3 +1,5 @@
-2.3.3
-2.4.5
-3.1.2
+3.3.1
+3.2.3
+3.1.3
+2.4.8
+2.3.4