Add assemblyUnzipDirectory to AssemblyOption to use a different directory for unzipping jars

README.md

@@ -393,6 +393,25 @@ lazy val app = (project in file("app"))
   )
 ```
 
+
+### Unzip Caching
+
+When assembling an über artifact, that has many library dependencies, the unzip process can be very IO intensive. These unzipped directories are very suitable for CI systems to persist in between job runs.
+
+```scala
+lazy val app = (project in file("app"))
+  .settings(
+    assemblyUnzipDirectory := Some(localCacheDirectory.value / "sbt-assembly" / "dependencies"),
+    assemblyCacheUnzip := true, // this is the default setting
+    assemblyCacheUseHardLinks := true, // this is experimental but will use a hard link between the files in assemblyUnzipDirectory to assemblyDirectory to avoid additional copy IO
+    // more settings here ...
+  )
+```
+
+To populate the assemblyUnzipDirectory without a full assembly:
+
+    sbt assemblyCacheDependency 
+
 Other Things
 ------------
 

src/main/contraband/AssemblyOption.contra

@@ -4,6 +4,8 @@ package sbtassembly
 type AssemblyOption {
   assemblyDirectory: java.io.File @since("0.15.0")
 
+  assemblyUnzipDirectory: java.io.File @since("1.2.0")
+
   ## include compiled class files from itself or subprojects
   includeBin: Boolean! = true @since("0.15.0")
 
@@ -22,6 +24,9 @@ type AssemblyOption {
 
   cacheUnzip: Boolean! = true @since("0.15.0")
 
+  ## this is experimental but will use a hard link between the files in assemblyUnzipDirectory to assemblyDirectory to avoid additional copy IO
+  cacheUseHardLinks: Boolean! = false @since("1.2.0")
+
   appendContentHash: Boolean! = false @since("0.15.0")
 
   prependShellScript: sbtassembly.Assembly.SeqString @since("0.15.0")

src/main/scala/sbtassembly/AssemblyKeys.scala

@@ -5,23 +5,26 @@ import Keys._
 import com.eed3si9n.jarjarabrams
 
 trait AssemblyKeys {
-  lazy val assembly                  = taskKey[File]("Builds a deployable über JAR")
-  lazy val assembleArtifact          = settingKey[Boolean]("Enables (true) or disables (false) assembling an artifact")
-  lazy val assemblyOption            = taskKey[AssemblyOption]("Configuration for making a deployable über JAR")
-  lazy val assembledMappings         = taskKey[Seq[MappingSet]]("Keeps track of jar origins for each source")
+  lazy val assembly                   = taskKey[File]("Builds a deployable über JAR")
+  lazy val assembleArtifact           = settingKey[Boolean]("Enables (true) or disables (false) assembling an artifact")
+  lazy val assemblyOption             = taskKey[AssemblyOption]("Configuration for making a deployable über JAR")
+  lazy val assembledMappings          = taskKey[Seq[MappingSet]]("Keeps track of jar origins for each source")
+  lazy val assemblyCacheDependency    = taskKey[Boolean]("Caches the unzipped products of the dependency JAR files.  Requires assemblyCacheUnzip (true) and AssemblyOption.assemblyUnzipCacheDir to be provided.")
 
-  lazy val assemblyPackageScala      = taskKey[File]("Produces the Scala artifact")
-  lazy val assemblyPackageDependency = taskKey[File]("Produces the dependency artifact")
-  lazy val assemblyJarName           = taskKey[String]("name of the über jar")
-  lazy val assemblyDefaultJarName    = taskKey[String]("default name of the über jar")
-  lazy val assemblyOutputPath        = taskKey[File]("output path of the über jar")
-  lazy val assemblyExcludedJars      = taskKey[Classpath]("list of excluded jars")
-  lazy val assemblyMergeStrategy     = settingKey[String => MergeStrategy]("mapping from archive member path to merge strategy")
-  lazy val assemblyShadeRules        = settingKey[Seq[jarjarabrams.ShadeRule]]("shading rules backed by jarjar")
-  lazy val assemblyAppendContentHash = settingKey[Boolean]("Appends SHA-1 fingerprint to the assembly file name")
-  lazy val assemblyMaxHashLength     = settingKey[Int]("Length of SHA-1 fingerprint used for the assembly file name")
-  lazy val assemblyCacheUnzip        = settingKey[Boolean]("Enables (true) or disables (false) cacheing the unzipped products of the dependency JAR files")
-  lazy val assemblyCacheOutput       = settingKey[Boolean]("Enables (true) or disables (false) cacheing the output if the content has not changed")
+  lazy val assemblyUnzipDirectory     = settingKey[Option[File]]("Specify a directory to unzip the products of dependency JAR files (e.g. assemblyUnzipDirectory := Some(localCacheDirectory.value / \"sbt-assembly\" / \"dependencies\").  Default None (uses default assembly directory).")
+  lazy val assemblyPackageScala       = taskKey[File]("Produces the Scala artifact")
+  lazy val assemblyPackageDependency  = taskKey[File]("Produces the dependency artifact")
+  lazy val assemblyJarName            = taskKey[String]("name of the über jar")
+  lazy val assemblyDefaultJarName     = taskKey[String]("default name of the über jar")
+  lazy val assemblyOutputPath         = taskKey[File]("output path of the über jar")
+  lazy val assemblyExcludedJars       = taskKey[Classpath]("list of excluded jars")
+  lazy val assemblyMergeStrategy      = settingKey[String => MergeStrategy]("mapping from archive member path to merge strategy")
+  lazy val assemblyShadeRules         = settingKey[Seq[jarjarabrams.ShadeRule]]("shading rules backed by jarjar")
+  lazy val assemblyAppendContentHash  = settingKey[Boolean]("Appends SHA-1 fingerprint to the assembly file name")
+  lazy val assemblyMaxHashLength      = settingKey[Int]("Length of SHA-1 fingerprint used for the assembly file name")
+  lazy val assemblyCacheUnzip         = settingKey[Boolean]("Enables (true) or disables (false) cacheing the unzipped products of the dependency JAR files")
+  lazy val assemblyCacheOutput        = settingKey[Boolean]("Enables (true) or disables (false) cacheing the output if the content has not changed")
+  lazy val assemblyCacheUseHardLinks  = settingKey[Boolean]("Experimental. Enables (true) or disables (false) using Files.createLink from the unzipped dependency cache to the assembly directory.  Requires both paths to be on the same physical filesystem.  Default false.")
   lazy val assemblyPrependShellScript = settingKey[Option[Seq[String]]]("A launch script to prepend to the über JAR")
 }
 

src/main/scala/sbtassembly/AssemblyOption.scala

@@ -7,9 +7,11 @@ package sbtassembly
 /**
  * @param includeBin include compiled class files from itself or subprojects
  * @param includeDependency include class files from external dependencies
+ * @param cacheUseHardLinks this is experimental but will use a hard link between the files in assemblyUnzipDirectory to assemblyDirectory to avoid additional copy IO
  */
 final class AssemblyOption private (
   val assemblyDirectory: Option[java.io.File],
+  val assemblyUnzipDirectory: Option[java.io.File],
   val includeBin: Boolean,
   val includeScala: Boolean,
   val includeDependency: Boolean,
@@ -18,34 +20,42 @@ final class AssemblyOption private (
   val mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy,
   val cacheOutput: Boolean,
   val cacheUnzip: Boolean,
+  val cacheUseHardLinks: Boolean,
   val appendContentHash: Boolean,
   val prependShellScript: Option[sbtassembly.Assembly.SeqString],
   val maxHashLength: Option[Int],
   val shadeRules: sbtassembly.Assembly.SeqShadeRules,
   val scalaVersion: String,
   val level: sbt.Level.Value) extends Serializable {
 
-  private def this() = this(None, true, true, true, Nil, sbtassembly.Assembly.defaultExcludedFiles, sbtassembly.MergeStrategy.defaultMergeStrategy, true, true, false, None, None, sbtassembly.Assembly.defaultShadeRules, "", sbt.Level.Info)
+  private def this() = this(None, None, true, true, true, Nil, sbtassembly.Assembly.defaultExcludedFiles, sbtassembly.MergeStrategy.defaultMergeStrategy, true, true, false, false, None, None, sbtassembly.Assembly.defaultShadeRules, "", sbt.Level.Info)
+  private def this(assemblyDirectory: Option[java.io.File], includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, appendContentHash: Boolean, prependShellScript: Option[sbtassembly.Assembly.SeqString], maxHashLength: Option[Int], shadeRules: sbtassembly.Assembly.SeqShadeRules, scalaVersion: String, level: sbt.Level.Value) = this(assemblyDirectory, None, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, false, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level)
 
   override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match {
-    case x: AssemblyOption => (this.assemblyDirectory == x.assemblyDirectory) && (this.includeBin == x.includeBin) && (this.includeScala == x.includeScala) && (this.includeDependency == x.includeDependency) && (this.excludedJars == x.excludedJars) && (this.excludedFiles == x.excludedFiles) && (this.mergeStrategy == x.mergeStrategy) && (this.cacheOutput == x.cacheOutput) && (this.cacheUnzip == x.cacheUnzip) && (this.appendContentHash == x.appendContentHash) && (this.prependShellScript == x.prependShellScript) && (this.maxHashLength == x.maxHashLength) && (this.shadeRules == x.shadeRules) && (this.scalaVersion == x.scalaVersion) && (this.level == x.level)
+    case x: AssemblyOption => (this.assemblyDirectory == x.assemblyDirectory) && (this.assemblyUnzipDirectory == x.assemblyUnzipDirectory) && (this.includeBin == x.includeBin) && (this.includeScala == x.includeScala) && (this.includeDependency == x.includeDependency) && (this.excludedJars == x.excludedJars) && (this.excludedFiles == x.excludedFiles) && (this.mergeStrategy == x.mergeStrategy) && (this.cacheOutput == x.cacheOutput) && (this.cacheUnzip == x.cacheUnzip) && (this.cacheUseHardLinks == x.cacheUseHardLinks) && (this.appendContentHash == x.appendContentHash) && (this.prependShellScript == x.prependShellScript) && (this.maxHashLength == x.maxHashLength) && (this.shadeRules == x.shadeRules) && (this.scalaVersion == x.scalaVersion) && (this.level == x.level)
     case _ => false
   })
   override def hashCode: Int = {
-    37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (17 + "sbtassembly.AssemblyOption".##) + assemblyDirectory.##) + includeBin.##) + includeScala.##) + includeDependency.##) + excludedJars.##) + excludedFiles.##) + mergeStrategy.##) + cacheOutput.##) + cacheUnzip.##) + appendContentHash.##) + prependShellScript.##) + maxHashLength.##) + shadeRules.##) + scalaVersion.##) + level.##)
+    37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (17 + "sbtassembly.AssemblyOption".##) + assemblyDirectory.##) + assemblyUnzipDirectory.##) + includeBin.##) + includeScala.##) + includeDependency.##) + excludedJars.##) + excludedFiles.##) + mergeStrategy.##) + cacheOutput.##) + cacheUnzip.##) + cacheUseHardLinks.##) + appendContentHash.##) + prependShellScript.##) + maxHashLength.##) + shadeRules.##) + scalaVersion.##) + level.##)
   }
   override def toString: String = {
-    "AssemblyOption(" + assemblyDirectory + ", " + includeBin + ", " + includeScala + ", " + includeDependency + ", " + excludedJars + ", " + excludedFiles + ", " + mergeStrategy + ", " + cacheOutput + ", " + cacheUnzip + ", " + appendContentHash + ", " + prependShellScript + ", " + maxHashLength + ", " + shadeRules + ", " + scalaVersion + ", " + level + ")"
+    "AssemblyOption(" + assemblyDirectory + ", " + assemblyUnzipDirectory + ", " + includeBin + ", " + includeScala + ", " + includeDependency + ", " + excludedJars + ", " + excludedFiles + ", " + mergeStrategy + ", " + cacheOutput + ", " + cacheUnzip + ", " + cacheUseHardLinks + ", " + appendContentHash + ", " + prependShellScript + ", " + maxHashLength + ", " + shadeRules + ", " + scalaVersion + ", " + level + ")"
   }
-  private[this] def copy(assemblyDirectory: Option[java.io.File] = assemblyDirectory, includeBin: Boolean = includeBin, includeScala: Boolean = includeScala, includeDependency: Boolean = includeDependency, excludedJars: sbt.Keys.Classpath = excludedJars, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile = excludedFiles, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy = mergeStrategy, cacheOutput: Boolean = cacheOutput, cacheUnzip: Boolean = cacheUnzip, appendContentHash: Boolean = appendContentHash, prependShellScript: Option[sbtassembly.Assembly.SeqString] = prependShellScript, maxHashLength: Option[Int] = maxHashLength, shadeRules: sbtassembly.Assembly.SeqShadeRules = shadeRules, scalaVersion: String = scalaVersion, level: sbt.Level.Value = level): AssemblyOption = {
-    new AssemblyOption(assemblyDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level)
+  private[this] def copy(assemblyDirectory: Option[java.io.File] = assemblyDirectory, assemblyUnzipDirectory: Option[java.io.File] = assemblyUnzipDirectory, includeBin: Boolean = includeBin, includeScala: Boolean = includeScala, includeDependency: Boolean = includeDependency, excludedJars: sbt.Keys.Classpath = excludedJars, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile = excludedFiles, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy = mergeStrategy, cacheOutput: Boolean = cacheOutput, cacheUnzip: Boolean = cacheUnzip, cacheUseHardLinks: Boolean = cacheUseHardLinks, appendContentHash: Boolean = appendContentHash, prependShellScript: Option[sbtassembly.Assembly.SeqString] = prependShellScript, maxHashLength: Option[Int] = maxHashLength, shadeRules: sbtassembly.Assembly.SeqShadeRules = shadeRules, scalaVersion: String = scalaVersion, level: sbt.Level.Value = level): AssemblyOption = {
+    new AssemblyOption(assemblyDirectory, assemblyUnzipDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, cacheUseHardLinks, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level)
   }
   def withAssemblyDirectory(assemblyDirectory: Option[java.io.File]): AssemblyOption = {
     copy(assemblyDirectory = assemblyDirectory)
   }
   def withAssemblyDirectory(assemblyDirectory: java.io.File): AssemblyOption = {
     copy(assemblyDirectory = Option(assemblyDirectory))
   }
+  def withAssemblyUnzipDirectory(assemblyUnzipDirectory: Option[java.io.File]): AssemblyOption = {
+    copy(assemblyUnzipDirectory = assemblyUnzipDirectory)
+  }
+  def withAssemblyUnzipDirectory(assemblyUnzipDirectory: java.io.File): AssemblyOption = {
+    copy(assemblyUnzipDirectory = Option(assemblyUnzipDirectory))
+  }
   def withIncludeBin(includeBin: Boolean): AssemblyOption = {
     copy(includeBin = includeBin)
   }
@@ -70,6 +80,9 @@ final class AssemblyOption private (
   def withCacheUnzip(cacheUnzip: Boolean): AssemblyOption = {
     copy(cacheUnzip = cacheUnzip)
   }
+  def withCacheUseHardLinks(cacheUseHardLinks: Boolean): AssemblyOption = {
+    copy(cacheUseHardLinks = cacheUseHardLinks)
+  }
   def withAppendContentHash(appendContentHash: Boolean): AssemblyOption = {
     copy(appendContentHash = appendContentHash)
   }
@@ -100,4 +113,6 @@ object AssemblyOption {
   def apply(): AssemblyOption = new AssemblyOption()
   def apply(assemblyDirectory: Option[java.io.File], includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, appendContentHash: Boolean, prependShellScript: Option[sbtassembly.Assembly.SeqString], maxHashLength: Option[Int], shadeRules: sbtassembly.Assembly.SeqShadeRules, scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(assemblyDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level)
   def apply(assemblyDirectory: java.io.File, includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, appendContentHash: Boolean, prependShellScript: sbtassembly.Assembly.SeqString, maxHashLength: Int, shadeRules: sbtassembly.Assembly.SeqShadeRules, scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(Option(assemblyDirectory), includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, appendContentHash, Option(prependShellScript), Option(maxHashLength), shadeRules, scalaVersion, level)
+  def apply(assemblyDirectory: Option[java.io.File], assemblyUnzipDirectory: Option[java.io.File], includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, cacheUseHardLinks: Boolean, appendContentHash: Boolean, prependShellScript: Option[sbtassembly.Assembly.SeqString], maxHashLength: Option[Int], shadeRules: sbtassembly.Assembly.SeqShadeRules, scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(assemblyDirectory, assemblyUnzipDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, cacheUseHardLinks, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level)
+  def apply(assemblyDirectory: java.io.File, assemblyUnzipDirectory: java.io.File, includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, cacheUseHardLinks: Boolean, appendContentHash: Boolean, prependShellScript: sbtassembly.Assembly.SeqString, maxHashLength: Int, shadeRules: sbtassembly.Assembly.SeqShadeRules, scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(Option(assemblyDirectory), Option(assemblyUnzipDirectory), includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, cacheUseHardLinks, appendContentHash, Option(prependShellScript), Option(maxHashLength), shadeRules, scalaVersion, level)
 }