From 772fb69d57cbe14257e9c58009d48cfa4332af1e Mon Sep 17 00:00:00 2001 From: Eric Peters Date: Thu, 21 Oct 2021 11:00:38 -0700 Subject: [PATCH 1/3] Add assemblyUnzipDirectory and cacheUseHardLinks to AssemblyOption.contra --- src/main/contraband/AssemblyOption.contra | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/contraband/AssemblyOption.contra b/src/main/contraband/AssemblyOption.contra index 083882e9..ca02819a 100644 --- a/src/main/contraband/AssemblyOption.contra +++ b/src/main/contraband/AssemblyOption.contra @@ -4,6 +4,8 @@ package sbtassembly type AssemblyOption { assemblyDirectory: java.io.File @since("0.15.0") + assemblyUnzipDirectory: java.io.File @since("1.2.0") + ## include compiled class files from itself or subprojects includeBin: Boolean! = true @since("0.15.0") @@ -22,6 +24,9 @@ type AssemblyOption { cacheUnzip: Boolean! = true @since("0.15.0") + ## this is experimental but will use a hard link between the files in assemblyUnzipDirectory to assemblyDirectory to avoid additional copy IO + cacheUseHardLinks: Boolean! = false @since("1.2.0") + appendContentHash: Boolean! = false @since("0.15.0") prependShellScript: sbtassembly.Assembly.SeqString @since("0.15.0") From b542e09484397830565fd618c9fc2fbbc7a5ff17 Mon Sep 17 00:00:00 2001 From: Eric Peters Date: Thu, 21 Oct 2021 11:02:28 -0700 Subject: [PATCH 2/3] Update generated AssemblyOption.scala --- .../scala/sbtassembly/AssemblyOption.scala | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/main/scala/sbtassembly/AssemblyOption.scala b/src/main/scala/sbtassembly/AssemblyOption.scala index 298080c2..64b6a951 100644 --- a/src/main/scala/sbtassembly/AssemblyOption.scala +++ b/src/main/scala/sbtassembly/AssemblyOption.scala @@ -7,9 +7,11 @@ package sbtassembly /** * @param includeBin include compiled class files from itself or subprojects * @param includeDependency include class files from external dependencies + * @param cacheUseHardLinks this is experimental but will use a hard link between the files in assemblyUnzipDirectory to assemblyDirectory to avoid additional copy IO */ final class AssemblyOption private ( val assemblyDirectory: Option[java.io.File], + val assemblyUnzipDirectory: Option[java.io.File], val includeBin: Boolean, val includeScala: Boolean, val includeDependency: Boolean, @@ -18,6 +20,7 @@ final class AssemblyOption private ( val mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, val cacheOutput: Boolean, val cacheUnzip: Boolean, + val cacheUseHardLinks: Boolean, val appendContentHash: Boolean, val prependShellScript: Option[sbtassembly.Assembly.SeqString], val maxHashLength: Option[Int], @@ -25,20 +28,21 @@ final class AssemblyOption private ( val scalaVersion: String, val level: sbt.Level.Value) extends Serializable { - private def this() = this(None, true, true, true, Nil, sbtassembly.Assembly.defaultExcludedFiles, sbtassembly.MergeStrategy.defaultMergeStrategy, true, true, false, None, None, sbtassembly.Assembly.defaultShadeRules, "", sbt.Level.Info) + private def this() = this(None, None, true, true, true, Nil, sbtassembly.Assembly.defaultExcludedFiles, sbtassembly.MergeStrategy.defaultMergeStrategy, true, true, false, false, None, None, sbtassembly.Assembly.defaultShadeRules, "", sbt.Level.Info) + private def this(assemblyDirectory: Option[java.io.File], includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, appendContentHash: Boolean, prependShellScript: Option[sbtassembly.Assembly.SeqString], maxHashLength: Option[Int], shadeRules: sbtassembly.Assembly.SeqShadeRules, scalaVersion: String, level: sbt.Level.Value) = this(assemblyDirectory, None, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, false, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level) override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { - case x: AssemblyOption => (this.assemblyDirectory == x.assemblyDirectory) && (this.includeBin == x.includeBin) && (this.includeScala == x.includeScala) && (this.includeDependency == x.includeDependency) && (this.excludedJars == x.excludedJars) && (this.excludedFiles == x.excludedFiles) && (this.mergeStrategy == x.mergeStrategy) && (this.cacheOutput == x.cacheOutput) && (this.cacheUnzip == x.cacheUnzip) && (this.appendContentHash == x.appendContentHash) && (this.prependShellScript == x.prependShellScript) && (this.maxHashLength == x.maxHashLength) && (this.shadeRules == x.shadeRules) && (this.scalaVersion == x.scalaVersion) && (this.level == x.level) + case x: AssemblyOption => (this.assemblyDirectory == x.assemblyDirectory) && (this.assemblyUnzipDirectory == x.assemblyUnzipDirectory) && (this.includeBin == x.includeBin) && (this.includeScala == x.includeScala) && (this.includeDependency == x.includeDependency) && (this.excludedJars == x.excludedJars) && (this.excludedFiles == x.excludedFiles) && (this.mergeStrategy == x.mergeStrategy) && (this.cacheOutput == x.cacheOutput) && (this.cacheUnzip == x.cacheUnzip) && (this.cacheUseHardLinks == x.cacheUseHardLinks) && (this.appendContentHash == x.appendContentHash) && (this.prependShellScript == x.prependShellScript) && (this.maxHashLength == x.maxHashLength) && (this.shadeRules == x.shadeRules) && (this.scalaVersion == x.scalaVersion) && (this.level == x.level) case _ => false }) override def hashCode: Int = { - 37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (17 + "sbtassembly.AssemblyOption".##) + assemblyDirectory.##) + includeBin.##) + includeScala.##) + includeDependency.##) + excludedJars.##) + excludedFiles.##) + mergeStrategy.##) + cacheOutput.##) + cacheUnzip.##) + appendContentHash.##) + prependShellScript.##) + maxHashLength.##) + shadeRules.##) + scalaVersion.##) + level.##) + 37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (17 + "sbtassembly.AssemblyOption".##) + assemblyDirectory.##) + assemblyUnzipDirectory.##) + includeBin.##) + includeScala.##) + includeDependency.##) + excludedJars.##) + excludedFiles.##) + mergeStrategy.##) + cacheOutput.##) + cacheUnzip.##) + cacheUseHardLinks.##) + appendContentHash.##) + prependShellScript.##) + maxHashLength.##) + shadeRules.##) + scalaVersion.##) + level.##) } override def toString: String = { - "AssemblyOption(" + assemblyDirectory + ", " + includeBin + ", " + includeScala + ", " + includeDependency + ", " + excludedJars + ", " + excludedFiles + ", " + mergeStrategy + ", " + cacheOutput + ", " + cacheUnzip + ", " + appendContentHash + ", " + prependShellScript + ", " + maxHashLength + ", " + shadeRules + ", " + scalaVersion + ", " + level + ")" + "AssemblyOption(" + assemblyDirectory + ", " + assemblyUnzipDirectory + ", " + includeBin + ", " + includeScala + ", " + includeDependency + ", " + excludedJars + ", " + excludedFiles + ", " + mergeStrategy + ", " + cacheOutput + ", " + cacheUnzip + ", " + cacheUseHardLinks + ", " + appendContentHash + ", " + prependShellScript + ", " + maxHashLength + ", " + shadeRules + ", " + scalaVersion + ", " + level + ")" } - private[this] def copy(assemblyDirectory: Option[java.io.File] = assemblyDirectory, includeBin: Boolean = includeBin, includeScala: Boolean = includeScala, includeDependency: Boolean = includeDependency, excludedJars: sbt.Keys.Classpath = excludedJars, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile = excludedFiles, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy = mergeStrategy, cacheOutput: Boolean = cacheOutput, cacheUnzip: Boolean = cacheUnzip, appendContentHash: Boolean = appendContentHash, prependShellScript: Option[sbtassembly.Assembly.SeqString] = prependShellScript, maxHashLength: Option[Int] = maxHashLength, shadeRules: sbtassembly.Assembly.SeqShadeRules = shadeRules, scalaVersion: String = scalaVersion, level: sbt.Level.Value = level): AssemblyOption = { - new AssemblyOption(assemblyDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level) + private[this] def copy(assemblyDirectory: Option[java.io.File] = assemblyDirectory, assemblyUnzipDirectory: Option[java.io.File] = assemblyUnzipDirectory, includeBin: Boolean = includeBin, includeScala: Boolean = includeScala, includeDependency: Boolean = includeDependency, excludedJars: sbt.Keys.Classpath = excludedJars, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile = excludedFiles, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy = mergeStrategy, cacheOutput: Boolean = cacheOutput, cacheUnzip: Boolean = cacheUnzip, cacheUseHardLinks: Boolean = cacheUseHardLinks, appendContentHash: Boolean = appendContentHash, prependShellScript: Option[sbtassembly.Assembly.SeqString] = prependShellScript, maxHashLength: Option[Int] = maxHashLength, shadeRules: sbtassembly.Assembly.SeqShadeRules = shadeRules, scalaVersion: String = scalaVersion, level: sbt.Level.Value = level): AssemblyOption = { + new AssemblyOption(assemblyDirectory, assemblyUnzipDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, cacheUseHardLinks, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level) } def withAssemblyDirectory(assemblyDirectory: Option[java.io.File]): AssemblyOption = { copy(assemblyDirectory = assemblyDirectory) @@ -46,6 +50,12 @@ final class AssemblyOption private ( def withAssemblyDirectory(assemblyDirectory: java.io.File): AssemblyOption = { copy(assemblyDirectory = Option(assemblyDirectory)) } + def withAssemblyUnzipDirectory(assemblyUnzipDirectory: Option[java.io.File]): AssemblyOption = { + copy(assemblyUnzipDirectory = assemblyUnzipDirectory) + } + def withAssemblyUnzipDirectory(assemblyUnzipDirectory: java.io.File): AssemblyOption = { + copy(assemblyUnzipDirectory = Option(assemblyUnzipDirectory)) + } def withIncludeBin(includeBin: Boolean): AssemblyOption = { copy(includeBin = includeBin) } @@ -70,6 +80,9 @@ final class AssemblyOption private ( def withCacheUnzip(cacheUnzip: Boolean): AssemblyOption = { copy(cacheUnzip = cacheUnzip) } + def withCacheUseHardLinks(cacheUseHardLinks: Boolean): AssemblyOption = { + copy(cacheUseHardLinks = cacheUseHardLinks) + } def withAppendContentHash(appendContentHash: Boolean): AssemblyOption = { copy(appendContentHash = appendContentHash) } @@ -100,4 +113,6 @@ object AssemblyOption { def apply(): AssemblyOption = new AssemblyOption() def apply(assemblyDirectory: Option[java.io.File], includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, appendContentHash: Boolean, prependShellScript: Option[sbtassembly.Assembly.SeqString], maxHashLength: Option[Int], shadeRules: sbtassembly.Assembly.SeqShadeRules, scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(assemblyDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level) def apply(assemblyDirectory: java.io.File, includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, appendContentHash: Boolean, prependShellScript: sbtassembly.Assembly.SeqString, maxHashLength: Int, shadeRules: sbtassembly.Assembly.SeqShadeRules, scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(Option(assemblyDirectory), includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, appendContentHash, Option(prependShellScript), Option(maxHashLength), shadeRules, scalaVersion, level) + def apply(assemblyDirectory: Option[java.io.File], assemblyUnzipDirectory: Option[java.io.File], includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, cacheUseHardLinks: Boolean, appendContentHash: Boolean, prependShellScript: Option[sbtassembly.Assembly.SeqString], maxHashLength: Option[Int], shadeRules: sbtassembly.Assembly.SeqShadeRules, scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(assemblyDirectory, assemblyUnzipDirectory, includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, cacheUseHardLinks, appendContentHash, prependShellScript, maxHashLength, shadeRules, scalaVersion, level) + def apply(assemblyDirectory: java.io.File, assemblyUnzipDirectory: java.io.File, includeBin: Boolean, includeScala: Boolean, includeDependency: Boolean, excludedJars: sbt.Keys.Classpath, excludedFiles: sbtassembly.Assembly.SeqFileToSeqFile, mergeStrategy: sbtassembly.MergeStrategy.StringToMergeStrategy, cacheOutput: Boolean, cacheUnzip: Boolean, cacheUseHardLinks: Boolean, appendContentHash: Boolean, prependShellScript: sbtassembly.Assembly.SeqString, maxHashLength: Int, shadeRules: sbtassembly.Assembly.SeqShadeRules, scalaVersion: String, level: sbt.Level.Value): AssemblyOption = new AssemblyOption(Option(assemblyDirectory), Option(assemblyUnzipDirectory), includeBin, includeScala, includeDependency, excludedJars, excludedFiles, mergeStrategy, cacheOutput, cacheUnzip, cacheUseHardLinks, appendContentHash, Option(prependShellScript), Option(maxHashLength), shadeRules, scalaVersion, level) } From f9e57eb007206df21e8d973cc06241f9e85c8f69 Mon Sep 17 00:00:00 2001 From: Eric Peters Date: Thu, 21 Oct 2021 11:02:42 -0700 Subject: [PATCH 3/3] * Add assemblyUnzipDirectory to AssemblyOption to use a different directory for unzipping jars * Add assemblyCacheDependency task to unzip jar dependencies as a separate task --- README.md | 19 + src/main/scala/sbtassembly/Assembly.scala | 357 ++++++++++++++---- src/main/scala/sbtassembly/AssemblyKeys.scala | 35 +- .../scala/sbtassembly/AssemblyPlugin.scala | 25 +- .../scala/sbtassembly/AssemblyUtils.scala | 123 ++++++ src/sbt-test/caching/caching/build.sbt | 2 + .../caching/caching/project/plugins.sbt | 2 + src/sbt-test/caching/caching/test | 16 + src/sbt-test/caching/unzip/build.sbt | 67 ++++ .../caching/unzip/project/plugins.sbt | 9 + .../caching/unzip/src/main/scala/hello.scala | 8 + src/sbt-test/caching/unzip/test | 63 ++++ 12 files changed, 630 insertions(+), 96 deletions(-) create mode 100644 src/sbt-test/caching/unzip/build.sbt create mode 100644 src/sbt-test/caching/unzip/project/plugins.sbt create mode 100644 src/sbt-test/caching/unzip/src/main/scala/hello.scala create mode 100644 src/sbt-test/caching/unzip/test diff --git a/README.md b/README.md index 9925e21f..2c09be77 100644 --- a/README.md +++ b/README.md @@ -393,6 +393,25 @@ lazy val app = (project in file("app")) ) ``` + +### Unzip Caching + +When assembling an über artifact, that has many library dependencies, the unzip process can be very IO intensive. These unzipped directories are very suitable for CI systems to persist in between job runs. + +```scala +lazy val app = (project in file("app")) + .settings( + assemblyUnzipDirectory := Some(localCacheDirectory.value / "sbt-assembly" / "dependencies"), + assemblyCacheUnzip := true, // this is the default setting + assemblyCacheUseHardLinks := true, // this is experimental but will use a hard link between the files in assemblyUnzipDirectory to assemblyDirectory to avoid additional copy IO + // more settings here ... + ) +``` + +To populate the assemblyUnzipDirectory without a full assembly: + + sbt assemblyCacheDependency + Other Things ------------ diff --git a/src/main/scala/sbtassembly/Assembly.scala b/src/main/scala/sbtassembly/Assembly.scala index 2c4f9cfc..296fbc55 100644 --- a/src/main/scala/sbtassembly/Assembly.scala +++ b/src/main/scala/sbtassembly/Assembly.scala @@ -3,13 +3,18 @@ package sbtassembly import sbt._ import Keys._ import Path.relativeTo + import java.security.MessageDigest -import java.io.{IOException, File} -import scala.collection.mutable +import java.io.{File, IOException} import Def.Initialize import PluginCompat._ import com.eed3si9n.jarjarabrams._ +import scala.collection.immutable.ListMap +import scala.collection.mutable +import scala.collection.parallel.immutable.ParVector + + object Assembly { import AssemblyPlugin.autoImport.{ Assembly => _, _ } @@ -219,48 +224,54 @@ object Assembly { // even though fullClasspath includes deps, dependencyClasspath is needed to figure out // which jars exactly belong to the deps for packageDependency option. - def assembleMappings(classpath: Classpath, dependencies: Classpath, - ao: AssemblyOption, log: Logger): Vector[MappingSet] = { - val tempDir = ao.assemblyDirectory.get - if (!ao.cacheUnzip) IO.delete(tempDir) - if (!tempDir.exists) tempDir.mkdir() + def assembleMappings( + classpath: Classpath, + dependencies: Classpath, + ao: AssemblyOption, + log: Logger, + state: State + ): Vector[MappingSet] = { + val assemblyDir = ao.assemblyDirectory.get + val assemblyUnzipDir = ao.assemblyUnzipDirectory.getOrElse(assemblyDir) + val projectIdMsg: String = getProjectIdMsg(state) - val shadeRules = ao.shadeRules + if (!ao.cacheOutput && assemblyDir.exists) { + log.info(s"AssemblyOption.cacheOutput set to false, deleting assemblyDirectory: $assemblyDir for project: $projectIdMsg") + IO.delete(assemblyDir) + } - val (libs, dirs) = classpath.toVector.sortBy(_.data.getCanonicalPath).partition(c => ClasspathUtilities.isArchive(c.data)) + for { + unzipDir <- ao.assemblyUnzipDirectory + if !ao.cacheUnzip + if unzipDir.exists + } { + log.info(s"AssemblyOption.cacheUnzip set to false, deleting assemblyUnzipDirectory: $unzipDir for project: $projectIdMsg") + IO.delete(unzipDir) + } - val depLibs = dependencies.map(_.data).toSet.filter(ClasspathUtilities.isArchive) - val excludedJars = ao.excludedJars map {_.data} + if (!assemblyDir.exists) IO.createDirectory(assemblyDir) + if (!assemblyUnzipDir.exists) IO.createDirectory(assemblyUnzipDir) - val scalaLibraries = { - val scalaVersionParts = VersionNumber(ao.scalaVersion) - val isScala213AndLater = scalaVersionParts.numbers.length>=2 && scalaVersionParts._1.get>=2 && scalaVersionParts._2.get>=13 - if (isScala213AndLater) scala213AndLaterLibraries else scalaPre213Libraries - } + val (libsFiltered: Vector[Attributed[File]], dirs: Vector[Attributed[File]]) = getFilteredLibsAndDirs( + classpath = classpath, + dependencies = dependencies, + assemblyOption = ao + ) - val libsFiltered = libs flatMap { - case jar if excludedJars contains jar.data.asFile => None - case jar if isScalaLibraryFile(scalaLibraries, jar.data.asFile) => - if (ao.includeScala) Some(jar) else None - case jar if depLibs contains jar.data.asFile => - if (ao.includeDependency) Some(jar) else None - case jar => - if (ao.includeBin) Some(jar) else None - } - val dirRules = shadeRules.filter(_.isApplicableToCompiling) - val dirsFiltered = + val dirRules: Seq[ShadeRule] = ao.shadeRules.filter(_.isApplicableToCompiling) + val dirsFiltered: ParVector[File] = dirs.par flatMap { case dir => if (ao.includeBin) Some(dir) else None } map { dir => val hash = sha1name(dir.data) - IO.write(tempDir / (hash + "_dir.dir"), dir.data.getCanonicalPath, IO.utf8, false) - val dest = tempDir / (hash + "_dir") + IO.write(assemblyDir / (hash + "_dir.dir"), dir.data.getCanonicalPath, IO.utf8, false) + val dest = assemblyDir / (hash + "_dir") if (dest.exists) { IO.delete(dest) } - dest.mkdir() + IO.createDirectory(dest) IO.copyDirectory(dir.data, dest) if (dirRules.nonEmpty) { val mappings = ((dest ** (-DirectoryFilter)).get pair relativeTo(dest)) map { @@ -270,62 +281,85 @@ object Assembly { } dest } - val jarDirs = - (for(jar <- libsFiltered.par) yield { - val jarName = jar.data.asFile.getName - val jarRules = shadeRules - .filter(r => (r.isApplicableToAll || - jar.metadata.get(moduleID.key) - .map(m => ModuleCoordinate(m.organization, m.name, m.revision)) - .exists(r.isApplicableTo))) - val hash = sha1name(jar.data) + "_" + sha1content(jar.data) + "_" + sha1rules(jarRules) - val jarNamePath = tempDir / (hash + ".jarName") - val dest = tempDir / hash - // If the jar name path does not exist, or is not for this jar, unzip the jar - if (!ao.cacheUnzip || !jarNamePath.exists || IO.read(jarNamePath) != jar.data.getCanonicalPath ) - { - log.debug("Including: %s".format(jarName)) - IO.delete(dest) - dest.mkdir() - AssemblyUtils.unzip(jar.data, dest, log) - IO.delete(ao.excludedFiles(Seq(dest))) - if (jarRules.nonEmpty) { - val mappings = ((dest ** (-DirectoryFilter)).get pair relativeTo(dest)) map { - case (k, v) => k.toPath -> v - } - Shader.shadeDirectory(dirRules, dest.toPath, mappings, ao.level == Level.Debug) - } - - // Write the jarNamePath at the end to minimise the chance of having a - // corrupt cache if the user aborts the build midway through - IO.write(jarNamePath, jar.data.getCanonicalPath, IO.utf8, false) - } - else log.debug("Including from cache: %s".format(jarName)) - (dest, jar.data) - }) + val jarDirs: ParVector[(File, File)] = processDependencyJars( + libsFiltered, + ao, + isCacheOnly = false, + log, + state + ) - log.debug("Calculate mappings...") + log.info("Calculate mappings...") val base: Vector[File] = dirsFiltered.seq ++ (jarDirs map { _._1 }) - val excluded = (ao.excludedFiles(base) ++ base).toSet - val retval = (dirsFiltered map { d => MappingSet(None, AssemblyUtils.getMappings(d, excluded)) }).seq ++ + val excluded: Set[File] = (ao.excludedFiles(base) ++ base).toSet + val retval: Vector[MappingSet] = (dirsFiltered map { d => MappingSet(None, AssemblyUtils.getMappings(d, excluded)) }).seq ++ (jarDirs map { case (d, j) => MappingSet(Some(j), AssemblyUtils.getMappings(d, excluded)) }) - retval.toVector + retval + } + + def assemblyCacheDependency( + classpath: Classpath, + dependencies: Classpath, + assemblyOption: AssemblyOption, + log: Logger, + state: State + ): Boolean = { + if (!assemblyOption.cacheUnzip) sys.error("AssemblyOption.cacheUnzip must be true") + if (assemblyOption.assemblyUnzipDirectory.isEmpty) sys.error("AssemblyOption.assemblyUnzipDirectory must be supplied") + + val (libsFiltered: Vector[Attributed[File]], _) = getFilteredLibsAndDirs( + classpath = classpath, + dependencies = dependencies, + assemblyOption = assemblyOption + ) + + processDependencyJars(libsFiltered, assemblyOption, isCacheOnly = true, log, state) + + true } def assemblyTask(key: TaskKey[File]): Initialize[Task[File]] = Def.task { - val t = (test in key).value + // Run tests if enabled before assembly task + val _ = (test in key).value + val s = (streams in key).value Assembly( - (assemblyOutputPath in key).value, (assemblyOption in key).value, - (packageOptions in key).value, (assembledMappings in key).value, - s.cacheDirectory, s.log) + (assemblyOutputPath in key).value, + (assemblyOption in key).value, + (packageOptions in key).value, + (assembledMappings in key).value, + s.cacheDirectory, + s.log + ) } + def assembledMappingsTask(key: TaskKey[File]): Initialize[Task[Seq[MappingSet]]] = Def.task { val s = (streams in key).value assembleMappings( - (fullClasspath in assembly).value, (externalDependencyClasspath in assembly).value, - (assemblyOption in key).value, s.log) + (fullClasspath in assembly).value, + (externalDependencyClasspath in assembly).value, + (assemblyOption in key).value, + s.log, + state.value + ) + } + + def assemblyCacheDependencyTask(key: TaskKey[File]): Initialize[Task[Boolean]] = Def.task { + val s = (streams in key).value + val ao = (assemblyOption in key).value + val cp = (fullClasspath in assembly).value + val deps = (externalDependencyClasspath in assembly).value + val currentState = state.value + val projectIdMsg: String = getProjectIdMsg(currentState) + + if (!ao.cacheUnzip || ao.assemblyUnzipDirectory.isEmpty) { + if (!ao.cacheUnzip) s.log.warn(s"AssemblyOption.cacheUnzip must be true. Skipping unzip task for projectID: $projectIdMsg.") + if (ao.assemblyUnzipDirectory.isEmpty) s.log.warn(s"AssemblyOption.assemblyUnzipDirectory must be be supplied. Skipping cache unzip task for projectID: $projectIdMsg") + false + } else { + assemblyCacheDependency(classpath = cp, dependencies = deps, ao, s.log, currentState) + } } def isSystemJunkFile(fileName: String): Boolean = @@ -359,6 +393,183 @@ object Assembly { def isScalaLibraryFile(scalaLibraries: Vector[String], file: File): Boolean = scalaLibraries exists { x => file.getName startsWith x } + private[sbtassembly] def getProjectIdMsg(state: State): String = { + val project = Project.extract(state) + + val projectName = project.get(Keys.projectID).name + val currentRefProjectName = project.currentRef.project + + if (projectName != currentRefProjectName) s"$projectName/$currentRefProjectName" + else projectName + } + + private[sbtassembly] def processDependencyJars( + libsFiltered: Vector[Attributed[File]], + assemblyOption: AssemblyOption, + isCacheOnly: Boolean, + log: Logger, + state: State + ): ParVector[(File, File)] = { + val defaultAssemblyDir = assemblyOption.assemblyDirectory.get + val assemblyUnzipDir: File = assemblyOption.assemblyUnzipDirectory.getOrElse(defaultAssemblyDir) + val assemblyDir: Option[File] = if (isCacheOnly) None else Some(defaultAssemblyDir) + val isSameDir: Boolean = assemblyDir.exists{ _ == assemblyUnzipDir } + + if (!assemblyUnzipDir.exists) IO.createDirectory(assemblyUnzipDir) + if (assemblyDir.isDefined && !assemblyDir.get.exists) IO.createDirectory(assemblyDir.get) + + state.locked(assemblyUnzipDir / "sbt-assembly.lock") { + + val projectIdMsg: String = getProjectIdMsg(state) + + if (!assemblyUnzipDir.exists) IO.createDirectory(assemblyUnzipDir) + if (assemblyDir.isDefined && !assemblyDir.get.exists) IO.createDirectory(assemblyDir.get) + + val unzippingIntoMessage: String = if (isCacheOnly && !isSameDir) "unzip cache" else "output cache" + + val useHardLinks: Boolean = assemblyOption.cacheUseHardLinks && !isCacheOnly && { + if (isSameDir) { + log.warn(s"cacheUseHardLinks is enabled for project ($projectIdMsg), but assemblyUnzipDirectory is the same as assemblyDirectory ($assemblyUnzipDirectory)") + false + } else { + val isHardLinkSupported = AssemblyUtils.isHardLinkSupported(sourceDir = assemblyUnzipDir, destDir = assemblyDir.get) + if (!isHardLinkSupported) log.warn(s"cacheUseHardLinks is enabled for project ($projectIdMsg), but file system doesn't support hardlinks between from $assemblyUnzipDir to ${assemblyDir.get}") + isHardLinkSupported + } + } + + // Ensure we are not processing the same File twice, retain original ordering + val jarToAttributedFiles: ListMap[File, Vector[Attributed[File]]] = + libsFiltered + .foldLeft(ListMap.empty[File, Vector[Attributed[File]]]) { + case (lm, f) => + val canonicalJar = f.data.getCanonicalFile + lm.updated(canonicalJar, f +: lm.getOrElse(canonicalJar, Vector.empty)) + } + + for { + jar: File <- jarToAttributedFiles.keys.toVector.par + jarName = jar.asFile.getName + jarRules = assemblyOption.shadeRules + .filter { r => + r.isApplicableToAll || + jarToAttributedFiles.getOrElse(jar, Vector.empty) + .flatMap(_.metadata.get(moduleID.key)) + .map(m => ModuleCoordinate(m.organization, m.name, m.revision)) + .exists(r.isApplicableTo) + } + hash = sha1name(jar) + "_" + sha1content(jar) + "_" + sha1rules(jarRules) + jarOutputDir = (assemblyDir.getOrElse(assemblyUnzipDir) / hash).getCanonicalFile + } yield { + // TODO: Canonical path might be problematic if mount points inside docker are different + val jarNameFinalPath = new File(jarOutputDir + ".jarName") + + val jarNameCachePath = assemblyUnzipDir / (hash + ".jarName") + val jarCacheDir = assemblyUnzipDir / hash + + // If the jar name path does not exist, or is not for this jar, unzip the jar + if (!jarNameFinalPath.exists || IO.read(jarNameFinalPath) != jar.getCanonicalPath) { + log.info("Including: %s, for project: %s".format(jarName, projectIdMsg)) + + // Copy/Link from cache location if cache exists and is current + if (assemblyOption.cacheUnzip && + jarNameCachePath.exists && IO.read(jarNameCachePath) == jar.getCanonicalPath && + !jarNameFinalPath.exists + //(!jarNameFinalPath.exists || IO.read(jarNameFinalPath) != jar.getCanonicalPath) + ) { + if (useHardLinks) { + log.info("Creating hardlinks of %s from unzip cache: %s, to: %s, for project: %s".format(jarName, jarCacheDir, jarOutputDir, projectIdMsg)) + AssemblyUtils.copyDirectory(jarCacheDir, jarOutputDir, hardLink = true) + } else { + log.info("Copying %s from unzip cache: %s, to: %s, for project: %s".format(jarName, jarCacheDir, jarOutputDir, projectIdMsg)) + AssemblyUtils.copyDirectory(jarCacheDir, jarOutputDir, hardLink = false) + } + AssemblyUtils.copyDirectory(jarCacheDir, jarOutputDir, hardLink = useHardLinks) + IO.delete(jarNameFinalPath) // write after merge/shade rules applied + // Unzip into cache dir and copy over + } else if (assemblyOption.cacheUnzip && jarNameFinalPath != jarNameCachePath) { + IO.delete(jarCacheDir) + IO.delete(jarOutputDir) + + IO.createDirectory(jarCacheDir) + IO.createDirectory(jarOutputDir) + + log.info("Unzipping %s into unzip cache: %s for project: %s".format(jarName, jarCacheDir, projectIdMsg)) + val files = AssemblyUtils.unzip(jar, jarCacheDir, log) + + // TODO: This is kind of a hack, but doing it seems to prevent a docker file system issue preventing + // FileNotFound exception after unzipping + files.foreach { f => + assert(f.exists(), s"File $f not found after unzipping $jar into $jarCacheDir!") + } + + if (useHardLinks) log.info("Creating hardlinks of %s from unzip cache: %s, to: %s, for project: %s".format(jarName, jarCacheDir, jarOutputDir, projectIdMsg)) + else log.info("Copying %s from unzip cache: %s, to: %s, for project: %s".format(jarName, jarCacheDir, jarOutputDir, projectIdMsg)) + AssemblyUtils.copyDirectory(jarCacheDir, jarOutputDir, hardLink = useHardLinks) + // Don't use cache dir, just unzip to output cache + } else { + IO.delete(jarOutputDir) + IO.createDirectory(jarOutputDir) + log.info("Unzipping %s into %s: %s, for project: %s".format(jarName, unzippingIntoMessage, jarOutputDir, projectIdMsg)) + AssemblyUtils.unzip(jar, jarOutputDir, log) + } + + if (!isCacheOnly) { + IO.delete(assemblyOption.excludedFiles(Seq(jarOutputDir))) + if (jarRules.nonEmpty) { + val mappings = ((jarOutputDir ** (-DirectoryFilter)).get pair relativeTo(jarOutputDir)) map { + case (k, v) => k.toPath -> v + } + val dirRules: Seq[ShadeRule] = assemblyOption.shadeRules.filter(_.isApplicableToCompiling) + Shader.shadeDirectory(dirRules, jarOutputDir.toPath, mappings, assemblyOption.level == Level.Debug) + } + } + + // Write the jarNamePath at the end to minimise the chance of having a + // corrupt cache if the user aborts the build midway through + if (jarNameFinalPath != jarNameCachePath && !jarNameCachePath.exists) + IO.write(jarNameCachePath, jar.getCanonicalPath, IO.utf8, false) + + IO.write(jarNameFinalPath, jar.getCanonicalPath, IO.utf8, false) + } else { + if (isCacheOnly) log.info("Unzip cache of %s is up to date, for project: %s".format(jarName, projectIdMsg)) + else log.info("Including %s from output cache: %s, for project: %s".format(jarName, jarOutputDir, projectIdMsg)) + } + (jarOutputDir, jar) + } + } + } + + private[sbtassembly] def getFilteredLibsAndDirs( + classpath: Classpath, + dependencies: Classpath, + assemblyOption: AssemblyOption + ): (Vector[Attributed[File]], Vector[Attributed[File]]) = { + val (libs: Vector[Attributed[File]], dirs: Vector[Attributed[File]]) = + classpath.toVector.sortBy(_.data.getCanonicalPath).partition(c => ClasspathUtilities.isArchive(c.data)) + + val depLibs: Set[File] = dependencies.map(_.data).toSet.filter(ClasspathUtilities.isArchive) + val excludedJars: Seq[File] = assemblyOption.excludedJars map {_.data} + + val scalaLibraries: Vector[String] = { + val scalaVersionParts = VersionNumber(assemblyOption.scalaVersion) + val isScala213AndLater = scalaVersionParts.numbers.length>=2 && scalaVersionParts._1.get>=2 && scalaVersionParts._2.get>=13 + if (isScala213AndLater) scala213AndLaterLibraries else scalaPre213Libraries + } + + val libsFiltered: Vector[Attributed[File]] = libs flatMap { + case jar if excludedJars contains jar.data.asFile => None + case jar if isScalaLibraryFile(scalaLibraries, jar.data.asFile) => + if (assemblyOption.includeScala) Some(jar) else None + case jar if depLibs contains jar.data.asFile => + if (assemblyOption.includeDependency) Some(jar) else None + case jar => + if (assemblyOption.includeBin) Some(jar) else None + } + + (libsFiltered, dirs) + } + private[sbtassembly] def sha1 = MessageDigest.getInstance("SHA-1") private[sbtassembly] def sha1content(f: File): String = { Using.fileInputStream(f) { in => diff --git a/src/main/scala/sbtassembly/AssemblyKeys.scala b/src/main/scala/sbtassembly/AssemblyKeys.scala index f6e4c7fa..8f27bfec 100644 --- a/src/main/scala/sbtassembly/AssemblyKeys.scala +++ b/src/main/scala/sbtassembly/AssemblyKeys.scala @@ -5,23 +5,26 @@ import Keys._ import com.eed3si9n.jarjarabrams trait AssemblyKeys { - lazy val assembly = taskKey[File]("Builds a deployable über JAR") - lazy val assembleArtifact = settingKey[Boolean]("Enables (true) or disables (false) assembling an artifact") - lazy val assemblyOption = taskKey[AssemblyOption]("Configuration for making a deployable über JAR") - lazy val assembledMappings = taskKey[Seq[MappingSet]]("Keeps track of jar origins for each source") + lazy val assembly = taskKey[File]("Builds a deployable über JAR") + lazy val assembleArtifact = settingKey[Boolean]("Enables (true) or disables (false) assembling an artifact") + lazy val assemblyOption = taskKey[AssemblyOption]("Configuration for making a deployable über JAR") + lazy val assembledMappings = taskKey[Seq[MappingSet]]("Keeps track of jar origins for each source") + lazy val assemblyCacheDependency = taskKey[Boolean]("Caches the unzipped products of the dependency JAR files. Requires assemblyCacheUnzip (true) and AssemblyOption.assemblyUnzipCacheDir to be provided.") - lazy val assemblyPackageScala = taskKey[File]("Produces the Scala artifact") - lazy val assemblyPackageDependency = taskKey[File]("Produces the dependency artifact") - lazy val assemblyJarName = taskKey[String]("name of the über jar") - lazy val assemblyDefaultJarName = taskKey[String]("default name of the über jar") - lazy val assemblyOutputPath = taskKey[File]("output path of the über jar") - lazy val assemblyExcludedJars = taskKey[Classpath]("list of excluded jars") - lazy val assemblyMergeStrategy = settingKey[String => MergeStrategy]("mapping from archive member path to merge strategy") - lazy val assemblyShadeRules = settingKey[Seq[jarjarabrams.ShadeRule]]("shading rules backed by jarjar") - lazy val assemblyAppendContentHash = settingKey[Boolean]("Appends SHA-1 fingerprint to the assembly file name") - lazy val assemblyMaxHashLength = settingKey[Int]("Length of SHA-1 fingerprint used for the assembly file name") - lazy val assemblyCacheUnzip = settingKey[Boolean]("Enables (true) or disables (false) cacheing the unzipped products of the dependency JAR files") - lazy val assemblyCacheOutput = settingKey[Boolean]("Enables (true) or disables (false) cacheing the output if the content has not changed") + lazy val assemblyUnzipDirectory = settingKey[Option[File]]("Specify a directory to unzip the products of dependency JAR files (e.g. assemblyUnzipDirectory := Some(localCacheDirectory.value / \"sbt-assembly\" / \"dependencies\"). Default None (uses default assembly directory).") + lazy val assemblyPackageScala = taskKey[File]("Produces the Scala artifact") + lazy val assemblyPackageDependency = taskKey[File]("Produces the dependency artifact") + lazy val assemblyJarName = taskKey[String]("name of the über jar") + lazy val assemblyDefaultJarName = taskKey[String]("default name of the über jar") + lazy val assemblyOutputPath = taskKey[File]("output path of the über jar") + lazy val assemblyExcludedJars = taskKey[Classpath]("list of excluded jars") + lazy val assemblyMergeStrategy = settingKey[String => MergeStrategy]("mapping from archive member path to merge strategy") + lazy val assemblyShadeRules = settingKey[Seq[jarjarabrams.ShadeRule]]("shading rules backed by jarjar") + lazy val assemblyAppendContentHash = settingKey[Boolean]("Appends SHA-1 fingerprint to the assembly file name") + lazy val assemblyMaxHashLength = settingKey[Int]("Length of SHA-1 fingerprint used for the assembly file name") + lazy val assemblyCacheUnzip = settingKey[Boolean]("Enables (true) or disables (false) cacheing the unzipped products of the dependency JAR files") + lazy val assemblyCacheOutput = settingKey[Boolean]("Enables (true) or disables (false) cacheing the output if the content has not changed") + lazy val assemblyCacheUseHardLinks = settingKey[Boolean]("Experimental. Enables (true) or disables (false) using Files.createLink from the unzipped dependency cache to the assembly directory. Requires both paths to be on the same physical filesystem. Default false.") lazy val assemblyPrependShellScript = settingKey[Option[Seq[String]]]("A launch script to prepend to the über JAR") } diff --git a/src/main/scala/sbtassembly/AssemblyPlugin.scala b/src/main/scala/sbtassembly/AssemblyPlugin.scala index d52922c4..608ac87d 100644 --- a/src/main/scala/sbtassembly/AssemblyPlugin.scala +++ b/src/main/scala/sbtassembly/AssemblyPlugin.scala @@ -23,6 +23,7 @@ object AssemblyPlugin extends sbt.AutoPlugin { import autoImport.{ Assembly => _, baseAssemblySettings => _, _ } override lazy val globalSettings: Seq[Def.Setting[_]] = Seq( + assemblyUnzipDirectory := None, assemblyMergeStrategy := MergeStrategy.defaultMergeStrategy, assemblyShadeRules := Nil, assemblyExcludedJars := Nil, @@ -32,6 +33,7 @@ object AssemblyPlugin extends sbt.AutoPlugin { assemblyAppendContentHash := false, assemblyCacheUnzip := true, assemblyCacheOutput := true, + assemblyCacheUseHardLinks := false, assemblyPrependShellScript := None ) @@ -50,12 +52,13 @@ object AssemblyPlugin extends sbt.AutoPlugin { ) lazy val baseAssemblySettings: Seq[sbt.Def.Setting[_]] = (Seq( - assembly := Assembly.assemblyTask(assembly).value, - assembledMappings in assembly := Assembly.assembledMappingsTask(assembly).value, - assemblyPackageScala := Assembly.assemblyTask(assemblyPackageScala).value, - assembledMappings in assemblyPackageScala := Assembly.assembledMappingsTask(assemblyPackageScala).value, - assemblyPackageDependency := Assembly.assemblyTask(assemblyPackageDependency).value, - assembledMappings in assemblyPackageDependency := Assembly.assembledMappingsTask(assemblyPackageDependency).value, + assembly := Assembly.assemblyTask(assembly).value, + assembledMappings in assembly := Assembly.assembledMappingsTask(assembly).value, + assemblyPackageScala := Assembly.assemblyTask(assemblyPackageScala).value, + assembledMappings in assemblyPackageScala := Assembly.assembledMappingsTask(assemblyPackageScala).value, + assemblyPackageDependency := Assembly.assemblyTask(assemblyPackageDependency).value, + assembledMappings in assemblyPackageDependency := Assembly.assembledMappingsTask(assemblyPackageDependency).value, + assemblyCacheDependency := Assembly.assemblyCacheDependencyTask(assemblyPackageDependency).value, // test test in assembly := { () }, @@ -94,6 +97,7 @@ object AssemblyPlugin extends sbt.AutoPlugin { ) ++ inTask(assembly)(assemblyOptionSettings) ++ inTask(assemblyPackageScala)(assemblyOptionSettings) ++ inTask(assemblyPackageDependency)(assemblyOptionSettings) + ++ inTask(assemblyCacheDependency)(assemblyOptionSettings) ++ Seq( assemblyOption in assemblyPackageScala ~= { _.withIncludeBin(false) @@ -104,7 +108,12 @@ object AssemblyPlugin extends sbt.AutoPlugin { _.withIncludeBin(false) .withIncludeScala(true) .withIncludeDependency(true) - } + }, + assemblyOption in assemblyCacheDependency ~= { + _.withIncludeBin(false) + .withIncludeScala(true) + .withIncludeDependency(true) + }, )) def assemblyOptionSettings: Seq[Setting[_]] = Seq( @@ -112,6 +121,7 @@ object AssemblyPlugin extends sbt.AutoPlugin { val s = streams.value AssemblyOption() .withAssemblyDirectory(s.cacheDirectory / "assembly") + .withAssemblyUnzipDirectory(assemblyUnzipDirectory.value) .withIncludeBin((assembleArtifact in packageBin).value) .withIncludeScala((assembleArtifact in assemblyPackageScala).value) .withIncludeDependency((assembleArtifact in assemblyPackageDependency).value) @@ -120,6 +130,7 @@ object AssemblyPlugin extends sbt.AutoPlugin { .withExcludedFiles(Assembly.defaultExcludedFiles) .withCacheOutput(assemblyCacheOutput.value) .withCacheUnzip(assemblyCacheUnzip.value) + .withCacheUseHardLinks(assemblyCacheUseHardLinks.value) .withAppendContentHash(assemblyAppendContentHash.value) .withPrependShellScript(assemblyPrependShellScript.value) .withMaxHashLength(assemblyMaxHashLength.?.value) diff --git a/src/main/scala/sbtassembly/AssemblyUtils.scala b/src/main/scala/sbtassembly/AssemblyUtils.scala index 8ffd74b9..7c2719a4 100644 --- a/src/main/scala/sbtassembly/AssemblyUtils.scala +++ b/src/main/scala/sbtassembly/AssemblyUtils.scala @@ -1,6 +1,7 @@ package sbtassembly import sbt._ + import java.io.{File, InputStream} import java.util.zip.ZipInputStream import scala.collection.mutable.HashSet @@ -8,6 +9,9 @@ import ErrorHandling.translate import PluginCompat._ import Using._ +import java.nio.file.{FileSystemException, Files} +import scala.Function.tupled + private[sbtassembly] object AssemblyUtils { private val PathRE = "([^/]+)/(.*)".r @@ -106,4 +110,123 @@ private[sbtassembly] object AssemblyUtils { } loop(rootDir, "", Nil).toVector } + + + def isHardLinkSupported(sourceDir: File, destDir: File): Boolean = { + assert(sourceDir.isDirectory) + assert(destDir.isDirectory) + + withTemporaryFileInDirectory("sbt-assembly", "file", sourceDir) { sourceFile => + try { + val destFile = destDir / sourceFile.getName + Files.createLink(destFile.toPath, sourceFile.toPath) + IO.delete(destFile) + true + } catch { + case ex: FileSystemException if ex.getMessage().contains("Invalid cross-device link") => false + } + } + } + + def withTemporaryFileInDirectory[T](prefix: String, postfix: String, dir: File)( + action: File => T + ): T = { + assert(dir.isDirectory) + val file = File.createTempFile(prefix, postfix, dir) + try { action(file) } finally { file.delete(); () } + } + + // region copyDirectory + + /** This is an experimental port of https://github.com/sbt/io/pull/326 */ + + def copyDirectory( + source: File, + target: File, + overwrite: Boolean = false, + preserveLastModified: Boolean = false, + preserveExecutable: Boolean = true, + hardLink: Boolean = false + ): Unit = { + val sources = PathFinder(source).allPaths pair Path.rebase(source, target) + copy(sources, overwrite, preserveLastModified, preserveExecutable, hardLink) + () + } + + def copy( + sources: Traversable[(File, File)], + overwrite: Boolean, + preserveLastModified: Boolean, + preserveExecutable: Boolean, + hardLink: Boolean + ): Set[File] = + sources + .map(tupled(copyImpl(overwrite, preserveLastModified, preserveExecutable, hardLink))) + .toSet + + private def copyImpl( + overwrite: Boolean, + preserveLastModified: Boolean, + preserveExecutable: Boolean, + hardLink: Boolean + )(from: File, to: File): File = { + if (overwrite || !to.exists || IO.getModifiedTimeOrZero(from) > IO.getModifiedTimeOrZero(to)) { + if (from.isDirectory) { + IO.createDirectory(to) + } else { + IO.createDirectory(to.getParentFile) + copyFile(from, to, preserveLastModified, preserveExecutable, hardLink) + } + } + to + } + + def copyFile( + sourceFile: File, + targetFile: File, + preserveLastModified: Boolean, + preserveExecutable: Boolean, + hardLink: Boolean + ): Unit = { + // NOTE: when modifying this code, test with larger values of CopySpec.MaxFileSizeBits than default + + require(sourceFile.exists, "Source file '" + sourceFile.getAbsolutePath + "' does not exist.") + require( + !sourceFile.isDirectory, + "Source file '" + sourceFile.getAbsolutePath + "' is a directory." + ) + if (hardLink) { + if (targetFile.exists) targetFile.delete() + Files.createLink(targetFile.toPath, sourceFile.toPath) + () + } else { + fileInputChannel(sourceFile) { in => + fileOutputChannel(targetFile) { out => + // maximum bytes per transfer according to from http://dzone.com/snippets/java-filecopy-using-nio + val max = (64L * 1024 * 1024) - (32 * 1024) + val total = in.size + def loop(offset: Long): Long = + if (offset < total) + loop(offset + out.transferFrom(in, offset, max)) + else + offset + val copied = loop(0) + if (copied != in.size) + sys.error( + "Could not copy '" + sourceFile + "' to '" + targetFile + "' (" + copied + "/" + in.size + " bytes copied)" + ) + } + } + if (preserveLastModified) { + IO.copyLastModified(sourceFile, targetFile) + () + } + if (preserveExecutable) { + IO.copyExecutable(sourceFile, targetFile) + () + } + } + } + + // endregion } diff --git a/src/sbt-test/caching/caching/build.sbt b/src/sbt-test/caching/caching/build.sbt index ea853c2c..a2d47db5 100644 --- a/src/sbt-test/caching/caching/build.sbt +++ b/src/sbt-test/caching/caching/build.sbt @@ -4,6 +4,8 @@ lazy val root = (project in file(".")). scalaVersion := "2.11.12", libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.1" % "test", libraryDependencies += "ch.qos.logback" % "logback-classic" % "0.9.29" % "runtime", + logLevel := sbt.Level.Info, + logBuffered := false, assembly / assemblyOption ~= { _.withCacheOutput(true) .withCacheUnzip(true) diff --git a/src/sbt-test/caching/caching/project/plugins.sbt b/src/sbt-test/caching/caching/project/plugins.sbt index e50b4e95..7c90d6d0 100644 --- a/src/sbt-test/caching/caching/project/plugins.sbt +++ b/src/sbt-test/caching/caching/project/plugins.sbt @@ -5,3 +5,5 @@ |Specify this property using the scriptedLaunchOpts -D.""".stripMargin) else addSbtPlugin("com.eed3si9n" % "sbt-assembly" % pluginVersion) } + +addSbtPlugin("io.github.er1c" % "sbt-scriptedutils" % "0.1.0") \ No newline at end of file diff --git a/src/sbt-test/caching/caching/test b/src/sbt-test/caching/caching/test index 7ee7b742..b14ddd83 100644 --- a/src/sbt-test/caching/caching/test +++ b/src/sbt-test/caching/caching/test @@ -1,6 +1,12 @@ # check if the file gets created > clean > assembly +# Ensure all warnings have time to be printed +$ sleep 500 +> checkLogContains Unzipping slf4j-api-1.6.1.jar into output cache +> checkLogContains Unzipping logback-classic-0.9.29.jar into output cache +> checkLogContains Unzipping logback-core-0.9.29.jar into output cache +> checkLogContains Unzipping scala-library-2.11.12.jar into output cache $ exists target/scala-2.11/foo.jar # run to cache the hash, then check it's consistent @@ -28,3 +34,13 @@ $ delete src/main/resources/foo.txt > genresource2 > assembly > check + +> clearLog +> assemblyCacheDependency +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains AssemblyOption.assemblyUnzipDirectory must be be supplied. Skipping cache unzip task +> checkLogNotContains Unzipping slf4j-api-1.6.1.jar into unzip cache +> checkLogNotContains Unzipping logback-classic-0.9.29.jar into unzip cache +> checkLogNotContains Unzipping logback-core-0.9.29.jar into unzip cache +> checkLogNotContains Unzipping scala-library-2.11.12.jar into unzip cache \ No newline at end of file diff --git a/src/sbt-test/caching/unzip/build.sbt b/src/sbt-test/caching/unzip/build.sbt new file mode 100644 index 00000000..feee6b9e --- /dev/null +++ b/src/sbt-test/caching/unzip/build.sbt @@ -0,0 +1,67 @@ +import org.apache.logging.log4j.Level +import org.apache.logging.log4j.core.{LogEvent => Log4JLogEvent, _} +import org.apache.logging.log4j.core.Filter.Result +import org.apache.logging.log4j.core.appender.AbstractAppender +import org.apache.logging.log4j.core.filter.LevelRangeFilter +import org.apache.logging.log4j.core.layout.PatternLayout + +lazy val tempUnzipDir = IO.createTemporaryDirectory + +lazy val root = (project in file(".")). + settings( + version := "0.1", + scalaVersion := "2.11.12", + libraryDependencies += "commons-io" % "commons-io" % "2.4", + libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.1" % "test", + libraryDependencies += "ch.qos.logback" % "logback-classic" % "0.9.29" % "runtime", + assembly / assemblyShadeRules := Seq( + ShadeRule + .rename("org.apache.commons.io.**" -> "shadeio.@1") + .inLibrary("commons-io" % "commons-io" % "2.4") + .inProject + ), + assemblyUnzipDirectory := Some(tempUnzipDir), + assemblyCacheUseHardLinks := true, + logLevel := sbt.Level.Info, + logBuffered := false, + assembly / assemblyJarName := "foo.jar", + TaskKey[Unit]("checkunzip") := { + val opt = (assembly / assemblyOption).value + val assemblyDir = opt.assemblyDirectory.get + val assemblyUnzipDir = opt.assemblyUnzipDirectory.get + val preShadePath = "org.apache.commons.io".replace('.', java.io.File.separatorChar) + val postShadePath = "shadeio" + + val sources = PathFinder(assemblyUnzipDir).allPaths pair Path.rebase(assemblyUnzipDir, assemblyDir) + val ioSources = sources.filter{ case (unzip, _) => unzip.getAbsolutePath.contains(preShadePath) && unzip.isFile } + + assert(ioSources.nonEmpty) + sources.map{ _._1 }.foreach{ f => assert(f.exists) } + + ioSources.foreach { case (unzipFile, origOutFile) => + val outputFile = new java.io.File( + origOutFile + .getAbsolutePath + .toString + .replace(preShadePath, postShadePath) + ) + + assert(unzipFile.exists) + assert(outputFile.exists) + assert(getHashString(unzipFile) != getHashString(outputFile)) + } + () + }, + TaskKey[Unit]("cleanunzip") := { + IO.delete(tempUnzipDir) + } + ) + +def getHashString(file: java.io.File): String = { + import java.security.MessageDigest + MessageDigest + .getInstance("SHA-1") + .digest(IO.readBytes(file)) + .map( b => "%02x".format(b) ) + .mkString +} diff --git a/src/sbt-test/caching/unzip/project/plugins.sbt b/src/sbt-test/caching/unzip/project/plugins.sbt new file mode 100644 index 00000000..7c90d6d0 --- /dev/null +++ b/src/sbt-test/caching/unzip/project/plugins.sbt @@ -0,0 +1,9 @@ +{ + val pluginVersion = System.getProperty("plugin.version") + if(pluginVersion == null) + throw new RuntimeException("""|The system property 'plugin.version' is not defined. + |Specify this property using the scriptedLaunchOpts -D.""".stripMargin) + else addSbtPlugin("com.eed3si9n" % "sbt-assembly" % pluginVersion) +} + +addSbtPlugin("io.github.er1c" % "sbt-scriptedutils" % "0.1.0") \ No newline at end of file diff --git a/src/sbt-test/caching/unzip/src/main/scala/hello.scala b/src/sbt-test/caching/unzip/src/main/scala/hello.scala new file mode 100644 index 00000000..5b2bde64 --- /dev/null +++ b/src/sbt-test/caching/unzip/src/main/scala/hello.scala @@ -0,0 +1,8 @@ +object Main { + def main(args: Array[String]) { + Option(getClass().getResource("foo.txt")) match { + case Some(_) => println("foo.txt") + case _ => println("hello") + } + } +} diff --git a/src/sbt-test/caching/unzip/test b/src/sbt-test/caching/unzip/test new file mode 100644 index 00000000..bfa84a49 --- /dev/null +++ b/src/sbt-test/caching/unzip/test @@ -0,0 +1,63 @@ +# check if the file gets created and unzips and creates hardlinks +> clean +> assembly +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains Unzipping slf4j-api-1.6.1.jar into unzip cache +> checkLogContains Unzipping commons-io-2.4.jar into unzip cache +> checkLogContains Unzipping logback-classic-0.9.29.jar into unzip cache +> checkLogContains Unzipping logback-core-0.9.29.jar into unzip cache +> checkLogContains Unzipping scala-library-2.11.12.jar into unzip cache +> checkLogContains Creating hardlinks of slf4j-api-1.6.1.jar from unzip cache +> checkLogContains Creating hardlinks of commons-io-2.4.jar from unzip cache +> checkLogContains Creating hardlinks of logback-classic-0.9.29.jar from unzip cache +> checkLogContains Creating hardlinks of logback-core-0.9.29.jar from unzip cache +> checkLogContains Creating hardlinks of scala-library-2.11.12.jar from unzip cache +$ exists target/scala-2.11/foo.jar + +# check if already cached +> clearLog +> assembly +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains Assembly up to date + +# check if creates from cache files +> clearLog +$ delete target/scala-2.11/foo.jar +> assembly +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains Including slf4j-api-1.6.1.jar from output cache +> checkLogContains Including commons-io-2.4.jar from output cache +> checkLogContains Including logback-classic-0.9.29.jar from output cache +> checkLogContains Including logback-core-0.9.29.jar from output cache +> checkLogContains Including scala-library-2.11.12.jar from output cache + +# check for using unzip cache +> clean +$ absent target/scala-2.11 +> clearLog +> assembly +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains Creating hardlinks of slf4j-api-1.6.1.jar from unzip cache +> checkLogContains Creating hardlinks of commons-io-2.4.jar from unzip cache +> checkLogContains Creating hardlinks of logback-classic-0.9.29.jar from unzip cache +> checkLogContains Creating hardlinks of logback-core-0.9.29.jar from unzip cache +> checkLogContains Creating hardlinks of scala-library-2.11.12.jar from unzip cache +> checkunzip + +> cleanunzip +> clean +> clearLog +> assemblyCacheDependency +# Ensure all warnings have time to be printed +$ sleep 1000 +> checkLogContains Unzipping slf4j-api-1.6.1.jar into unzip cache +> checkLogContains Unzipping commons-io-2.4.jar into unzip cache +> checkLogContains Unzipping logback-classic-0.9.29.jar into unzip cache +> checkLogContains Unzipping logback-core-0.9.29.jar into unzip cache +> checkLogContains Unzipping scala-library-2.11.12.jar into unzip cache + +> cleanunzip \ No newline at end of file