Accelerating the coveralls file generation by using `DigestInputStrea…

…m` and parallelizing reports' generation for source files

src/main/scala/org/scoverage/coveralls/CoverallPayloadWriter.scala

@@ -1,8 +1,7 @@
 package org.scoverage.coveralls
 
-import java.io.File
-import java.security.MessageDigest
-import scala.io.Source
+import java.io.{File, FileInputStream}
+import java.security.{DigestInputStream, MessageDigest}
 
 import sbt.Logger
 import com.fasterxml.jackson.core.{ JsonFactory, JsonEncoding }
@@ -13,7 +12,6 @@ class CoverallPayloadWriter(
     repoToken: Option[String],
     travisJobId: Option[String],
     serviceName: Option[String],
-    sourceEncoding: Option[String],
     gitClient: GitClient) {
 
   val repoRootDirStr = repoRootDir.getCanonicalPath.replace(File.separator, "/") + "/"
@@ -92,14 +90,7 @@ class CoverallPayloadWriter(
     gen.writeStartObject()
     gen.writeStringField("name", fileName)
 
-    val source = sourceEncoding match {
-     case Some(enc) => Source.fromFile(report.file, enc)
-     case None => Source.fromFile(report.file)
-    }
-    val sourceCode = source.getLines().mkString("\n")
-    source.close()
-
-    val sourceDigest = CoverallPayloadWriter.md5.digest(sourceCode.getBytes).map("%02X" format _).mkString
+    val sourceDigest = computeSourceDigest(report.file)
 
     gen.writeStringField("source_digest", sourceDigest)
 
@@ -113,6 +104,16 @@ class CoverallPayloadWriter(
     gen.writeEndObject()
   }
 
+  private def computeSourceDigest(path: String) = {
+    val buffer = new Array[Byte](8192)
+    val md5 = MessageDigest.getInstance("MD5")
+
+    val dis = new DigestInputStream(new FileInputStream(new File(path)), md5)
+    try { while (dis.read(buffer) != -1) { } } finally { dis.close() }
+
+    md5.digest.map("%02x".format(_)).mkString.toUpperCase
+  }
+
   def end(): Unit = {
     gen.writeEndArray()
     gen.writeEndObject()
@@ -124,9 +125,3 @@ class CoverallPayloadWriter(
     gen.flush()
   }
 }
-
-object CoverallPayloadWriter {
-
-  val md5: MessageDigest = MessageDigest.getInstance("MD5")
-
-}

src/main/scala/org/scoverage/coveralls/CoverallsPlugin.scala

@@ -83,7 +83,6 @@ object CoverallsPlugin extends AutoPlugin {
       repoToken,
       travisJobIdent,
       coverallsServiceName.value,
-      sourcesEnc,
       new GitClient(repoRootDirectory)(log)
     )
 
@@ -93,12 +92,19 @@ object CoverallsPlugin extends AutoPlugin {
     val allSources = sourceDirectories.all(aggregateFilter).value.flatten.filter(_.isDirectory()).distinct
 
     val reader = new CoberturaMultiSourceReader(coberturaFile.value, allSources, sourcesEnc)
-    reader.sourceFilenames
-          .map(reader.reportForSource(_))
-          .foreach(writer.addSourceFile(_))
+
+    log.info(s"Generating reports for ${reader.sourceFilenames.size} files")
+
+    val fileReports = reader.sourceFilenames.par.map(reader.reportForSource(_)).seq
+
+    log.info(s"Adding file reports to the coveralls file (${coverallsFile.value.getName})")
+
+    fileReports.foreach(writer.addSourceFile(_))
 
     writer.end()
 
+    log.info(s"Uploading the coveralls file (${coverallsFile.value.getName})")
+
     val res = coverallsClient.postFile(coverallsFile.value)
     val failBuildOnError = coverallsFailBuildOnError.value
 

src/test/scala/com/github/theon/coveralls/CoverallPayloadWriterTest.scala

@@ -21,8 +21,8 @@ class CoverallPayloadWriterTest extends WordSpec with BeforeAndAfterAll with Mat
     }
   }
 
-  def coverallsWriter(writer: Writer, tokenIn: Option[String], travisJobIdIn: Option[String], serviceName: Option[String], enc: Option[String]) =
-    new CoverallPayloadWriter(new File("").getAbsoluteFile, new File(""), tokenIn, travisJobIdIn, serviceName, enc, testGitClient) {
+  def coverallsWriter(writer: Writer, tokenIn: Option[String], travisJobIdIn: Option[String], serviceName: Option[String]) =
+    new CoverallPayloadWriter(new File("").getAbsoluteFile, new File(""), tokenIn, travisJobIdIn, serviceName, testGitClient) {
       override def generator(file: File) = {
         val factory = new JsonFactory()
         factory.createGenerator(writer)
@@ -36,7 +36,7 @@ class CoverallPayloadWriterTest extends WordSpec with BeforeAndAfterAll with Mat
 
       "generate a correct starting payload with travis job id" in {
         val w = new StringWriter()
-        val coverallsW = coverallsWriter(w, Some("testRepoToken"), Some("testTravisJob"), Some("travis-ci"), Some("UTF-8"))
+        val coverallsW = coverallsWriter(w, Some("testRepoToken"), Some("testTravisJob"), Some("travis-ci"))
 
         coverallsW.start
         coverallsW.flush()
@@ -50,7 +50,7 @@ class CoverallPayloadWriterTest extends WordSpec with BeforeAndAfterAll with Mat
 
       "generate a correct starting payload without travis job id" in {
         val w = new StringWriter()
-        val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, None, Some("UTF-8"))
+        val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, None)
 
         coverallsW.start
         coverallsW.flush()
@@ -64,7 +64,7 @@ class CoverallPayloadWriterTest extends WordSpec with BeforeAndAfterAll with Mat
 
       "add source files correctly" in {
         val w = new StringWriter()
-        val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, Some("travis-ci"), Some("UTF-8"))
+        val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, Some("travis-ci"))
 
         val projectRoot = new File("").getAbsolutePath.replace(File.separator, "/") + "/"
 
@@ -80,7 +80,7 @@ class CoverallPayloadWriterTest extends WordSpec with BeforeAndAfterAll with Mat
 
       "end the file correctly" in {
         val w = new StringWriter()
-        val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, Some("travis-ci"), Some("UTF-8"))
+        val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, Some("travis-ci"))
 
         coverallsW.start
         coverallsW.end()