Skip to content

Commit

Permalink
Merge pull request #123 from vuspenskiy/faster-md5-and-parallel-reports
Browse files Browse the repository at this point in the history
Faster MD5 and Parallel reports generation
  • Loading branch information
gslowikowski authored Oct 28, 2018
2 parents e4ca3c5 + bc65f7f commit c88bc5c
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 28 deletions.
31 changes: 13 additions & 18 deletions src/main/scala/org/scoverage/coveralls/CoverallPayloadWriter.scala
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
package org.scoverage.coveralls

import java.io.File
import java.security.MessageDigest
import scala.io.Source
import java.io.{File, FileInputStream}
import java.security.{DigestInputStream, MessageDigest}

import sbt.Logger
import com.fasterxml.jackson.core.{ JsonFactory, JsonEncoding }
Expand All @@ -13,7 +12,6 @@ class CoverallPayloadWriter(
repoToken: Option[String],
travisJobId: Option[String],
serviceName: Option[String],
sourceEncoding: Option[String],
gitClient: GitClient) {

val repoRootDirStr = repoRootDir.getCanonicalPath.replace(File.separator, "/") + "/"
Expand Down Expand Up @@ -92,14 +90,7 @@ class CoverallPayloadWriter(
gen.writeStartObject()
gen.writeStringField("name", fileName)

val source = sourceEncoding match {
case Some(enc) => Source.fromFile(report.file, enc)
case None => Source.fromFile(report.file)
}
val sourceCode = source.getLines().mkString("\n")
source.close()

val sourceDigest = CoverallPayloadWriter.md5.digest(sourceCode.getBytes).map("%02X" format _).mkString
val sourceDigest = computeSourceDigest(report.file)

gen.writeStringField("source_digest", sourceDigest)

Expand All @@ -113,6 +104,16 @@ class CoverallPayloadWriter(
gen.writeEndObject()
}

private def computeSourceDigest(path: String) = {
val buffer = new Array[Byte](8192)
val md5 = MessageDigest.getInstance("MD5")

val dis = new DigestInputStream(new FileInputStream(new File(path)), md5)
try { while (dis.read(buffer) != -1) { } } finally { dis.close() }

md5.digest.map("%02x".format(_)).mkString.toUpperCase
}

def end(): Unit = {
gen.writeEndArray()
gen.writeEndObject()
Expand All @@ -124,9 +125,3 @@ class CoverallPayloadWriter(
gen.flush()
}
}

object CoverallPayloadWriter {

val md5: MessageDigest = MessageDigest.getInstance("MD5")

}
14 changes: 10 additions & 4 deletions src/main/scala/org/scoverage/coveralls/CoverallsPlugin.scala
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@ object CoverallsPlugin extends AutoPlugin {
repoToken,
travisJobIdent,
coverallsServiceName.value,
sourcesEnc,
new GitClient(repoRootDirectory)(log)
)

Expand All @@ -93,12 +92,19 @@ object CoverallsPlugin extends AutoPlugin {
val allSources = sourceDirectories.all(aggregateFilter).value.flatten.filter(_.isDirectory()).distinct

val reader = new CoberturaMultiSourceReader(coberturaFile.value, allSources, sourcesEnc)
reader.sourceFilenames
.map(reader.reportForSource(_))
.foreach(writer.addSourceFile(_))

log.info(s"Generating reports for ${reader.sourceFilenames.size} files")

val fileReports = reader.sourceFilenames.par.map(reader.reportForSource(_)).seq

log.info(s"Adding file reports to the coveralls file (${coverallsFile.value.getName})")

fileReports.foreach(writer.addSourceFile(_))

writer.end()

log.info(s"Uploading the coveralls file (${coverallsFile.value.getName})")

val res = coverallsClient.postFile(coverallsFile.value)
val failBuildOnError = coverallsFailBuildOnError.value

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ class CoverallPayloadWriterTest extends WordSpec with BeforeAndAfterAll with Mat
}
}

def coverallsWriter(writer: Writer, tokenIn: Option[String], travisJobIdIn: Option[String], serviceName: Option[String], enc: Option[String]) =
new CoverallPayloadWriter(new File("").getAbsoluteFile, new File(""), tokenIn, travisJobIdIn, serviceName, enc, testGitClient) {
def coverallsWriter(writer: Writer, tokenIn: Option[String], travisJobIdIn: Option[String], serviceName: Option[String]) =
new CoverallPayloadWriter(new File("").getAbsoluteFile, new File(""), tokenIn, travisJobIdIn, serviceName, testGitClient) {
override def generator(file: File) = {
val factory = new JsonFactory()
factory.createGenerator(writer)
Expand All @@ -36,7 +36,7 @@ class CoverallPayloadWriterTest extends WordSpec with BeforeAndAfterAll with Mat

"generate a correct starting payload with travis job id" in {
val w = new StringWriter()
val coverallsW = coverallsWriter(w, Some("testRepoToken"), Some("testTravisJob"), Some("travis-ci"), Some("UTF-8"))
val coverallsW = coverallsWriter(w, Some("testRepoToken"), Some("testTravisJob"), Some("travis-ci"))

coverallsW.start
coverallsW.flush()
Expand All @@ -50,7 +50,7 @@ class CoverallPayloadWriterTest extends WordSpec with BeforeAndAfterAll with Mat

"generate a correct starting payload without travis job id" in {
val w = new StringWriter()
val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, None, Some("UTF-8"))
val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, None)

coverallsW.start
coverallsW.flush()
Expand All @@ -64,7 +64,7 @@ class CoverallPayloadWriterTest extends WordSpec with BeforeAndAfterAll with Mat

"add source files correctly" in {
val w = new StringWriter()
val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, Some("travis-ci"), Some("UTF-8"))
val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, Some("travis-ci"))

val projectRoot = new File("").getAbsolutePath.replace(File.separator, "/") + "/"

Expand All @@ -80,7 +80,7 @@ class CoverallPayloadWriterTest extends WordSpec with BeforeAndAfterAll with Mat

"end the file correctly" in {
val w = new StringWriter()
val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, Some("travis-ci"), Some("UTF-8"))
val coverallsW = coverallsWriter(w, Some("testRepoToken"), None, Some("travis-ci"))

coverallsW.start
coverallsW.end()
Expand Down

0 comments on commit c88bc5c

Please sign in to comment.