diff --git a/.scalafmt.conf b/.scalafmt.conf index 43d15ca..c7429c1 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,6 +1,10 @@ -align = more +version = 3.8.2 + +align.preset = more +runner.dialect = scala212 maxColumn = 150 -docstrings = JavaDoc +docstrings.style = Asterisk + //style = defaultWithAlign //align = true //danglingParentheses = false diff --git a/build.sbt b/build.sbt index 7ea13df..cd410d5 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,7 @@ scalafmtOnCompile in Compile := true organization := "com.github.mrpowers" name := "spark-fast-tests" -version := "1.3.0" +version := "1.10.1" val versionRegex = """^(.*)\.(.*)\.(.*)$""".r @@ -34,11 +34,11 @@ fork in Test := true javaOptions ++= Seq("-Xms512M", "-Xmx2048M", "-XX:+CMSClassUnloadingEnabled", "-Duser.timezone=GMT") licenses := Seq("MIT" -> url("http://opensource.org/licenses/MIT")) -homepage := Some(url("https://github.com/MrPowers/spark-fast-tests")) +homepage := Some(url("https://github.com/mrpowers-io/spark-fast-tests")) developers ++= List( Developer("MrPowers", "Matthew Powers", "@MrPowers", url("https://github.com/MrPowers")) ) -scmInfo := Some(ScmInfo(url("https://github.com/MrPowers/spark-fast-tests"), "git@github.com:MrPowers/spark-fast-tests.git")) +scmInfo := Some(ScmInfo(url("https://github.com/mrpowers-io/spark-fast-tests"), "git@github.com:MrPowers/spark-fast-tests.git")) updateOptions := updateOptions.value.withLatestSnapshots(false) diff --git a/project/plugins.sbt b/project/plugins.sbt index 1d30f43..fc3a0ff 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,7 +4,7 @@ resolvers += Resolver.bintrayIvyRepo("s22s", "sbt-plugins") resolvers += Resolver.typesafeRepo("releases") -addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.15") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2") addSbtPlugin("com.geirsson" % "sbt-ci-release" % "1.5.3") diff --git a/src/main/scala/com/github/mrpowers/spark/fast/tests/ArrayUtil.scala b/src/main/scala/com/github/mrpowers/spark/fast/tests/ArrayUtil.scala index febd7b1..98b0514 100644 --- a/src/main/scala/com/github/mrpowers/spark/fast/tests/ArrayUtil.scala +++ b/src/main/scala/com/github/mrpowers/spark/fast/tests/ArrayUtil.scala @@ -56,15 +56,13 @@ object ArrayUtil { // column names val h: Seq[(String, Int)] = rows.head.zipWithIndex - h.map { - case (cell, i) => - if (truncate > 0) { - StringUtils.leftPad(cell, colWidths(i)) - } else { - StringUtils.rightPad(cell, colWidths(i)) - } + h.map { case (cell, i) => + if (truncate > 0) { + StringUtils.leftPad(cell, colWidths(i)) + } else { + StringUtils.rightPad(cell, colWidths(i)) } - .addString(sb, "|", "|", "|\n") + }.addString(sb, "|", "|", "|\n") sb.append(sep) @@ -72,18 +70,17 @@ object ArrayUtil { rows.tail.map { row => val color = if (row(0) == row(1)) "blue" else "red" row.zipWithIndex - .map { - case (cell, i) => - val r = if (truncate > 0) { - StringUtils.leftPad(cell.toString, colWidths(i)) - } else { - StringUtils.rightPad(cell.toString, colWidths(i)) - } - if (color == "blue") { - ufansi.Color.DarkGray(r) - } else { - ufansi.Color.Red(r) - } + .map { case (cell, i) => + val r = if (truncate > 0) { + StringUtils.leftPad(cell.toString, colWidths(i)) + } else { + StringUtils.rightPad(cell.toString, colWidths(i)) + } + if (color == "blue") { + ufansi.Color.DarkGray(r) + } else { + ufansi.Color.Red(r) + } } .addString(sb, "|", "|", "|\n") } @@ -123,36 +120,32 @@ object ArrayUtil { // column names val h: Seq[(String, Int)] = rows.head.zipWithIndex - h.map { - case (cell, i) => - if (truncate > 0) { - StringUtils.leftPad(cell, colWidths(i)) - } else { - StringUtils.rightPad(cell, colWidths(i)) - } + h.map { case (cell, i) => + if (truncate > 0) { + StringUtils.leftPad(cell, colWidths(i)) + } else { + StringUtils.rightPad(cell, colWidths(i)) } - .addString(sb, "|", "|", "|\n") + }.addString(sb, "|", "|", "|\n") sb.append(sep) // data - rows.tail.zipWithIndex.map { - case (row, j) => - row.zipWithIndex - .map { - case (cell, i) => - val r = if (truncate > 0) { - StringUtils.leftPad(cell.toString, colWidths(i)) - } else { - StringUtils.rightPad(cell.toString, colWidths(i)) - } - if (rowEqual(j)) { - equalColor(r) - } else { - unequalColor(r) - } + rows.tail.zipWithIndex.map { case (row, j) => + row.zipWithIndex + .map { case (cell, i) => + val r = if (truncate > 0) { + StringUtils.leftPad(cell.toString, colWidths(i)) + } else { + StringUtils.rightPad(cell.toString, colWidths(i)) } - .addString(sb, "|", "|", "|\n") + if (rowEqual(j)) { + equalColor(r) + } else { + unequalColor(r) + } + } + .addString(sb, "|", "|", "|\n") } sb.append(sep) diff --git a/src/main/scala/com/github/mrpowers/spark/fast/tests/ColumnComparer.scala b/src/main/scala/com/github/mrpowers/spark/fast/tests/ColumnComparer.scala index 68c2af1..003ac17 100644 --- a/src/main/scala/com/github/mrpowers/spark/fast/tests/ColumnComparer.scala +++ b/src/main/scala/com/github/mrpowers/spark/fast/tests/ColumnComparer.scala @@ -151,7 +151,8 @@ trait ColumnComparer { if (sf1.dataType != sf2.dataType) { throw ColumnMismatch( - s"The column dataTypes are different. The `${colName1}` column has a `${sf1.dataType}` dataType and the `${colName2}` column has a `${sf2.dataType}` dataType.") + s"The column dataTypes are different. The `${colName1}` column has a `${sf1.dataType}` dataType and the `${colName2}` column has a `${sf2.dataType}` dataType." + ) } val r = df diff --git a/src/main/scala/com/github/mrpowers/spark/fast/tests/DataFrameComparer.scala b/src/main/scala/com/github/mrpowers/spark/fast/tests/DataFrameComparer.scala index 21b2bdd..4ba6d25 100644 --- a/src/main/scala/com/github/mrpowers/spark/fast/tests/DataFrameComparer.scala +++ b/src/main/scala/com/github/mrpowers/spark/fast/tests/DataFrameComparer.scala @@ -7,12 +7,14 @@ trait DataFrameComparer extends DatasetComparer { /** * Raises an error unless `actualDF` and `expectedDF` are equal */ - def assertSmallDataFrameEquality(actualDF: DataFrame, - expectedDF: DataFrame, - ignoreNullable: Boolean = false, - ignoreColumnNames: Boolean = false, - orderedComparison: Boolean = true, - truncate: Int = 500): Unit = { + def assertSmallDataFrameEquality( + actualDF: DataFrame, + expectedDF: DataFrame, + ignoreNullable: Boolean = false, + ignoreColumnNames: Boolean = false, + orderedComparison: Boolean = true, + truncate: Int = 500 + ): Unit = { assertSmallDatasetEquality( actualDF, expectedDF, @@ -26,11 +28,13 @@ trait DataFrameComparer extends DatasetComparer { /** * Raises an error unless `actualDF` and `expectedDF` are equal */ - def assertLargeDataFrameEquality(actualDF: DataFrame, - expectedDF: DataFrame, - ignoreNullable: Boolean = false, - ignoreColumnNames: Boolean = false, - orderedComparison: Boolean = true): Unit = { + def assertLargeDataFrameEquality( + actualDF: DataFrame, + expectedDF: DataFrame, + ignoreNullable: Boolean = false, + ignoreColumnNames: Boolean = false, + orderedComparison: Boolean = true + ): Unit = { assertLargeDatasetEquality( actualDF, expectedDF, diff --git a/src/main/scala/com/github/mrpowers/spark/fast/tests/DataFramePrettyPrint.scala b/src/main/scala/com/github/mrpowers/spark/fast/tests/DataFramePrettyPrint.scala index d61b518..3722020 100644 --- a/src/main/scala/com/github/mrpowers/spark/fast/tests/DataFramePrettyPrint.scala +++ b/src/main/scala/com/github/mrpowers/spark/fast/tests/DataFramePrettyPrint.scala @@ -93,45 +93,42 @@ object DataFramePrettyPrint { // column names val h: Seq[(String, Int)] = rows.head.zipWithIndex - h.map { - case (cell, i) => + h.map { case (cell, i) => + if (truncate > 0) { + StringUtils.leftPad( + cell, + colWidths(i) + ) + } else { + StringUtils.rightPad( + cell, + colWidths(i) + ) + } + }.addString( + sb, + "|", + "|", + "|\n" + ) + + sb.append(sep) + + // data + rows.tail.map { + _.zipWithIndex + .map { case (cell, i) => if (truncate > 0) { StringUtils.leftPad( - cell, + cell.toString, colWidths(i) ) } else { StringUtils.rightPad( - cell, + cell.toString, colWidths(i) ) } - } - .addString( - sb, - "|", - "|", - "|\n" - ) - - sb.append(sep) - - // data - rows.tail.map { - _.zipWithIndex - .map { - case (cell, i) => - if (truncate > 0) { - StringUtils.leftPad( - cell.toString, - colWidths(i) - ) - } else { - StringUtils.rightPad( - cell.toString, - colWidths(i) - ) - } } .addString( sb, diff --git a/src/main/scala/com/github/mrpowers/spark/fast/tests/DatasetComparer.scala b/src/main/scala/com/github/mrpowers/spark/fast/tests/DatasetComparer.scala index 736b829..793001e 100644 --- a/src/main/scala/com/github/mrpowers/spark/fast/tests/DatasetComparer.scala +++ b/src/main/scala/com/github/mrpowers/spark/fast/tests/DatasetComparer.scala @@ -66,9 +66,8 @@ Expected DataFrame Row Count: '${expectedCount}' private def unequalRDDMessage[T](unequalRDD: RDD[(Long, (T, T))], length: Int): String = { "\nRow Index | Actual Row | Expected Row\n" + unequalRDD .take(length) - .map { - case (idx, (left, right)) => - ufansi.Color.Red(s"$idx | $left | $right") + .map { case (idx, (left, right)) => + ufansi.Color.Red(s"$idx | $left | $right") } .mkString("\n") } @@ -76,12 +75,14 @@ Expected DataFrame Row Count: '${expectedCount}' /** * Raises an error unless `actualDS` and `expectedDS` are equal */ - def assertSmallDatasetEquality[T](actualDS: Dataset[T], - expectedDS: Dataset[T], - ignoreNullable: Boolean = false, - ignoreColumnNames: Boolean = false, - orderedComparison: Boolean = true, - truncate: Int = 500): Unit = { + def assertSmallDatasetEquality[T]( + actualDS: Dataset[T], + expectedDS: Dataset[T], + ignoreNullable: Boolean = false, + ignoreColumnNames: Boolean = false, + orderedComparison: Boolean = true, + truncate: Int = 500 + ): Unit = { if (!SchemaComparer.equals(actualDS.schema, expectedDS.schema, ignoreNullable, ignoreColumnNames)) { throw DatasetSchemaMismatch( betterSchemaMismatchMessage(actualDS, expectedDS) @@ -121,12 +122,14 @@ Expected DataFrame Row Count: '${expectedCount}' /** * Raises an error unless `actualDS` and `expectedDS` are equal */ - def assertLargeDatasetEquality[T: ClassTag](actualDS: Dataset[T], - expectedDS: Dataset[T], - equals: (T, T) => Boolean = naiveEquality _, - ignoreNullable: Boolean = false, - ignoreColumnNames: Boolean = false, - orderedComparison: Boolean = true): Unit = { + def assertLargeDatasetEquality[T: ClassTag]( + actualDS: Dataset[T], + expectedDS: Dataset[T], + equals: (T, T) => Boolean = naiveEquality _, + ignoreNullable: Boolean = false, + ignoreColumnNames: Boolean = false, + orderedComparison: Boolean = true + ): Unit = { // first check if the schemas are equal if (!SchemaComparer.equals(actualDS.schema, expectedDS.schema, ignoreNullable, ignoreColumnNames)) { throw DatasetSchemaMismatch(betterSchemaMismatchMessage(actualDS, expectedDS)) @@ -148,8 +151,8 @@ Expected DataFrame Row Count: '${expectedCount}' val resultIndexValue: RDD[(Long, T)] = RddHelpers.zipWithIndex(ds2.rdd) val unequalRDD = expectedIndexValue .join(resultIndexValue) - .filter { - case (idx, (o1, o2)) => !equals(o1, o2) + .filter { case (idx, (o1, o2)) => + !equals(o1, o2) } val maxUnequalRowsToShow = 10 if (!unequalRDD.isEmpty()) { @@ -172,12 +175,14 @@ Expected DataFrame Row Count: '${expectedCount}' } } - def assertApproximateDataFrameEquality(actualDF: DataFrame, - expectedDF: DataFrame, - precision: Double, - ignoreNullable: Boolean = false, - ignoreColumnNames: Boolean = false, - orderedComparison: Boolean = true): Unit = { + def assertApproximateDataFrameEquality( + actualDF: DataFrame, + expectedDF: DataFrame, + precision: Double, + ignoreNullable: Boolean = false, + ignoreColumnNames: Boolean = false, + orderedComparison: Boolean = true + ): Unit = { val e = (r1: Row, r2: Row) => { r1.equals(r2) || RowComparer.areRowsEqual(r1, r2, precision) } diff --git a/src/main/scala/com/github/mrpowers/spark/fast/tests/RddHelpers.scala b/src/main/scala/com/github/mrpowers/spark/fast/tests/RddHelpers.scala index 3a7204b..30993dd 100644 --- a/src/main/scala/com/github/mrpowers/spark/fast/tests/RddHelpers.scala +++ b/src/main/scala/com/github/mrpowers/spark/fast/tests/RddHelpers.scala @@ -5,14 +5,12 @@ import org.apache.spark.rdd.RDD object RddHelpers { /** - * Zip RDD's with precise indexes. This is used so we can join two DataFrame's - * Rows together regardless of if the source is different but still compare based on - * the order. + * Zip RDD's with precise indexes. This is used so we can join two DataFrame's Rows together regardless of if the source is different but still + * compare based on the order. */ def zipWithIndex[T](rdd: RDD[T]): RDD[(Long, T)] = { - rdd.zipWithIndex().map { - case (row, idx) => - (idx, row) + rdd.zipWithIndex().map { case (row, idx) => + (idx, row) } } diff --git a/src/main/scala/com/github/mrpowers/spark/fast/tests/RowComparer.scala b/src/main/scala/com/github/mrpowers/spark/fast/tests/RowComparer.scala index f839b40..5a2e5cd 100644 --- a/src/main/scala/com/github/mrpowers/spark/fast/tests/RowComparer.scala +++ b/src/main/scala/com/github/mrpowers/spark/fast/tests/RowComparer.scala @@ -22,16 +22,20 @@ object RowComparer { val o2 = r2.get(idx) o1 match { case b1: Array[Byte] => - if (!java.util.Arrays.equals( - b1, - o2.asInstanceOf[Array[Byte]] - )) { + if ( + !java.util.Arrays.equals( + b1, + o2.asInstanceOf[Array[Byte]] + ) + ) { return false } case f1: Float => - if (java.lang.Float.isNaN(f1) != - java.lang.Float.isNaN(o2.asInstanceOf[Float])) { + if ( + java.lang.Float.isNaN(f1) != + java.lang.Float.isNaN(o2.asInstanceOf[Float]) + ) { return false } if (abs(f1 - o2.asInstanceOf[Float]) > tol) { @@ -39,8 +43,10 @@ object RowComparer { } case d1: Double => - if (java.lang.Double.isNaN(d1) != - java.lang.Double.isNaN(o2.asInstanceOf[Double])) { + if ( + java.lang.Double.isNaN(d1) != + java.lang.Double.isNaN(o2.asInstanceOf[Double]) + ) { return false } if (abs(d1 - o2.asInstanceOf[Double]) > tol) { diff --git a/src/main/scala/com/github/mrpowers/spark/fast/tests/ufansi/Fansi.scala b/src/main/scala/com/github/mrpowers/spark/fast/tests/ufansi/Fansi.scala index 8ffabe4..1891367 100644 --- a/src/main/scala/com/github/mrpowers/spark/fast/tests/ufansi/Fansi.scala +++ b/src/main/scala/com/github/mrpowers/spark/fast/tests/ufansi/Fansi.scala @@ -21,17 +21,13 @@ object sourcecode { /** * Encapsulates a string with associated ANSI colors and text decorations. * - * This is your primary data-type when you are dealing with colored fansi - * strings. + * This is your primary data-type when you are dealing with colored fansi strings. * - * Contains some basic string methods, as well as some ansi methods to e.g. - * apply particular colors or other decorations to particular sections of - * the [[ufansi.Str]]. [[render]] flattens it out into a `java.lang.String` - * with all the colors present as ANSI escapes. + * Contains some basic string methods, as well as some ansi methods to e.g. apply particular colors or other decorations to particular sections of the + * [[ufansi.Str]]. [[render]] flattens it out into a `java.lang.String` with all the colors present as ANSI escapes. * - * Avoids using Scala collections operations in favor of util.Arrays, - * giving 20% (on `++`) to >1000% (on `splitAt`, `subString` - * and `Str.parse`) speedups + * Avoids using Scala collections operations in favor of util.Arrays, giving 20% (on `++`) to >1000% (on `splitAt`, `subString` and `Str.parse`) + * speedups */ case class Str private (private val chars: Array[Char], private val colors: Array[Str.State]) { require(chars.length == colors.length) @@ -50,8 +46,7 @@ case class Str private (private val chars: Array[Char], private val colors: Arra } /** - * Concatenates two [[ufansi.Str]]s, preserving the colors in each one and - * avoiding any interference between them + * Concatenates two [[ufansi.Str]]s, preserving the colors in each one and avoiding any interference between them */ def ++(other: Str) = { val chars2 = new Array[Char](length + other.length) @@ -115,11 +110,10 @@ case class Str private (private val chars: Array[Char], private val colors: Arra } /** - * Splits an [[ufansi.Str]] into two sub-strings, preserving the colors in - * each one. + * Splits an [[ufansi.Str]] into two sub-strings, preserving the colors in each one. * - * @param index the plain-text index of the point within the [[ufansi.Str]] - * you want to use to split it. + * @param index + * the plain-text index of the point within the [[ufansi.Str]] you want to use to split it. */ def splitAt(index: Int) = ( @@ -150,9 +144,7 @@ case class Str private (private val chars: Array[Char], private val colors: Arra ) /** - * Returns an [[ufansi.Str]] which is a substring of this string, - * and has the same colors as the original section of this string - * did + * Returns an [[ufansi.Str]] which is a substring of this string, and has the same colors as the original section of this string did */ def substring(start: Int = 0, end: Int = length) = { require( @@ -178,8 +170,7 @@ case class Str private (private val chars: Array[Char], private val colors: Arra } /** - * The plain-text length of this [[ufansi.Str]], in UTF-16 characters (same - * as `.length` on a `java.lang.String`). If you want fancy UTF-8 lengths, + * The plain-text length of this [[ufansi.Str]], in UTF-16 characters (same as `.length` on a `java.lang.String`). If you want fancy UTF-8 lengths, * use `.plainText` */ def length = chars.length @@ -187,14 +178,12 @@ case class Str private (private val chars: Array[Char], private val colors: Arra override def toString = render /** - * The plain-text `java.lang.String` represented by this [[ufansi.Str]], - * without all the fansi colors or other decorations + * The plain-text `java.lang.String` represented by this [[ufansi.Str]], without all the fansi colors or other decorations */ def plainText = new String(chars) /** - * Returns a copy of the colors array backing this `fansi.Str`, in case - * you want to use it to + * Returns a copy of the colors array backing this `fansi.Str`, in case you want to use it to */ def getColors = colors.clone() @@ -204,8 +193,7 @@ case class Str private (private val chars: Array[Char], private val colors: Arra def getColor(i: Int) = colors(i) /** - * Returns a copy of the character array backing this `fansi.Str`, in case - * you want to use it to + * Returns a copy of the character array backing this `fansi.Str`, in case you want to use it to */ def getChars = chars.clone() @@ -215,10 +203,8 @@ case class Str private (private val chars: Array[Char], private val colors: Arra def getChar(i: Int) = chars(i) /** - * Converts this [[ufansi.Str]] into a `java.lang.String`, including all - * the fancy fansi colors or decorations as fansi escapes embedded within - * the string. "Terminates" colors at the right-most end of the resultant - * `java.lang.String`, making it safe to concat-with or embed-inside other + * Converts this [[ufansi.Str]] into a `java.lang.String`, including all the fancy fansi colors or decorations as fansi escapes embedded within the + * string. "Terminates" colors at the right-most end of the resultant `java.lang.String`, making it safe to concat-with or embed-inside other * `java.lang.String` without worrying about fansi colors leaking out of it. */ def render = { @@ -270,9 +256,8 @@ case class Str private (private val chars: Array[Char], private val colors: Arra } /** - * Batch version of [[overlay]], letting you apply a bunch of [[Attrs]] onto - * various parts of the same string in one operation, avoiding the unnecessary - * copying that would happen if you applied them with [[overlay]] one by one. + * Batch version of [[overlay]], letting you apply a bunch of [[Attrs]] onto various parts of the same string in one operation, avoiding the + * unnecessary copying that would happen if you applied them with [[overlay]] one by one. * * The input sequence of overlay-tuples is applied from left to right */ @@ -310,35 +295,29 @@ case class Str private (private val chars: Array[Char], private val colors: Arra object Str { /** - * An [[ufansi.Str]]'s `color`s array is filled with Long, each representing - * the ANSI state of one character encoded in its bits. Each [[Attr]] belongs - * to a [[Category]] that occupies a range of bits within each long: - * - * 61... 55 54 53 52 51 .... 31 30 29 28 27 26 25 ..... 6 5 4 3 2 1 0 - * |--------| |-----------------------| |-----------------------| | | |bold - * | | | | |reversed - * | | | |underlined - * | | |foreground-color - * | |background-color - * |unused + * An [[ufansi.Str]]'s `color`s array is filled with Long, each representing the ANSI state of one character encoded in its bits. Each [[Attr]] + * belongs to a [[Category]] that occupies a range of bits within each long: * + * 61... 55 54 53 52 51 .... 31 30 29 28 27 26 25 ..... 6 5 4 3 2 1 0 + * \|--------| |-----------------------| |-----------------------| | | |bold + * \| | | | |reversed + * \| | | |underlined + * \| | |foreground-color + * \| |background-color + * \|unused * * The `0000 0000 0000 0000` long corresponds to plain text with no decoration - * */ type State = Long /** - * Make the construction of [[ufansi.Str]]s from `String`s and other - * `CharSequence`s automatic + * Make the construction of [[ufansi.Str]]s from `String`s and other `CharSequence`s automatic */ implicit def implicitApply(raw: CharSequence): ufansi.Str = apply(raw) /** * Regex that can be used to identify Ansi escape patterns in a string. * - * - * * Found from: http://stackoverflow.com/a/33925425/871202 * * Which references: @@ -350,16 +329,13 @@ object Str { val ansiRegex = "(\u009b|\u001b\\[)[0-?]*[ -\\/]*[@-~]".r.pattern /** - * Creates an [[ufansi.Str]] from a non-fansi `java.lang.String` or other - * `CharSequence`. + * Creates an [[ufansi.Str]] from a non-fansi `java.lang.String` or other `CharSequence`. * - * Note that this method is implicit, meaning you can pass in a - * `java.lang.String` anywhere an `fansi.Str` is required and it will be - * automatically parsed and converted for you. + * Note that this method is implicit, meaning you can pass in a `java.lang.String` anywhere an `fansi.Str` is required and it will be automatically + * parsed and converted for you. * - * @param errorMode Used to control what kind of behavior you get if the - * input `CharSequence` contains an Ansi escape not - * recognized by Fansi as a valid color. + * @param errorMode + * Used to control what kind of behavior you get if the input `CharSequence` contains an Ansi escape not recognized by Fansi as a valid color. */ def apply(raw: CharSequence, errorMode: ErrorMode = ErrorMode.Throw): ufansi.Str = { // Pre-allocate some arrays for us to fill up. They will probably be @@ -368,7 +344,7 @@ object Str { val chars = new Array[Char](raw.length) val colors = new Array[Str.State](raw.length) - var currentColor = 0l + var currentColor = 0L var sourceIndex = 0 var destIndex = 0 val length = raw.length @@ -419,25 +395,31 @@ object Str { if (!isDigit(sourceIndex)) fail() else { val r = getNumber() - if (!checkChar( - sourceIndex, - ';' - ) || !isDigit(sourceIndex + 1)) + if ( + !checkChar( + sourceIndex, + ';' + ) || !isDigit(sourceIndex + 1) + ) fail() else { sourceIndex += 1 val g = getNumber() - if (!checkChar( - sourceIndex, - ';' - ) || !isDigit(sourceIndex + 1)) fail() + if ( + !checkChar( + sourceIndex, + ';' + ) || !isDigit(sourceIndex + 1) + ) fail() else { sourceIndex += 1 val b = getNumber() - if (!checkChar( - sourceIndex, - 'm' - )) fail() + if ( + !checkChar( + sourceIndex, + 'm' + ) + ) fail() else { sourceIndex += 1 // Manually perform the `transform` for perf to avoid @@ -483,12 +465,10 @@ object Str { } /** - * Constructs a [[ufansi.Str]] from an array of characters and an array - * of colors. Performs a defensive copy of the arrays, and validates that - * they both have the same length + * Constructs a [[ufansi.Str]] from an array of characters and an array of colors. Performs a defensive copy of the arrays, and validates that they + * both have the same length * - * Useful together with `getChars` and `getColors` if you want to do manual - * work on the two mutable arrays before stitching them back together into + * Useful together with `getChars` and `getColors` if you want to do manual work on the two mutable arrays before stitching them back together into * one immutable [[ufansi.Str]] */ def fromArrays(chars: Array[Char], colors: Array[Str.State]) = { @@ -533,15 +513,13 @@ object Str { } /** - * Used to control what kind of behavior you get if the a `CharSequence` you - * are trying to parse into a [[ufansi.Str]] contains an Ansi escape not + * Used to control what kind of behavior you get if the a `CharSequence` you are trying to parse into a [[ufansi.Str]] contains an Ansi escape not * recognized by Fansi as a valid color. */ sealed trait ErrorMode { /** - * Given an unknown Ansi escape was found at `sourceIndex` inside your - * `raw: CharSequence`, what index should you resume parsing at? + * Given an unknown Ansi escape was found at `sourceIndex` inside your `raw: CharSequence`, what index should you resume parsing at? */ def handle(sourceIndex: Int, raw: CharSequence): Int } @@ -573,9 +551,8 @@ object ErrorMode { } /** - * Skip the `\u001b` that kicks off the unknown Ansi escape but leave - * subsequent characters in place, so the end-user can see that an Ansi - * escape was entered e.g. via the [A[B[A[C that appears in the result + * Skip the `\u001b` that kicks off the unknown Ansi escape but leave subsequent characters in place, so the end-user can see that an Ansi escape + * was entered e.g. via the [A[B[A[C that appears in the result */ case object Sanitize extends ErrorMode { @@ -585,8 +562,7 @@ object ErrorMode { } /** - * Find the end of the unknown Ansi escape and skip over it's characters - * entirely, so no trace of them appear in the parsed fansi.Str. + * Find the end of the unknown Ansi escape and skip over it's characters entirely, so no trace of them appear in the parsed fansi.Str. */ case object Strip extends ErrorMode { @@ -599,18 +575,15 @@ object ErrorMode { } /** - * Represents one or more [[ufansi.Attr]]s, that can be passed around - * as a set or combined with other sets of [[ufansi.Attr]]s. + * Represents one or more [[ufansi.Attr]]s, that can be passed around as a set or combined with other sets of [[ufansi.Attr]]s. * - * Note that a single [[Attr]] is a subclass of [[Attrs]]. If you want to - * know if this contains multiple [[Attr]]s, you should check for + * Note that a single [[Attr]] is a subclass of [[Attrs]]. If you want to know if this contains multiple [[Attr]]s, you should check for * [[Attrs.Multiple]]. */ sealed trait Attrs { /** - * Apply these [[Attrs]] to the given [[ufansi.Str]], making it take effect - * across the entire length of that string. + * Apply these [[Attrs]] to the given [[ufansi.Str]], making it take effect across the entire length of that string. */ def apply(s: ufansi.Str) = s.overlay( @@ -620,28 +593,23 @@ sealed trait Attrs { ) /** - * Which bits of the [[Str.State]] integer these [[Attrs]] will - * override when it is applied + * Which bits of the [[Str.State]] integer these [[Attrs]] will override when it is applied */ def resetMask: Long /** - * Which bits of the [[Str.State]] integer these [[Attrs]] will - * set to `1` when it is applied + * Which bits of the [[Str.State]] integer these [[Attrs]] will set to `1` when it is applied */ def applyMask: Long /** - * Apply the current [[Attrs]] to the [[Str.State]] integer, - * modifying it to represent the state after all changes have taken - * effect + * Apply the current [[Attrs]] to the [[Str.State]] integer, modifying it to represent the state after all changes have taken effect */ def transform(state: Str.State) = (state & ~resetMask) | applyMask /** - * Combine this [[ufansi.Attrs]] with other [[ufansi.Attrs]]s, returning one - * which when applied is equivalent to applying this one and then the `other` - * one in series. + * Combine this [[ufansi.Attrs]] with other [[ufansi.Attrs]]s, returning one which when applied is equivalent to applying this one and then the + * `other` one in series. */ def ++(other: ufansi.Attrs): ufansi.Attrs @@ -652,8 +620,7 @@ object Attrs { val Empty = Attrs() /** - * Emit the ansi escapes necessary to transition - * between two states, if necessary, as a `java.lang.String` + * Emit the ansi escapes necessary to transition between two states, if necessary, as a `java.lang.String` */ def emitAnsiCodes(currentState: Str.State, nextState: Str.State) = { val output = new StringBuilder @@ -668,8 +635,7 @@ object Attrs { } /** - * Messy-but-fast version of [[emitAnsiCodes]] that avoids allocating things - * unnecessarily. Reads it's category listing from a fast Array version of + * Messy-but-fast version of [[emitAnsiCodes]] that avoids allocating things unnecessarily. Reads it's category listing from a fast Array version of * Attrs.categories and writes it's output to a mutable `StringBuilder` */ def emitAnsiCodes0(currentState: Str.State, nextState: Str.State, output: StringBuilder, categoryArray: Array[Category]) = { @@ -683,7 +649,7 @@ object Attrs { val currentState2 = if ((currentState & ~nextState & hardOffMask) != 0) { output.append(Console.RESET) - 0l + 0L } else { currentState } @@ -702,8 +668,8 @@ object Attrs { def apply(attrs: Attr*): Attrs = { var output = List.empty[Attr] - var resetMask = 0l - var applyMask = 0l + var resetMask = 0L + var applyMask = 0L // Walk the list of attributes backwards, and aggregate only those whose // `resetMask` is not going to get totally covered by the union of all // `resetMask`s that come after it. @@ -754,12 +720,9 @@ object Attrs { } /** - * Represents a single, atomic ANSI escape sequence that results in a - * color, background or decoration being added to the output. May or may not - * have an escape sequence (`escapeOpt`), as some attributes (e.g. [[Bold.Off]]) - * are not widely/directly supported by terminals and so fansi.Str supports them - * by rendering a hard [[Attr.Reset]] and then re-rendering other [[Attr]]s that are - * active. + * Represents a single, atomic ANSI escape sequence that results in a color, background or decoration being added to the output. May or may not have + * an escape sequence (`escapeOpt`), as some attributes (e.g. [[Bold.Off]]) are not widely/directly supported by terminals and so fansi.Str supports + * them by rendering a hard [[Attr.Reset]] and then re-rendering other [[Attr]]s that are active. * * Many of the codes were stolen shamelessly from * @@ -776,8 +739,7 @@ sealed trait Attr extends Attrs { def name: String /** - * Combine this [[ufansi.Attr]] with one or more other [[ufansi.Attr]]s - * so they can be passed around together + * Combine this [[ufansi.Attr]] with one or more other [[ufansi.Attr]]s so they can be passed around together */ def ++(other: ufansi.Attrs): Attrs = Attrs(Array(this) ++ Attrs.toSeq(other): _*) @@ -786,8 +748,7 @@ sealed trait Attr extends Attrs { object Attr { /** - * Represents the removal of all ansi text decoration. Doesn't fit into any - * convenient category, since it applies to them all. + * Represents the removal of all ansi text decoration. Doesn't fit into any convenient category, since it applies to them all. */ val Reset = new EscapeAttr( Console.RESET, @@ -826,8 +787,7 @@ case class ResetAttr private[ufansi] (resetMask: Long, applyMask: Long)(implicit } /** - * Represents a set of [[ufansi.Attr]]s all occupying the same bit-space - * in the state `Int` + * Represents a set of [[ufansi.Attr]]s all occupying the same bit-space in the state `Int` */ sealed abstract class Category(val offset: Int, val width: Int)(implicit catName: sourcecode.Name) { def mask = ((1 << width) - 1) << offset @@ -895,8 +855,7 @@ object Bold } /** - * [[Attr]]s to reverse the background/foreground colors of your text, - * or un-reverse them + * [[Attr]]s to reverse the background/foreground colors of your text, or un-reverse them */ object Reversed extends Category( @@ -1179,8 +1138,7 @@ object Back } /** - * An string trie for quickly looking up values of type [[T]] - * using string-keys. Used to speed up + * An string trie for quickly looking up values of type [[T]] using string-keys. Used to speed up */ private[this] final class Trie[T](strings: Seq[(String, T)]) { @@ -1239,11 +1197,7 @@ private[this] final class Trie[T](strings: Seq[(String, T)]) { } /** - * * Color a encoded on 25 bit as follow : - * 0 : reset value - * 1 - 16 : 3 bit colors - * 17 - 272 : 8 bit colors - * 273 - 16 777 388 : 24 bit colors + * * Color a encoded on 25 bit as follow : 0 : reset value 1 - 16 : 3 bit colors 17 - 272 : 8 bit colors 273 - 16 777 388 : 24 bit colors */ abstract class ColorCategory(offset: Int, width: Int, val colorCode: Int)(implicit catName: sourcecode.Name) extends Category( @@ -1256,11 +1210,10 @@ abstract class ColorCategory(offset: Int, width: Int, val colorCode: Int)(implic */ val Full = for (x <- 0 until 256) - yield - makeAttr( - s"\u001b[$colorCode;5;${x}m", - 17 + x - )(s"Full($x)") + yield makeAttr( + s"\u001b[$colorCode;5;${x}m", + 17 + x + )(s"Full($x)") private[this] def True0(r: Int, g: Int, b: Int, index: Int) = { makeAttr( @@ -1278,8 +1231,7 @@ abstract class ColorCategory(offset: Int, width: Int, val colorCode: Int)(implic } /** - * Create a TrueColor color, from a given index within the 16-million-color - * TrueColor range + * Create a TrueColor color, from a given index within the 16-million-color TrueColor range */ def True(index: Int) = { require( @@ -1287,8 +1239,8 @@ abstract class ColorCategory(offset: Int, width: Int, val colorCode: Int)(implic "True parameter `index` must be 273 <= index <= 16777488, not " + index ) val r = index >> 16 - val g = (index & 0x00FF00) >> 8 - val b = index & 0x0000FF + val g = (index & 0x00ff00) >> 8 + val b = index & 0x0000ff True0( r, g, @@ -1298,8 +1250,7 @@ abstract class ColorCategory(offset: Int, width: Int, val colorCode: Int)(implic } /** - * Create a TrueColor color, from a given (r, g, b) within the 16-million-color - * TrueColor range + * Create a TrueColor color, from a given (r, g, b) within the 16-million-color TrueColor range */ def True(r: Int, g: Int, b: Int) = True0( @@ -1336,8 +1287,8 @@ abstract class ColorCategory(offset: Int, width: Int, val colorCode: Int)(implic val index = rawIndex - 273 trueRgbEscape( r = index >> 16, - g = (index & 0x00FF00) >> 8, - b = index & 0x0000FF + g = (index & 0x00ff00) >> 8, + b = index & 0x0000ff ) } } diff --git a/src/test/scala/com/github/mrpowers/spark/fast/tests/DatasetComparerTest.scala b/src/test/scala/com/github/mrpowers/spark/fast/tests/DatasetComparerTest.scala index bc5e592..1a91bbb 100644 --- a/src/test/scala/com/github/mrpowers/spark/fast/tests/DatasetComparerTest.scala +++ b/src/test/scala/com/github/mrpowers/spark/fast/tests/DatasetComparerTest.scala @@ -161,19 +161,11 @@ class DatasetComparerTest extends FreeSpec with DatasetComparer with SparkSessio "throws an error if the DataFrames content is different" in { val sourceDF = Seq( - (1), - (5), - (7), - (1), - (1) + (1), (5), (7), (1), (1) ).toDF("number") val expectedDF = Seq( - (10), - (5), - (3), - (7), - (1) + (10), (5), (3), (7), (1) ).toDF("number") val e = intercept[DatasetContentMismatch] { diff --git a/src/test/scala/com/github/mrpowers/spark/fast/tests/SchemaComparerTest.scala b/src/test/scala/com/github/mrpowers/spark/fast/tests/SchemaComparerTest.scala index 1fe696e..d1d898b 100644 --- a/src/test/scala/com/github/mrpowers/spark/fast/tests/SchemaComparerTest.scala +++ b/src/test/scala/com/github/mrpowers/spark/fast/tests/SchemaComparerTest.scala @@ -82,7 +82,8 @@ class SchemaComparerTest extends FreeSpec { StructField("something", StringType, false), StructField("mood", ArrayType(StringType, containsNull = false), true) ) - )), + ) + ), true ) ) @@ -100,7 +101,8 @@ class SchemaComparerTest extends FreeSpec { StructField("something", StringType, false), StructField("mood", ArrayType(StringType, containsNull = true), true) ) - )), + ) + ), false ) ) diff --git a/src/test/scala/com/github/mrpowers/spark/fast/tests/SparkSessionExt.scala b/src/test/scala/com/github/mrpowers/spark/fast/tests/SparkSessionExt.scala index d845b57..e8b5b9b 100644 --- a/src/test/scala/com/github/mrpowers/spark/fast/tests/SparkSessionExt.scala +++ b/src/test/scala/com/github/mrpowers/spark/fast/tests/SparkSessionExt.scala @@ -24,32 +24,15 @@ object SparkSessionExt { } /** - * Creates a DataFrame, similar to createDataFrame, but with better syntax - * spark-daria defined a createDF method that allows for the terse syntax of `toDF` and the control of `createDataFrame`. + * Creates a DataFrame, similar to createDataFrame, but with better syntax spark-daria defined a createDF method that allows for the terse syntax + * of `toDF` and the control of `createDataFrame`. * - * spark.createDF( - * List( - * ("bob", 45), - * ("liz", 25), - * ("freeman", 32) - * ), List( - * ("name", StringType, true), - * ("age", IntegerType, false) - * ) - * ) + * spark.createDF( List( ("bob", 45), ("liz", 25), ("freeman", 32) ), List( ("name", StringType, true), ("age", IntegerType, false) ) ) * * The `createDF` method can also be used with lists of `Row` and `StructField` objects. * - * spark.createDF( - * List( - * Row("bob", 45), - * Row("liz", 25), - * Row("freeman", 32) - * ), List( - * StructField("name", StringType, true), - * StructField("age", IntegerType, false) - * ) - * ) + * spark.createDF( List( Row("bob", 45), Row("liz", 25), Row("freeman", 32) ), List( StructField("name", StringType, true), StructField("age", + * IntegerType, false) ) ) */ def createDF[U, T](rowData: List[U], fields: List[T]): DataFrame = { spark.createDataFrame(