diff --git a/core/src/test/scala/com/github/mrpowers/spark/fast/tests/DataFrameComparerTest.scala b/core/src/test/scala/com/github/mrpowers/spark/fast/tests/DataFrameComparerTest.scala index d8620b3..3cda19c 100644 --- a/core/src/test/scala/com/github/mrpowers/spark/fast/tests/DataFrameComparerTest.scala +++ b/core/src/test/scala/com/github/mrpowers/spark/fast/tests/DataFrameComparerTest.scala @@ -310,6 +310,27 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar ) assertLargeDataFrameEquality(sourceDF, expectedDF, ignoreColumnOrder = true) } + + "should not ignore nullable if ignoreNullable is false" in { + val sourceDF = spark.createDF( + List( + 1.2, + 5.1 + ), + List(("number", DoubleType, false)) + ) + val expectedDF = spark.createDF( + List( + 1.2, + 5.1 + ), + List(("number", DoubleType, true)) + ) + + intercept[DatasetSchemaMismatch] { + assertLargeDataFrameEquality(sourceDF, expectedDF) + } + } } "assertApproximateDataFrameEquality" - { @@ -375,6 +396,27 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar } } + "should not ignore nullable if ignoreNullable is false" in { + val sourceDF = spark.createDF( + List( + 1.2, + 5.1 + ), + List(("number", DoubleType, false)) + ) + val expectedDF = spark.createDF( + List( + 1.2, + 5.1 + ), + List(("number", DoubleType, true)) + ) + + intercept[DatasetSchemaMismatch] { + assertApproximateDataFrameEquality(sourceDF, expectedDF, 0.01) + } + } + "can ignore the nullable property" in { val sourceDF = spark.createDF( List( @@ -540,6 +582,27 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar assertApproximateSmallDataFrameEquality(sourceDF, expectedDF, 0.01, ignoreNullable = true) } + "should not ignore nullable if ignoreNullable is false" in { + val sourceDF = spark.createDF( + List( + 1.2, + 5.1 + ), + List(("number", DoubleType, false)) + ) + val expectedDF = spark.createDF( + List( + 1.2, + 5.1 + ), + List(("number", DoubleType, true)) + ) + + intercept[DatasetSchemaMismatch] { + assertApproximateSmallDataFrameEquality(sourceDF, expectedDF, 0.01) + } + } + "can ignore the column names" in { val sourceDF = spark.createDF( List( diff --git a/core/src/test/scala/com/github/mrpowers/spark/fast/tests/DatasetComparerTest.scala b/core/src/test/scala/com/github/mrpowers/spark/fast/tests/DatasetComparerTest.scala index 0ab6b27..f7a9f57 100644 --- a/core/src/test/scala/com/github/mrpowers/spark/fast/tests/DatasetComparerTest.scala +++ b/core/src/test/scala/com/github/mrpowers/spark/fast/tests/DatasetComparerTest.scala @@ -280,6 +280,29 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes assertLargeDatasetEquality(sourceDF, expectedDF, ignoreNullable = true) } + "should not ignore nullable if ignoreNullable is false" in { + + val sourceDF = spark.createDF( + List( + (1), + (5) + ), + List(("number", IntegerType, false)) + ) + + val expectedDF = spark.createDF( + List( + (1), + (5) + ), + List(("number", IntegerType, true)) + ) + + intercept[DatasetSchemaMismatch] { + assertLargeDatasetEquality(sourceDF, expectedDF) + } + } + "can performed unordered DataFrame comparisons" in { val sourceDF = spark.createDF( List( @@ -432,6 +455,28 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes assertSmallDatasetEquality(sourceDF, expectedDF, ignoreNullable = true) } + "should not ignore nullable if ignoreNullable is false" in { + val sourceDF = spark.createDF( + List( + (1), + (5) + ), + List(("number", IntegerType, false)) + ) + + val expectedDF = spark.createDF( + List( + (1), + (5) + ), + List(("number", IntegerType, true)) + ) + + intercept[DatasetSchemaMismatch] { + assertSmallDatasetEquality(sourceDF, expectedDF) + } + } + "can performed unordered DataFrame comparisons" in { val sourceDF = spark.createDF( List( diff --git a/core/src/test/scala/com/github/mrpowers/spark/fast/tests/SchemaComparerTest.scala b/core/src/test/scala/com/github/mrpowers/spark/fast/tests/SchemaComparerTest.scala index c09f7b1..714d001 100644 --- a/core/src/test/scala/com/github/mrpowers/spark/fast/tests/SchemaComparerTest.scala +++ b/core/src/test/scala/com/github/mrpowers/spark/fast/tests/SchemaComparerTest.scala @@ -68,6 +68,22 @@ class SchemaComparerTest extends AnyFreeSpec { assert(SchemaComparer.equals(s1, s2, ignoreNullable = true)) } + "do not ignore nullable when determining equality if ignoreNullable is true" in { + val s1 = StructType( + Seq( + StructField("something", StringType, true), + StructField("mood", StringType, true) + ) + ) + val s2 = StructType( + Seq( + StructField("something", StringType, false), + StructField("mood", StringType, true) + ) + ) + assert(!SchemaComparer.equals(s1, s2)) + } + "can ignore the nullable flag when determining equality on complex data types" in { val s1 = StructType( Seq( @@ -110,6 +126,48 @@ class SchemaComparerTest extends AnyFreeSpec { assert(SchemaComparer.equals(s1, s2, ignoreNullable = true)) } + "do not ignore nullable when determining equality on complex data types if ignoreNullable is true" in { + val s1 = StructType( + Seq( + StructField("something", StringType, true), + StructField("array", ArrayType(StringType, containsNull = true), true), + StructField("map", MapType(StringType, StringType, valueContainsNull = false), true), + StructField( + "struct", + StructType( + StructType( + Seq( + StructField("something", StringType, false), + StructField("mood", ArrayType(StringType, containsNull = false), true) + ) + ) + ), + true + ) + ) + ) + val s2 = StructType( + Seq( + StructField("something", StringType, false), + StructField("array", ArrayType(StringType, containsNull = false), true), + StructField("map", MapType(StringType, StringType, valueContainsNull = true), true), + StructField( + "struct", + StructType( + StructType( + Seq( + StructField("something", StringType, false), + StructField("mood", ArrayType(StringType, containsNull = true), true) + ) + ) + ), + false + ) + ) + ) + assert(!SchemaComparer.equals(s1, s2)) + } + "can ignore the column names flag when determining equality" in { val s1 = StructType( Seq(