Skip to content

Commit

Permalink
Add test to verify ignoreNullable work when false (#167)
Browse files Browse the repository at this point in the history
* Add test to verify ignoreNullable work when false

* Remove unused import
  • Loading branch information
zeotuan authored Oct 14, 2024
1 parent 716be99 commit c53aeff
Show file tree
Hide file tree
Showing 3 changed files with 166 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,27 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
)
assertLargeDataFrameEquality(sourceDF, expectedDF, ignoreColumnOrder = true)
}

"should not ignore nullable if ignoreNullable is false" in {
val sourceDF = spark.createDF(
List(
1.2,
5.1
),
List(("number", DoubleType, false))
)
val expectedDF = spark.createDF(
List(
1.2,
5.1
),
List(("number", DoubleType, true))
)

intercept[DatasetSchemaMismatch] {
assertLargeDataFrameEquality(sourceDF, expectedDF)
}
}
}

"assertApproximateDataFrameEquality" - {
Expand Down Expand Up @@ -375,6 +396,27 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
}
}

"should not ignore nullable if ignoreNullable is false" in {
val sourceDF = spark.createDF(
List(
1.2,
5.1
),
List(("number", DoubleType, false))
)
val expectedDF = spark.createDF(
List(
1.2,
5.1
),
List(("number", DoubleType, true))
)

intercept[DatasetSchemaMismatch] {
assertApproximateDataFrameEquality(sourceDF, expectedDF, 0.01)
}
}

"can ignore the nullable property" in {
val sourceDF = spark.createDF(
List(
Expand Down Expand Up @@ -540,6 +582,27 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
assertApproximateSmallDataFrameEquality(sourceDF, expectedDF, 0.01, ignoreNullable = true)
}

"should not ignore nullable if ignoreNullable is false" in {
val sourceDF = spark.createDF(
List(
1.2,
5.1
),
List(("number", DoubleType, false))
)
val expectedDF = spark.createDF(
List(
1.2,
5.1
),
List(("number", DoubleType, true))
)

intercept[DatasetSchemaMismatch] {
assertApproximateSmallDataFrameEquality(sourceDF, expectedDF, 0.01)
}
}

"can ignore the column names" in {
val sourceDF = spark.createDF(
List(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,29 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes
assertLargeDatasetEquality(sourceDF, expectedDF, ignoreNullable = true)
}

"should not ignore nullable if ignoreNullable is false" in {

val sourceDF = spark.createDF(
List(
(1),
(5)
),
List(("number", IntegerType, false))
)

val expectedDF = spark.createDF(
List(
(1),
(5)
),
List(("number", IntegerType, true))
)

intercept[DatasetSchemaMismatch] {
assertLargeDatasetEquality(sourceDF, expectedDF)
}
}

"can performed unordered DataFrame comparisons" in {
val sourceDF = spark.createDF(
List(
Expand Down Expand Up @@ -432,6 +455,28 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes
assertSmallDatasetEquality(sourceDF, expectedDF, ignoreNullable = true)
}

"should not ignore nullable if ignoreNullable is false" in {
val sourceDF = spark.createDF(
List(
(1),
(5)
),
List(("number", IntegerType, false))
)

val expectedDF = spark.createDF(
List(
(1),
(5)
),
List(("number", IntegerType, true))
)

intercept[DatasetSchemaMismatch] {
assertSmallDatasetEquality(sourceDF, expectedDF)
}
}

"can performed unordered DataFrame comparisons" in {
val sourceDF = spark.createDF(
List(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,22 @@ class SchemaComparerTest extends AnyFreeSpec {
assert(SchemaComparer.equals(s1, s2, ignoreNullable = true))
}

"do not ignore nullable when determining equality if ignoreNullable is true" in {
val s1 = StructType(
Seq(
StructField("something", StringType, true),
StructField("mood", StringType, true)
)
)
val s2 = StructType(
Seq(
StructField("something", StringType, false),
StructField("mood", StringType, true)
)
)
assert(!SchemaComparer.equals(s1, s2))
}

"can ignore the nullable flag when determining equality on complex data types" in {
val s1 = StructType(
Seq(
Expand Down Expand Up @@ -110,6 +126,48 @@ class SchemaComparerTest extends AnyFreeSpec {
assert(SchemaComparer.equals(s1, s2, ignoreNullable = true))
}

"do not ignore nullable when determining equality on complex data types if ignoreNullable is true" in {
val s1 = StructType(
Seq(
StructField("something", StringType, true),
StructField("array", ArrayType(StringType, containsNull = true), true),
StructField("map", MapType(StringType, StringType, valueContainsNull = false), true),
StructField(
"struct",
StructType(
StructType(
Seq(
StructField("something", StringType, false),
StructField("mood", ArrayType(StringType, containsNull = false), true)
)
)
),
true
)
)
)
val s2 = StructType(
Seq(
StructField("something", StringType, false),
StructField("array", ArrayType(StringType, containsNull = false), true),
StructField("map", MapType(StringType, StringType, valueContainsNull = true), true),
StructField(
"struct",
StructType(
StructType(
Seq(
StructField("something", StringType, false),
StructField("mood", ArrayType(StringType, containsNull = true), true)
)
)
),
false
)
)
)
assert(!SchemaComparer.equals(s1, s2))
}

"can ignore the column names flag when determining equality" in {
val s1 = StructType(
Seq(
Expand Down

0 comments on commit c53aeff

Please sign in to comment.