Skip to content

Commit f181fc5

Browse files
authored
fix: fixes complex objects not working insize lists
fixes #86
1 parent aa11744 commit f181fc5

File tree

2 files changed

+23
-1
lines changed

2 files changed

+23
-1
lines changed

core/3.0/src/main/scala/org/apache/spark/sql/KotlinReflection.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,7 @@ object KotlinReflection extends KotlinReflection {
421421
"toJavaMap",
422422
keyData :: valueData :: Nil,
423423
returnNullable = false)
424+
424425
case ArrayType(elementType, containsNull) =>
425426
val dataTypeWithClass = elementType.asInstanceOf[DataTypeWithClass]
426427
val mapFunction: Expression => Expression = element => {
@@ -434,7 +435,7 @@ object KotlinReflection extends KotlinReflection {
434435
nullable = dataTypeWithClass.nullable,
435436
newTypePath,
436437
(casted, typePath) => {
437-
deserializerFor(et, casted, typePath, Some(dataTypeWithClass.dt).filter(_.isInstanceOf[ComplexWrapper]).map(_.asInstanceOf[ComplexWrapper]))
438+
deserializerFor(et, casted, typePath, Some(dataTypeWithClass).filter(_.isInstanceOf[ComplexWrapper]).map(_.asInstanceOf[ComplexWrapper]))
438439
})
439440
}
440441

kotlin-spark-api/3.0/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,27 @@ class ApiTest : ShouldSpec({
347347
val asList = dataset.takeAsList(2)
348348
asList.first().tuple shouldBe Tuple3(5L, "test", Tuple1(""))
349349
}
350+
should("Be able to serialize lists of data classes") {
351+
val dataset = dsOf(
352+
listOf(SomeClass(intArrayOf(1, 2, 3), 4)),
353+
listOf(SomeClass(intArrayOf(3, 2, 1), 0)),
354+
)
355+
dataset.show()
356+
}
357+
should("Be able to serialize arrays of data classes") {
358+
val dataset = dsOf(
359+
arrayOf(SomeClass(intArrayOf(1, 2, 3), 4)),
360+
arrayOf(SomeClass(intArrayOf(3, 2, 1), 0)),
361+
)
362+
dataset.show()
363+
}
364+
should("Be able to serialize lists of tuples") {
365+
val dataset = dsOf(
366+
listOf(Tuple2(intArrayOf(1, 2, 3), 4)),
367+
listOf(Tuple2(intArrayOf(3, 2, 1), 0)),
368+
)
369+
dataset.show()
370+
}
350371
}
351372
}
352373
})

0 commit comments

Comments
 (0)