Add some missing Date encoders and data types

kotlin-spark-api/2.4/src/main/kotlin/org/jetbrains/kotlinx/spark/api/ApiV1.kt

@@ -364,6 +364,7 @@ private val knownDataTypes = mapOf(
         Double::class to DataTypes.DoubleType,
         String::class to DataTypes.StringType,
         LocalDate::class to `DateType$`.`MODULE$`,
+        Date::class to `DateType$`.`MODULE$`,
         Instant::class to `TimestampType$`.`MODULE$`
 )
 

kotlin-spark-api/2.4/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Conversions.kt

@@ -0,0 +1,139 @@
+@file:Suppress("NOTHING_TO_INLINE", "RemoveExplicitTypeArguments")
+
+package org.jetbrains.kotlinx.spark.api
+
+import scala.collection.JavaConversions
+import java.util.*
+import java.util.concurrent.ConcurrentMap
+import scala.collection.Iterable as ScalaIterable
+import scala.collection.Iterator as ScalaIterator
+import scala.collection.Map as ScalaMap
+import scala.collection.Seq as ScalaSeq
+import scala.collection.Set as ScalaSet
+import scala.collection.concurrent.Map as ScalaConcurrentMap
+import scala.collection.mutable.Buffer as ScalaMutableBuffer
+import scala.collection.mutable.Map as ScalaMutableMap
+import scala.collection.mutable.Seq as ScalaMutableSeq
+import scala.collection.mutable.Set as ScalaMutableSet
+
+/**
+ * @see JavaConversions.asScalaIterator for more information.
+ */
+fun <A> Iterator<A>.asScalaIterator(): ScalaIterator<A> = JavaConversions.asScalaIterator<A>(this)
+
+/**
+ * @see JavaConversions.enumerationAsScalaIterator for more information.
+ */
+fun <A> Enumeration<A>.asScalaIterator(): ScalaIterator<A> = JavaConversions.enumerationAsScalaIterator<A>(this)
+
+/**
+ * @see JavaConversions.iterableAsScalaIterable for more information.
+ */
+fun <A> Iterable<A>.asScalaIterable(): ScalaIterable<A> = JavaConversions.iterableAsScalaIterable<A>(this)
+
+/**
+ * @see JavaConversions.collectionAsScalaIterable for more information.
+ */
+fun <A> Collection<A>.asScalaIterable(): ScalaIterable<A> = JavaConversions.collectionAsScalaIterable<A>(this)
+
+/**
+ * @see JavaConversions.asScalaBuffer for more information.
+ */
+fun <A> MutableList<A>.asScalaMutableBuffer(): ScalaMutableBuffer<A> = JavaConversions.asScalaBuffer<A>(this)
+
+/**
+ * @see JavaConversions.asScalaSet for more information.
+ */
+fun <A> MutableSet<A>.asScalaMutableSet(): ScalaMutableSet<A> = JavaConversions.asScalaSet<A>(this)
+
+/**
+ * @see JavaConversions.mapAsScalaMap for more information.
+ */
+fun <A, B> MutableMap<A, B>.asScalaMutableMap(): ScalaMutableMap<A, B> = JavaConversions.mapAsScalaMap<A, B>(this)
+
+/**
+ * @see JavaConversions.dictionaryAsScalaMap for more information.
+ */
+fun <A, B> Map<A, B>.asScalaMap(): ScalaMap<A, B> = JavaConversions.mapAsScalaMap<A, B>(this)
+
+/**
+ * @see JavaConversions.mapAsScalaConcurrentMap for more information.
+ */
+fun <A, B> ConcurrentMap<A, B>.asScalaConcurrentMap(): ScalaConcurrentMap<A, B> = JavaConversions.mapAsScalaConcurrentMap<A, B>(this)
+
+/**
+ * @see JavaConversions.dictionaryAsScalaMap for more information.
+ */
+fun <A, B> Dictionary<A, B>.asScalaMap(): ScalaMutableMap<A, B> = JavaConversions.dictionaryAsScalaMap<A, B>(this)
+
+/**
+ * @see JavaConversions.propertiesAsScalaMap for more information.
+ */
+fun Properties.asScalaMap(): ScalaMutableMap<String, String> = JavaConversions.propertiesAsScalaMap(this)
+
+
+/**
+ * @see JavaConversions.asJavaIterator for more information.
+ */
+fun <A> ScalaIterator<A>.asKotlinIterator(): Iterator<A> = JavaConversions.asJavaIterator<A>(this)
+
+/**
+ * @see JavaConversions.asJavaEnumeration for more information.
+ */
+fun <A> ScalaIterator<A>.asKotlinEnumeration(): Enumeration<A> = JavaConversions.asJavaEnumeration<A>(this)
+
+/**
+ * @see JavaConversions.asJavaIterable for more information.
+ */
+fun <A> ScalaIterable<A>.asKotlinIterable(): Iterable<A> = JavaConversions.asJavaIterable<A>(this)
+
+/**
+ * @see JavaConversions.asJavaCollection for more information.
+ */
+fun <A> ScalaIterable<A>.asKotlinCollection(): Collection<A> = JavaConversions.asJavaCollection<A>(this)
+
+/**
+ * @see JavaConversions.bufferAsJavaList for more information.
+ */
+fun <A> ScalaMutableBuffer<A>.asKotlinMutableList(): MutableList<A> = JavaConversions.bufferAsJavaList<A>(this)
+
+/**
+ * @see JavaConversions.mutableSeqAsJavaList for more information.
+ */
+fun <A> ScalaMutableSeq<A>.asKotlinMutableList(): MutableList<A> = JavaConversions.mutableSeqAsJavaList<A>(this)
+
+/**
+ * @see JavaConversions.seqAsJavaList for more information.
+ */
+fun <A> ScalaSeq<A>.asKotlinList(): List<A> = JavaConversions.seqAsJavaList<A>(this)
+
+/**
+ * @see JavaConversions.mutableSetAsJavaSet for more information.
+ */
+fun <A> ScalaMutableSet<A>.asKotlinMutableSet(): MutableSet<A> = JavaConversions.mutableSetAsJavaSet<A>(this)
+
+/**
+ * @see JavaConversions.setAsJavaSet for more information.
+ */
+fun <A> ScalaSet<A>.asKotlinSet(): Set<A> = JavaConversions.setAsJavaSet<A>(this)
+
+/**
+ * @see JavaConversions.mutableMapAsJavaMap for more information.
+ */
+fun <A, B> ScalaMutableMap<A, B>.asKotlinMutableMap(): MutableMap<A, B> = JavaConversions.mutableMapAsJavaMap<A, B>(this)
+
+/**
+ * @see JavaConversions.asJavaDictionary for more information.
+ */
+fun <A, B> ScalaMutableMap<A, B>.asKotlinDictionary(): Dictionary<A, B> = JavaConversions.asJavaDictionary<A, B>(this)
+
+/**
+ * @see JavaConversions.mapAsJavaMap for more information.
+ */
+fun <A, B> ScalaMap<A, B>.asKotlinMap(): Map<A, B> = JavaConversions.mapAsJavaMap<A, B>(this)
+
+/**
+ * @see JavaConversions.mapAsJavaConcurrentMap for more information.
+ */
+fun <A, B> ScalaConcurrentMap<A, B>.asKotlinConcurrentMap(): ConcurrentMap<A, B> = JavaConversions.mapAsJavaConcurrentMap<A, B>(this)
+

kotlin-spark-api/2.4/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt

@@ -21,8 +21,15 @@ import ch.tutteli.atrium.api.fluent.en_GB.*
 import ch.tutteli.atrium.domain.builders.migration.asExpect
 import ch.tutteli.atrium.verbs.expect
 import io.kotest.core.spec.style.ShouldSpec
+import org.apache.spark.sql.Dataset
+import io.kotest.matchers.shouldBe
+import scala.collection.Seq
 import java.io.Serializable
+import java.sql.Date
 import java.time.LocalDate
+import scala.collection.Iterator as ScalaIterator
+import scala.collection.Map as ScalaMap
+import scala.collection.mutable.Map as ScalaMutableMap
 
 class ApiTest : ShouldSpec({
     context("integration tests") {
@@ -156,6 +163,47 @@ class ApiTest : ShouldSpec({
 
                 expect(result).asExpect().contains.inOrder.only.values(3, 5, 7, 9, 11)
             }
+
+            should("be able to serialize Date") {
+                val dataset: Dataset<Pair<Date, Int>> = dsOf(Date.valueOf("2020-02-10") to 5)
+                dataset.show()
+            }
+            should("Handle JavaConversions in Kotlin") {
+                // Test the iterator conversion
+                val scalaIterator: ScalaIterator<String> = listOf("test1", "test2").iterator().asScalaIterator()
+                scalaIterator.next() shouldBe "test1"
+
+                val kotlinIterator: Iterator<String> = scalaIterator.asKotlinIterator()
+                kotlinIterator.next() shouldBe "test2"
+
+
+                val scalaMap: ScalaMap<Int, String> = mapOf(1 to "a", 2 to "b").asScalaMap()
+                scalaMap.get(1).get() shouldBe "a"
+                scalaMap.get(2).get() shouldBe "b"
+
+                val kotlinMap: Map<Int, String> = scalaMap.asKotlinMap()
+                kotlinMap[1] shouldBe "a"
+                kotlinMap[2] shouldBe "b"
+
+
+                val scalaMutableMap: ScalaMutableMap<Int, String> = mutableMapOf(1 to "a").asScalaMutableMap()
+                scalaMutableMap.get(1).get() shouldBe "a"
+
+                scalaMutableMap.put(2, "b")
+
+                val kotlinMutableMap: MutableMap<Int, String> = scalaMutableMap.asKotlinMutableMap()
+                kotlinMutableMap[1] shouldBe "a"
+                kotlinMutableMap[2] shouldBe "b"
+
+                val scalaSeq: Seq<String> = listOf("a", "b").iterator().asScalaIterator().toSeq()
+                scalaSeq.take(1).toList().last() shouldBe "a"
+                scalaSeq.take(2).toList().last() shouldBe "b"
+
+                val kotlinList: List<String> = scalaSeq.asKotlinList()
+                kotlinList.first() shouldBe "a"
+                kotlinList.last() shouldBe "b"
+
+            }
         }
     }
 })

kotlin-spark-api/3.0/src/main/kotlin/org/jetbrains/kotlinx/spark/api/ApiV1.kt

@@ -56,7 +56,9 @@ val ENCODERS = mapOf<KClass<*>, Encoder<*>>(
         String::class to STRING(),
         BigDecimal::class to DECIMAL(),
         Date::class to DATE(),
+        LocalDate::class to LOCALDATE(), // 3.0 only
         Timestamp::class to TIMESTAMP(),
+        Instant::class to INSTANT(), // 3.0 only
         ByteArray::class to BINARY()
 )
 
@@ -351,6 +353,7 @@ private val knownDataTypes = mapOf(
         Double::class to DataTypes.DoubleType,
         String::class to DataTypes.StringType,
         LocalDate::class to `DateType$`.`MODULE$`,
+        Date::class to `DateType$`.`MODULE$`,
         Instant::class to `TimestampType$`.`MODULE$`
 )
 

kotlin-spark-api/3.0/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Conversions.kt

@@ -0,0 +1,139 @@
+@file:Suppress("NOTHING_TO_INLINE", "RemoveExplicitTypeArguments")
+
+package org.jetbrains.kotlinx.spark.api
+
+import scala.collection.JavaConverters
+import java.util.*
+import java.util.concurrent.ConcurrentMap
+import scala.collection.Iterable as ScalaIterable
+import scala.collection.Iterator as ScalaIterator
+import scala.collection.Map as ScalaMap
+import scala.collection.Seq as ScalaSeq
+import scala.collection.Set as ScalaSet
+import scala.collection.concurrent.Map as ScalaConcurrentMap
+import scala.collection.mutable.Buffer as ScalaMutableBuffer
+import scala.collection.mutable.Map as ScalaMutableMap
+import scala.collection.mutable.Seq as ScalaMutableSeq
+import scala.collection.mutable.Set as ScalaMutableSet
+
+/**
+ * @see JavaConverters.asScalaIterator for more information.
+ */
+fun <A> Iterator<A>.asScalaIterator(): ScalaIterator<A> = JavaConverters.asScalaIterator<A>(this)
+
+/**
+ * @see JavaConverters.enumerationAsScalaIterator for more information.
+ */
+fun <A> Enumeration<A>.asScalaIterator(): ScalaIterator<A> = JavaConverters.enumerationAsScalaIterator<A>(this)
+
+/**
+ * @see JavaConverters.iterableAsScalaIterable for more information.
+ */
+fun <A> Iterable<A>.asScalaIterable(): ScalaIterable<A> = JavaConverters.iterableAsScalaIterable<A>(this)
+
+/**
+ * @see JavaConverters.collectionAsScalaIterable for more information.
+ */
+fun <A> Collection<A>.asScalaIterable(): ScalaIterable<A> = JavaConverters.collectionAsScalaIterable<A>(this)
+
+/**
+ * @see JavaConverters.asScalaBuffer for more information.
+ */
+fun <A> MutableList<A>.asScalaMutableBuffer(): ScalaMutableBuffer<A> = JavaConverters.asScalaBuffer<A>(this)
+
+/**
+ * @see JavaConverters.asScalaSet for more information.
+ */
+fun <A> MutableSet<A>.asScalaMutableSet(): ScalaMutableSet<A> = JavaConverters.asScalaSet<A>(this)
+
+/**
+ * @see JavaConverters.mapAsScalaMap for more information.
+ */
+fun <A, B> MutableMap<A, B>.asScalaMutableMap(): ScalaMutableMap<A, B> = JavaConverters.mapAsScalaMap<A, B>(this)
+
+/**
+ * @see JavaConverters.dictionaryAsScalaMap for more information.
+ */
+fun <A, B> Map<A, B>.asScalaMap(): ScalaMap<A, B> = JavaConverters.mapAsScalaMap<A, B>(this)
+
+/**
+ * @see JavaConverters.mapAsScalaConcurrentMap for more information.
+ */
+fun <A, B> ConcurrentMap<A, B>.asScalaConcurrentMap(): ScalaConcurrentMap<A, B> = JavaConverters.mapAsScalaConcurrentMap<A, B>(this)
+
+/**
+ * @see JavaConverters.dictionaryAsScalaMap for more information.
+ */
+fun <A, B> Dictionary<A, B>.asScalaMap(): ScalaMutableMap<A, B> = JavaConverters.dictionaryAsScalaMap<A, B>(this)
+
+/**
+ * @see JavaConverters.propertiesAsScalaMap for more information.
+ */
+fun Properties.asScalaMap(): ScalaMutableMap<String, String> = JavaConverters.propertiesAsScalaMap(this)
+
+
+/**
+ * @see JavaConverters.asJavaIterator for more information.
+ */
+fun <A> ScalaIterator<A>.asKotlinIterator(): Iterator<A> = JavaConverters.asJavaIterator<A>(this)
+
+/**
+ * @see JavaConverters.asJavaEnumeration for more information.
+ */
+fun <A> ScalaIterator<A>.asKotlinEnumeration(): Enumeration<A> = JavaConverters.asJavaEnumeration<A>(this)
+
+/**
+ * @see JavaConverters.asJavaIterable for more information.
+ */
+fun <A> ScalaIterable<A>.asKotlinIterable(): Iterable<A> = JavaConverters.asJavaIterable<A>(this)
+
+/**
+ * @see JavaConverters.asJavaCollection for more information.
+ */
+fun <A> ScalaIterable<A>.asKotlinCollection(): Collection<A> = JavaConverters.asJavaCollection<A>(this)
+
+/**
+ * @see JavaConverters.bufferAsJavaList for more information.
+ */
+fun <A> ScalaMutableBuffer<A>.asKotlinMutableList(): MutableList<A> = JavaConverters.bufferAsJavaList<A>(this)
+
+/**
+ * @see JavaConverters.mutableSeqAsJavaList for more information.
+ */
+fun <A> ScalaMutableSeq<A>.asKotlinMutableList(): MutableList<A> = JavaConverters.mutableSeqAsJavaList<A>(this)
+
+/**
+ * @see JavaConverters.seqAsJavaList for more information.
+ */
+fun <A> ScalaSeq<A>.asKotlinList(): List<A> = JavaConverters.seqAsJavaList<A>(this)
+
+/**
+ * @see JavaConverters.mutableSetAsJavaSet for more information.
+ */
+fun <A> ScalaMutableSet<A>.asKotlinMutableSet(): MutableSet<A> = JavaConverters.mutableSetAsJavaSet<A>(this)
+
+/**
+ * @see JavaConverters.setAsJavaSet for more information.
+ */
+fun <A> ScalaSet<A>.asKotlinSet(): Set<A> = JavaConverters.setAsJavaSet<A>(this)
+
+/**
+ * @see JavaConverters.mutableMapAsJavaMap for more information.
+ */
+fun <A, B> ScalaMutableMap<A, B>.asKotlinMutableMap(): MutableMap<A, B> = JavaConverters.mutableMapAsJavaMap<A, B>(this)
+
+/**
+ * @see JavaConverters.asJavaDictionary for more information.
+ */
+fun <A, B> ScalaMutableMap<A, B>.asKotlinDictionary(): Dictionary<A, B> = JavaConverters.asJavaDictionary<A, B>(this)
+
+/**
+ * @see JavaConverters.mapAsJavaMap for more information.
+ */
+fun <A, B> ScalaMap<A, B>.asKotlinMap(): Map<A, B> = JavaConverters.mapAsJavaMap<A, B>(this)
+
+/**
+ * @see JavaConverters.mapAsJavaConcurrentMap for more information.
+ */
+fun <A, B> ScalaConcurrentMap<A, B>.asKotlinConcurrentMap(): ConcurrentMap<A, B> = JavaConverters.mapAsJavaConcurrentMap<A, B>(this)
+