Adds Kotlin Jupyter notebook support

.github/workflows/publish_dev_version.yml

@@ -0,0 +1,31 @@
+name: Generate and publish docs
+
+on:
+  push:
+    branches:
+      - "jupyter-test" # TODO change to spark-3.2
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          distributions: adopt
+          java-version: 11
+          check-latest: true
+      - name: Cache Maven packages
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+      - name: Deploy to GH Packages
+        run: ./mvnw --batch-mode deploy -Dkotest.tags="!Kafka"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+

README.md

@@ -73,6 +73,32 @@ Once you have configured the dependency, you only need to add the following impo
 import org.jetbrains.kotlinx.spark.api.*
 ```   
 
+### Jupyter
+
+The Kotlin Spark API also supports Kotlin Jupyter notebooks.
+To it, simply add
+
+```jupyterpython
+%use kotlin-spark-api
+```
+to the top of your notebook. This will get the latest version of the API, together with the latest version of Spark.
+To define a certain version of Spark or the API itself, simply add it like this:
+```jupyterpython
+%use kotlin-spark-api(spark=3.2, version=1.0.4)
+```
+
+Inside the notebook a Spark session will be initiated automatically. This can be accessed via the `spark` value.
+`sc: JavaSparkContext` can also be accessed directly. The API operates pretty similarly.
+
+There is also support for HTML rendering of Datasets and simple (Java)RDDs.
+
+To use Spark Streaming abilities, instead use
+```jupyterpython
+%use kotlin-spark-api-streaming
+```
+This does not start a Spark session right away, meaning you can call `withSparkStreaming(batchDuration) {}` 
+in whichever cell you want.
+
 ## Kotlin for Apache Spark features
 
 ### Creating a SparkSession in Kotlin
@@ -81,12 +107,13 @@ val spark = SparkSession
         .builder()
         .master("local[2]")
         .appName("Simple Application").orCreate
-
 ```
 
+This is not needed when running the Kotlin Spark API from a Jupyter notebook.
+
 ### Creating a Dataset in Kotlin
 ```kotlin
-spark.toDS("a" to 1, "b" to 2)
+spark.dsOf("a" to 1, "b" to 2)
 ```
 The example above produces `Dataset<Pair<String, Int>>`. While Kotlin Pairs and Triples are supported, Scala Tuples are reccomended for better support.
 
@@ -102,6 +129,8 @@ We provide you with useful function `withSpark`, which accepts everything that m
 
 After work block ends, `spark.stop()` is called automatically.
 
+Do not use this when running the Kotlin Spark API from a Jupyter notebook.
+
 ```kotlin
 withSpark {
     dsOf(1, 2)

core/3.2/pom_2.12.xml

@@ -27,8 +27,13 @@
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-sql_${scala.compat.version}</artifactId>
       <version>${spark3.version}</version>
-      <scope>provided</scope>
     </dependency>
+<!--    <dependency>-->
+<!--      <groupId>org.apache.spark</groupId>-->
+<!--      <artifactId>spark-core_${scala.compat.version}</artifactId>-->
+<!--      <version>${spark3.version}</version>-->
+<!--&lt;!&ndash;      <scope>provided</scope>&ndash;&gt;-->
+<!--    </dependency>-->
   </dependencies>
 
   <build>

examples/pom-3.2_2.12.xml

@@ -1,108 +1,114 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 
-  <modelVersion>4.0.0</modelVersion>
+    <modelVersion>4.0.0</modelVersion>
 
-  <name>Kotlin Spark API: Examples for Spark 3.2+ (Scala 2.12)</name>
-  <description>Example of usage</description>
-  <artifactId>examples-3.2_2.12</artifactId>
-  <parent>
-    <groupId>org.jetbrains.kotlinx.spark</groupId>
-    <artifactId>kotlin-spark-api-parent_2.12</artifactId>
-    <version>1.0.4-SNAPSHOT</version>
-    <relativePath>../pom_2.12.xml</relativePath>
-  </parent>
+    <name>Kotlin Spark API: Examples for Spark 3.2+ (Scala 2.12)</name>
+    <description>Example of usage</description>
+    <artifactId>examples-3.2_2.12</artifactId>
+    <parent>
+        <groupId>org.jetbrains.kotlinx.spark</groupId>
+        <artifactId>kotlin-spark-api-parent_2.12</artifactId>
+        <version>1.0.4-SNAPSHOT</version>
+        <relativePath>../pom_2.12.xml</relativePath>
+    </parent>
 
-  <dependencies>
-    <dependency>
-      <groupId>org.jetbrains.kotlinx.spark</groupId>
-      <artifactId>kotlin-spark-api-3.2</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-sql_${scala.compat.version}</artifactId>
-      <version>${spark3.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-streaming_${scala.compat.version}</artifactId>
-      <version>${spark3.version}</version>
-    </dependency>
-    <dependency><!-- Only needed for Qodana -->
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-streaming-kafka-0-10_${scala.compat.version}</artifactId>
-      <version>${spark3.version}</version>
-    </dependency>
-  </dependencies>
+    <dependencies>
+        <dependency>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-reflect</artifactId>
+        </dependency><!-- TODO this shouldn't be needed -->
+        <dependency>
+            <groupId>org.jetbrains.kotlinx.spark</groupId>
+            <artifactId>kotlin-spark-api-3.2</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.compat.version}</artifactId>
+            <version>${spark3.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-streaming_${scala.compat.version}</artifactId>
+            <version>${spark3.version}</version>
+        </dependency>
+        <dependency><!-- Only needed for Qodana -->
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-streaming-kafka-0-10_${scala.compat.version}</artifactId>
+            <version>${spark3.version}</version>
+        </dependency>
+    </dependencies>
 
-  <build>
-    <sourceDirectory>src/main/kotlin</sourceDirectory>
-    <testSourceDirectory>src/test/kotlin</testSourceDirectory>
-    <directory>target/3.2/${scala.compat.version}</directory>
-    <plugins>
-      <plugin>
-        <groupId>org.jetbrains.kotlin</groupId>
-        <artifactId>kotlin-maven-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>compile</id>
-            <goals>
-              <goal>compile</goal>
-            </goals>
-          </execution>
-          <execution>
-            <id>test-compile</id>
-            <goals>
-              <goal>test-compile</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-assembly-plugin</artifactId>
-        <version>${maven-assembly-plugin.version}</version>
-        <configuration>
-          <descriptorRefs>
-            <descriptorRef>jar-with-dependencies</descriptorRef>
-          </descriptorRefs>
-          <archive>
-            <manifest>
-              <mainClass>org.jetbrains.spark.api.examples.WordCountKt</mainClass>
-            </manifest>
-          </archive>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-site-plugin</artifactId>
-        <configuration>
-          <skip>true</skip>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-deploy-plugin</artifactId>
-        <configuration>
-          <skip>true</skip>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.sonatype.plugins</groupId>
-        <artifactId>nexus-staging-maven-plugin</artifactId>
-        <configuration>
-          <skipNexusStagingDeployMojo>true</skipNexusStagingDeployMojo>
-        </configuration>
-      </plugin>
-        <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-compiler-plugin</artifactId>
-            <configuration>
-                <source>8</source>
-                <target>8</target>
-            </configuration>
-        </plugin>
-    </plugins>
-  </build>
+    <build>
+        <sourceDirectory>src/main/kotlin</sourceDirectory>
+        <testSourceDirectory>src/test/kotlin</testSourceDirectory>
+        <directory>target/3.2/${scala.compat.version}</directory>
+        <plugins>
+            <plugin>
+                <groupId>org.jetbrains.kotlin</groupId>
+                <artifactId>kotlin-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>compile</id>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>test-compile</id>
+                        <goals>
+                            <goal>test-compile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>${maven-assembly-plugin.version}</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                    <archive>
+                        <manifest>
+                            <mainClass>org.jetbrains.spark.api.examples.WordCountKt</mainClass>
+                        </manifest>
+                    </archive>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-site-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <configuration>
+                    <skipNexusStagingDeployMojo>true</skipNexusStagingDeployMojo>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>${maven-compiler-plugin.version}</version>
+                <configuration>
+                    <source>9</source>
+                    <target>9</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
 </project>

kotlin-spark-api-streaming.json

@@ -0,0 +1,14 @@
+{
+  "description": "Kotlin for Apache® Spark™",
+  "properties": {
+    "spark": "3.2",
+    "v": "2.0.0"
+  },
+  "link": "https://github.com/JetBrains/kotlin-spark-api",
+  "dependencies": [
+    "org.jetbrains.kotlinx.spark:kotlin-spark-api-$spark:$v"
+  ],
+  "init": [
+    "%dumpClassesForSpark"
+  ]
+}

kotlin-spark-api.json

@@ -0,0 +1,30 @@
+{
+  "description": "Kotlin for Apache® Spark™",
+  "properties": {
+    "spark": "3.2",
+    "v": "2.0.0"
+  },
+  "link": "https://github.com/JetBrains/kotlin-spark-api",
+  "dependencies": [
+    "org.jetbrains.kotlinx.spark:kotlin-spark-api-$spark:$v"
+  ],
+  "init": [
+    "%dumpClassesForSpark",
+    "val spark = org.jetbrains.kotlinx.spark.api.SparkSession.builder().master(SparkConf().get(\"spark.master\", \"local[*]\")).appName(\"Jupyter\").getOrCreate()",
+    "spark.sparkContext.setLogLevel(org.jetbrains.kotlinx.spark.api.SparkLogLevel.ERROR)",
+    "val sc by lazy { org.apache.spark.api.java.JavaSparkContext(spark.sparkContext) }",
+    "println(\"Spark session has been started and is running. No `withSpark { }` necessary, you can access `spark` and `sc` directly. To use Spark streaming, use `%use kotlin-spark-api-streaming` instead.\")",
+
+    "inline fun <reified T> List<T>.toDS(): Dataset<T> = toDS(spark)",
+    "inline fun <reified T> Array<T>.toDS(): Dataset<T> = spark.dsOf(*this)",
+    "inline fun <reified T> dsOf(vararg arg: T): Dataset<T> = spark.dsOf(*arg)",
+    "inline fun <reified T> RDD<T>.toDS(): Dataset<T> = toDS(spark)",
+    "inline fun <reified T> JavaRDDLike<T, *>.toDS(): Dataset<T> = toDS(spark)",
+    "inline fun <reified T> RDD<T>.toDF(): Dataset<Row> = toDF(spark)",
+    "inline fun <reified T> JavaRDDLike<T, *>.toDF(): Dataset<Row> = toDF(spark)",
+    "val udf: UDFRegistration get() = spark.udf()"
+  ],
+  "shutdown": [
+    "spark.stop()"
+  ]
+}