diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 98a7ac04..7d87d712 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -3,34 +3,71 @@ name: Build and test
on:
push:
branches: '*'
- pull_request:
- branches: '*'
+# pull_request:
+# branches: '*'
jobs:
- build-scala-12:
+ build-all-versions:
+ strategy:
+ matrix:
+ scala: [ "2.12.15", "2.13.8" ]
+ spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
+ exclude:
+ - scala: "2.13.8"
+ spark: "3.1.3"
+ - scala: "2.13.8"
+ spark: "3.1.2"
+ - scala: "2.13.8"
+ spark: "3.1.1"
+ - scala: "2.13.8"
+ spark: "3.1.0"
+ - scala: "2.13.8"
+ spark: "3.0.3"
+ - scala: "2.13.8"
+ spark: "3.0.2"
+ - scala: "2.13.8"
+ spark: "3.0.1"
+ - scala: "2.13.8"
+ spark: "3.0.0"
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
+
- name: Set up JDK 11
- uses: actions/setup-java@v1
+ uses: actions/setup-java@v3
with:
- distributions: adopt
+ distribution: adopt
java-version: 11
check-latest: true
- - name: Cache Maven packages
- uses: actions/cache@v2
+
+ - name: Cache Gradle packages
+ uses: actions/cache@v3
+ with:
+ path: |
+ ~/.gradle/caches
+ ~/.gradle/wrapper
+ ~/.gradle/jdks
+ key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}
+ restore-keys: |
+ ${{ runner.os }}-gradle-
+
+ - name: Build with Gradle
+ uses: gradle/gradle-build-action@v2
with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2
- - name: Build with Maven
- run: ./mvnw -B package --file pom.xml -Pscala-2.12
-# qodana:
-# runs-on: ubuntu-latest
-# steps:
-# - uses: actions/checkout@v3
-# - name: 'Qodana Scan'
-# uses: JetBrains/qodana-action@v5.0.2
+ arguments: |
+ -Pspark=${{ matrix.spark }}
+ -Pscala=${{ matrix.scala }}
+ clean
+ build
+
+ # qodana:
+ # runs-on: ubuntu-latest
+ # steps:
+ # - uses: actions/checkout@v3
+ # - name: 'Qodana Scan'
+ # uses: JetBrains/qodana-action@v5.0.2
+
+
# vim: ts=2:sts=2:sw=2:expandtab
diff --git a/.github/workflows/generate_docs.yml b/.github/workflows/generate_docs.yml
index cdbd1949..1709b0ae 100644
--- a/.github/workflows/generate_docs.yml
+++ b/.github/workflows/generate_docs.yml
@@ -3,28 +3,45 @@ name: Generate and publish docs
on:
push:
branches:
- - "spark-3.2"
+ - "release" # todo make release branch
jobs:
generate-and-publish-docs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
+
- name: Set up JDK 11
- uses: actions/setup-java@v1
+ uses: actions/setup-java@v3
with:
- distributions: adopt
+ distribution: adopt
java-version: 11
check-latest: true
- - name: Generate docs
- run: ./mvnw clean package site -Dmaven.test.skip=true
+
+ - name: Cache Gradle packages
+ uses: actions/cache@v3
+ with:
+ path: |
+ ~/.gradle/caches
+ ~/.gradle/wrapper
+ ~/.gradle/jdks
+ key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}
+ restore-keys: |
+ ${{ runner.os }}-gradle-
+
+ - name: Generate docs with Gradle
+ uses: gradle/gradle-build-action@v2
+ with:
+ arguments: |
+ dokkaHtmlMultiModule
+
- name: Copy docs to "docs" branch
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_branch: docs
- publish_dir: ./kotlin-spark-api/3.2/target/dokka
+ publish_dir: ./build/dokka/htmlMultiModule
force_orphan: true
diff --git a/.github/workflows/publish_dev_version.yml b/.github/workflows/publish_dev_version.yml
index b292cf15..660878dd 100644
--- a/.github/workflows/publish_dev_version.yml
+++ b/.github/workflows/publish_dev_version.yml
@@ -7,24 +7,65 @@ on:
jobs:
build-and-deploy:
+ strategy:
+ matrix:
+ scala: [ "2.12.15", "2.13.8" ]
+ spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
+ exclude:
+ - scala: "2.13.8"
+ spark: "3.1.3"
+ - scala: "2.13.8"
+ spark: "3.1.2"
+ - scala: "2.13.8"
+ spark: "3.1.1"
+ - scala: "2.13.8"
+ spark: "3.1.0"
+ - scala: "2.13.8"
+ spark: "3.0.3"
+ - scala: "2.13.8"
+ spark: "3.0.2"
+ - scala: "2.13.8"
+ spark: "3.0.1"
+ - scala: "2.13.8"
+ spark: "3.0.0"
runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
+
- name: Set up JDK 11
- uses: actions/setup-java@v1
+ uses: actions/setup-java@v3
with:
- distributions: adopt
+ distribution: adopt
java-version: 11
check-latest: true
- - name: Cache Maven packages
- uses: actions/cache@v2
+
+ - name: Cache Gradle packages
+ uses: actions/cache@v3
+ with:
+ path: |
+ ~/.gradle/caches
+ ~/.gradle/wrapper
+ ~/.gradle/jdks
+ key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}
+ restore-keys: |
+ ${{ runner.os }}-gradle-
+
+ - name: Validate Gradle wrapper
+ uses: gradle/wrapper-validation-action@v1
+
+ - name: Deploy to GH Packages with Gradle
+ uses: gradle/gradle-build-action@v2
with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2
- - name: Deploy to GH Packages
- run: ./mvnw --batch-mode deploy
+ arguments: |
+ -Pspark=${{ matrix.spark }}
+ -Pscala=${{ matrix.scala }}
+ -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }}
+ clean
+ publishMavenPublicationToGitHubPackagesRepository
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
index f7521c9b..d1f280d2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -376,3 +376,4 @@ orcpath/
.env
**/.allure/
**/allure-results/
+/generated_*
diff --git a/.mvn/wrapper/MavenWrapperDownloader.java b/.mvn/wrapper/MavenWrapperDownloader.java
deleted file mode 100644
index b901097f..00000000
--- a/.mvn/wrapper/MavenWrapperDownloader.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright 2007-present the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.net.*;
-import java.io.*;
-import java.nio.channels.*;
-import java.util.Properties;
-
-public class MavenWrapperDownloader {
-
- private static final String WRAPPER_VERSION = "0.5.6";
- /**
- * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
- */
- private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
- + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
-
- /**
- * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
- * use instead of the default one.
- */
- private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
- ".mvn/wrapper/maven-wrapper.properties";
-
- /**
- * Path where the maven-wrapper.jar will be saved to.
- */
- private static final String MAVEN_WRAPPER_JAR_PATH =
- ".mvn/wrapper/maven-wrapper.jar";
-
- /**
- * Name of the property which should be used to override the default download url for the wrapper.
- */
- private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
-
- public static void main(String args[]) {
- System.out.println("- Downloader started");
- File baseDirectory = new File(args[0]);
- System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
-
- // If the maven-wrapper.properties exists, read it and check if it contains a custom
- // wrapperUrl parameter.
- File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
- String url = DEFAULT_DOWNLOAD_URL;
- if(mavenWrapperPropertyFile.exists()) {
- FileInputStream mavenWrapperPropertyFileInputStream = null;
- try {
- mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
- Properties mavenWrapperProperties = new Properties();
- mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
- url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
- } catch (IOException e) {
- System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
- } finally {
- try {
- if(mavenWrapperPropertyFileInputStream != null) {
- mavenWrapperPropertyFileInputStream.close();
- }
- } catch (IOException e) {
- // Ignore ...
- }
- }
- }
- System.out.println("- Downloading from: " + url);
-
- File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
- if(!outputFile.getParentFile().exists()) {
- if(!outputFile.getParentFile().mkdirs()) {
- System.out.println(
- "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
- }
- }
- System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
- try {
- downloadFileFromURL(url, outputFile);
- System.out.println("Done");
- System.exit(0);
- } catch (Throwable e) {
- System.out.println("- Error downloading");
- e.printStackTrace();
- System.exit(1);
- }
- }
-
- private static void downloadFileFromURL(String urlString, File destination) throws Exception {
- if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
- String username = System.getenv("MVNW_USERNAME");
- char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
- Authenticator.setDefault(new Authenticator() {
- @Override
- protected PasswordAuthentication getPasswordAuthentication() {
- return new PasswordAuthentication(username, password);
- }
- });
- }
- URL website = new URL(urlString);
- ReadableByteChannel rbc;
- rbc = Channels.newChannel(website.openStream());
- FileOutputStream fos = new FileOutputStream(destination);
- fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
- fos.close();
- rbc.close();
- }
-
-}
diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties
deleted file mode 100644
index ffdc10e5..00000000
--- a/.mvn/wrapper/maven-wrapper.properties
+++ /dev/null
@@ -1,2 +0,0 @@
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.1/apache-maven-3.8.1-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar
diff --git a/README.md b/README.md
index f54416f0..b89a04cb 100644
--- a/README.md
+++ b/README.md
@@ -30,20 +30,34 @@ We have opened a Spark Project Improvement Proposal: [Kotlin support for Apache
- [Streaming](#streaming)
- [User Defined Functions](#user-defined-functions)
- [Examples](#examples)
-- [Reporting issues/Support](#reporting-issuessupport)
+- [Reporting issues/Support](#reporting-issues--support)
- [Code of Conduct](#code-of-conduct)
- [License](#license)
## Supported versions of Apache Spark
+| Apache Spark | Scala | Kotlin for Apache Spark |
+|:------------:|:-----:|:-----------------------------------:|
+| 3.3.0 | 2.13 | kotlin-spark-api_3.3.0_2.13:VERSION |
+| | 2.12 | kotlin-spark-api_3.3.0_2.12:VERSION |
+| 3.2.1 | 2.13 | kotlin-spark-api_3.2.1_2.13:VERSION |
+| | 2.12 | kotlin-spark-api_3.2.1_2.12:VERSION |
+| 3.2.0 | 2.13 | kotlin-spark-api_3.2.0_2.13:VERSION |
+| | 2.12 | kotlin-spark-api_3.2.0_2.12:VERSION |
+| 3.1.3 | 2.12 | kotlin-spark-api_3.1.3_2.12:VERSION |
+| 3.1.2 | 2.12 | kotlin-spark-api_3.1.2_2.12:VERSION |
+| 3.1.1 | 2.12 | kotlin-spark-api_3.1.1_2.12:VERSION |
+| 3.1.0 | 2.12 | kotlin-spark-api_3.1.0_2.12:VERSION |
+| 3.0.3 | 2.12 | kotlin-spark-api_3.0.3_2.12:VERSION |
+| 3.0.2 | 2.12 | kotlin-spark-api_3.0.2_2.12:VERSION |
+| 3.0.1 | 2.12 | kotlin-spark-api_3.0.1_2.12:VERSION |
+| 3.0.0 | 2.12 | kotlin-spark-api_3.0.0_2.12:VERSION |
+
+## Deprecated versions
| Apache Spark | Scala | Kotlin for Apache Spark |
|:------------:|:-----:|:-------------------------------:|
-| 3.2.1+ | 2.12 | kotlin-spark-api-3.2:1.1.0 |
-| 3.1.3+ | 2.12 | kotlin-spark-api-3.1:1.1.0 |
-| 3.0.3+ | 2.12 | kotlin-spark-api-3.0:1.1.0 |
| 2.4.1+ | 2.12 | kotlin-spark-api-2.4_2.12:1.0.2 |
| 2.4.1+ | 2.11 | kotlin-spark-api-2.4_2.11:1.0.2 |
-
## Releases
The list of Kotlin for Apache Spark releases is available [here](https://github.com/JetBrains/kotlin-spark-api/releases/).
@@ -61,19 +75,23 @@ Here's an example `pom.xml`:
```xml
org.jetbrains.kotlinx.spark
- kotlin-spark-api-3.2
+ kotlin-spark-api_3.3.0_2.13
${kotlin-spark-api.version}
org.apache.spark
- spark-sql_2.12
+ spark-sql_2.13
${spark.version}
```
-Note that `core` is being compiled against Scala version `2.12`.
+Note that you must match the version of the Kotlin for Apache Spark API to the Spark- and Scala version of your project.
You can find a complete example with `pom.xml` and `build.gradle` in the [Quick Start Guide](https://github.com/JetBrains/kotlin-spark-api/wiki/Quick-Start-Guide).
+If you want to try a development version. You can use the versions published to [GH Packages](https://github.com/orgs/Kotlin/packages?tab=packages&q=kotlin-spark-api_3.3.0_2.13).
+They typically have the same version as the release version, but with a `-SNAPSHOT` suffix. See the [GitHub Docs](https://docs.github.com/en/packages/learn-github-packages/installing-a-package)
+for more information.
+
Once you have configured the dependency, you only need to add the following import to your Kotlin file:
```kotlin
import org.jetbrains.kotlinx.spark.api.*
@@ -90,7 +108,7 @@ To it, simply add
to the top of your notebook. This will get the latest version of the API, together with the latest version of Spark.
To define a certain version of Spark or the API itself, simply add it like this:
```jupyterpython
-%use spark(spark=3.2, v=1.1.0)
+%use spark(spark=3.3.0, scala=2.13, v=1.2.0)
```
Inside the notebook a Spark session will be initiated automatically. This can be accessed via the `spark` value.
@@ -331,13 +349,18 @@ You are also welcome to join [kotlin-spark channel](https://kotlinlang.slack.com
Contributions are more than welcome! Pull requests can be created for the [main](https://github.com/Kotlin/kotlin-spark-api/tree/main) branch
and will be considered as soon as possible. Be sure to add the necessary tests for any new feature you add. The [main](https://github.com/Kotlin/kotlin-spark-api/tree/main)
branch always aims to target the latest available [Apache Spark version](https://spark.apache.org/downloads.html).
+Note that we use [Java Comment Preprocessor](https://github.com/raydac/java-comment-preprocessor) to build the library
+for all different supported versions of Apache Spark and Scala.
+The current values of these versions can be edited in `gradle.properties` and should always be the latest versions for commits.
+For testing, all versions need a pass for the request to be accepted.
+We use GitHub Actions to test and deploy the library for all versions, but locally you can also use the `gradlew_all_versions` file.
+
Of the [main](https://github.com/Kotlin/kotlin-spark-api/tree/main) branch, development versions of the library are published to
-[Github Packages](https://github.com/orgs/Kotlin/packages?tab=packages&q=kotlin-spark-api). This way, new features can be
+[GitHub Packages](https://github.com/orgs/Kotlin/packages?tab=packages&q=kotlin-spark-api). This way, new features can be
tested quickly without having to wait for a full release.
-For full releases, all `spark-X.x` branches are updated. At the moment, this includes Spark 3.0, 3.1, and 3.2. For this we also
-follow the [supported versions of Apache](https://spark.apache.org/downloads.html).
+For full releases, the `release` branch is updated.
## Code of Conduct
This project and the corresponding community is governed by the [JetBrains Open Source and Community Code of Conduct](https://confluence.jetbrains.com/display/ALL/JetBrains+Open+Source+and+Community+Code+of+Conduct). Please make sure you read it.
diff --git a/build.gradle.kts b/build.gradle.kts
new file mode 100644
index 00000000..d943cb25
--- /dev/null
+++ b/build.gradle.kts
@@ -0,0 +1,112 @@
+@file:Suppress("UnstableApiUsage")
+
+buildscript {
+ repositories {
+ mavenCentral()
+ }
+ dependencies {
+ classpath(jcp)
+ classpath(mavenPublish)
+ }
+}
+
+
+plugins {
+ mavenPublish version Versions.mavenPublish
+ dokka version Versions.dokka
+ idea
+}
+
+group = Versions.groupID
+version = Versions.project
+
+tasks.withType().configureEach {
+ useJUnitPlatform()
+}
+
+repositories {
+ mavenCentral()
+}
+
+
+allprojects {
+ plugins.withId(mavenPublishBase) {
+ group = Versions.groupID
+ version = Versions.project
+
+ publishing {
+ repositories {
+ maven {
+ name = "GitHubPackages"
+ url = uri("https://maven.pkg.github.com/Kotlin/kotlin-spark-api")
+ credentials {
+ username = project.findProperty("gpr.user") as String?
+ ?: System.getenv("GITHUB_ACTOR")
+ password = project.findProperty("gpr.key") as String?
+ ?: System.getenv("GITHUB_TOKEN")
+ }
+ }
+ }
+ }
+
+ mavenPublishing {
+ pomFromGradleProperties()
+ publishToMavenCentral()
+ // The username and password for Sonatype OSS can be provided as Gradle properties
+ // called mavenCentralUsername and mavenCentralPassword to avoid having to commit them.
+ // You can also supply them as environment variables called
+ // ORG_GRADLE_PROJECT_mavenCentralUsername and
+ // ORG_GRADLE_PROJECT_mavenCentralPassword.
+
+ signAllPublications()
+ pom {
+ name.set("Kotlin Spark API")
+ description.set("Kotlin for Apache Spark")
+ packaging = "pom"
+
+ url.set("https://maven.apache.org")
+ inceptionYear.set("2019")
+
+ organization {
+ name.set("JetBrains")
+ url.set("https://www.jetbrains.com/")
+ }
+
+ licenses {
+ license {
+ name.set("Apache License, Version 2.0")
+ url.set("https://www.apache.org/licenses/LICENSE-2.0.txt")
+ }
+ }
+
+ developers {
+ developer {
+ id.set("asm0dey")
+ name.set("Pasha Finkelshteyn")
+ email.set("asm0dey@jetbrains.com")
+ timezone.set("GMT+3")
+ }
+ developer {
+ id.set("vitaly.khudobakhshov")
+ name.set("Vitaly Khudobakhshov")
+ email.set("vitaly.khudobakhshov@jetbrains.com")
+ timezone.set("GMT+3")
+ }
+ developer {
+ id.set("Jolanrensen")
+ name.set("Jolan Rensen")
+ email.set("jolan.rensen@jetbrains.com")
+ timezone.set("GMT+1")
+ }
+ }
+
+ scm {
+ connection.set("scm:git:https://github.com/JetBrains/kotlin-spark-api.git")
+ url.set("https://github.com/JetBrains/kotlin-spark-api")
+ tag.set("HEAD")
+ }
+ }
+
+ }
+ }
+}
\ No newline at end of file
diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts
new file mode 100644
index 00000000..181a9870
--- /dev/null
+++ b/buildSrc/build.gradle.kts
@@ -0,0 +1,9 @@
+import org.gradle.kotlin.dsl.`kotlin-dsl`
+
+plugins {
+ `kotlin-dsl`
+}
+
+repositories {
+ mavenCentral()
+}
diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt
new file mode 100644
index 00000000..70f2761d
--- /dev/null
+++ b/buildSrc/src/main/kotlin/Dependencies.kt
@@ -0,0 +1,27 @@
+object Dependencies {
+ inline val kotlinStdLib get() = "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${Versions.kotlin}"
+ inline val reflect get() = "org.jetbrains.kotlin:kotlin-reflect:${Versions.kotlin}"
+ inline val scalaLibrary get() = "org.scala-lang:scala-library:${Versions.scala}"
+ inline val kotlinxHtml get() = "org.jetbrains.kotlinx:kotlinx-html-jvm:${Versions.kotlinxHtml}"
+ inline val sparkSql get() = "org.apache.spark:spark-sql_${Versions.scalaCompat}:${Versions.spark}"
+ inline val sparkMl get() = "org.apache.spark:spark-mllib_${Versions.scalaCompat}:${Versions.spark}"
+ inline val sparkStreaming get() = "org.apache.spark:spark-streaming_${Versions.scalaCompat}:${Versions.spark}"
+ inline val hadoopClient get() = "org.apache.hadoop:hadoop-client:${Versions.hadoop}"
+ inline val sparkRepl get() = "org.apache.spark:spark-repl_${Versions.scalaCompat}:${Versions.spark}"
+ inline val jupyter get() = "org.jetbrains.kotlinx:kotlin-jupyter-api:${Versions.jupyter}"
+ inline val junit get() = "org.junit.jupiter:junit-jupiter-engine:5.8.1"
+ inline val sparkStreamingKafka get() = "org.apache.spark:spark-streaming-kafka-0-10_${Versions.scalaCompat}:${Versions.spark}"
+ inline val kotest get() = "io.kotest:kotest-runner-junit5:${Versions.kotest}"
+ inline val kotestTestcontainers get() = "io.kotest.extensions:kotest-extensions-testcontainers:${Versions.kotestTestContainers}"
+ inline val klaxon get() = "com.beust:klaxon:${Versions.klaxon}"
+ inline val atrium get() = "ch.tutteli.atrium:atrium-fluent-en_GB:${Versions.atrium}"
+ inline val kafkaStreamsTestUtils get() = "org.apache.kafka:kafka-streams-test-utils:${Versions.kafkaStreamsTestUtils}"
+ inline val jupyterTest get() = "org.jetbrains.kotlinx:kotlin-jupyter-test-kit:${Versions.jupyter}"
+ inline val kotlinTest get() = "org.jetbrains.kotlin:kotlin-test:${Versions.kotlin}"
+ inline val kotlinScriptingCommon get() = "org.jetbrains.kotlin:kotlin-scripting-common"
+ inline val kotlinScriptingJvm get() = "org.jetbrains.kotlin:kotlin-scripting-jvm"
+}
+
+
+
+
diff --git a/buildSrc/src/main/kotlin/Helpers.kt b/buildSrc/src/main/kotlin/Helpers.kt
new file mode 100644
index 00000000..e62a8dae
--- /dev/null
+++ b/buildSrc/src/main/kotlin/Helpers.kt
@@ -0,0 +1,37 @@
+import org.gradle.api.artifacts.Dependency
+import org.gradle.api.artifacts.ProjectDependency
+import org.gradle.api.artifacts.dsl.DependencyHandler
+
+fun DependencyHandler.testApi(vararg dependencyNotations: Any): List =
+ dependencyNotations.map {
+ add("testApi", it)
+ }
+
+fun DependencyHandler.api(vararg dependencyNotations: Any): List =
+ dependencyNotations.map {
+ add("api", it)
+ }
+
+
+fun DependencyHandler.testImplementation(vararg dependencyNotations: Any): List =
+ dependencyNotations.map {
+ add("testImplementation", it)
+ }
+
+fun DependencyHandler.implementation(vararg dependencyNotations: Any): List =
+ dependencyNotations.map {
+ add("implementation", it)
+ }
+
+fun DependencyHandler.runtimeOnly(vararg dependencyNotations: Any): List =
+ dependencyNotations.map {
+ add("runtimeOnly", it)
+ }
+
+fun DependencyHandler.project(
+ path: String,
+ configuration: String? = null
+): ProjectDependency = project(
+ if (configuration != null) mapOf("path" to path, "configuration" to configuration)
+ else mapOf("path" to path)
+) as ProjectDependency
diff --git a/buildSrc/src/main/kotlin/Plugins.kt b/buildSrc/src/main/kotlin/Plugins.kt
new file mode 100644
index 00000000..5bf098b1
--- /dev/null
+++ b/buildSrc/src/main/kotlin/Plugins.kt
@@ -0,0 +1,36 @@
+import org.gradle.api.Project
+import org.gradle.kotlin.dsl.*
+import org.gradle.plugin.use.PluginDependenciesSpec
+
+
+inline val PluginDependenciesSpec.kotlin
+ get() = kotlin("jvm") version Versions.kotlin
+
+inline val PluginDependenciesSpec.dokka
+ get() = id("org.jetbrains.dokka")
+
+inline val PluginDependenciesSpec.license
+ get() = id("com.github.hierynomus.license") version Versions.licenseGradlePluginVersion
+
+inline val PluginDependenciesSpec.jcp
+ get() = id("com.igormaznitsa.jcp")
+
+
+inline val DependencyHandlerScope.jcp
+ get() = "com.igormaznitsa:jcp:${Versions.jcp}"
+
+inline val DependencyHandlerScope.mavenPublish
+ get() = "com.vanniktech:gradle-maven-publish-plugin:${Versions.mavenPublish}"
+
+inline val PluginDependenciesSpec.mavenPublish
+ get() = id("com.vanniktech.maven.publish")
+
+inline val PluginDependenciesSpec.mavenPublishBase
+ get() = id("com.vanniktech.maven.publish.base")
+
+inline val Project.mavenPublishBase
+ get() = "com.vanniktech.maven.publish.base"
+
+inline val PluginDependenciesSpec.jupyter
+ get() = kotlin("jupyter.api") version Versions.jupyter
+
diff --git a/buildSrc/src/main/kotlin/Projects.kt b/buildSrc/src/main/kotlin/Projects.kt
new file mode 100644
index 00000000..62578196
--- /dev/null
+++ b/buildSrc/src/main/kotlin/Projects.kt
@@ -0,0 +1,29 @@
+import org.gradle.api.Project
+import org.gradle.api.artifacts.dsl.DependencyHandler
+import org.gradle.kotlin.dsl.support.delegates.ProjectDelegate
+
+object Projects {
+
+ inline fun Project.searchProject(name: String): Project =
+ rootProject
+ .childProjects
+ .filterKeys { name in it }
+ .entries
+ .singleOrNull()
+ ?.value ?: error("Project $name not found")
+
+ inline val Project.kotlinSparkApi
+ get() = searchProject("kotlin-spark-api")
+
+ inline val Project.core
+ get() = searchProject("core")
+
+ inline val Project.examples
+ get() = searchProject("examples")
+
+ inline val Project.jupyter
+ get() = searchProject("jupyter")
+
+ inline val Project.scalaTuplesInKotlin
+ get() = searchProject("scala-tuples-in-kotlin")
+}
\ No newline at end of file
diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt
new file mode 100644
index 00000000..c657a108
--- /dev/null
+++ b/buildSrc/src/main/kotlin/Versions.kt
@@ -0,0 +1,34 @@
+object Versions {
+ const val project = "1.2.0-SNAPSHOT"
+ const val groupID = "org.jetbrains.kotlinx.spark"
+ const val kotlin = "1.7.10"
+
+ inline val spark get() = System.getProperty("spark") as String
+ inline val scala get() = System.getProperty("scala") as String
+ inline val sparkMinor get() = spark.substringBeforeLast('.')
+ inline val scalaCompat get() = scala.substringBeforeLast('.')
+
+ const val jupyter = "0.11.0-95"
+ const val kotest = "5.3.2"
+ const val kotestTestContainers = "1.3.3"
+ const val dokka = "1.7.0"
+ const val jcp = "7.0.5"
+ const val mavenPublish = "0.20.0"
+ const val atrium = "0.17.0"
+ const val kotestExtensionAllure = "1.1.0"
+ const val licenseGradlePluginVersion = "0.15.0"
+ const val kafkaStreamsTestUtils = "3.1.0"
+ const val hadoop = "3.3.1"
+ const val kotlinxHtml = "0.7.5"
+ const val klaxon = "5.5"
+
+ inline val versionMap
+ get() = mapOf(
+ "kotlin" to kotlin,
+ "scala" to scala,
+ "scalaCompat" to scalaCompat,
+ "spark" to spark,
+ "sparkMinor" to sparkMinor,
+ )
+
+}
\ No newline at end of file
diff --git a/core/3.2/pom_2.12.xml b/core/3.2/pom_2.12.xml
deleted file mode 100644
index 807c2c81..00000000
--- a/core/3.2/pom_2.12.xml
+++ /dev/null
@@ -1,80 +0,0 @@
-
-
- 4.0.0
-
- Kotlin Spark API: Scala core for Spark 3.2+ (Scala 2.12)
- Scala-Spark 3.2+ compatibility layer for Kotlin for Apache Spark
- core-3.2_2.12
-
- org.jetbrains.kotlinx.spark
- kotlin-spark-api-parent-3.2_2.12
- 1.1.1-SNAPSHOT
- ../../pom_2.12.xml
-
-
-
-
- org.scala-lang
- scala-library
- ${scala.version}
-
-
- org.jetbrains.kotlin
- kotlin-reflect
-
-
-
-
-
- org.apache.spark
- spark-sql_${scala.compat.version}
- ${spark3.version}
- provided
-
-
-
-
-
- src/main/scala
- src/test/scala
- target/${scala.compat.version}
-
-
- net.alchim31.maven
- scala-maven-plugin
- ${scala-maven-plugin.version}
-
-
- compile
-
- compile
- testCompile
-
-
-
- -dependencyfile
- ${project.build.directory}/.scala_dependencies
-
-
-
-
- docjar
-
- doc-jar
-
- pre-integration-test
-
-
-
-
- org.apache.maven.plugins
- maven-site-plugin
-
- true
-
-
-
-
-
-
-
diff --git a/core/build.gradle.kts b/core/build.gradle.kts
new file mode 100644
index 00000000..42f2c247
--- /dev/null
+++ b/core/build.gradle.kts
@@ -0,0 +1,75 @@
+@file:Suppress("UnstableApiUsage")
+
+import com.igormaznitsa.jcp.gradle.JcpTask
+import com.vanniktech.maven.publish.JavaLibrary
+import com.vanniktech.maven.publish.JavadocJar
+import com.vanniktech.maven.publish.JavadocJar.Javadoc
+
+plugins {
+ scala
+ `java-library`
+ jcp
+ mavenPublishBase
+}
+
+group = Versions.groupID
+version = Versions.project
+
+repositories {
+ mavenCentral()
+}
+
+dependencies {
+
+ with(Dependencies) {
+ api(
+ scalaLibrary,
+ reflect,
+ )
+
+ implementation(
+ sparkSql,
+ )
+ }
+}
+
+val scalaMainSources = sourceSets.main.get().scala.sourceDirectories
+
+val preprocessMain by tasks.creating(JcpTask::class) {
+ sources.set(scalaMainSources)
+ clearTarget.set(true)
+ fileExtensions.set(listOf("scala"))
+ vars.set(Versions.versionMap)
+ outputs.upToDateWhen { target.get().exists() }
+}
+
+tasks.compileScala {
+ dependsOn(preprocessMain)
+ outputs.upToDateWhen {
+ preprocessMain.outcomingFiles.files.isEmpty()
+ }
+ doFirst {
+ scala {
+ sourceSets {
+ main {
+ scala.setSrcDirs(listOf(preprocessMain.target.get()))
+ }
+ }
+ }
+ }
+
+ doLast {
+ scala {
+ sourceSets {
+ main {
+ scala.setSrcDirs(scalaMainSources)
+ }
+ }
+ }
+ }
+}
+
+mavenPublishing {
+ configure(JavaLibrary(Javadoc()))
+}
+
diff --git a/core/3.2/src/main/scala/org/apache/spark/sql/KotlinReflection.scala b/core/src/main/scala/org/apache/spark/sql/KotlinReflection.scala
similarity index 93%
rename from core/3.2/src/main/scala/org/apache/spark/sql/KotlinReflection.scala
rename to core/src/main/scala/org/apache/spark/sql/KotlinReflection.scala
index cbc30be3..f11012ef 100644
--- a/core/3.2/src/main/scala/org/apache/spark/sql/KotlinReflection.scala
+++ b/core/src/main/scala/org/apache/spark/sql/KotlinReflection.scala
@@ -20,9 +20,10 @@
package org.apache.spark.sql
+import org.apache.commons.lang3.reflect.ConstructorUtils
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.DeserializerBuildHelper._
-import org.apache.spark.sql.catalyst.ScalaReflection.{Schema, dataTypeFor, getClassFromType, isSubtype, javaBoxedType, localTypeOf}
+import org.apache.spark.sql.catalyst.ScalaReflection.{Schema, dataTypeFor, getClassFromType, isSubtype, javaBoxedType, localTypeOf, mirror, universe}
import org.apache.spark.sql.catalyst.SerializerBuildHelper._
import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal
import org.apache.spark.sql.catalyst.expressions.objects._
@@ -120,7 +121,7 @@ object KotlinReflection extends KotlinReflection {
val className = getClassNameFromType(tpe)
className match {
case "scala.Array" => {
- val TypeRef(_, _, Seq(elementType)) = tpe
+ val TypeRef(_, _, Seq(elementType)) = tpe.dealias
arrayClassFor(elementType)
}
case _ => {
@@ -290,13 +291,25 @@ object KotlinReflection extends KotlinReflection {
createDeserializerForSqlTimestamp(path)
}
case t if isSubtype(t, localTypeOf[java.time.LocalDateTime]) => {
+ //#if sparkMinor >= 3.2
createDeserializerForLocalDateTime(path)
+ //#else
+ //$throw new IllegalArgumentException("TimestampNTZType is supported in spark 3.2+")
+ //#endif
}
case t if isSubtype(t, localTypeOf[java.time.Duration]) => {
+ //#if sparkMinor >= 3.2
createDeserializerForDuration(path)
+ //#else
+ //$throw new IllegalArgumentException("java.time.Duration is supported in spark 3.2+")
+ //#endif
}
case t if isSubtype(t, localTypeOf[java.time.Period]) => {
+ //#if sparkMinor >= 3.2
createDeserializerForPeriod(path)
+ //#else
+ //$throw new IllegalArgumentException("java.time.Period is supported in spark 3.2+")
+ //#endif
}
case t if isSubtype(t, localTypeOf[java.lang.String]) => {
createDeserializerForString(path, returnNullable = false)
@@ -828,7 +841,11 @@ object KotlinReflection extends KotlinReflection {
createSerializerForSqlTimestamp(inputObject)
}
case t if isSubtype(t, localTypeOf[java.time.LocalDateTime]) => {
+ //#if sparkMinor >= 3.2
createSerializerForLocalDateTime(inputObject)
+ //#else
+ //$throw new IllegalArgumentException("TimestampNTZType is supported in spark 3.2+")
+ //#endif
}
case t if isSubtype(t, localTypeOf[java.time.LocalDate]) => {
createSerializerForJavaLocalDate(inputObject)
@@ -837,10 +854,18 @@ object KotlinReflection extends KotlinReflection {
createSerializerForSqlDate(inputObject)
}
case t if isSubtype(t, localTypeOf[java.time.Duration]) => {
+ //#if sparkMinor >= 3.2
createSerializerForJavaDuration(inputObject)
+ //#else
+ //$throw new IllegalArgumentException("java.time.Duration is supported in spark 3.2+")
+ //#endif
}
case t if isSubtype(t, localTypeOf[java.time.Period]) => {
+ //#if sparkMinor >= 3.2
createSerializerForJavaPeriod(inputObject)
+ //#else
+ //$throw new IllegalArgumentException("java.time.Period is supported in spark 3.2+")
+ //#endif
}
case t if isSubtype(t, localTypeOf[BigDecimal]) => {
createSerializerForScalaBigDecimal(inputObject)
@@ -1178,7 +1203,11 @@ object KotlinReflection extends KotlinReflection {
}
// SPARK-36227: Remove TimestampNTZ type support in Spark 3.2 with minimal code changes.
case t if isSubtype(t, localTypeOf[java.time.LocalDateTime]) && Utils.isTesting => {
+ //#if sparkMinor >= 3.2
Schema(TimestampNTZType, nullable = true)
+ //#else
+ //$throw new IllegalArgumentException("java.time.LocalDateTime is supported in Spark 3.2+")
+ //#endif
}
case t if isSubtype(t, localTypeOf[java.time.LocalDate]) => {
Schema(DateType, nullable = true)
@@ -1190,10 +1219,18 @@ object KotlinReflection extends KotlinReflection {
Schema(CalendarIntervalType, nullable = true)
}
case t if isSubtype(t, localTypeOf[java.time.Duration]) => {
+ //#if sparkMinor >= 3.2
Schema(DayTimeIntervalType(), nullable = true)
+ //#else
+ //$throw new IllegalArgumentException("DayTimeIntervalType for java.time.Duration is supported in Spark 3.2+")
+ //#endif
}
case t if isSubtype(t, localTypeOf[java.time.Period]) => {
+ //#if sparkMinor >= 3.2
Schema(YearMonthIntervalType(), nullable = true)
+ //#else
+ //$throw new IllegalArgumentException("YearMonthIntervalType for java.time.Period is supported in Spark 3.2+")
+ //#endif
}
case t if isSubtype(t, localTypeOf[BigDecimal]) => {
Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
@@ -1241,6 +1278,40 @@ object KotlinReflection extends KotlinReflection {
}
}
+ /**
+ * Finds an accessible constructor with compatible parameters. This is a more flexible search than
+ * the exact matching algorithm in `Class.getConstructor`. The first assignment-compatible
+ * matching constructor is returned if it exists. Otherwise, we check for additional compatible
+ * constructors defined in the companion object as `apply` methods. Otherwise, it returns `None`.
+ */
+ def findConstructor[T](cls: Class[T], paramTypes: Seq[Class[_]]): Option[Seq[AnyRef] => T] = {
+ Option(ConstructorUtils.getMatchingAccessibleConstructor(cls, paramTypes: _*)) match {
+ case Some(c) => Some(x => c.newInstance(x: _*))
+ case None =>
+ val companion = mirror.staticClass(cls.getName).companion
+ val moduleMirror = mirror.reflectModule(companion.asModule)
+ val applyMethods = companion.asTerm.typeSignature
+ .member(universe.TermName("apply")).asTerm.alternatives
+ applyMethods.find { method =>
+ val params = method.typeSignature.paramLists.head
+ // Check that the needed params are the same length and of matching types
+ params.size == paramTypes.tail.size &&
+ params.zip(paramTypes.tail).forall { case(ps, pc) =>
+ ps.typeSignature.typeSymbol == mirror.classSymbol(pc)
+ }
+ }.map { applyMethodSymbol =>
+ val expectedArgsCount = applyMethodSymbol.typeSignature.paramLists.head.size
+ val instanceMirror = mirror.reflect(moduleMirror.instance)
+ val method = instanceMirror.reflectMethod(applyMethodSymbol.asMethod)
+ (_args: Seq[AnyRef]) => {
+ // Drop the "outer" argument if it is provided
+ val args = if (_args.size == expectedArgsCount) _args else _args.tail
+ method.apply(args: _*).asInstanceOf[T]
+ }
+ }
+ }
+ }
+
/**
* Whether the fields of the given type is defined entirely by its constructor parameters.
*/
@@ -1268,8 +1339,10 @@ object KotlinReflection extends KotlinReflection {
@scala.annotation.tailrec
def javaBoxedType(dt: DataType): Class[_] = dt match {
case _: DecimalType => classOf[Decimal]
+ //#if sparkMinor >= 3.2
case _: DayTimeIntervalType => classOf[java.lang.Long]
case _: YearMonthIntervalType => classOf[java.lang.Integer]
+ //#endif
case BinaryType => classOf[Array[Byte]]
case StringType => classOf[UTF8String]
case CalendarIntervalType => classOf[CalendarInterval]
@@ -1325,6 +1398,15 @@ trait KotlinReflection extends Logging {
tag.in(mirror).tpe.dealias
}
+ private def isValueClass(tpe: Type): Boolean = {
+ tpe.typeSymbol.isClass && tpe.typeSymbol.asClass.isDerivedValueClass
+ }
+
+ /** Returns the name and type of the underlying parameter of value class `tpe`. */
+ private def getUnderlyingTypeOfValueClass(tpe: `Type`): Type = {
+ getConstructorParameters(tpe).head._2
+ }
+
/**
* Returns the full class name for a type. The returned name is the canonical
* Scala name, where each component is separated by a period. It is NOT the
@@ -1350,15 +1432,13 @@ trait KotlinReflection extends Logging {
val formalTypeArgs = dealiasedTpe.typeSymbol.asClass.typeParams
val TypeRef(_, _, actualTypeArgs) = dealiasedTpe
val params = constructParams(dealiasedTpe)
- // if there are type variables to fill in, do the substitution (SomeClass[T] -> SomeClass[Int])
- if (actualTypeArgs.nonEmpty) {
- params.map { p =>
- p.name.decodedName.toString ->
- p.typeSignature.substituteTypes(formalTypeArgs, actualTypeArgs)
- }
- } else {
- params.map { p =>
- p.name.decodedName.toString -> p.typeSignature
+ params.map { p =>
+ val paramTpe = p.typeSignature
+ if (isValueClass(paramTpe)) {
+ // Replace value class with underlying type
+ p.name.decodedName.toString -> getUnderlyingTypeOfValueClass(paramTpe)
+ } else {
+ p.name.decodedName.toString -> paramTpe.substituteTypes(formalTypeArgs, actualTypeArgs)
}
}
}
diff --git a/core/3.2/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala b/core/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala
similarity index 99%
rename from core/3.2/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala
rename to core/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala
index 9395019c..76da9016 100644
--- a/core/3.2/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala
+++ b/core/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala
@@ -77,6 +77,9 @@ class KDataTypeWrapper(
override private[ sql ] def getFieldIndex(name: String) = dt.getFieldIndex(name)
+ //#if sparkMinor < 3.2
+ //$override
+ //#endif
private[ sql ] def findNestedField(fieldNames: Seq[ String ], includeCollections: Boolean, resolver: Resolver) =
dt.findNestedField(fieldNames, includeCollections, resolver)
diff --git a/core/3.2/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala b/core/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
similarity index 100%
rename from core/3.2/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
rename to core/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
diff --git a/core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/DemoCaseClass.scala b/core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/DemoCaseClass.scala
similarity index 100%
rename from core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/DemoCaseClass.scala
rename to core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/DemoCaseClass.scala
diff --git a/core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala b/core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala
similarity index 83%
rename from core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala
rename to core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala
index 6b4935f1..2092d83e 100644
--- a/core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala
+++ b/core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala
@@ -26,6 +26,13 @@ import java.util
import scala.collection.JavaConverters
object KSparkExtensions {
+
+ val kotlinVersion = /*$"\""+kotlin+"\""$*/ /*-*/ ""
+ val scalaVersion = /*$"\""+scala+"\""$*/ /*-*/ ""
+ val scalaCompatVersion = /*$"\""+scalaCompat+"\""$*/ /*-*/ ""
+ val sparkVersion = /*$"\""+spark+"\""$*/ /*-*/ ""
+ val sparkMinorVersion = /*$"\""+sparkMinor+"\""$*/ /*-*/ ""
+
def col(d: Dataset[_], name: String): Column = d.col(name)
def col(name: String): Column = functions.col(name)
diff --git a/core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/VarargUnwrapper.scala b/core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/VarargUnwrapper.scala
similarity index 100%
rename from core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/VarargUnwrapper.scala
rename to core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/VarargUnwrapper.scala
diff --git a/dummy/pom.xml b/dummy/pom.xml
deleted file mode 100644
index 1614d4bd..00000000
--- a/dummy/pom.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
- kotlin-spark-api-parent-3.2
- org.jetbrains.kotlinx.spark
- 1.1.1-SNAPSHOT
-
- pom
- 4.0.0
- Kotlin API for Apache Spark: not-needed
- Module to workaround https://issues.sonatype.org/browse/NEXUS-9138
- dummy-3.2
-
-
- scala-2.12
-
-
- org.jetbrains.kotlinx.spark
- examples-3.2_2.12
- ${project.parent.version}
-
-
-
-
-
-
\ No newline at end of file
diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts
new file mode 100644
index 00000000..535cd17a
--- /dev/null
+++ b/examples/build.gradle.kts
@@ -0,0 +1,30 @@
+plugins {
+ kotlin
+ idea
+}
+
+group = Versions.groupID
+version = Versions.project
+
+repositories {
+ mavenCentral()
+}
+
+dependencies {
+
+ with(Projects) {
+ implementation(
+ kotlinSparkApi,
+ )
+ }
+
+ with(Dependencies) {
+ implementation(
+ sparkSql,
+ sparkMl,
+ sparkStreaming,
+ sparkStreamingKafka,
+ )
+
+ }
+}
diff --git a/examples/pom-3.2_2.12.xml b/examples/pom-3.2_2.12.xml
deleted file mode 100644
index 92fba9f2..00000000
--- a/examples/pom-3.2_2.12.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-
-
-
- 4.0.0
-
- Kotlin Spark API: Examples for Spark 3.2+ (Scala 2.12)
- Example of usage
- examples-3.2_2.12
-
- org.jetbrains.kotlinx.spark
- kotlin-spark-api-parent-3.2_2.12
- 1.1.1-SNAPSHOT
- ../pom_2.12.xml
-
-
-
-
- org.jetbrains.kotlinx.spark
- kotlin-spark-api-3.2
- ${project.version}
-
-
- org.apache.spark
- spark-sql_${scala.compat.version}
- ${spark3.version}
-
-
- org.apache.spark
- spark-streaming_${scala.compat.version}
- ${spark3.version}
-
-
- org.apache.spark
- spark-mllib_${scala.compat.version}
- ${spark3.version}
-
-
- org.apache.spark
- spark-streaming-kafka-0-10_${scala.compat.version}
- ${spark3.version}
-
-
-
-
- src/main/kotlin
- src/test/kotlin
- target/3.2/${scala.compat.version}
-
-
- org.jetbrains.kotlin
- kotlin-maven-plugin
-
-
- compile
-
- compile
-
-
-
- test-compile
-
- test-compile
-
-
-
-
-
- org.apache.maven.plugins
- maven-assembly-plugin
- ${maven-assembly-plugin.version}
-
-
- jar-with-dependencies
-
-
-
- org.jetbrains.spark.api.examples.WordCountKt
-
-
-
-
-
- org.apache.maven.plugins
- maven-site-plugin
-
- true
-
-
-
- org.apache.maven.plugins
- maven-deploy-plugin
-
- true
-
-
-
- org.sonatype.plugins
- nexus-staging-maven-plugin
-
- true
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
- ${maven-compiler-plugin.version}
-
- 9
- 9
-
-
-
-
-
diff --git a/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UDFs.kt b/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UDFs.kt
index 39636a18..1a306a50 100644
--- a/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UDFs.kt
+++ b/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UDFs.kt
@@ -202,7 +202,8 @@ private fun strongTypingInDatasets() = withSpark {
// We can thus provide TypedColumns instead of normal ones which the select function takes
// advantage of!
- // NOTE: In UDFs, iterables, lists, arrays and such need to be represented as WrappedArray
+
+ // NOTE: In UDFs, iterables, lists, arrays and such need to be represented as Seq
val toJson by udf { age: Int, name: String, pets: Seq ->
"""{ "age" : $age, "name" : "$name", "pets" : [${pets.asKotlinIterable().joinToString { "\"$it\"" }}] }"""
}
@@ -228,7 +229,7 @@ private fun strongTypingInDatasets() = withSpark {
col<_, Int>("age"),
col<_, String>("name"),
col>("pets").asSeq(),
-// or `col<_, WrappedArray>("pets")` if you want to be less strict
+// or `col<_, Seq>("pets")` if you want to be less strict
)
).showDS(truncate = false)
// +-------------------------------------------------------+
@@ -437,7 +438,7 @@ private fun varargUDFs() = withSpark {
// As you can see, it just works :), up to 22 parameters!
- // In fact, since UDFs don't support arrays (only scala's WrappedArray), any udf that contains just an array
+ // In fact, since UDFs don't support arrays (only scala's Seq), any udf that contains just an array
// as parameter will become a vararg udf:
udf.register("joinToString") { strings: Array -> strings.joinToString(separator = "-") }
spark.sql("""SELECT joinToString("a", "hi there", "test"), joinToString(), joinToString("b", "c")""")
diff --git a/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UdtRegistration.kt b/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UdtRegistration.kt
index 0c468d91..19db2c41 100644
--- a/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UdtRegistration.kt
+++ b/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UdtRegistration.kt
@@ -17,11 +17,12 @@
* limitations under the License.
* =LICENSEEND=
*/
-import org.apache.hadoop.shaded.com.google.common.base.MoreObjects
+
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
import org.apache.spark.sql.types.*
import org.apache.spark.unsafe.types.UTF8String
+import org.glassfish.jersey.internal.guava.MoreObjects
import org.jetbrains.kotlinx.spark.api.*
import org.jetbrains.kotlinx.spark.api.tuples.tupleOf
import java.io.Serializable
diff --git a/gradle.properties b/gradle.properties
new file mode 100644
index 00000000..6a0c214a
--- /dev/null
+++ b/gradle.properties
@@ -0,0 +1,16 @@
+kotlin.daemon.jvmargs=-Xmx10G
+org.gradle.jvmargs=-Xmx4G -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8
+mavenCentralUsername=dummy
+mavenCentralPassword=dummy
+
+GROUP=org.jetbrains.kotlinx.spark
+
+# Controls the spark and scala version for the entire project
+# can also be defined like ./gradlew -Pspark=X.X.X -Pscala=X.X.X build
+spark=3.3.0
+scala=2.13.8
+#scala=2.12.15
+skipScalaTuplesInKotlin=false
+
+org.gradle.caching=true
+org.gradle.parallel=false
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
new file mode 100644
index 00000000..41d9927a
Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
new file mode 100644
index 00000000..41dfb879
--- /dev/null
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
new file mode 100755
index 00000000..1b6c7873
--- /dev/null
+++ b/gradlew
@@ -0,0 +1,234 @@
+#!/bin/sh
+
+#
+# Copyright © 2015-2021 the original authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##############################################################################
+#
+# Gradle start up script for POSIX generated by Gradle.
+#
+# Important for running:
+#
+# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+# noncompliant, but you have some other compliant shell such as ksh or
+# bash, then to run this script, type that shell name before the whole
+# command line, like:
+#
+# ksh Gradle
+#
+# Busybox and similar reduced shells will NOT work, because this script
+# requires all of these POSIX shell features:
+# * functions;
+# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+# * compound commands having a testable exit status, especially «case»;
+# * various built-in commands including «command», «set», and «ulimit».
+#
+# Important for patching:
+#
+# (2) This script targets any POSIX shell, so it avoids extensions provided
+# by Bash, Ksh, etc; in particular arrays are avoided.
+#
+# The "traditional" practice of packing multiple parameters into a
+# space-separated string is a well documented source of bugs and security
+# problems, so this is (mostly) avoided, by progressively accumulating
+# options in "$@", and eventually passing that to Java.
+#
+# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+# see the in-line comments for details.
+#
+# There are tweaks for specific operating systems such as AIX, CygWin,
+# Darwin, MinGW, and NonStop.
+#
+# (3) This script is generated from the Groovy template
+# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+# within the Gradle project.
+#
+# You can find Gradle at https://github.com/gradle/gradle/.
+#
+##############################################################################
+
+# Attempt to set APP_HOME
+
+# Resolve links: $0 may be a link
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+ APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
+ [ -h "$app_path" ]
+do
+ ls=$( ls -ld "$app_path" )
+ link=${ls#*' -> '}
+ case $link in #(
+ /*) app_path=$link ;; #(
+ *) app_path=$APP_HOME$link ;;
+ esac
+done
+
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
+
+APP_NAME="Gradle"
+APP_BASE_NAME=${0##*/}
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD=maximum
+
+warn () {
+ echo "$*"
+} >&2
+
+die () {
+ echo
+ echo "$*"
+ echo
+ exit 1
+} >&2
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "$( uname )" in #(
+ CYGWIN* ) cygwin=true ;; #(
+ Darwin* ) darwin=true ;; #(
+ MSYS* | MINGW* ) msys=true ;; #(
+ NONSTOP* ) nonstop=true ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD=$JAVA_HOME/jre/sh/java
+ else
+ JAVACMD=$JAVA_HOME/bin/java
+ fi
+ if [ ! -x "$JAVACMD" ] ; then
+ die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+ fi
+else
+ JAVACMD=java
+ which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+ case $MAX_FD in #(
+ max*)
+ MAX_FD=$( ulimit -H -n ) ||
+ warn "Could not query maximum file descriptor limit"
+ esac
+ case $MAX_FD in #(
+ '' | soft) :;; #(
+ *)
+ ulimit -n "$MAX_FD" ||
+ warn "Could not set maximum file descriptor limit to $MAX_FD"
+ esac
+fi
+
+# Collect all arguments for the java command, stacking in reverse order:
+# * args from the command line
+# * the main class name
+# * -classpath
+# * -D...appname settings
+# * --module-path (only if needed)
+# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if "$cygwin" || "$msys" ; then
+ APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+ CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+ JAVACMD=$( cygpath --unix "$JAVACMD" )
+
+ # Now convert the arguments - kludge to limit ourselves to /bin/sh
+ for arg do
+ if
+ case $arg in #(
+ -*) false ;; # don't mess with options #(
+ /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
+ [ -e "$t" ] ;; #(
+ *) false ;;
+ esac
+ then
+ arg=$( cygpath --path --ignore --mixed "$arg" )
+ fi
+ # Roll the args list around exactly as many times as the number of
+ # args, so each arg winds up back in the position where it started, but
+ # possibly modified.
+ #
+ # NB: a `for` loop captures its iteration list before it begins, so
+ # changing the positional parameters here affects neither the number of
+ # iterations, nor the values presented in `arg`.
+ shift # remove old arg
+ set -- "$@" "$arg" # push replacement arg
+ done
+fi
+
+# Collect all arguments for the java command;
+# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
+# shell script including quotes and variable substitutions, so put them in
+# double quotes to make sure that they get re-expanded; and
+# * put everything else in single quotes, so that it's not re-expanded.
+
+set -- \
+ "-Dorg.gradle.appname=$APP_BASE_NAME" \
+ -classpath "$CLASSPATH" \
+ org.gradle.wrapper.GradleWrapperMain \
+ "$@"
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+# set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+ printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+ xargs -n1 |
+ sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+ tr '\n' ' '
+ )" '"$@"'
+
+exec "$JAVACMD" "$@"
diff --git a/gradlew.bat b/gradlew.bat
new file mode 100644
index 00000000..107acd32
--- /dev/null
+++ b/gradlew.bat
@@ -0,0 +1,89 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega
diff --git a/gradlew_all_versions b/gradlew_all_versions
new file mode 100755
index 00000000..3fab16ca
--- /dev/null
+++ b/gradlew_all_versions
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Run like you would `./gradlew arguments`
+# but now like `./gradlew_all_versions arguments`.
+
+DRY_RUN=false
+SCALA2_12VERSION="2.12.15"
+SCALA2_13VERSION="2.13.8"
+SparkVersionsForBothScalaVersions=("3.3.0" "3.2.1" "3.2.0")
+SparkVersionsForScala2_12=("3.1.3" "3.1.2" "3.1.1" "3.1.0" "3.0.3" "3.0.2" "3.0.1" "3.0.0")
+
+echo Running for "$(expr ${#SparkVersionsForBothScalaVersions[@]} \* 2 + ${#SparkVersionsForScala2_12[@]}) versions of the library."
+
+echo "Cleaning the project first."
+if [ "$DRY_RUN" = false ]; then
+ ./gradlew clean
+fi
+
+ARGS=("$@")
+execute() {
+ echo "running ./gradlew -Pspark=$SPARK -Pscala=$SCALA -PskipScalaTuplesInKotlin=$SKIP_SCALA_TUPLES -PenforceCleanJCP=true ${ARGS[*]}"
+ if [ "$DRY_RUN" = false ]; then
+ ./gradlew -Pspark="$SPARK" -Pscala="$SCALA" -PskipScalaTuplesInKotlin="$SKIP_SCALA_TUPLES" "${ARGS[@]}"
+ fi
+}
+
+SCALA="$SCALA2_12VERSION"
+SKIP_SCALA_TUPLES=false
+for spark in "${SparkVersionsForScala2_12[@]}"; do
+ SPARK="$spark"
+ execute
+ SKIP_SCALA_TUPLES=true
+done
+
+
+execute_for_both_scala_versions() {
+ for spark in "${SparkVersionsForBothScalaVersions[@]}"; do
+ SPARK="$spark"
+ execute
+ SKIP_SCALA_TUPLES=true
+ done
+}
+SCALA="$SCALA2_12VERSION"
+execute_for_both_scala_versions
+
+SCALA="$SCALA2_13VERSION"
+SKIP_SCALA_TUPLES=false
+execute_for_both_scala_versions
+
+
+
diff --git a/jupyter/build.gradle.kts b/jupyter/build.gradle.kts
new file mode 100644
index 00000000..229d6a34
--- /dev/null
+++ b/jupyter/build.gradle.kts
@@ -0,0 +1,149 @@
+@file:Suppress("UnstableApiUsage")
+
+import com.igormaznitsa.jcp.gradle.JcpTask
+import com.vanniktech.maven.publish.JavadocJar.Dokka
+import com.vanniktech.maven.publish.KotlinJvm
+import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask
+
+plugins {
+ scala
+ kotlin
+ dokka
+ mavenPublishBase
+ jupyter
+ jcp
+}
+
+group = Versions.groupID
+version = Versions.project
+
+repositories {
+ mavenCentral()
+ maven(url = "https://maven.pkg.jetbrains.space/public/p/kotlinx-html/maven")
+ maven(url = "https://maven.pkg.jetbrains.space/kotlin/p/kotlin/dev")
+}
+
+tasks.withType().configureEach {
+ useJUnitPlatform()
+}
+
+tasks.processJupyterApiResources {
+ libraryProducers = listOf(
+ "org.jetbrains.kotlinx.spark.api.jupyter.SparkIntegration",
+ "org.jetbrains.kotlinx.spark.api.jupyter.SparkStreamingIntegration",
+ )
+}
+
+dependencies {
+ with(Projects) {
+ api(
+ kotlinSparkApi,
+ )
+ }
+
+ with(Dependencies) {
+ api(
+ kotlinxHtml,
+ sparkSql,
+ sparkRepl,
+ sparkStreaming,
+ hadoopClient,
+ )
+
+ implementation(
+ kotlinStdLib,
+ )
+
+ testImplementation(
+ kotest,
+ kotlinScriptingCommon,
+ kotlinScriptingJvm,
+ )
+
+ }
+}
+
+// Setup preprocessing with JCP for main sources
+
+val kotlinMainSources = kotlin.sourceSets.main.get().kotlin.sourceDirectories
+
+val preprocessMain by tasks.creating(JcpTask::class) {
+ sources.set(kotlinMainSources)
+ clearTarget.set(true)
+ fileExtensions.set(listOf("kt"))
+ vars.set(Versions.versionMap)
+ outputs.upToDateWhen { target.get().exists() }
+}
+
+tasks.compileKotlin {
+ dependsOn(preprocessMain)
+ outputs.upToDateWhen { preprocessMain.outcomingFiles.files.isEmpty() }
+ doFirst {
+ kotlin {
+ sourceSets {
+ main {
+ kotlin.setSrcDirs(listOf(preprocessMain.target.get()))
+ }
+ }
+ }
+ }
+
+ doLast {
+ kotlin {
+ sourceSets {
+ main {
+ kotlin.setSrcDirs(kotlinMainSources)
+ }
+ }
+ }
+ }
+}
+
+// Setup preprocessing with JCP for test sources
+
+val kotlinTestSources = kotlin.sourceSets.test.get().kotlin.sourceDirectories
+
+val preprocessTest by tasks.creating(JcpTask::class) {
+ sources.set(kotlinTestSources)
+ clearTarget.set(true)
+ fileExtensions.set(listOf("java", "kt"))
+ vars.set(Versions.versionMap)
+ outputs.upToDateWhen { target.get().exists() }
+}
+
+tasks.compileTestKotlin {
+ dependsOn(preprocessTest)
+ outputs.upToDateWhen { preprocessTest.outcomingFiles.files.isEmpty() }
+ doFirst {
+ kotlin {
+ sourceSets {
+ test {
+ kotlin.setSrcDirs(listOf(preprocessTest.target.get()))
+ }
+ }
+ }
+ }
+
+ doLast {
+ kotlin {
+ sourceSets {
+ test {
+ kotlin.setSrcDirs(kotlinTestSources)
+ }
+ }
+ }
+ }
+}
+
+tasks.withType {
+ dokkaSourceSets {
+ create("jupyter") {
+ sourceRoot(preprocessMain.target.get())
+ }
+ }
+}
+
+
+mavenPublishing {
+ configure(KotlinJvm(Dokka("dokkaHtml")))
+}
\ No newline at end of file
diff --git a/jupyter/pom.xml b/jupyter/pom.xml
deleted file mode 100644
index cde0c15d..00000000
--- a/jupyter/pom.xml
+++ /dev/null
@@ -1,182 +0,0 @@
-
-
-
- 4.0.0
-
- Kotlin Spark API: Jupyter integration for Spark 3.2+ (Scala 2.12)
- kotlin-spark-api-jupyter-3.2
- Jupyter integration
-
- org.jetbrains.kotlinx.spark
- kotlin-spark-api-parent-3.2_2.12
- 1.1.1-SNAPSHOT
- ../pom_2.12.xml
-
- jar
-
-
-
- 11
- 11
-
-
-
-
- kotlinx-html
- kotlinx-html
- https://maven.pkg.jetbrains.space/public/p/kotlinx-html/maven
-
-
- kotlin
- kotlin
- https://maven.pkg.jetbrains.space/kotlin/p/kotlin/dev
-
-
-
-
-
- org.jetbrains.kotlinx.spark
- kotlin-spark-api-3.2
- ${project.version}
-
-
- org.jetbrains.kotlinx
- kotlinx-html-jvm
- ${kotlinx.html.version}
-
-
- org.apache.spark
- spark-sql_${scala.compat.version}
- ${spark3.version}
-
-
- org.apache.spark
- spark-repl_${scala.compat.version}
- ${spark3.version}
-
-
- org.apache.spark
- spark-streaming_${scala.compat.version}
- ${spark3.version}
-
-
- org.apache.hadoop
- hadoop-client
- ${hadoop.version}
-
-
- org.jetbrains.kotlinx
- kotlin-jupyter-api
- ${kotlin-jupyter-api.version}
-
-
-
-
- io.kotest
- kotest-runner-junit5-jvm
- ${kotest.version}
- test
-
-
- org.jetbrains.kotlinx
- kotlin-jupyter-test-kit
- ${kotlin-jupyter-api.version}
- test
-
-
-
-
- src/main/kotlin
- src/test/kotlin
- target/${scala.compat.version}
-
-
- org.jetbrains.dokka
- dokka-maven-plugin
- ${dokka.version}
-
- 8
-
-
-
- dokka
-
- dokka
-
- pre-site
-
-
- javadocjar
-
- javadocJar
-
- pre-integration-test
-
-
-
-
- org.jetbrains.kotlin
- kotlin-maven-plugin
-
-
- compile
-
- compile
-
-
-
- test-compile
-
- test-compile
-
-
-
-
-
- org.apache.maven.plugins
- maven-assembly-plugin
- ${maven-assembly-plugin.version}
-
-
- jar-with-dependencies
-
-
-
- org.jetbrains.spark.api.examples.WordCountKt
-
-
-
-
-
- org.apache.maven.plugins
- maven-site-plugin
-
- true
-
-
-
- org.apache.maven.plugins
- maven-deploy-plugin
-
- false
-
-
-
- org.sonatype.plugins
- nexus-staging-maven-plugin
-
- false
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
- ${maven-compiler-plugin.version}
-
- 9
- 9
-
-
-
-
-
\ No newline at end of file
diff --git a/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/Integration.kt b/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/Integration.kt
index 19ddda50..db364e72 100644
--- a/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/Integration.kt
+++ b/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/Integration.kt
@@ -24,14 +24,16 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Dataset
import org.jetbrains.kotlinx.jupyter.api.*
import org.jetbrains.kotlinx.jupyter.api.libraries.JupyterIntegration
+import org.jetbrains.kotlinx.spark.api.tuples.map
+import org.jetbrains.kotlinx.spark.api.tuples.t
import kotlin.reflect.typeOf
abstract class Integration : JupyterIntegration() {
- private val kotlinVersion = "1.6.21"
- private val scalaCompatVersion = "2.12"
- private val scalaVersion = "2.12.15"
- private val spark3Version = "3.2.1"
+ private val kotlinVersion = /*$"\""+kotlin+"\""$*/ /*-*/ ""
+ private val scalaCompatVersion = /*$"\""+scalaCompat+"\""$*/ /*-*/ ""
+ private val scalaVersion = /*$"\""+scala+"\""$*/ /*-*/ ""
+ private val sparkVersion = /*$"\""+spark+"\""$*/ /*-*/ ""
private val displayLimit = "DISPLAY_LIMIT"
private val displayLimitDefault = 20
@@ -54,18 +56,18 @@ abstract class Integration : JupyterIntegration() {
open fun Builder.onLoadedAlsoDo() = Unit
open val dependencies: Array = arrayOf(
- "org.apache.spark:spark-repl_$scalaCompatVersion:$spark3Version",
+ "org.apache.spark:spark-repl_$scalaCompatVersion:$sparkVersion",
"org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion",
"org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion",
- "org.apache.spark:spark-sql_$scalaCompatVersion:$spark3Version",
- "org.apache.spark:spark-streaming_$scalaCompatVersion:$spark3Version",
- "org.apache.spark:spark-mllib_$scalaCompatVersion:$spark3Version",
- "org.apache.spark:spark-sql_$scalaCompatVersion:$spark3Version",
- "org.apache.spark:spark-graphx_$scalaCompatVersion:$spark3Version",
- "org.apache.spark:spark-launcher_$scalaCompatVersion:$spark3Version",
- "org.apache.spark:spark-catalyst_$scalaCompatVersion:$spark3Version",
- "org.apache.spark:spark-streaming_$scalaCompatVersion:$spark3Version",
- "org.apache.spark:spark-core_$scalaCompatVersion:$spark3Version",
+ "org.apache.spark:spark-sql_$scalaCompatVersion:$sparkVersion",
+ "org.apache.spark:spark-streaming_$scalaCompatVersion:$sparkVersion",
+ "org.apache.spark:spark-mllib_$scalaCompatVersion:$sparkVersion",
+ "org.apache.spark:spark-sql_$scalaCompatVersion:$sparkVersion",
+ "org.apache.spark:spark-graphx_$scalaCompatVersion:$sparkVersion",
+ "org.apache.spark:spark-launcher_$scalaCompatVersion:$sparkVersion",
+ "org.apache.spark:spark-catalyst_$scalaCompatVersion:$sparkVersion",
+ "org.apache.spark:spark-streaming_$scalaCompatVersion:$sparkVersion",
+ "org.apache.spark:spark-core_$scalaCompatVersion:$sparkVersion",
"org.scala-lang:scala-library:$scalaVersion",
"org.scala-lang.modules:scala-xml_$scalaCompatVersion:2.0.1",
"org.scala-lang:scala-reflect:$scalaVersion",
@@ -113,7 +115,10 @@ abstract class Integration : JupyterIntegration() {
}
beforeCellExecution {
- execute("""scala.Console.setOut(System.out)""")
+ if (scalaCompatVersion.toDouble() >= 2.13)
+ execute("scala.`Console\$`.`MODULE\$`.setOutDirect(System.out)")
+ else
+ execute("""scala.Console.setOut(System.out)""")
beforeCellExecution()
}
diff --git a/jupyter/src/main/resources/META-INF/kotlin-jupyter-libraries/libraries.json b/jupyter/src/main/resources/META-INF/kotlin-jupyter-libraries/libraries.json
deleted file mode 100644
index 82c7354e..00000000
--- a/jupyter/src/main/resources/META-INF/kotlin-jupyter-libraries/libraries.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "definitions": [],
- "producers": [
- {
- "fqn": "org.jetbrains.kotlinx.spark.api.jupyter.SparkIntegration"
- },
- {
- "fqn": "org.jetbrains.kotlinx.spark.api.jupyter.SparkStreamingIntegration"
- }
- ]
-}
diff --git a/kotlin-spark-api/3.2/pom_2.12.xml b/kotlin-spark-api/3.2/pom_2.12.xml
deleted file mode 100644
index 6bd490ab..00000000
--- a/kotlin-spark-api/3.2/pom_2.12.xml
+++ /dev/null
@@ -1,185 +0,0 @@
-
-
-
- 4.0.0
-
- Kotlin Spark API: API for Spark 3.2+ (Scala 2.12)
- kotlin-spark-api-3.2
- Kotlin API compatible with spark 3.2.0 Kotlin for Apache Spark
-
- org.jetbrains.kotlinx.spark
- kotlin-spark-api-parent-3.2_2.12
- 1.1.1-SNAPSHOT
- ../../pom_2.12.xml
-
- jar
-
-
-
-
- org.jetbrains.kotlin
- kotlin-stdlib-jdk8
-
-
- org.jetbrains.kotlin
- kotlin-reflect
-
-
- org.jetbrains.kotlinx.spark
- core-3.2_${scala.compat.version}
-
-
- org.jetbrains.kotlinx.spark
- scala-tuples-in-kotlin-3.2
-
-
-
-
- org.apache.spark
- spark-sql_${scala.compat.version}
- ${spark3.version}
- provided
-
-
- org.apache.spark
- spark-streaming_${scala.compat.version}
- ${spark3.version}
- provided
-
-
- org.apache.hadoop
- hadoop-client
- ${hadoop.version}
- provided
-
-
-
-
- org.apache.spark
- spark-streaming-kafka-0-10_${scala.compat.version}
- ${spark3.version}
- test
-
-
- org.apache.spark
- spark-mllib_${scala.compat.version}
- ${spark3.version}
- test
-
-
- io.kotest
- kotest-runner-junit5-jvm
- ${kotest.version}
- test
-
-
- io.kotest.extensions
- kotest-extensions-testcontainers
- ${kotest-extensions-testcontainers.version}
- test
-
-
- io.github.embeddedkafka
- embedded-kafka_${scala.compat.version}
- ${embedded-kafka.version}
- test
-
-
- com.beust
- klaxon
- ${klaxon.version}
- test
-
-
- ch.tutteli.atrium
- atrium-fluent-en_GB
- ${atrium.version}
- test
-
-
- org.apache.spark
- spark-streaming_${scala.compat.version}
- ${spark3.version}
- tests
- test
-
-
- org.apache.kafka
- kafka-streams-test-utils
- 3.1.0
- test
-
-
-
-
- src/main/kotlin
- src/test/kotlin
- target/${scala.compat.version}
-
-
-
- org.jetbrains.kotlin
- kotlin-maven-plugin
-
-
- compile
-
- compile
-
-
-
- test-compile
-
- test-compile
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-surefire-plugin
-
-
-
- org.jetbrains.dokka
- dokka-maven-plugin
- ${dokka.version}
-
- 8
-
-
-
- dokka
-
- dokka
-
- pre-site
-
-
- javadocjar
-
- javadocJar
-
- pre-integration-test
-
-
-
-
-
- org.jacoco
- jacoco-maven-plugin
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
- ${maven-compiler-plugin.version}
-
- 8
- 8
-
-
-
-
-
-
diff --git a/kotlin-spark-api/build.gradle.kts b/kotlin-spark-api/build.gradle.kts
new file mode 100644
index 00000000..5b947d75
--- /dev/null
+++ b/kotlin-spark-api/build.gradle.kts
@@ -0,0 +1,152 @@
+@file:Suppress("UnstableApiUsage")
+
+import com.igormaznitsa.jcp.gradle.JcpTask
+import com.vanniktech.maven.publish.JavadocJar.Dokka
+import com.vanniktech.maven.publish.KotlinJvm
+import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask
+
+plugins {
+ kotlin
+ dokka
+ mavenPublishBase
+ jcp
+ idea
+}
+
+group = Versions.groupID
+version = Versions.project
+
+
+repositories {
+ mavenCentral()
+}
+
+tasks.withType().configureEach {
+ useJUnitPlatform()
+}
+
+dependencies {
+
+ with(Projects) {
+ api(
+ core,
+ scalaTuplesInKotlin,
+ )
+ }
+
+ with(Dependencies) {
+ implementation(
+ kotlinStdLib,
+ reflect,
+ sparkSql,
+ sparkStreaming,
+ hadoopClient,
+ )
+
+ testImplementation(
+ sparkStreamingKafka,
+ kotest,
+ kotestTestcontainers,
+ klaxon,
+ atrium,
+ sparkStreaming,
+ kafkaStreamsTestUtils,
+ sparkMl,
+ )
+ }
+}
+
+// Setup preprocessing with JCP for main sources
+
+val kotlinMainSources = kotlin.sourceSets.main.get().kotlin.sourceDirectories
+
+val preprocessMain by tasks.creating(JcpTask::class) {
+ sources.set(kotlinMainSources)
+ clearTarget.set(true)
+ fileExtensions.set(listOf("kt"))
+ vars.set(Versions.versionMap)
+ outputs.upToDateWhen { target.get().exists() }
+}
+
+tasks.compileKotlin {
+ dependsOn(preprocessMain)
+ outputs.upToDateWhen {
+ preprocessMain.outcomingFiles.files.isEmpty()
+ }
+
+ doFirst {
+ kotlin {
+ sourceSets {
+ main {
+ kotlin.setSrcDirs(listOf(preprocessMain.target.get()))
+ }
+ }
+ }
+ }
+
+ doLast {
+ kotlin {
+ sourceSets {
+ main {
+ kotlin.setSrcDirs(kotlinMainSources)
+ }
+ }
+ }
+ }
+}
+
+// Setup preprocessing with JCP for test sources
+
+val kotlinTestSources = kotlin.sourceSets.test.get().kotlin.sourceDirectories
+
+val preprocessTest by tasks.creating(JcpTask::class) {
+ sources.set(kotlinTestSources)
+ clearTarget.set(true)
+ fileExtensions.set(listOf("kt"))
+ vars.set(Versions.versionMap)
+ outputs.upToDateWhen { target.get().exists() }
+}
+
+tasks.compileTestKotlin {
+ dependsOn(preprocessTest)
+ outputs.upToDateWhen {
+ preprocessTest.outcomingFiles.files.isEmpty()
+ }
+
+ doFirst {
+ kotlin {
+ sourceSets {
+ test {
+ kotlin.setSrcDirs(listOf(preprocessTest.target.get()))
+ }
+ }
+ }
+ }
+
+ doLast {
+ kotlin {
+ sourceSets {
+ test {
+ kotlin.setSrcDirs(kotlinTestSources)
+ }
+ }
+ }
+ }
+}
+
+tasks.withType {
+ dokkaSourceSets {
+ create("kotlin-spark-api") {
+ sourceRoot(preprocessMain.target.get())
+ }
+ }
+}
+
+mavenPublishing {
+ configure(KotlinJvm(Dokka("dokkaHtml")))
+}
+
+
+
+
+
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Arities.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Arities.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Arities.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Arities.kt
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Column.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Column.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Column.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Column.kt
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Conversions.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Conversions.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Conversions.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Conversions.kt
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/DataStreamWriter.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/DataStreamWriter.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/DataStreamWriter.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/DataStreamWriter.kt
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt
similarity index 99%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt
index d44c62dd..fe936e58 100644
--- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt
+++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt
@@ -371,11 +371,13 @@ fun Dataset>.sortByKey(): Dataset> = sort
fun Dataset>.sortByValue(): Dataset> = sort("_2")
/** Returns a dataset sorted by the first (`_1`) value of each [Arity2] inside. */
+@Suppress("DEPRECATION")
@Deprecated("Use Scala tuples instead.", ReplaceWith(""))
@JvmName("sortByArity2Key")
fun Dataset>.sortByKey(): Dataset> = sort("_1")
/** Returns a dataset sorted by the second (`_2`) value of each [Arity2] inside. */
+@Suppress("DEPRECATION")
@Deprecated("Use Scala tuples instead.", ReplaceWith(""))
@JvmName("sortByArity2Value")
fun Dataset>.sortByValue(): Dataset> = sort("_2")
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt
similarity index 96%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt
index 8f2e2159..b37674c8 100644
--- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt
+++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt
@@ -41,10 +41,7 @@ import java.beans.PropertyDescriptor
import java.math.BigDecimal
import java.sql.Date
import java.sql.Timestamp
-import java.time.Duration
-import java.time.Instant
-import java.time.LocalDate
-import java.time.Period
+import java.time.*
import java.util.*
import java.util.concurrent.ConcurrentHashMap
import kotlin.Any
@@ -88,14 +85,24 @@ val ENCODERS: Map, Encoder<*>> = mapOf(
String::class to STRING(),
BigDecimal::class to DECIMAL(),
Date::class to DATE(),
- LocalDate::class to LOCALDATE(), // 3.0+
+ LocalDate::class to LOCALDATE(),
Timestamp::class to TIMESTAMP(),
- Instant::class to INSTANT(), // 3.0+
+ Instant::class to INSTANT(),
ByteArray::class to BINARY(),
- Duration::class to DURATION(), // 3.2+
- Period::class to PERIOD(), // 3.2+
+ //#if sparkMinor >= 3.2
+ Duration::class to DURATION(),
+ Period::class to PERIOD(),
+ //#endif
)
+private fun checkIfEncoderRequiresNewerVersion(kClass: KClass<*>) {
+ when (kClass) {
+ //#if sparkMinor < 3.2
+ //$Duration::class, Period::class -> throw IllegalArgumentException("$kClass is supported in Spark 3.2+")
+ //#endif
+ }
+}
+
private val knownDataTypes: Map, DataType> = mapOf(
Byte::class to DataTypes.ByteType,
Short::class to DataTypes.ShortType,
@@ -131,11 +138,13 @@ inline fun encoder(): Encoder = generateEncoder(typeOf(), T::c
* @see encoder
*/
@Suppress("UNCHECKED_CAST")
-fun generateEncoder(type: KType, cls: KClass<*>): Encoder =
- when {
+fun generateEncoder(type: KType, cls: KClass<*>): Encoder {
+ checkIfEncoderRequiresNewerVersion(cls)
+ return when {
isSupportedByKotlinClassEncoder(cls) -> kotlinClassEncoder(schema = memoizedSchema(type), kClass = cls)
else -> ENCODERS[cls] as? Encoder? ?: bean(cls.java)
} as Encoder
+}
private fun isSupportedByKotlinClassEncoder(cls: KClass<*>): Boolean =
when {
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/GroupState.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/GroupState.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/GroupState.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/GroupState.kt
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Iterators.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Iterators.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Iterators.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Iterators.kt
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/KeyValueGroupedDataset.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/KeyValueGroupedDataset.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/KeyValueGroupedDataset.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/KeyValueGroupedDataset.kt
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt
similarity index 99%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt
index 94971f43..27a7294c 100644
--- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt
+++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt
@@ -54,6 +54,8 @@ import java.io.Serializable
*/
class KSparkSession(val spark: SparkSession) {
+ val sparkVersion = /*$"\""+spark+"\""$*/ /*-*/ "nope"
+
/** Lazy instance of [JavaSparkContext] wrapper around [sparkContext]. */
val sc: JavaSparkContext by lazy { JavaSparkContext(spark.sparkContext) }
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/StreamingKeyValues.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/StreamingKeyValues.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/StreamingKeyValues.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/StreamingKeyValues.kt
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt
similarity index 99%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt
index ccbdc01b..bd08d92c 100644
--- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt
+++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt
@@ -22,7 +22,7 @@
* This file contains functions to register UDFs easily from Kotlin.
*/
-@file:Suppress("DuplicatedCode")
+@file:Suppress("DuplicatedCode", "DEPRECATION")
package org.jetbrains.kotlinx.spark.api
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedAggregateFunction.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedAggregateFunction.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedAggregateFunction.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedAggregateFunction.kt
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt
similarity index 97%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt
index d9013334..fe5279e2 100644
--- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt
+++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt
@@ -24,7 +24,6 @@ package org.jetbrains.kotlinx.spark.api
import org.apache.spark.sql.*
import org.apache.spark.sql.types.DataType
import scala.collection.Seq
-import scala.collection.mutable.WrappedArray
import kotlin.reflect.KClass
import kotlin.reflect.KProperty
import kotlin.reflect.full.isSubclassOf
@@ -43,7 +42,11 @@ fun DataType.unWrap(): DataType =
*/
@PublishedApi
internal fun KClass<*>.checkForValidType(parameterName: String) {
- if (this == String::class || isSubclassOf(WrappedArray::class) || isSubclassOf(Seq::class))
+ if (this == String::class || isSubclassOf(Seq::class)
+ //#if scalaCompat < 2.13
+ //$|| isSubclassOf(scala.collection.mutable.WrappedArray::class)
+ //#endif
+ )
return // Most of the time we need strings or WrappedArrays/Seqs
if (isSubclassOf(Iterable::class)
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt
similarity index 99%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt
index cf0c066e..e23aa160 100644
--- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt
+++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt
@@ -17,6 +17,8 @@
* limitations under the License.
* =LICENSEEND=
*/
+@file:Suppress("DEPRECATION", "unused", "UNCHECKED_CAST")
+
package org.jetbrains.kotlinx.spark.api
diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt
similarity index 99%
rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt
rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt
index 9b006e35..c5fa749a 100644
--- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt
+++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt
@@ -17,6 +17,7 @@
* limitations under the License.
* =LICENSEEND=
*/
+@file:Suppress("DEPRECATION", "UNCHECKED_CAST", "unused")
package org.jetbrains.kotlinx.spark.api
import org.apache.spark.sql.*
diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt
similarity index 97%
rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt
rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt
index e95a65f7..c3362e91 100644
--- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt
+++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt
@@ -53,7 +53,7 @@ class ApiTest : ShouldSpec({
}
.collectAsList()
- expect(result).contains.inOrder.only.values(3.0, 5.0, 7.0, 9.0, 11.0)
+ expect(result).toContain.inOrder.only.values(3.0, 5.0, 7.0, 9.0, 11.0)
}
should("Handle JavaConversions in Kotlin") {
diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/DatasetFunctionTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/DatasetFunctionTest.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/DatasetFunctionTest.kt
rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/DatasetFunctionTest.kt
diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt
similarity index 95%
rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt
rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt
index afb87435..1b7dec29 100644
--- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt
+++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt
@@ -64,11 +64,14 @@ class EncodingTest : ShouldSpec({
dataset.collectAsList() shouldBe timeStamps
}
+ //#if sparkMinor >= 3.2
should("handle Duration Datasets") {
val dataset = dsOf(Duration.ZERO)
dataset.collectAsList() shouldBe listOf(Duration.ZERO)
}
+ //#endif
+ //#if sparkMinor >= 3.2
should("handle Period Datasets") {
val periods = listOf(Period.ZERO, Period.ofDays(2))
val dataset = periods.toDS()
@@ -82,6 +85,7 @@ class EncodingTest : ShouldSpec({
it[1] shouldBe Period.ofDays(0)
}
}
+ //#endif
should("handle binary datasets") {
val byteArray = "Hello there".encodeToByteArray()
@@ -272,21 +276,21 @@ class EncodingTest : ShouldSpec({
val ll1 = LonLat(1.0, 2.0)
val ll2 = LonLat(3.0, 4.0)
val lonlats = dsOf(ll1, ll2).collectAsList()
- expect(lonlats).contains.inAnyOrder.only.values(ll1.copy(), ll2.copy())
+ expect(lonlats).toContain.inAnyOrder.only.values(ll1.copy(), ll2.copy())
}
should("contain all generic primitives with complex schema") {
val primitives = t(1, 1.0, 1.toFloat(), 1.toByte(), LocalDate.now(), true)
val primitives2 = t(2, 2.0, 2.toFloat(), 2.toByte(), LocalDate.now().plusDays(1), false)
val tuples = dsOf(primitives, primitives2).collectAsList()
- expect(tuples).contains.inAnyOrder.only.values(primitives, primitives2)
+ expect(tuples).toContain.inAnyOrder.only.values(primitives, primitives2)
}
should("contain all generic primitives with complex nullable schema") {
val primitives = t(1, 1.0, 1.toFloat(), 1.toByte(), LocalDate.now(), true)
val nulls = t(null, null, null, null, null, null)
val tuples = dsOf(primitives, nulls).collectAsList()
- expect(tuples).contains.inAnyOrder.only.values(primitives, nulls)
+ expect(tuples).toContain.inAnyOrder.only.values(primitives, nulls)
}
should("Be able to serialize lists of data classes") {
@@ -387,7 +391,7 @@ class EncodingTest : ShouldSpec({
.filterNotNull()
.distinct()
.collectAsList()
- expect(result).contains.inOrder.only.value("y")
+ expect(result).toContain.inOrder.only.value("y")
}
should("work with lists of lists") {
@@ -399,7 +403,7 @@ class EncodingTest : ShouldSpec({
.map { it.last() }
.map { it.first() }
.reduceK { a, b -> a + b }
- expect(result).toBe(3)
+ expect(result).toEqual(3)
}
should("Generate schema correctly with nullalble list and map") {
@@ -417,7 +421,7 @@ class EncodingTest : ShouldSpec({
.map { MovieExpanded(it.id, it.genres.split("|").toList()) }
.filter { it.genres.contains("Comedy") }
.collectAsList()
- expect(comedies).contains.inAnyOrder.only.values(
+ expect(comedies).toContain.inAnyOrder.only.values(
MovieExpanded(
1,
listOf("Comedy", "Romance")
@@ -449,7 +453,7 @@ class EncodingTest : ShouldSpec({
.filter { it.genres.contains("Comedy") }
.collectAsList()
- expect(comedies).contains.inAnyOrder.only.values(
+ expect(comedies).toContain.inAnyOrder.only.values(
MovieExpanded(
1,
arrayOf("Comedy", "Romance")
@@ -465,7 +469,7 @@ class EncodingTest : ShouldSpec({
.map { it.id to it.data.firstOrNull { liEl -> liEl.first < 6 } }
.map { it.second }
.collectAsList()
- expect(result).contains.inOrder.only.values(5.1 to 6)
+ expect(result).toContain.inOrder.only.values(5.1 to 6)
}
should("handle lists of generics") {
@@ -476,7 +480,7 @@ class EncodingTest : ShouldSpec({
.map { it.id to it.data.firstOrNull { liEl -> liEl.first < 6 } }
.map { it.second }
.collectAsList()
- expect(result).contains.inOrder.only.values(5.1 to 6)
+ expect(result).toContain.inOrder.only.values(5.1 to 6)
}
should("!handle primitive arrays") {
@@ -485,7 +489,7 @@ class EncodingTest : ShouldSpec({
.map { it.map { ai -> ai + 1 } }
.collectAsList()
.flatten()
- expect(result).contains.inOrder.only.values(2, 3, 4, 5)
+ expect(result).toContain.inOrder.only.values(2, 3, 4, 5)
}
}
}
diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/KafkaStreamingTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/KafkaStreamingTest.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/KafkaStreamingTest.kt
rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/KafkaStreamingTest.kt
diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ProjectConfig.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ProjectConfig.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ProjectConfig.kt
rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ProjectConfig.kt
diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt
similarity index 93%
rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt
rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt
index 9719e8fc..4cc8b9c8 100644
--- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt
+++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt
@@ -36,6 +36,8 @@ import org.jetbrains.kotlinx.spark.api.tuples.X
import org.jetbrains.kotlinx.spark.api.tuples.component1
import org.jetbrains.kotlinx.spark.api.tuples.component2
import org.jetbrains.kotlinx.spark.api.tuples.t
+import org.jetbrains.kotlinx.spark.extensions.KSparkExtensions
+import org.jetbrains.kotlinx.spark.extensions.`KSparkExtensions$`
import scala.Tuple2
import java.io.File
import java.io.Serializable
@@ -199,13 +201,18 @@ class StreamingTest : ShouldSpec({
}
})
-private fun createTempDir() = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark")
- .apply { deleteOnExit() }
+
+private val scalaCompatVersion = `KSparkExtensions$`.`MODULE$`.scalaCompatVersion()
+private val sparkVersion = `KSparkExtensions$`.`MODULE$`.sparkVersion()
+private fun createTempDir() = Utils.createTempDir(
+ System.getProperty("java.io.tmpdir"),
+ "spark_${scalaCompatVersion}_${sparkVersion}"
+).apply { deleteOnExit() }
private fun createCorruptedCheckpoint(): String {
val checkpointDirectory = createTempDir().absolutePath
val fakeCheckpointFile = Checkpoint.checkpointFile(checkpointDirectory, Time(1000))
- FileUtils.write(File(fakeCheckpointFile.toString()), "blablabla", StandardCharsets.UTF_8)
+ FileUtils.write(File(fakeCheckpointFile.toString()), "spark_corrupt_${scalaCompatVersion}_${sparkVersion}", StandardCharsets.UTF_8)
assert(Checkpoint.getCheckpointFiles(checkpointDirectory, (null as FileSystem?).toOption()).nonEmpty())
return checkpointDirectory
}
diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt
similarity index 74%
rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt
rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt
index 2d481e79..1eb54ba4 100644
--- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt
+++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt
@@ -40,15 +40,19 @@ class TypeInferenceTest : ShouldSpec({
val struct = Struct.fromJson(schema(typeOf>>()).prettyJson())!!
should("contain correct typings") {
- expect(struct.fields).notToBeNull().contains.inAnyOrder.only.entries(
+ expect(struct.fields).notToEqualNull().toContain.inAnyOrder.only.entries(
hasField("first", "string"),
- hasStruct("second",
+ hasStruct(
+ "second",
hasField("vala", "integer"),
- hasStruct("tripl1",
+ hasStruct(
+ "tripl1",
hasField("first", "integer"),
- hasStruct("second",
+ hasStruct(
+ "second",
hasField("vala2", "long"),
- hasStruct("para2",
+ hasStruct(
+ "para2",
hasField("first", "long"),
hasField("second", "string")
)
@@ -66,17 +70,22 @@ class TypeInferenceTest : ShouldSpec({
val struct = Struct.fromJson(schema(typeOf>>()).prettyJson())!!
should("contain correct typings") {
- expect(struct.fields).notToBeNull().contains.inAnyOrder.only.entries(
+ expect(struct.fields).notToEqualNull().toContain.inAnyOrder.only.entries(
hasField("first", "string"),
- hasStruct("second",
+ hasStruct(
+ "second",
hasField("vala", "integer"),
- hasStruct("tripl1",
+ hasStruct(
+ "tripl1",
hasField("first", "integer"),
- hasStruct("second",
+ hasStruct(
+ "second",
hasField("vala2", "long"),
- hasStruct("para2",
+ hasStruct(
+ "para2",
hasField("first", "long"),
- hasStruct("second",
+ hasStruct(
+ "second",
hasField("vala3", "double")
)
)
@@ -92,7 +101,7 @@ class TypeInferenceTest : ShouldSpec({
val struct = Struct.fromJson(schema(typeOf()).prettyJson())!!
should("return correct types too") {
- expect(struct.fields).notToBeNull().contains.inAnyOrder.only.entries(
+ expect(struct.fields).notToEqualNull().toContain.inAnyOrder.only.entries(
hasField("a", "string"),
hasField("b", "integer"),
hasField("c", "double")
@@ -104,7 +113,7 @@ class TypeInferenceTest : ShouldSpec({
should("return correct types too") {
expect(struct) {
isOfType("array")
- feature { f(it::elementType) }.toBe(SimpleElement("integer"))
+ feature { f(it::elementType) }.toEqual(SimpleElement("integer"))
}
}
}
@@ -113,12 +122,12 @@ class TypeInferenceTest : ShouldSpec({
should("return correct types too") {
expect(struct) {
isOfType("array")
- feature { f(it::elementType) }.notToBeNull().isA {
- feature { f(it.value::fields) }.notToBeNull().contains.inAnyOrder.only.entries(
+ feature { f(it::elementType) }.notToEqualNull().toBeAnInstanceOf(fun Expect.() {
+ feature { f(it.value::fields) }.notToEqualNull().toContain.inAnyOrder.only.entries(
hasField("first", "integer"),
hasField("second", "long")
)
- }
+ })
}
}
}
@@ -129,11 +138,11 @@ class TypeInferenceTest : ShouldSpec({
should("return correct types too") {
expect(struct) {
isOfType("array")
- feature { f(it::elementType) }.notToBeNull().isA {
- feature { f(it.value::fields) }.notToBeNull().contains.inAnyOrder.only.entries(
+ feature { f(it::elementType) }.notToEqualNull().toBeAnInstanceOf(fun Expect.() {
+ feature { f(it.value::fields) }.notToEqualNull().toContain.inAnyOrder.only.entries(
hasField("e", "string")
)
- }
+ })
}
}
}
@@ -142,9 +151,9 @@ class TypeInferenceTest : ShouldSpec({
should("return correct types too") {
expect(struct) {
isOfType("array")
- feature { f(it::elementType) }.notToBeNull().isA {
- feature { f(it.value::elementType) }.toBe(SimpleElement("integer"))
- }
+ feature { f(it::elementType) }.notToEqualNull().toBeAnInstanceOf(fun Expect.() {
+ feature { f(it.value::elementType) }.toEqual(SimpleElement("integer"))
+ })
}
}
}
@@ -153,8 +162,8 @@ class TypeInferenceTest : ShouldSpec({
should("return correct types too") {
expect(struct) {
isOfType("array")
- feature { f(it::elementType) }.toBe(SimpleElement("integer"))
- feature { f(it::containsNull) }.toBe(false)
+ feature { f(it::elementType) }.toEqual(SimpleElement("integer"))
+ feature { f(it::containsNull) }.toEqual(false)
}
}
}
@@ -163,8 +172,8 @@ class TypeInferenceTest : ShouldSpec({
should("return correct types too") {
expect(struct) {
isOfType("array")
- feature { f(it::elementType) }.toBe(SimpleElement("integer"))
- feature { f(it::containsNull) }.toBe(true)
+ feature { f(it::elementType) }.toEqual(SimpleElement("integer"))
+ feature { f(it::containsNull) }.toEqual(true)
}
}
}
@@ -173,7 +182,7 @@ class TypeInferenceTest : ShouldSpec({
val struct = Struct.fromJson(schema(typeOf()).prettyJson())!!
should("Not change order of fields") {
- expect(struct.fields).notToBeNull().containsExactly(
+ expect(struct.fields).notToEqualNull().containsExactly(
hasField("lon", "double"),
hasField("lat", "double")
)
@@ -187,16 +196,16 @@ class TypeInferenceTest : ShouldSpec({
should("show that list is nullable and element is not") {
expect(struct)
.feature("some", { fields }) {
- notToBeNull().contains.inOrder.only.entry {
+ notToEqualNull().toContain.inOrder.only.entry {
this
- .feature("field name", { name }) { toBe("optionList") }
- .feature("optionList is nullable", { nullable }) { toBe(true) }
+ .feature("field name", { name }) { toEqual("optionList") }
+ .feature("optionList is nullable", { nullable }) { toEqual(true) }
.feature("optionList", { type }) {
this
.isA()
.feature("element type of optionList",
- { value.elementType }) { toBe(SimpleElement("integer")) }
- .feature("optionList contains null", { value.containsNull }) { toBe(false) }
+ { value.elementType }) { toEqual(SimpleElement("integer")) }
+ .feature("optionList contains null", { value.containsNull }) { toEqual(false) }
.feature("optionList type", { value }) { isOfType("array") }
}
}
@@ -207,16 +216,16 @@ class TypeInferenceTest : ShouldSpec({
expect(encoder().schema()) {
this
.feature("data type", { this.fields()?.toList() }) {
- this.notToBeNull().contains.inOrder.only.entry {
+ this.notToEqualNull().toContain.inOrder.only.entry {
this
- .feature("element name", { name() }) { toBe("optionList") }
+ .feature("element name", { name() }) { toEqual("optionList") }
.feature("field type", { dataType() }, {
this
.isA()
.feature("element type", { elementType() }) { isA() }
- .feature("element nullable", { containsNull() }) { toBe(expected = false) }
+ .feature("element nullable", { containsNull() }) { toEqual(expected = false) }
})
- .feature("optionList nullable", { nullable() }) { toBe(true) }
+ .feature("optionList nullable", { nullable() }) { toEqual(true) }
}
}
}
@@ -226,7 +235,7 @@ class TypeInferenceTest : ShouldSpec({
})
private fun Expect.isOfType(type: String) {
- feature { f(it::type) }.toBe(type)
+ feature { f(it::type) }.toEqual(type)
}
private fun hasStruct(
@@ -235,15 +244,17 @@ private fun hasStruct(
vararg expectedFields: Expect.() -> Unit,
): Expect.() -> Unit {
return {
- feature { f(it::name) }.toBe(name)
- feature { f(it::type) }.isA {
- feature { f(it.value::fields) }.notToBeNull().contains.inAnyOrder.only.entries(expectedField,
- *expectedFields)
- }
+ feature { f(it::name) }.toEqual(name)
+ feature { f(it::type) }.toBeAnInstanceOf(fun Expect.() {
+ feature { f(it.value::fields) }.notToEqualNull().toContain.inAnyOrder.only.entries(
+ expectedField,
+ *expectedFields
+ )
+ })
}
}
private fun hasField(name: String, type: String): Expect.() -> Unit = {
- feature { f(it::name) }.toBe(name)
- feature { f(it::type) }.isA().feature { f(it::value) }.toBe(type)
+ feature { f(it::name) }.toEqual(name)
+ feature { f(it::type) }.isA().feature { f(it::value) }.toEqual(type)
}
diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt
similarity index 97%
rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt
rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt
index a4848d40..ad142c00 100644
--- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt
+++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt
@@ -35,7 +35,6 @@ import org.apache.spark.sql.expressions.Aggregator
import org.intellij.lang.annotations.Language
import org.junit.jupiter.api.assertThrows
import scala.collection.Seq
-import scala.collection.mutable.WrappedArray
import java.io.Serializable
import kotlin.random.Random
@@ -204,11 +203,13 @@ class UDFTest : ShouldSpec({
}
}
- should("succeed when using a WrappedArray") {
- udf.register("shouldSucceed") { array: WrappedArray ->
- array.asKotlinIterable().joinToString(" ")
- }
- }
+ //#if scalaCompat <= 2.12
+ //$should("succeed when using a WrappedArray") {
+ //$ udf.register("shouldSucceed") { array: scala.collection.mutable.WrappedArray ->
+ //$ array.asKotlinIterable().joinToString(" ")
+ //$ }
+ //$}
+ //#endif
should("succeed when using a Seq") {
udf.register("shouldSucceed") { array: Seq ->
@@ -241,25 +242,28 @@ class UDFTest : ShouldSpec({
context("calling the UDF-Wrapper") {
withSpark(logLevel = SparkLogLevel.DEBUG) {
- should("succeed in withColumn") {
-
- val stringArrayMerger = udf { it: WrappedArray ->
- it.asKotlinIterable().joinToString(" ")
- }
- val testData = dsOf(arrayOf("a", "b"))
- val newData = testData.withColumn(
- "text",
- stringArrayMerger(
- testData.singleCol().asSeq()
- ),
- )
-
- (newData.select("text").collectAsList() zip newData.select("value").collectAsList())
- .forEach { (text, textArray) ->
- assert(text.getString(0) == textArray.getList(0).joinToString(" "))
- }
- }
+ //#if scalaCompat <= 2.12
+ //$should("succeed in withColumn with WrappedArray") {
+ //$
+ //$ val stringArrayMerger = udf { it: scala.collection.mutable.WrappedArray ->
+ //$ it.asKotlinIterable().joinToString(" ")
+ //$ }
+ //$
+ //$ val testData = dsOf(arrayOf("a", "b"))
+ //$ val newData = testData.withColumn(
+ //$ "text",
+ //$ stringArrayMerger(
+ //$ testData.singleCol().typed()
+ //$ ),
+ //$ )
+ //$
+ //$ (newData.select("text").collectAsList() zip newData.select("value").collectAsList())
+ //$ .forEach { (text, textArray) ->
+ //$ assert(text.getString(0) == textArray.getList(0).joinToString(" "))
+ //$ }
+ //$}
+ //#endif
should("succeed in withColumn using Seq") {
diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt
similarity index 98%
rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt
rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt
index 411bee2c..a0edddeb 100644
--- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt
+++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt
@@ -21,7 +21,7 @@ package org.jetbrains.kotlinx.spark.api
import io.kotest.core.spec.style.ShouldSpec
import io.kotest.matchers.shouldBe
-import org.apache.hadoop.shaded.com.google.common.base.MoreObjects
+import org.glassfish.jersey.internal.guava.MoreObjects
import org.apache.spark.ml.linalg.*
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/struct/model/models.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/struct/model/models.kt
similarity index 100%
rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/struct/model/models.kt
rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/struct/model/models.kt
diff --git a/mvnw b/mvnw
deleted file mode 100755
index 41c0f0c2..00000000
--- a/mvnw
+++ /dev/null
@@ -1,310 +0,0 @@
-#!/bin/sh
-# ----------------------------------------------------------------------------
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# ----------------------------------------------------------------------------
-
-# ----------------------------------------------------------------------------
-# Maven Start Up Batch script
-#
-# Required ENV vars:
-# ------------------
-# JAVA_HOME - location of a JDK home dir
-#
-# Optional ENV vars
-# -----------------
-# M2_HOME - location of maven2's installed home dir
-# MAVEN_OPTS - parameters passed to the Java VM when running Maven
-# e.g. to debug Maven itself, use
-# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
-# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
-# ----------------------------------------------------------------------------
-
-if [ -z "$MAVEN_SKIP_RC" ] ; then
-
- if [ -f /etc/mavenrc ] ; then
- . /etc/mavenrc
- fi
-
- if [ -f "$HOME/.mavenrc" ] ; then
- . "$HOME/.mavenrc"
- fi
-
-fi
-
-# OS specific support. $var _must_ be set to either true or false.
-cygwin=false;
-darwin=false;
-mingw=false
-case "`uname`" in
- CYGWIN*) cygwin=true ;;
- MINGW*) mingw=true;;
- Darwin*) darwin=true
- # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
- # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
- if [ -z "$JAVA_HOME" ]; then
- if [ -x "/usr/libexec/java_home" ]; then
- export JAVA_HOME="`/usr/libexec/java_home`"
- else
- export JAVA_HOME="/Library/Java/Home"
- fi
- fi
- ;;
-esac
-
-if [ -z "$JAVA_HOME" ] ; then
- if [ -r /etc/gentoo-release ] ; then
- JAVA_HOME=`java-config --jre-home`
- fi
-fi
-
-if [ -z "$M2_HOME" ] ; then
- ## resolve links - $0 may be a link to maven's home
- PRG="$0"
-
- # need this for relative symlinks
- while [ -h "$PRG" ] ; do
- ls=`ls -ld "$PRG"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '/.*' > /dev/null; then
- PRG="$link"
- else
- PRG="`dirname "$PRG"`/$link"
- fi
- done
-
- saveddir=`pwd`
-
- M2_HOME=`dirname "$PRG"`/..
-
- # make it fully qualified
- M2_HOME=`cd "$M2_HOME" && pwd`
-
- cd "$saveddir"
- # echo Using m2 at $M2_HOME
-fi
-
-# For Cygwin, ensure paths are in UNIX format before anything is touched
-if $cygwin ; then
- [ -n "$M2_HOME" ] &&
- M2_HOME=`cygpath --unix "$M2_HOME"`
- [ -n "$JAVA_HOME" ] &&
- JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
- [ -n "$CLASSPATH" ] &&
- CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
-fi
-
-# For Mingw, ensure paths are in UNIX format before anything is touched
-if $mingw ; then
- [ -n "$M2_HOME" ] &&
- M2_HOME="`(cd "$M2_HOME"; pwd)`"
- [ -n "$JAVA_HOME" ] &&
- JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
-fi
-
-if [ -z "$JAVA_HOME" ]; then
- javaExecutable="`which javac`"
- if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
- # readlink(1) is not available as standard on Solaris 10.
- readLink=`which readlink`
- if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
- if $darwin ; then
- javaHome="`dirname \"$javaExecutable\"`"
- javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
- else
- javaExecutable="`readlink -f \"$javaExecutable\"`"
- fi
- javaHome="`dirname \"$javaExecutable\"`"
- javaHome=`expr "$javaHome" : '\(.*\)/bin'`
- JAVA_HOME="$javaHome"
- export JAVA_HOME
- fi
- fi
-fi
-
-if [ -z "$JAVACMD" ] ; then
- if [ -n "$JAVA_HOME" ] ; then
- if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
- # IBM's JDK on AIX uses strange locations for the executables
- JAVACMD="$JAVA_HOME/jre/sh/java"
- else
- JAVACMD="$JAVA_HOME/bin/java"
- fi
- else
- JAVACMD="`which java`"
- fi
-fi
-
-if [ ! -x "$JAVACMD" ] ; then
- echo "Error: JAVA_HOME is not defined correctly." >&2
- echo " We cannot execute $JAVACMD" >&2
- exit 1
-fi
-
-if [ -z "$JAVA_HOME" ] ; then
- echo "Warning: JAVA_HOME environment variable is not set."
-fi
-
-CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
-
-# traverses directory structure from process work directory to filesystem root
-# first directory with .mvn subdirectory is considered project base directory
-find_maven_basedir() {
-
- if [ -z "$1" ]
- then
- echo "Path not specified to find_maven_basedir"
- return 1
- fi
-
- basedir="$1"
- wdir="$1"
- while [ "$wdir" != '/' ] ; do
- if [ -d "$wdir"/.mvn ] ; then
- basedir=$wdir
- break
- fi
- # workaround for JBEAP-8937 (on Solaris 10/Sparc)
- if [ -d "${wdir}" ]; then
- wdir=`cd "$wdir/.."; pwd`
- fi
- # end of workaround
- done
- echo "${basedir}"
-}
-
-# concatenates all lines of a file
-concat_lines() {
- if [ -f "$1" ]; then
- echo "$(tr -s '\n' ' ' < "$1")"
- fi
-}
-
-BASE_DIR=`find_maven_basedir "$(pwd)"`
-if [ -z "$BASE_DIR" ]; then
- exit 1;
-fi
-
-##########################################################################################
-# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
-# This allows using the maven wrapper in projects that prohibit checking in binary data.
-##########################################################################################
-if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
- if [ "$MVNW_VERBOSE" = true ]; then
- echo "Found .mvn/wrapper/maven-wrapper.jar"
- fi
-else
- if [ "$MVNW_VERBOSE" = true ]; then
- echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
- fi
- if [ -n "$MVNW_REPOURL" ]; then
- jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
- else
- jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
- fi
- while IFS="=" read key value; do
- case "$key" in (wrapperUrl) jarUrl="$value"; break ;;
- esac
- done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
- if [ "$MVNW_VERBOSE" = true ]; then
- echo "Downloading from: $jarUrl"
- fi
- wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
- if $cygwin; then
- wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"`
- fi
-
- if command -v wget > /dev/null; then
- if [ "$MVNW_VERBOSE" = true ]; then
- echo "Found wget ... using wget"
- fi
- if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
- wget "$jarUrl" -O "$wrapperJarPath"
- else
- wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath"
- fi
- elif command -v curl > /dev/null; then
- if [ "$MVNW_VERBOSE" = true ]; then
- echo "Found curl ... using curl"
- fi
- if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
- curl -o "$wrapperJarPath" "$jarUrl" -f
- else
- curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
- fi
-
- else
- if [ "$MVNW_VERBOSE" = true ]; then
- echo "Falling back to using Java to download"
- fi
- javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
- # For Cygwin, switch paths to Windows format before running javac
- if $cygwin; then
- javaClass=`cygpath --path --windows "$javaClass"`
- fi
- if [ -e "$javaClass" ]; then
- if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
- if [ "$MVNW_VERBOSE" = true ]; then
- echo " - Compiling MavenWrapperDownloader.java ..."
- fi
- # Compiling the Java class
- ("$JAVA_HOME/bin/javac" "$javaClass")
- fi
- if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
- # Running the downloader
- if [ "$MVNW_VERBOSE" = true ]; then
- echo " - Running MavenWrapperDownloader.java ..."
- fi
- ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
- fi
- fi
- fi
-fi
-##########################################################################################
-# End of extension
-##########################################################################################
-
-export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
-if [ "$MVNW_VERBOSE" = true ]; then
- echo $MAVEN_PROJECTBASEDIR
-fi
-MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
-
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin; then
- [ -n "$M2_HOME" ] &&
- M2_HOME=`cygpath --path --windows "$M2_HOME"`
- [ -n "$JAVA_HOME" ] &&
- JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
- [ -n "$CLASSPATH" ] &&
- CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
- [ -n "$MAVEN_PROJECTBASEDIR" ] &&
- MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
-fi
-
-# Provide a "standardized" way to retrieve the CLI args that will
-# work with both Windows and non-Windows executions.
-MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
-export MAVEN_CMD_LINE_ARGS
-
-WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
-
-exec "$JAVACMD" \
- $MAVEN_OPTS \
- -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
- "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
- ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
diff --git a/mvnw.cmd b/mvnw.cmd
deleted file mode 100644
index 86115719..00000000
--- a/mvnw.cmd
+++ /dev/null
@@ -1,182 +0,0 @@
-@REM ----------------------------------------------------------------------------
-@REM Licensed to the Apache Software Foundation (ASF) under one
-@REM or more contributor license agreements. See the NOTICE file
-@REM distributed with this work for additional information
-@REM regarding copyright ownership. The ASF licenses this file
-@REM to you under the Apache License, Version 2.0 (the
-@REM "License"); you may not use this file except in compliance
-@REM with the License. You may obtain a copy of the License at
-@REM
-@REM http://www.apache.org/licenses/LICENSE-2.0
-@REM
-@REM Unless required by applicable law or agreed to in writing,
-@REM software distributed under the License is distributed on an
-@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-@REM KIND, either express or implied. See the License for the
-@REM specific language governing permissions and limitations
-@REM under the License.
-@REM ----------------------------------------------------------------------------
-
-@REM ----------------------------------------------------------------------------
-@REM Maven Start Up Batch script
-@REM
-@REM Required ENV vars:
-@REM JAVA_HOME - location of a JDK home dir
-@REM
-@REM Optional ENV vars
-@REM M2_HOME - location of maven2's installed home dir
-@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
-@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
-@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
-@REM e.g. to debug Maven itself, use
-@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
-@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
-@REM ----------------------------------------------------------------------------
-
-@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
-@echo off
-@REM set title of command window
-title %0
-@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
-@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
-
-@REM set %HOME% to equivalent of $HOME
-if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
-
-@REM Execute a user defined script before this one
-if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
-@REM check for pre script, once with legacy .bat ending and once with .cmd ending
-if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
-if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
-:skipRcPre
-
-@setlocal
-
-set ERROR_CODE=0
-
-@REM To isolate internal variables from possible post scripts, we use another setlocal
-@setlocal
-
-@REM ==== START VALIDATION ====
-if not "%JAVA_HOME%" == "" goto OkJHome
-
-echo.
-echo Error: JAVA_HOME not found in your environment. >&2
-echo Please set the JAVA_HOME variable in your environment to match the >&2
-echo location of your Java installation. >&2
-echo.
-goto error
-
-:OkJHome
-if exist "%JAVA_HOME%\bin\java.exe" goto init
-
-echo.
-echo Error: JAVA_HOME is set to an invalid directory. >&2
-echo JAVA_HOME = "%JAVA_HOME%" >&2
-echo Please set the JAVA_HOME variable in your environment to match the >&2
-echo location of your Java installation. >&2
-echo.
-goto error
-
-@REM ==== END VALIDATION ====
-
-:init
-
-@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
-@REM Fallback to current working directory if not found.
-
-set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
-IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
-
-set EXEC_DIR=%CD%
-set WDIR=%EXEC_DIR%
-:findBaseDir
-IF EXIST "%WDIR%"\.mvn goto baseDirFound
-cd ..
-IF "%WDIR%"=="%CD%" goto baseDirNotFound
-set WDIR=%CD%
-goto findBaseDir
-
-:baseDirFound
-set MAVEN_PROJECTBASEDIR=%WDIR%
-cd "%EXEC_DIR%"
-goto endDetectBaseDir
-
-:baseDirNotFound
-set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
-cd "%EXEC_DIR%"
-
-:endDetectBaseDir
-
-IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
-
-@setlocal EnableExtensions EnableDelayedExpansion
-for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
-@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
-
-:endReadAdditionalConfig
-
-SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
-set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
-set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
-
-set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
-
-FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
- IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B
-)
-
-@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
-@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
-if exist %WRAPPER_JAR% (
- if "%MVNW_VERBOSE%" == "true" (
- echo Found %WRAPPER_JAR%
- )
-) else (
- if not "%MVNW_REPOURL%" == "" (
- SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
- )
- if "%MVNW_VERBOSE%" == "true" (
- echo Couldn't find %WRAPPER_JAR%, downloading it ...
- echo Downloading from: %DOWNLOAD_URL%
- )
-
- powershell -Command "&{"^
- "$webclient = new-object System.Net.WebClient;"^
- "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
- "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
- "}"^
- "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^
- "}"
- if "%MVNW_VERBOSE%" == "true" (
- echo Finished downloading %WRAPPER_JAR%
- )
-)
-@REM End of extension
-
-@REM Provide a "standardized" way to retrieve the CLI args that will
-@REM work with both Windows and non-Windows executions.
-set MAVEN_CMD_LINE_ARGS=%*
-
-%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
-if ERRORLEVEL 1 goto error
-goto end
-
-:error
-set ERROR_CODE=1
-
-:end
-@endlocal & set ERROR_CODE=%ERROR_CODE%
-
-if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
-@REM check for post script, once with legacy .bat ending and once with .cmd ending
-if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
-if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
-:skipRcPost
-
-@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
-if "%MAVEN_BATCH_PAUSE%" == "on" pause
-
-if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
-
-exit /B %ERROR_CODE%
diff --git a/pom.xml b/pom.xml
deleted file mode 100644
index 6e08e4d2..00000000
--- a/pom.xml
+++ /dev/null
@@ -1,337 +0,0 @@
-
-
- 4.0.0
-
- Kotlin Spark API: Parent for Spark 3.2+
- Parent project for Kotlin for Apache Spark
- org.jetbrains.kotlinx.spark
- kotlin-spark-api-parent-3.2
- 1.1.1-SNAPSHOT
- pom
-
-
- 0.17.0
- 1.6.10
- 3.1.0
- 3.3.1
- 1.3.1
- 5.3.1
- 0.11.0-95
- 1.7.0
- 0.7.5
- 3.2.1
-
-
- 0.8.7
- 5.5
- official
- 2.0.0
- 3.3.0
- 3.10.1
- 3.0.0-M1
- 3.0.0-M3
- 3.0.1
- 3.2.0
- 3.9.1
- 3.2.1
- 3.0.0-M6
- 1.6.8
- 4.5.6
-
-
-
- dummy
-
-
-
-
-
- org.jetbrains.kotlin
- kotlin-stdlib-jdk8
- ${kotlin.version}
-
-
- org.jetbrains.kotlin
- kotlin-reflect
- ${kotlin.version}
-
-
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-release-plugin
- 3.0.0-M1
-
-
- org.sonatype.plugins
- nexus-staging-maven-plugin
- ${nexus-staging-plugin.version}
-
-
- org.jacoco
- jacoco-maven-plugin
- ${jacoco-maven-plugin.version}
-
-
-
- prepare-agent
-
-
-
- report
-
- report
-
- prepare-package
-
-
-
-
- org.jetbrains.kotlin
- kotlin-maven-plugin
- ${kotlin.version}
-
-
- compile
-
- compile
-
- compile
-
- true
- 1.8
-
- -Xopt-in=kotlin.RequiresOptIn
- -XXLanguage:+InlineClasses
-
-
-
-
- test-compile
-
- test-compile
-
- test-compile
-
- true
- 1.8
-
- -Xopt-in=kotlin.RequiresOptIn
- -XXLanguage:+InlineClasses
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
- ${maven-jar-plugin.version}
-
-
- org.apache.maven.plugins
- maven-site-plugin
- ${maven-site-plugin.version}
-
-
- org.apache.maven.plugins
- maven-surefire-plugin
- ${maven-surefire-plugin.version}
-
- native
-
-
-
-
- net.alchim31.maven
- scala-maven-plugin
- ${scala-maven-plugin.version}
-
-
-
-
-
- org.codehaus.mojo
- license-maven-plugin
- ${license-maven-plugin.version}
-
-
- src/main/
- src/test/
-
- apache_v2
- =LICENSE=
- =LICENSE END=
- ----------
-
-
-
- first
-
- update-file-header
-
-
-
- **/*.json
- **/*.css
-
-
- process-sources
-
-
-
-
- org.apache.maven.plugins
- maven-source-plugin
- ${maven-source-plugin.version}
-
-
- attach-sources
-
- jar
-
-
-
-
-
- org.apache.maven.plugins
- maven-enforcer-plugin
- ${maven-enforcer-plugin.version}
-
-
- enforce-maven
-
- enforce
-
-
-
-
- 3.5.4
-
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-deploy-plugin
- ${maven-deploy-plugin.version}
-
-
- org.apache.maven.plugins
- maven-release-plugin
-
- true
- false
- forked-path
- scala-2.12,central-deploy
-
-
-
-
-
-
- http://maven.apache.org
- 2019
-
- JetBrains
- https://www.jetbrains.com/
-
-
-
-
- Apache License, Version 2.0
- https://www.apache.org/licenses/LICENSE-2.0.txt
-
-
-
-
-
- asm0dey
- Pasha Finkelshteyn
- asm0dey@jetbrains.com
- GMT+3
-
-
- vitaly.khudobakhshov
- Vitaly Khudobakhshov
- vitaly.khudobakhshov@jetbrains.com
- GMT+3
-
-
-
- scm:git:https://github.com/JetBrains/kotlin-spark-api.git
- https://github.com/JetBrains/kotlin-spark-api
- HEAD
-
-
-
- ossrh
- https://oss.sonatype.org/service/local/staging/deploy/maven2/
-
-
- github
- GitHub JetBrains Apache Maven Packages
- https://maven.pkg.github.com/Kotlin/kotlin-spark-api
-
-
-
-
-
-
-
- scala-2.12
-
- true
-
-
- pom_2.12.xml
-
-
-
- central-deploy
-
-
- performRelease
- true
-
-
-
-
-
- org.sonatype.plugins
- nexus-staging-maven-plugin
- ${nexus-staging-plugin.version}
- true
-
- ossrh
- https://oss.sonatype.org/
- false
- 20
-
-
-
- org.apache.maven.plugins
- maven-gpg-plugin
- ${maven-gpg-plugin.version}
-
-
- sign-artifacts
-
- sign
-
- verify
-
-
-
-
-
-
-
-
-
diff --git a/pom_2.12.xml b/pom_2.12.xml
deleted file mode 100644
index f569bbb7..00000000
--- a/pom_2.12.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-
-
- 4.0.0
-
- Kotlin Spark API: Parent (Scala 2.12)
- Parent project for Kotlin for Apache Spark
- kotlin-spark-api-parent-3.2_2.12
-
- org.jetbrains.kotlinx.spark
- kotlin-spark-api-parent-3.2
- 1.1.1-SNAPSHOT
- pom.xml
-
- pom
-
-
- true
- 2.12.15
- 2.12
-
-
-
- core/3.2/pom_2.12.xml
- scala-tuples-in-kotlin/pom_2.12.xml
- kotlin-spark-api/3.2/pom_2.12.xml
- examples/pom-3.2_2.12.xml
- jupyter
-
-
-
-
-
- org.jetbrains.kotlinx.spark
- core-3.2_${scala.compat.version}
- ${project.version}
-
-
- org.jetbrains.kotlinx.spark
- scala-tuples-in-kotlin-3.2
- ${project.version}
-
-
-
-
diff --git a/scala-tuples-in-kotlin/build.gradle.kts b/scala-tuples-in-kotlin/build.gradle.kts
new file mode 100644
index 00000000..5209a43f
--- /dev/null
+++ b/scala-tuples-in-kotlin/build.gradle.kts
@@ -0,0 +1,65 @@
+@file:Suppress("UnstableApiUsage")
+
+import com.vanniktech.maven.publish.JavadocJar.Dokka
+import com.vanniktech.maven.publish.KotlinJvm
+import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask
+import org.jetbrains.dokka.gradle.DokkaTask
+import org.jetbrains.dokka.gradle.DokkaTaskPartial
+
+plugins {
+ scala
+ kotlin
+ dokka
+ mavenPublishBase
+}
+
+group = Versions.groupID
+version = Versions.project
+
+repositories {
+ mavenCentral()
+}
+
+tasks.withType().configureEach {
+ useJUnitPlatform()
+}
+
+dependencies {
+ with(Dependencies) {
+ implementation(
+ kotlinStdLib,
+ scalaLibrary,
+ )
+ testImplementation(
+ kotest,
+ atrium,
+ kotlinTest,
+ )
+ }
+}
+
+tasks.withType {
+ dokkaSourceSets {
+ create("scala-tuples-in-kotlin") {
+ sourceRoot(
+ kotlin.sourceSets
+ .main.get()
+ .kotlin
+ .srcDirs
+ .first { it.path.endsWith("kotlin") }
+ )
+ }
+ }
+}
+
+mavenPublishing {
+ configure(KotlinJvm(Dokka("dokkaHtml")))
+}
+
+
+// Publishing of scala-tuples-in-kotlin can be skipped since it's only dependent on the Scala version
+val skipScalaTuplesInKotlin = System.getProperty("skipScalaTuplesInKotlin").toBoolean()
+tasks
+ .filter { "publish" in it.name }
+ .forEach { it.onlyIf { !skipScalaTuplesInKotlin } }
+
diff --git a/scala-tuples-in-kotlin/pom_2.12.xml b/scala-tuples-in-kotlin/pom_2.12.xml
deleted file mode 100644
index a5169fbb..00000000
--- a/scala-tuples-in-kotlin/pom_2.12.xml
+++ /dev/null
@@ -1,153 +0,0 @@
-
-
- 4.0.0
-
- Kotlin Spark API: Scala Tuples in Kotlin
- Scala Tuple helper functions for kotlin
- scala-tuples-in-kotlin-3.2
-
- org.jetbrains.kotlinx.spark
- kotlin-spark-api-parent-3.2_2.12
- 1.1.1-SNAPSHOT
- ../pom_2.12.xml
-
-
-
-
- org.scala-lang
- scala-library
- ${scala.version}
-
-
- org.jetbrains.kotlin
- kotlin-stdlib-jdk8
- ${kotlin.version}
-
-
-
-
- io.kotest
- kotest-runner-junit5-jvm
- ${kotest.version}
- test
-
-
- com.beust
- klaxon
- ${klaxon.version}
- test
-
-
- ch.tutteli.atrium
- atrium-fluent-en_GB
- ${atrium.version}
- test
-
-
-
- org.jetbrains.kotlin
- kotlin-test
- ${kotlin.version}
- test
-
-
-
-
- src/main/kotlin
- src/test/kotlin
- target/${scala.compat.version}
-
-
-
- org.jetbrains.dokka
- dokka-maven-plugin
- ${dokka.version}
-
- 8
-
-
-
- dokka
-
- dokka
-
- pre-site
-
-
- javadocjar
-
- javadocJar
-
- pre-integration-test
-
-
-
-
-
- org.jetbrains.kotlin
- kotlin-maven-plugin
- ${kotlin.version}
-
-
- compile
- compile
-
- compile
-
-
-
- test-compile
- test-compile
-
- test-compile
-
-
-
-
- 1.8
-
-
-
-
- org.apache.maven.plugins
- maven-assembly-plugin
- ${maven-assembly-plugin.version}
-
-
- jar-with-dependencies
-
-
-
- org.jetbrains.spark.api.examples.WordCountKt
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-site-plugin
-
- true
-
-
-
-
- org.apache.maven.plugins
- maven-deploy-plugin
-
- false
-
-
-
-
- org.sonatype.plugins
- nexus-staging-maven-plugin
-
- false
-
-
-
-
-
-
diff --git a/settings.gradle.kts b/settings.gradle.kts
new file mode 100644
index 00000000..b94c65ba
--- /dev/null
+++ b/settings.gradle.kts
@@ -0,0 +1,32 @@
+val spark: String by settings
+val scala: String by settings
+val skipScalaTuplesInKotlin: String by settings
+System.setProperty("spark", spark)
+System.setProperty("scala", scala)
+System.setProperty("skipScalaTuplesInKotlin", skipScalaTuplesInKotlin)
+
+
+val scalaCompat
+ get() = scala.substringBeforeLast('.')
+
+val versions = "${spark}_${scalaCompat}"
+
+rootProject.name = "kotlin-spark-api-parent_$versions"
+
+include("core")
+include("scala-tuples-in-kotlin")
+include("kotlin-spark-api")
+include("jupyter")
+include("examples")
+
+project(":core").name = "core_$versions"
+project(":scala-tuples-in-kotlin").name = "scala-tuples-in-kotlin_$scalaCompat"
+project(":kotlin-spark-api").name = "kotlin-spark-api_$versions"
+project(":jupyter").name = "jupyter_$versions"
+project(":examples").name = "examples_$versions"
+
+buildCache {
+ local {
+ removeUnusedEntriesAfterDays = 30
+ }
+}
\ No newline at end of file