diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 98a7ac04..7d87d712 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,34 +3,71 @@ name: Build and test on: push: branches: '*' - pull_request: - branches: '*' +# pull_request: +# branches: '*' jobs: - build-scala-12: + build-all-versions: + strategy: + matrix: + scala: [ "2.12.15", "2.13.8" ] + spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ] + exclude: + - scala: "2.13.8" + spark: "3.1.3" + - scala: "2.13.8" + spark: "3.1.2" + - scala: "2.13.8" + spark: "3.1.1" + - scala: "2.13.8" + spark: "3.1.0" + - scala: "2.13.8" + spark: "3.0.3" + - scala: "2.13.8" + spark: "3.0.2" + - scala: "2.13.8" + spark: "3.0.1" + - scala: "2.13.8" + spark: "3.0.0" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - distributions: adopt + distribution: adopt java-version: 11 check-latest: true - - name: Cache Maven packages - uses: actions/cache@v2 + + - name: Cache Gradle packages + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + ~/.gradle/jdks + key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Build with Gradle + uses: gradle/gradle-build-action@v2 with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - name: Build with Maven - run: ./mvnw -B package --file pom.xml -Pscala-2.12 -# qodana: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v3 -# - name: 'Qodana Scan' -# uses: JetBrains/qodana-action@v5.0.2 + arguments: | + -Pspark=${{ matrix.spark }} + -Pscala=${{ matrix.scala }} + clean + build + + # qodana: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - name: 'Qodana Scan' + # uses: JetBrains/qodana-action@v5.0.2 + + # vim: ts=2:sts=2:sw=2:expandtab diff --git a/.github/workflows/generate_docs.yml b/.github/workflows/generate_docs.yml index cdbd1949..1709b0ae 100644 --- a/.github/workflows/generate_docs.yml +++ b/.github/workflows/generate_docs.yml @@ -3,28 +3,45 @@ name: Generate and publish docs on: push: branches: - - "spark-3.2" + - "release" # todo make release branch jobs: generate-and-publish-docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - distributions: adopt + distribution: adopt java-version: 11 check-latest: true - - name: Generate docs - run: ./mvnw clean package site -Dmaven.test.skip=true + + - name: Cache Gradle packages + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + ~/.gradle/jdks + key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Generate docs with Gradle + uses: gradle/gradle-build-action@v2 + with: + arguments: | + dokkaHtmlMultiModule + - name: Copy docs to "docs" branch uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_branch: docs - publish_dir: ./kotlin-spark-api/3.2/target/dokka + publish_dir: ./build/dokka/htmlMultiModule force_orphan: true diff --git a/.github/workflows/publish_dev_version.yml b/.github/workflows/publish_dev_version.yml index b292cf15..660878dd 100644 --- a/.github/workflows/publish_dev_version.yml +++ b/.github/workflows/publish_dev_version.yml @@ -7,24 +7,65 @@ on: jobs: build-and-deploy: + strategy: + matrix: + scala: [ "2.12.15", "2.13.8" ] + spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ] + exclude: + - scala: "2.13.8" + spark: "3.1.3" + - scala: "2.13.8" + spark: "3.1.2" + - scala: "2.13.8" + spark: "3.1.1" + - scala: "2.13.8" + spark: "3.1.0" + - scala: "2.13.8" + spark: "3.0.3" + - scala: "2.13.8" + spark: "3.0.2" + - scala: "2.13.8" + spark: "3.0.1" + - scala: "2.13.8" + spark: "3.0.0" runs-on: ubuntu-latest + permissions: + contents: read + packages: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - distributions: adopt + distribution: adopt java-version: 11 check-latest: true - - name: Cache Maven packages - uses: actions/cache@v2 + + - name: Cache Gradle packages + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + ~/.gradle/jdks + key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Validate Gradle wrapper + uses: gradle/wrapper-validation-action@v1 + + - name: Deploy to GH Packages with Gradle + uses: gradle/gradle-build-action@v2 with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - name: Deploy to GH Packages - run: ./mvnw --batch-mode deploy + arguments: | + -Pspark=${{ matrix.spark }} + -Pscala=${{ matrix.scala }} + -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }} + clean + publishMavenPublicationToGitHubPackagesRepository env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index f7521c9b..d1f280d2 100644 --- a/.gitignore +++ b/.gitignore @@ -376,3 +376,4 @@ orcpath/ .env **/.allure/ **/allure-results/ +/generated_* diff --git a/.mvn/wrapper/MavenWrapperDownloader.java b/.mvn/wrapper/MavenWrapperDownloader.java deleted file mode 100644 index b901097f..00000000 --- a/.mvn/wrapper/MavenWrapperDownloader.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2007-present the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.net.*; -import java.io.*; -import java.nio.channels.*; -import java.util.Properties; - -public class MavenWrapperDownloader { - - private static final String WRAPPER_VERSION = "0.5.6"; - /** - * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. - */ - private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" - + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; - - /** - * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to - * use instead of the default one. - */ - private static final String MAVEN_WRAPPER_PROPERTIES_PATH = - ".mvn/wrapper/maven-wrapper.properties"; - - /** - * Path where the maven-wrapper.jar will be saved to. - */ - private static final String MAVEN_WRAPPER_JAR_PATH = - ".mvn/wrapper/maven-wrapper.jar"; - - /** - * Name of the property which should be used to override the default download url for the wrapper. - */ - private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; - - public static void main(String args[]) { - System.out.println("- Downloader started"); - File baseDirectory = new File(args[0]); - System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); - - // If the maven-wrapper.properties exists, read it and check if it contains a custom - // wrapperUrl parameter. - File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); - String url = DEFAULT_DOWNLOAD_URL; - if(mavenWrapperPropertyFile.exists()) { - FileInputStream mavenWrapperPropertyFileInputStream = null; - try { - mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); - Properties mavenWrapperProperties = new Properties(); - mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); - url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); - } catch (IOException e) { - System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); - } finally { - try { - if(mavenWrapperPropertyFileInputStream != null) { - mavenWrapperPropertyFileInputStream.close(); - } - } catch (IOException e) { - // Ignore ... - } - } - } - System.out.println("- Downloading from: " + url); - - File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); - if(!outputFile.getParentFile().exists()) { - if(!outputFile.getParentFile().mkdirs()) { - System.out.println( - "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); - } - } - System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); - try { - downloadFileFromURL(url, outputFile); - System.out.println("Done"); - System.exit(0); - } catch (Throwable e) { - System.out.println("- Error downloading"); - e.printStackTrace(); - System.exit(1); - } - } - - private static void downloadFileFromURL(String urlString, File destination) throws Exception { - if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { - String username = System.getenv("MVNW_USERNAME"); - char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); - Authenticator.setDefault(new Authenticator() { - @Override - protected PasswordAuthentication getPasswordAuthentication() { - return new PasswordAuthentication(username, password); - } - }); - } - URL website = new URL(urlString); - ReadableByteChannel rbc; - rbc = Channels.newChannel(website.openStream()); - FileOutputStream fos = new FileOutputStream(destination); - fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); - fos.close(); - rbc.close(); - } - -} diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties deleted file mode 100644 index ffdc10e5..00000000 --- a/.mvn/wrapper/maven-wrapper.properties +++ /dev/null @@ -1,2 +0,0 @@ -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.1/apache-maven-3.8.1-bin.zip -wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar diff --git a/README.md b/README.md index f54416f0..b89a04cb 100644 --- a/README.md +++ b/README.md @@ -30,20 +30,34 @@ We have opened a Spark Project Improvement Proposal: [Kotlin support for Apache - [Streaming](#streaming) - [User Defined Functions](#user-defined-functions) - [Examples](#examples) -- [Reporting issues/Support](#reporting-issuessupport) +- [Reporting issues/Support](#reporting-issues--support) - [Code of Conduct](#code-of-conduct) - [License](#license) ## Supported versions of Apache Spark +| Apache Spark | Scala | Kotlin for Apache Spark | +|:------------:|:-----:|:-----------------------------------:| +| 3.3.0 | 2.13 | kotlin-spark-api_3.3.0_2.13:VERSION | +| | 2.12 | kotlin-spark-api_3.3.0_2.12:VERSION | +| 3.2.1 | 2.13 | kotlin-spark-api_3.2.1_2.13:VERSION | +| | 2.12 | kotlin-spark-api_3.2.1_2.12:VERSION | +| 3.2.0 | 2.13 | kotlin-spark-api_3.2.0_2.13:VERSION | +| | 2.12 | kotlin-spark-api_3.2.0_2.12:VERSION | +| 3.1.3 | 2.12 | kotlin-spark-api_3.1.3_2.12:VERSION | +| 3.1.2 | 2.12 | kotlin-spark-api_3.1.2_2.12:VERSION | +| 3.1.1 | 2.12 | kotlin-spark-api_3.1.1_2.12:VERSION | +| 3.1.0 | 2.12 | kotlin-spark-api_3.1.0_2.12:VERSION | +| 3.0.3 | 2.12 | kotlin-spark-api_3.0.3_2.12:VERSION | +| 3.0.2 | 2.12 | kotlin-spark-api_3.0.2_2.12:VERSION | +| 3.0.1 | 2.12 | kotlin-spark-api_3.0.1_2.12:VERSION | +| 3.0.0 | 2.12 | kotlin-spark-api_3.0.0_2.12:VERSION | + +## Deprecated versions | Apache Spark | Scala | Kotlin for Apache Spark | |:------------:|:-----:|:-------------------------------:| -| 3.2.1+ | 2.12 | kotlin-spark-api-3.2:1.1.0 | -| 3.1.3+ | 2.12 | kotlin-spark-api-3.1:1.1.0 | -| 3.0.3+ | 2.12 | kotlin-spark-api-3.0:1.1.0 | | 2.4.1+ | 2.12 | kotlin-spark-api-2.4_2.12:1.0.2 | | 2.4.1+ | 2.11 | kotlin-spark-api-2.4_2.11:1.0.2 | - ## Releases The list of Kotlin for Apache Spark releases is available [here](https://github.com/JetBrains/kotlin-spark-api/releases/). @@ -61,19 +75,23 @@ Here's an example `pom.xml`: ```xml org.jetbrains.kotlinx.spark - kotlin-spark-api-3.2 + kotlin-spark-api_3.3.0_2.13 ${kotlin-spark-api.version} org.apache.spark - spark-sql_2.12 + spark-sql_2.13 ${spark.version} ``` -Note that `core` is being compiled against Scala version `2.12`. +Note that you must match the version of the Kotlin for Apache Spark API to the Spark- and Scala version of your project. You can find a complete example with `pom.xml` and `build.gradle` in the [Quick Start Guide](https://github.com/JetBrains/kotlin-spark-api/wiki/Quick-Start-Guide). +If you want to try a development version. You can use the versions published to [GH Packages](https://github.com/orgs/Kotlin/packages?tab=packages&q=kotlin-spark-api_3.3.0_2.13). +They typically have the same version as the release version, but with a `-SNAPSHOT` suffix. See the [GitHub Docs](https://docs.github.com/en/packages/learn-github-packages/installing-a-package) +for more information. + Once you have configured the dependency, you only need to add the following import to your Kotlin file: ```kotlin import org.jetbrains.kotlinx.spark.api.* @@ -90,7 +108,7 @@ To it, simply add to the top of your notebook. This will get the latest version of the API, together with the latest version of Spark. To define a certain version of Spark or the API itself, simply add it like this: ```jupyterpython -%use spark(spark=3.2, v=1.1.0) +%use spark(spark=3.3.0, scala=2.13, v=1.2.0) ``` Inside the notebook a Spark session will be initiated automatically. This can be accessed via the `spark` value. @@ -331,13 +349,18 @@ You are also welcome to join [kotlin-spark channel](https://kotlinlang.slack.com Contributions are more than welcome! Pull requests can be created for the [main](https://github.com/Kotlin/kotlin-spark-api/tree/main) branch and will be considered as soon as possible. Be sure to add the necessary tests for any new feature you add. The [main](https://github.com/Kotlin/kotlin-spark-api/tree/main) branch always aims to target the latest available [Apache Spark version](https://spark.apache.org/downloads.html). +Note that we use [Java Comment Preprocessor](https://github.com/raydac/java-comment-preprocessor) to build the library +for all different supported versions of Apache Spark and Scala. +The current values of these versions can be edited in `gradle.properties` and should always be the latest versions for commits. +For testing, all versions need a pass for the request to be accepted. +We use GitHub Actions to test and deploy the library for all versions, but locally you can also use the `gradlew_all_versions` file. + Of the [main](https://github.com/Kotlin/kotlin-spark-api/tree/main) branch, development versions of the library are published to -[Github Packages](https://github.com/orgs/Kotlin/packages?tab=packages&q=kotlin-spark-api). This way, new features can be +[GitHub Packages](https://github.com/orgs/Kotlin/packages?tab=packages&q=kotlin-spark-api). This way, new features can be tested quickly without having to wait for a full release. -For full releases, all `spark-X.x` branches are updated. At the moment, this includes Spark 3.0, 3.1, and 3.2. For this we also -follow the [supported versions of Apache](https://spark.apache.org/downloads.html). +For full releases, the `release` branch is updated. ## Code of Conduct This project and the corresponding community is governed by the [JetBrains Open Source and Community Code of Conduct](https://confluence.jetbrains.com/display/ALL/JetBrains+Open+Source+and+Community+Code+of+Conduct). Please make sure you read it. diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 00000000..d943cb25 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,112 @@ +@file:Suppress("UnstableApiUsage") + +buildscript { + repositories { + mavenCentral() + } + dependencies { + classpath(jcp) + classpath(mavenPublish) + } +} + + +plugins { + mavenPublish version Versions.mavenPublish + dokka version Versions.dokka + idea +} + +group = Versions.groupID +version = Versions.project + +tasks.withType().configureEach { + useJUnitPlatform() +} + +repositories { + mavenCentral() +} + + +allprojects { + plugins.withId(mavenPublishBase) { + group = Versions.groupID + version = Versions.project + + publishing { + repositories { + maven { + name = "GitHubPackages" + url = uri("https://maven.pkg.github.com/Kotlin/kotlin-spark-api") + credentials { + username = project.findProperty("gpr.user") as String? + ?: System.getenv("GITHUB_ACTOR") + password = project.findProperty("gpr.key") as String? + ?: System.getenv("GITHUB_TOKEN") + } + } + } + } + + mavenPublishing { + pomFromGradleProperties() + publishToMavenCentral() + // The username and password for Sonatype OSS can be provided as Gradle properties + // called mavenCentralUsername and mavenCentralPassword to avoid having to commit them. + // You can also supply them as environment variables called + // ORG_GRADLE_PROJECT_mavenCentralUsername and + // ORG_GRADLE_PROJECT_mavenCentralPassword. + + signAllPublications() + pom { + name.set("Kotlin Spark API") + description.set("Kotlin for Apache Spark") + packaging = "pom" + + url.set("https://maven.apache.org") + inceptionYear.set("2019") + + organization { + name.set("JetBrains") + url.set("https://www.jetbrains.com/") + } + + licenses { + license { + name.set("Apache License, Version 2.0") + url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") + } + } + + developers { + developer { + id.set("asm0dey") + name.set("Pasha Finkelshteyn") + email.set("asm0dey@jetbrains.com") + timezone.set("GMT+3") + } + developer { + id.set("vitaly.khudobakhshov") + name.set("Vitaly Khudobakhshov") + email.set("vitaly.khudobakhshov@jetbrains.com") + timezone.set("GMT+3") + } + developer { + id.set("Jolanrensen") + name.set("Jolan Rensen") + email.set("jolan.rensen@jetbrains.com") + timezone.set("GMT+1") + } + } + + scm { + connection.set("scm:git:https://github.com/JetBrains/kotlin-spark-api.git") + url.set("https://github.com/JetBrains/kotlin-spark-api") + tag.set("HEAD") + } + } + + } + } +} \ No newline at end of file diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts new file mode 100644 index 00000000..181a9870 --- /dev/null +++ b/buildSrc/build.gradle.kts @@ -0,0 +1,9 @@ +import org.gradle.kotlin.dsl.`kotlin-dsl` + +plugins { + `kotlin-dsl` +} + +repositories { + mavenCentral() +} diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt new file mode 100644 index 00000000..70f2761d --- /dev/null +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -0,0 +1,27 @@ +object Dependencies { + inline val kotlinStdLib get() = "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${Versions.kotlin}" + inline val reflect get() = "org.jetbrains.kotlin:kotlin-reflect:${Versions.kotlin}" + inline val scalaLibrary get() = "org.scala-lang:scala-library:${Versions.scala}" + inline val kotlinxHtml get() = "org.jetbrains.kotlinx:kotlinx-html-jvm:${Versions.kotlinxHtml}" + inline val sparkSql get() = "org.apache.spark:spark-sql_${Versions.scalaCompat}:${Versions.spark}" + inline val sparkMl get() = "org.apache.spark:spark-mllib_${Versions.scalaCompat}:${Versions.spark}" + inline val sparkStreaming get() = "org.apache.spark:spark-streaming_${Versions.scalaCompat}:${Versions.spark}" + inline val hadoopClient get() = "org.apache.hadoop:hadoop-client:${Versions.hadoop}" + inline val sparkRepl get() = "org.apache.spark:spark-repl_${Versions.scalaCompat}:${Versions.spark}" + inline val jupyter get() = "org.jetbrains.kotlinx:kotlin-jupyter-api:${Versions.jupyter}" + inline val junit get() = "org.junit.jupiter:junit-jupiter-engine:5.8.1" + inline val sparkStreamingKafka get() = "org.apache.spark:spark-streaming-kafka-0-10_${Versions.scalaCompat}:${Versions.spark}" + inline val kotest get() = "io.kotest:kotest-runner-junit5:${Versions.kotest}" + inline val kotestTestcontainers get() = "io.kotest.extensions:kotest-extensions-testcontainers:${Versions.kotestTestContainers}" + inline val klaxon get() = "com.beust:klaxon:${Versions.klaxon}" + inline val atrium get() = "ch.tutteli.atrium:atrium-fluent-en_GB:${Versions.atrium}" + inline val kafkaStreamsTestUtils get() = "org.apache.kafka:kafka-streams-test-utils:${Versions.kafkaStreamsTestUtils}" + inline val jupyterTest get() = "org.jetbrains.kotlinx:kotlin-jupyter-test-kit:${Versions.jupyter}" + inline val kotlinTest get() = "org.jetbrains.kotlin:kotlin-test:${Versions.kotlin}" + inline val kotlinScriptingCommon get() = "org.jetbrains.kotlin:kotlin-scripting-common" + inline val kotlinScriptingJvm get() = "org.jetbrains.kotlin:kotlin-scripting-jvm" +} + + + + diff --git a/buildSrc/src/main/kotlin/Helpers.kt b/buildSrc/src/main/kotlin/Helpers.kt new file mode 100644 index 00000000..e62a8dae --- /dev/null +++ b/buildSrc/src/main/kotlin/Helpers.kt @@ -0,0 +1,37 @@ +import org.gradle.api.artifacts.Dependency +import org.gradle.api.artifacts.ProjectDependency +import org.gradle.api.artifacts.dsl.DependencyHandler + +fun DependencyHandler.testApi(vararg dependencyNotations: Any): List = + dependencyNotations.map { + add("testApi", it) + } + +fun DependencyHandler.api(vararg dependencyNotations: Any): List = + dependencyNotations.map { + add("api", it) + } + + +fun DependencyHandler.testImplementation(vararg dependencyNotations: Any): List = + dependencyNotations.map { + add("testImplementation", it) + } + +fun DependencyHandler.implementation(vararg dependencyNotations: Any): List = + dependencyNotations.map { + add("implementation", it) + } + +fun DependencyHandler.runtimeOnly(vararg dependencyNotations: Any): List = + dependencyNotations.map { + add("runtimeOnly", it) + } + +fun DependencyHandler.project( + path: String, + configuration: String? = null +): ProjectDependency = project( + if (configuration != null) mapOf("path" to path, "configuration" to configuration) + else mapOf("path" to path) +) as ProjectDependency diff --git a/buildSrc/src/main/kotlin/Plugins.kt b/buildSrc/src/main/kotlin/Plugins.kt new file mode 100644 index 00000000..5bf098b1 --- /dev/null +++ b/buildSrc/src/main/kotlin/Plugins.kt @@ -0,0 +1,36 @@ +import org.gradle.api.Project +import org.gradle.kotlin.dsl.* +import org.gradle.plugin.use.PluginDependenciesSpec + + +inline val PluginDependenciesSpec.kotlin + get() = kotlin("jvm") version Versions.kotlin + +inline val PluginDependenciesSpec.dokka + get() = id("org.jetbrains.dokka") + +inline val PluginDependenciesSpec.license + get() = id("com.github.hierynomus.license") version Versions.licenseGradlePluginVersion + +inline val PluginDependenciesSpec.jcp + get() = id("com.igormaznitsa.jcp") + + +inline val DependencyHandlerScope.jcp + get() = "com.igormaznitsa:jcp:${Versions.jcp}" + +inline val DependencyHandlerScope.mavenPublish + get() = "com.vanniktech:gradle-maven-publish-plugin:${Versions.mavenPublish}" + +inline val PluginDependenciesSpec.mavenPublish + get() = id("com.vanniktech.maven.publish") + +inline val PluginDependenciesSpec.mavenPublishBase + get() = id("com.vanniktech.maven.publish.base") + +inline val Project.mavenPublishBase + get() = "com.vanniktech.maven.publish.base" + +inline val PluginDependenciesSpec.jupyter + get() = kotlin("jupyter.api") version Versions.jupyter + diff --git a/buildSrc/src/main/kotlin/Projects.kt b/buildSrc/src/main/kotlin/Projects.kt new file mode 100644 index 00000000..62578196 --- /dev/null +++ b/buildSrc/src/main/kotlin/Projects.kt @@ -0,0 +1,29 @@ +import org.gradle.api.Project +import org.gradle.api.artifacts.dsl.DependencyHandler +import org.gradle.kotlin.dsl.support.delegates.ProjectDelegate + +object Projects { + + inline fun Project.searchProject(name: String): Project = + rootProject + .childProjects + .filterKeys { name in it } + .entries + .singleOrNull() + ?.value ?: error("Project $name not found") + + inline val Project.kotlinSparkApi + get() = searchProject("kotlin-spark-api") + + inline val Project.core + get() = searchProject("core") + + inline val Project.examples + get() = searchProject("examples") + + inline val Project.jupyter + get() = searchProject("jupyter") + + inline val Project.scalaTuplesInKotlin + get() = searchProject("scala-tuples-in-kotlin") +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt new file mode 100644 index 00000000..c657a108 --- /dev/null +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -0,0 +1,34 @@ +object Versions { + const val project = "1.2.0-SNAPSHOT" + const val groupID = "org.jetbrains.kotlinx.spark" + const val kotlin = "1.7.10" + + inline val spark get() = System.getProperty("spark") as String + inline val scala get() = System.getProperty("scala") as String + inline val sparkMinor get() = spark.substringBeforeLast('.') + inline val scalaCompat get() = scala.substringBeforeLast('.') + + const val jupyter = "0.11.0-95" + const val kotest = "5.3.2" + const val kotestTestContainers = "1.3.3" + const val dokka = "1.7.0" + const val jcp = "7.0.5" + const val mavenPublish = "0.20.0" + const val atrium = "0.17.0" + const val kotestExtensionAllure = "1.1.0" + const val licenseGradlePluginVersion = "0.15.0" + const val kafkaStreamsTestUtils = "3.1.0" + const val hadoop = "3.3.1" + const val kotlinxHtml = "0.7.5" + const val klaxon = "5.5" + + inline val versionMap + get() = mapOf( + "kotlin" to kotlin, + "scala" to scala, + "scalaCompat" to scalaCompat, + "spark" to spark, + "sparkMinor" to sparkMinor, + ) + +} \ No newline at end of file diff --git a/core/3.2/pom_2.12.xml b/core/3.2/pom_2.12.xml deleted file mode 100644 index 807c2c81..00000000 --- a/core/3.2/pom_2.12.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - 4.0.0 - - Kotlin Spark API: Scala core for Spark 3.2+ (Scala 2.12) - Scala-Spark 3.2+ compatibility layer for Kotlin for Apache Spark - core-3.2_2.12 - - org.jetbrains.kotlinx.spark - kotlin-spark-api-parent-3.2_2.12 - 1.1.1-SNAPSHOT - ../../pom_2.12.xml - - - - - org.scala-lang - scala-library - ${scala.version} - - - org.jetbrains.kotlin - kotlin-reflect - - - - - - org.apache.spark - spark-sql_${scala.compat.version} - ${spark3.version} - provided - - - - - - src/main/scala - src/test/scala - target/${scala.compat.version} - - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - compile - - compile - testCompile - - - - -dependencyfile - ${project.build.directory}/.scala_dependencies - - - - - docjar - - doc-jar - - pre-integration-test - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - - - - - - diff --git a/core/build.gradle.kts b/core/build.gradle.kts new file mode 100644 index 00000000..42f2c247 --- /dev/null +++ b/core/build.gradle.kts @@ -0,0 +1,75 @@ +@file:Suppress("UnstableApiUsage") + +import com.igormaznitsa.jcp.gradle.JcpTask +import com.vanniktech.maven.publish.JavaLibrary +import com.vanniktech.maven.publish.JavadocJar +import com.vanniktech.maven.publish.JavadocJar.Javadoc + +plugins { + scala + `java-library` + jcp + mavenPublishBase +} + +group = Versions.groupID +version = Versions.project + +repositories { + mavenCentral() +} + +dependencies { + + with(Dependencies) { + api( + scalaLibrary, + reflect, + ) + + implementation( + sparkSql, + ) + } +} + +val scalaMainSources = sourceSets.main.get().scala.sourceDirectories + +val preprocessMain by tasks.creating(JcpTask::class) { + sources.set(scalaMainSources) + clearTarget.set(true) + fileExtensions.set(listOf("scala")) + vars.set(Versions.versionMap) + outputs.upToDateWhen { target.get().exists() } +} + +tasks.compileScala { + dependsOn(preprocessMain) + outputs.upToDateWhen { + preprocessMain.outcomingFiles.files.isEmpty() + } + doFirst { + scala { + sourceSets { + main { + scala.setSrcDirs(listOf(preprocessMain.target.get())) + } + } + } + } + + doLast { + scala { + sourceSets { + main { + scala.setSrcDirs(scalaMainSources) + } + } + } + } +} + +mavenPublishing { + configure(JavaLibrary(Javadoc())) +} + diff --git a/core/3.2/src/main/scala/org/apache/spark/sql/KotlinReflection.scala b/core/src/main/scala/org/apache/spark/sql/KotlinReflection.scala similarity index 93% rename from core/3.2/src/main/scala/org/apache/spark/sql/KotlinReflection.scala rename to core/src/main/scala/org/apache/spark/sql/KotlinReflection.scala index cbc30be3..f11012ef 100644 --- a/core/3.2/src/main/scala/org/apache/spark/sql/KotlinReflection.scala +++ b/core/src/main/scala/org/apache/spark/sql/KotlinReflection.scala @@ -20,9 +20,10 @@ package org.apache.spark.sql +import org.apache.commons.lang3.reflect.ConstructorUtils import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.DeserializerBuildHelper._ -import org.apache.spark.sql.catalyst.ScalaReflection.{Schema, dataTypeFor, getClassFromType, isSubtype, javaBoxedType, localTypeOf} +import org.apache.spark.sql.catalyst.ScalaReflection.{Schema, dataTypeFor, getClassFromType, isSubtype, javaBoxedType, localTypeOf, mirror, universe} import org.apache.spark.sql.catalyst.SerializerBuildHelper._ import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal import org.apache.spark.sql.catalyst.expressions.objects._ @@ -120,7 +121,7 @@ object KotlinReflection extends KotlinReflection { val className = getClassNameFromType(tpe) className match { case "scala.Array" => { - val TypeRef(_, _, Seq(elementType)) = tpe + val TypeRef(_, _, Seq(elementType)) = tpe.dealias arrayClassFor(elementType) } case _ => { @@ -290,13 +291,25 @@ object KotlinReflection extends KotlinReflection { createDeserializerForSqlTimestamp(path) } case t if isSubtype(t, localTypeOf[java.time.LocalDateTime]) => { + //#if sparkMinor >= 3.2 createDeserializerForLocalDateTime(path) + //#else + //$throw new IllegalArgumentException("TimestampNTZType is supported in spark 3.2+") + //#endif } case t if isSubtype(t, localTypeOf[java.time.Duration]) => { + //#if sparkMinor >= 3.2 createDeserializerForDuration(path) + //#else + //$throw new IllegalArgumentException("java.time.Duration is supported in spark 3.2+") + //#endif } case t if isSubtype(t, localTypeOf[java.time.Period]) => { + //#if sparkMinor >= 3.2 createDeserializerForPeriod(path) + //#else + //$throw new IllegalArgumentException("java.time.Period is supported in spark 3.2+") + //#endif } case t if isSubtype(t, localTypeOf[java.lang.String]) => { createDeserializerForString(path, returnNullable = false) @@ -828,7 +841,11 @@ object KotlinReflection extends KotlinReflection { createSerializerForSqlTimestamp(inputObject) } case t if isSubtype(t, localTypeOf[java.time.LocalDateTime]) => { + //#if sparkMinor >= 3.2 createSerializerForLocalDateTime(inputObject) + //#else + //$throw new IllegalArgumentException("TimestampNTZType is supported in spark 3.2+") + //#endif } case t if isSubtype(t, localTypeOf[java.time.LocalDate]) => { createSerializerForJavaLocalDate(inputObject) @@ -837,10 +854,18 @@ object KotlinReflection extends KotlinReflection { createSerializerForSqlDate(inputObject) } case t if isSubtype(t, localTypeOf[java.time.Duration]) => { + //#if sparkMinor >= 3.2 createSerializerForJavaDuration(inputObject) + //#else + //$throw new IllegalArgumentException("java.time.Duration is supported in spark 3.2+") + //#endif } case t if isSubtype(t, localTypeOf[java.time.Period]) => { + //#if sparkMinor >= 3.2 createSerializerForJavaPeriod(inputObject) + //#else + //$throw new IllegalArgumentException("java.time.Period is supported in spark 3.2+") + //#endif } case t if isSubtype(t, localTypeOf[BigDecimal]) => { createSerializerForScalaBigDecimal(inputObject) @@ -1178,7 +1203,11 @@ object KotlinReflection extends KotlinReflection { } // SPARK-36227: Remove TimestampNTZ type support in Spark 3.2 with minimal code changes. case t if isSubtype(t, localTypeOf[java.time.LocalDateTime]) && Utils.isTesting => { + //#if sparkMinor >= 3.2 Schema(TimestampNTZType, nullable = true) + //#else + //$throw new IllegalArgumentException("java.time.LocalDateTime is supported in Spark 3.2+") + //#endif } case t if isSubtype(t, localTypeOf[java.time.LocalDate]) => { Schema(DateType, nullable = true) @@ -1190,10 +1219,18 @@ object KotlinReflection extends KotlinReflection { Schema(CalendarIntervalType, nullable = true) } case t if isSubtype(t, localTypeOf[java.time.Duration]) => { + //#if sparkMinor >= 3.2 Schema(DayTimeIntervalType(), nullable = true) + //#else + //$throw new IllegalArgumentException("DayTimeIntervalType for java.time.Duration is supported in Spark 3.2+") + //#endif } case t if isSubtype(t, localTypeOf[java.time.Period]) => { + //#if sparkMinor >= 3.2 Schema(YearMonthIntervalType(), nullable = true) + //#else + //$throw new IllegalArgumentException("YearMonthIntervalType for java.time.Period is supported in Spark 3.2+") + //#endif } case t if isSubtype(t, localTypeOf[BigDecimal]) => { Schema(DecimalType.SYSTEM_DEFAULT, nullable = true) @@ -1241,6 +1278,40 @@ object KotlinReflection extends KotlinReflection { } } + /** + * Finds an accessible constructor with compatible parameters. This is a more flexible search than + * the exact matching algorithm in `Class.getConstructor`. The first assignment-compatible + * matching constructor is returned if it exists. Otherwise, we check for additional compatible + * constructors defined in the companion object as `apply` methods. Otherwise, it returns `None`. + */ + def findConstructor[T](cls: Class[T], paramTypes: Seq[Class[_]]): Option[Seq[AnyRef] => T] = { + Option(ConstructorUtils.getMatchingAccessibleConstructor(cls, paramTypes: _*)) match { + case Some(c) => Some(x => c.newInstance(x: _*)) + case None => + val companion = mirror.staticClass(cls.getName).companion + val moduleMirror = mirror.reflectModule(companion.asModule) + val applyMethods = companion.asTerm.typeSignature + .member(universe.TermName("apply")).asTerm.alternatives + applyMethods.find { method => + val params = method.typeSignature.paramLists.head + // Check that the needed params are the same length and of matching types + params.size == paramTypes.tail.size && + params.zip(paramTypes.tail).forall { case(ps, pc) => + ps.typeSignature.typeSymbol == mirror.classSymbol(pc) + } + }.map { applyMethodSymbol => + val expectedArgsCount = applyMethodSymbol.typeSignature.paramLists.head.size + val instanceMirror = mirror.reflect(moduleMirror.instance) + val method = instanceMirror.reflectMethod(applyMethodSymbol.asMethod) + (_args: Seq[AnyRef]) => { + // Drop the "outer" argument if it is provided + val args = if (_args.size == expectedArgsCount) _args else _args.tail + method.apply(args: _*).asInstanceOf[T] + } + } + } + } + /** * Whether the fields of the given type is defined entirely by its constructor parameters. */ @@ -1268,8 +1339,10 @@ object KotlinReflection extends KotlinReflection { @scala.annotation.tailrec def javaBoxedType(dt: DataType): Class[_] = dt match { case _: DecimalType => classOf[Decimal] + //#if sparkMinor >= 3.2 case _: DayTimeIntervalType => classOf[java.lang.Long] case _: YearMonthIntervalType => classOf[java.lang.Integer] + //#endif case BinaryType => classOf[Array[Byte]] case StringType => classOf[UTF8String] case CalendarIntervalType => classOf[CalendarInterval] @@ -1325,6 +1398,15 @@ trait KotlinReflection extends Logging { tag.in(mirror).tpe.dealias } + private def isValueClass(tpe: Type): Boolean = { + tpe.typeSymbol.isClass && tpe.typeSymbol.asClass.isDerivedValueClass + } + + /** Returns the name and type of the underlying parameter of value class `tpe`. */ + private def getUnderlyingTypeOfValueClass(tpe: `Type`): Type = { + getConstructorParameters(tpe).head._2 + } + /** * Returns the full class name for a type. The returned name is the canonical * Scala name, where each component is separated by a period. It is NOT the @@ -1350,15 +1432,13 @@ trait KotlinReflection extends Logging { val formalTypeArgs = dealiasedTpe.typeSymbol.asClass.typeParams val TypeRef(_, _, actualTypeArgs) = dealiasedTpe val params = constructParams(dealiasedTpe) - // if there are type variables to fill in, do the substitution (SomeClass[T] -> SomeClass[Int]) - if (actualTypeArgs.nonEmpty) { - params.map { p => - p.name.decodedName.toString -> - p.typeSignature.substituteTypes(formalTypeArgs, actualTypeArgs) - } - } else { - params.map { p => - p.name.decodedName.toString -> p.typeSignature + params.map { p => + val paramTpe = p.typeSignature + if (isValueClass(paramTpe)) { + // Replace value class with underlying type + p.name.decodedName.toString -> getUnderlyingTypeOfValueClass(paramTpe) + } else { + p.name.decodedName.toString -> paramTpe.substituteTypes(formalTypeArgs, actualTypeArgs) } } } diff --git a/core/3.2/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala b/core/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala similarity index 99% rename from core/3.2/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala rename to core/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala index 9395019c..76da9016 100644 --- a/core/3.2/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala +++ b/core/src/main/scala/org/apache/spark/sql/KotlinWrappers.scala @@ -77,6 +77,9 @@ class KDataTypeWrapper( override private[ sql ] def getFieldIndex(name: String) = dt.getFieldIndex(name) + //#if sparkMinor < 3.2 + //$override + //#endif private[ sql ] def findNestedField(fieldNames: Seq[ String ], includeCollections: Boolean, resolver: Resolver) = dt.findNestedField(fieldNames, includeCollections, resolver) diff --git a/core/3.2/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala b/core/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala similarity index 100% rename from core/3.2/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala rename to core/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala diff --git a/core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/DemoCaseClass.scala b/core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/DemoCaseClass.scala similarity index 100% rename from core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/DemoCaseClass.scala rename to core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/DemoCaseClass.scala diff --git a/core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala b/core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala similarity index 83% rename from core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala rename to core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala index 6b4935f1..2092d83e 100644 --- a/core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala +++ b/core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala @@ -26,6 +26,13 @@ import java.util import scala.collection.JavaConverters object KSparkExtensions { + + val kotlinVersion = /*$"\""+kotlin+"\""$*/ /*-*/ "" + val scalaVersion = /*$"\""+scala+"\""$*/ /*-*/ "" + val scalaCompatVersion = /*$"\""+scalaCompat+"\""$*/ /*-*/ "" + val sparkVersion = /*$"\""+spark+"\""$*/ /*-*/ "" + val sparkMinorVersion = /*$"\""+sparkMinor+"\""$*/ /*-*/ "" + def col(d: Dataset[_], name: String): Column = d.col(name) def col(name: String): Column = functions.col(name) diff --git a/core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/VarargUnwrapper.scala b/core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/VarargUnwrapper.scala similarity index 100% rename from core/3.2/src/main/scala/org/jetbrains/kotlinx/spark/extensions/VarargUnwrapper.scala rename to core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/VarargUnwrapper.scala diff --git a/dummy/pom.xml b/dummy/pom.xml deleted file mode 100644 index 1614d4bd..00000000 --- a/dummy/pom.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - kotlin-spark-api-parent-3.2 - org.jetbrains.kotlinx.spark - 1.1.1-SNAPSHOT - - pom - 4.0.0 - Kotlin API for Apache Spark: not-needed - Module to workaround https://issues.sonatype.org/browse/NEXUS-9138 - dummy-3.2 - - - scala-2.12 - - - org.jetbrains.kotlinx.spark - examples-3.2_2.12 - ${project.parent.version} - - - - - - \ No newline at end of file diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts new file mode 100644 index 00000000..535cd17a --- /dev/null +++ b/examples/build.gradle.kts @@ -0,0 +1,30 @@ +plugins { + kotlin + idea +} + +group = Versions.groupID +version = Versions.project + +repositories { + mavenCentral() +} + +dependencies { + + with(Projects) { + implementation( + kotlinSparkApi, + ) + } + + with(Dependencies) { + implementation( + sparkSql, + sparkMl, + sparkStreaming, + sparkStreamingKafka, + ) + + } +} diff --git a/examples/pom-3.2_2.12.xml b/examples/pom-3.2_2.12.xml deleted file mode 100644 index 92fba9f2..00000000 --- a/examples/pom-3.2_2.12.xml +++ /dev/null @@ -1,114 +0,0 @@ - - - - 4.0.0 - - Kotlin Spark API: Examples for Spark 3.2+ (Scala 2.12) - Example of usage - examples-3.2_2.12 - - org.jetbrains.kotlinx.spark - kotlin-spark-api-parent-3.2_2.12 - 1.1.1-SNAPSHOT - ../pom_2.12.xml - - - - - org.jetbrains.kotlinx.spark - kotlin-spark-api-3.2 - ${project.version} - - - org.apache.spark - spark-sql_${scala.compat.version} - ${spark3.version} - - - org.apache.spark - spark-streaming_${scala.compat.version} - ${spark3.version} - - - org.apache.spark - spark-mllib_${scala.compat.version} - ${spark3.version} - - - org.apache.spark - spark-streaming-kafka-0-10_${scala.compat.version} - ${spark3.version} - - - - - src/main/kotlin - src/test/kotlin - target/3.2/${scala.compat.version} - - - org.jetbrains.kotlin - kotlin-maven-plugin - - - compile - - compile - - - - test-compile - - test-compile - - - - - - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - - - jar-with-dependencies - - - - org.jetbrains.spark.api.examples.WordCountKt - - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - org.sonatype.plugins - nexus-staging-maven-plugin - - true - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 9 - 9 - - - - - diff --git a/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UDFs.kt b/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UDFs.kt index 39636a18..1a306a50 100644 --- a/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UDFs.kt +++ b/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UDFs.kt @@ -202,7 +202,8 @@ private fun strongTypingInDatasets() = withSpark { // We can thus provide TypedColumns instead of normal ones which the select function takes // advantage of! - // NOTE: In UDFs, iterables, lists, arrays and such need to be represented as WrappedArray + + // NOTE: In UDFs, iterables, lists, arrays and such need to be represented as Seq val toJson by udf { age: Int, name: String, pets: Seq -> """{ "age" : $age, "name" : "$name", "pets" : [${pets.asKotlinIterable().joinToString { "\"$it\"" }}] }""" } @@ -228,7 +229,7 @@ private fun strongTypingInDatasets() = withSpark { col<_, Int>("age"), col<_, String>("name"), col>("pets").asSeq(), -// or `col<_, WrappedArray>("pets")` if you want to be less strict +// or `col<_, Seq>("pets")` if you want to be less strict ) ).showDS(truncate = false) // +-------------------------------------------------------+ @@ -437,7 +438,7 @@ private fun varargUDFs() = withSpark { // As you can see, it just works :), up to 22 parameters! - // In fact, since UDFs don't support arrays (only scala's WrappedArray), any udf that contains just an array + // In fact, since UDFs don't support arrays (only scala's Seq), any udf that contains just an array // as parameter will become a vararg udf: udf.register("joinToString") { strings: Array -> strings.joinToString(separator = "-") } spark.sql("""SELECT joinToString("a", "hi there", "test"), joinToString(), joinToString("b", "c")""") diff --git a/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UdtRegistration.kt b/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UdtRegistration.kt index 0c468d91..19db2c41 100644 --- a/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UdtRegistration.kt +++ b/examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/UdtRegistration.kt @@ -17,11 +17,12 @@ * limitations under the License. * =LICENSEEND= */ -import org.apache.hadoop.shaded.com.google.common.base.MoreObjects + import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.GenericInternalRow import org.apache.spark.sql.types.* import org.apache.spark.unsafe.types.UTF8String +import org.glassfish.jersey.internal.guava.MoreObjects import org.jetbrains.kotlinx.spark.api.* import org.jetbrains.kotlinx.spark.api.tuples.tupleOf import java.io.Serializable diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 00000000..6a0c214a --- /dev/null +++ b/gradle.properties @@ -0,0 +1,16 @@ +kotlin.daemon.jvmargs=-Xmx10G +org.gradle.jvmargs=-Xmx4G -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 +mavenCentralUsername=dummy +mavenCentralPassword=dummy + +GROUP=org.jetbrains.kotlinx.spark + +# Controls the spark and scala version for the entire project +# can also be defined like ./gradlew -Pspark=X.X.X -Pscala=X.X.X build +spark=3.3.0 +scala=2.13.8 +#scala=2.12.15 +skipScalaTuplesInKotlin=false + +org.gradle.caching=true +org.gradle.parallel=false diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..41d9927a Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..41dfb879 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 00000000..1b6c7873 --- /dev/null +++ b/gradlew @@ -0,0 +1,234 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 00000000..107acd32 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/gradlew_all_versions b/gradlew_all_versions new file mode 100755 index 00000000..3fab16ca --- /dev/null +++ b/gradlew_all_versions @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Run like you would `./gradlew arguments` +# but now like `./gradlew_all_versions arguments`. + +DRY_RUN=false +SCALA2_12VERSION="2.12.15" +SCALA2_13VERSION="2.13.8" +SparkVersionsForBothScalaVersions=("3.3.0" "3.2.1" "3.2.0") +SparkVersionsForScala2_12=("3.1.3" "3.1.2" "3.1.1" "3.1.0" "3.0.3" "3.0.2" "3.0.1" "3.0.0") + +echo Running for "$(expr ${#SparkVersionsForBothScalaVersions[@]} \* 2 + ${#SparkVersionsForScala2_12[@]}) versions of the library." + +echo "Cleaning the project first." +if [ "$DRY_RUN" = false ]; then + ./gradlew clean +fi + +ARGS=("$@") +execute() { + echo "running ./gradlew -Pspark=$SPARK -Pscala=$SCALA -PskipScalaTuplesInKotlin=$SKIP_SCALA_TUPLES -PenforceCleanJCP=true ${ARGS[*]}" + if [ "$DRY_RUN" = false ]; then + ./gradlew -Pspark="$SPARK" -Pscala="$SCALA" -PskipScalaTuplesInKotlin="$SKIP_SCALA_TUPLES" "${ARGS[@]}" + fi +} + +SCALA="$SCALA2_12VERSION" +SKIP_SCALA_TUPLES=false +for spark in "${SparkVersionsForScala2_12[@]}"; do + SPARK="$spark" + execute + SKIP_SCALA_TUPLES=true +done + + +execute_for_both_scala_versions() { + for spark in "${SparkVersionsForBothScalaVersions[@]}"; do + SPARK="$spark" + execute + SKIP_SCALA_TUPLES=true + done +} +SCALA="$SCALA2_12VERSION" +execute_for_both_scala_versions + +SCALA="$SCALA2_13VERSION" +SKIP_SCALA_TUPLES=false +execute_for_both_scala_versions + + + diff --git a/jupyter/build.gradle.kts b/jupyter/build.gradle.kts new file mode 100644 index 00000000..229d6a34 --- /dev/null +++ b/jupyter/build.gradle.kts @@ -0,0 +1,149 @@ +@file:Suppress("UnstableApiUsage") + +import com.igormaznitsa.jcp.gradle.JcpTask +import com.vanniktech.maven.publish.JavadocJar.Dokka +import com.vanniktech.maven.publish.KotlinJvm +import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask + +plugins { + scala + kotlin + dokka + mavenPublishBase + jupyter + jcp +} + +group = Versions.groupID +version = Versions.project + +repositories { + mavenCentral() + maven(url = "https://maven.pkg.jetbrains.space/public/p/kotlinx-html/maven") + maven(url = "https://maven.pkg.jetbrains.space/kotlin/p/kotlin/dev") +} + +tasks.withType().configureEach { + useJUnitPlatform() +} + +tasks.processJupyterApiResources { + libraryProducers = listOf( + "org.jetbrains.kotlinx.spark.api.jupyter.SparkIntegration", + "org.jetbrains.kotlinx.spark.api.jupyter.SparkStreamingIntegration", + ) +} + +dependencies { + with(Projects) { + api( + kotlinSparkApi, + ) + } + + with(Dependencies) { + api( + kotlinxHtml, + sparkSql, + sparkRepl, + sparkStreaming, + hadoopClient, + ) + + implementation( + kotlinStdLib, + ) + + testImplementation( + kotest, + kotlinScriptingCommon, + kotlinScriptingJvm, + ) + + } +} + +// Setup preprocessing with JCP for main sources + +val kotlinMainSources = kotlin.sourceSets.main.get().kotlin.sourceDirectories + +val preprocessMain by tasks.creating(JcpTask::class) { + sources.set(kotlinMainSources) + clearTarget.set(true) + fileExtensions.set(listOf("kt")) + vars.set(Versions.versionMap) + outputs.upToDateWhen { target.get().exists() } +} + +tasks.compileKotlin { + dependsOn(preprocessMain) + outputs.upToDateWhen { preprocessMain.outcomingFiles.files.isEmpty() } + doFirst { + kotlin { + sourceSets { + main { + kotlin.setSrcDirs(listOf(preprocessMain.target.get())) + } + } + } + } + + doLast { + kotlin { + sourceSets { + main { + kotlin.setSrcDirs(kotlinMainSources) + } + } + } + } +} + +// Setup preprocessing with JCP for test sources + +val kotlinTestSources = kotlin.sourceSets.test.get().kotlin.sourceDirectories + +val preprocessTest by tasks.creating(JcpTask::class) { + sources.set(kotlinTestSources) + clearTarget.set(true) + fileExtensions.set(listOf("java", "kt")) + vars.set(Versions.versionMap) + outputs.upToDateWhen { target.get().exists() } +} + +tasks.compileTestKotlin { + dependsOn(preprocessTest) + outputs.upToDateWhen { preprocessTest.outcomingFiles.files.isEmpty() } + doFirst { + kotlin { + sourceSets { + test { + kotlin.setSrcDirs(listOf(preprocessTest.target.get())) + } + } + } + } + + doLast { + kotlin { + sourceSets { + test { + kotlin.setSrcDirs(kotlinTestSources) + } + } + } + } +} + +tasks.withType { + dokkaSourceSets { + create("jupyter") { + sourceRoot(preprocessMain.target.get()) + } + } +} + + +mavenPublishing { + configure(KotlinJvm(Dokka("dokkaHtml"))) +} \ No newline at end of file diff --git a/jupyter/pom.xml b/jupyter/pom.xml deleted file mode 100644 index cde0c15d..00000000 --- a/jupyter/pom.xml +++ /dev/null @@ -1,182 +0,0 @@ - - - - 4.0.0 - - Kotlin Spark API: Jupyter integration for Spark 3.2+ (Scala 2.12) - kotlin-spark-api-jupyter-3.2 - Jupyter integration - - org.jetbrains.kotlinx.spark - kotlin-spark-api-parent-3.2_2.12 - 1.1.1-SNAPSHOT - ../pom_2.12.xml - - jar - - - - 11 - 11 - - - - - kotlinx-html - kotlinx-html - https://maven.pkg.jetbrains.space/public/p/kotlinx-html/maven - - - kotlin - kotlin - https://maven.pkg.jetbrains.space/kotlin/p/kotlin/dev - - - - - - org.jetbrains.kotlinx.spark - kotlin-spark-api-3.2 - ${project.version} - - - org.jetbrains.kotlinx - kotlinx-html-jvm - ${kotlinx.html.version} - - - org.apache.spark - spark-sql_${scala.compat.version} - ${spark3.version} - - - org.apache.spark - spark-repl_${scala.compat.version} - ${spark3.version} - - - org.apache.spark - spark-streaming_${scala.compat.version} - ${spark3.version} - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - - - org.jetbrains.kotlinx - kotlin-jupyter-api - ${kotlin-jupyter-api.version} - - - - - io.kotest - kotest-runner-junit5-jvm - ${kotest.version} - test - - - org.jetbrains.kotlinx - kotlin-jupyter-test-kit - ${kotlin-jupyter-api.version} - test - - - - - src/main/kotlin - src/test/kotlin - target/${scala.compat.version} - - - org.jetbrains.dokka - dokka-maven-plugin - ${dokka.version} - - 8 - - - - dokka - - dokka - - pre-site - - - javadocjar - - javadocJar - - pre-integration-test - - - - - org.jetbrains.kotlin - kotlin-maven-plugin - - - compile - - compile - - - - test-compile - - test-compile - - - - - - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - - - jar-with-dependencies - - - - org.jetbrains.spark.api.examples.WordCountKt - - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - - - org.apache.maven.plugins - maven-deploy-plugin - - false - - - - org.sonatype.plugins - nexus-staging-maven-plugin - - false - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 9 - 9 - - - - - \ No newline at end of file diff --git a/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/Integration.kt b/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/Integration.kt index 19ddda50..db364e72 100644 --- a/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/Integration.kt +++ b/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/Integration.kt @@ -24,14 +24,16 @@ import org.apache.spark.rdd.RDD import org.apache.spark.sql.Dataset import org.jetbrains.kotlinx.jupyter.api.* import org.jetbrains.kotlinx.jupyter.api.libraries.JupyterIntegration +import org.jetbrains.kotlinx.spark.api.tuples.map +import org.jetbrains.kotlinx.spark.api.tuples.t import kotlin.reflect.typeOf abstract class Integration : JupyterIntegration() { - private val kotlinVersion = "1.6.21" - private val scalaCompatVersion = "2.12" - private val scalaVersion = "2.12.15" - private val spark3Version = "3.2.1" + private val kotlinVersion = /*$"\""+kotlin+"\""$*/ /*-*/ "" + private val scalaCompatVersion = /*$"\""+scalaCompat+"\""$*/ /*-*/ "" + private val scalaVersion = /*$"\""+scala+"\""$*/ /*-*/ "" + private val sparkVersion = /*$"\""+spark+"\""$*/ /*-*/ "" private val displayLimit = "DISPLAY_LIMIT" private val displayLimitDefault = 20 @@ -54,18 +56,18 @@ abstract class Integration : JupyterIntegration() { open fun Builder.onLoadedAlsoDo() = Unit open val dependencies: Array = arrayOf( - "org.apache.spark:spark-repl_$scalaCompatVersion:$spark3Version", + "org.apache.spark:spark-repl_$scalaCompatVersion:$sparkVersion", "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion", "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion", - "org.apache.spark:spark-sql_$scalaCompatVersion:$spark3Version", - "org.apache.spark:spark-streaming_$scalaCompatVersion:$spark3Version", - "org.apache.spark:spark-mllib_$scalaCompatVersion:$spark3Version", - "org.apache.spark:spark-sql_$scalaCompatVersion:$spark3Version", - "org.apache.spark:spark-graphx_$scalaCompatVersion:$spark3Version", - "org.apache.spark:spark-launcher_$scalaCompatVersion:$spark3Version", - "org.apache.spark:spark-catalyst_$scalaCompatVersion:$spark3Version", - "org.apache.spark:spark-streaming_$scalaCompatVersion:$spark3Version", - "org.apache.spark:spark-core_$scalaCompatVersion:$spark3Version", + "org.apache.spark:spark-sql_$scalaCompatVersion:$sparkVersion", + "org.apache.spark:spark-streaming_$scalaCompatVersion:$sparkVersion", + "org.apache.spark:spark-mllib_$scalaCompatVersion:$sparkVersion", + "org.apache.spark:spark-sql_$scalaCompatVersion:$sparkVersion", + "org.apache.spark:spark-graphx_$scalaCompatVersion:$sparkVersion", + "org.apache.spark:spark-launcher_$scalaCompatVersion:$sparkVersion", + "org.apache.spark:spark-catalyst_$scalaCompatVersion:$sparkVersion", + "org.apache.spark:spark-streaming_$scalaCompatVersion:$sparkVersion", + "org.apache.spark:spark-core_$scalaCompatVersion:$sparkVersion", "org.scala-lang:scala-library:$scalaVersion", "org.scala-lang.modules:scala-xml_$scalaCompatVersion:2.0.1", "org.scala-lang:scala-reflect:$scalaVersion", @@ -113,7 +115,10 @@ abstract class Integration : JupyterIntegration() { } beforeCellExecution { - execute("""scala.Console.setOut(System.out)""") + if (scalaCompatVersion.toDouble() >= 2.13) + execute("scala.`Console\$`.`MODULE\$`.setOutDirect(System.out)") + else + execute("""scala.Console.setOut(System.out)""") beforeCellExecution() } diff --git a/jupyter/src/main/resources/META-INF/kotlin-jupyter-libraries/libraries.json b/jupyter/src/main/resources/META-INF/kotlin-jupyter-libraries/libraries.json deleted file mode 100644 index 82c7354e..00000000 --- a/jupyter/src/main/resources/META-INF/kotlin-jupyter-libraries/libraries.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "definitions": [], - "producers": [ - { - "fqn": "org.jetbrains.kotlinx.spark.api.jupyter.SparkIntegration" - }, - { - "fqn": "org.jetbrains.kotlinx.spark.api.jupyter.SparkStreamingIntegration" - } - ] -} diff --git a/kotlin-spark-api/3.2/pom_2.12.xml b/kotlin-spark-api/3.2/pom_2.12.xml deleted file mode 100644 index 6bd490ab..00000000 --- a/kotlin-spark-api/3.2/pom_2.12.xml +++ /dev/null @@ -1,185 +0,0 @@ - - - - 4.0.0 - - Kotlin Spark API: API for Spark 3.2+ (Scala 2.12) - kotlin-spark-api-3.2 - Kotlin API compatible with spark 3.2.0 Kotlin for Apache Spark - - org.jetbrains.kotlinx.spark - kotlin-spark-api-parent-3.2_2.12 - 1.1.1-SNAPSHOT - ../../pom_2.12.xml - - jar - - - - - org.jetbrains.kotlin - kotlin-stdlib-jdk8 - - - org.jetbrains.kotlin - kotlin-reflect - - - org.jetbrains.kotlinx.spark - core-3.2_${scala.compat.version} - - - org.jetbrains.kotlinx.spark - scala-tuples-in-kotlin-3.2 - - - - - org.apache.spark - spark-sql_${scala.compat.version} - ${spark3.version} - provided - - - org.apache.spark - spark-streaming_${scala.compat.version} - ${spark3.version} - provided - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - provided - - - - - org.apache.spark - spark-streaming-kafka-0-10_${scala.compat.version} - ${spark3.version} - test - - - org.apache.spark - spark-mllib_${scala.compat.version} - ${spark3.version} - test - - - io.kotest - kotest-runner-junit5-jvm - ${kotest.version} - test - - - io.kotest.extensions - kotest-extensions-testcontainers - ${kotest-extensions-testcontainers.version} - test - - - io.github.embeddedkafka - embedded-kafka_${scala.compat.version} - ${embedded-kafka.version} - test - - - com.beust - klaxon - ${klaxon.version} - test - - - ch.tutteli.atrium - atrium-fluent-en_GB - ${atrium.version} - test - - - org.apache.spark - spark-streaming_${scala.compat.version} - ${spark3.version} - tests - test - - - org.apache.kafka - kafka-streams-test-utils - 3.1.0 - test - - - - - src/main/kotlin - src/test/kotlin - target/${scala.compat.version} - - - - org.jetbrains.kotlin - kotlin-maven-plugin - - - compile - - compile - - - - test-compile - - test-compile - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - org.jetbrains.dokka - dokka-maven-plugin - ${dokka.version} - - 8 - - - - dokka - - dokka - - pre-site - - - javadocjar - - javadocJar - - pre-integration-test - - - - - - org.jacoco - jacoco-maven-plugin - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 8 - 8 - - - - - - diff --git a/kotlin-spark-api/build.gradle.kts b/kotlin-spark-api/build.gradle.kts new file mode 100644 index 00000000..5b947d75 --- /dev/null +++ b/kotlin-spark-api/build.gradle.kts @@ -0,0 +1,152 @@ +@file:Suppress("UnstableApiUsage") + +import com.igormaznitsa.jcp.gradle.JcpTask +import com.vanniktech.maven.publish.JavadocJar.Dokka +import com.vanniktech.maven.publish.KotlinJvm +import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask + +plugins { + kotlin + dokka + mavenPublishBase + jcp + idea +} + +group = Versions.groupID +version = Versions.project + + +repositories { + mavenCentral() +} + +tasks.withType().configureEach { + useJUnitPlatform() +} + +dependencies { + + with(Projects) { + api( + core, + scalaTuplesInKotlin, + ) + } + + with(Dependencies) { + implementation( + kotlinStdLib, + reflect, + sparkSql, + sparkStreaming, + hadoopClient, + ) + + testImplementation( + sparkStreamingKafka, + kotest, + kotestTestcontainers, + klaxon, + atrium, + sparkStreaming, + kafkaStreamsTestUtils, + sparkMl, + ) + } +} + +// Setup preprocessing with JCP for main sources + +val kotlinMainSources = kotlin.sourceSets.main.get().kotlin.sourceDirectories + +val preprocessMain by tasks.creating(JcpTask::class) { + sources.set(kotlinMainSources) + clearTarget.set(true) + fileExtensions.set(listOf("kt")) + vars.set(Versions.versionMap) + outputs.upToDateWhen { target.get().exists() } +} + +tasks.compileKotlin { + dependsOn(preprocessMain) + outputs.upToDateWhen { + preprocessMain.outcomingFiles.files.isEmpty() + } + + doFirst { + kotlin { + sourceSets { + main { + kotlin.setSrcDirs(listOf(preprocessMain.target.get())) + } + } + } + } + + doLast { + kotlin { + sourceSets { + main { + kotlin.setSrcDirs(kotlinMainSources) + } + } + } + } +} + +// Setup preprocessing with JCP for test sources + +val kotlinTestSources = kotlin.sourceSets.test.get().kotlin.sourceDirectories + +val preprocessTest by tasks.creating(JcpTask::class) { + sources.set(kotlinTestSources) + clearTarget.set(true) + fileExtensions.set(listOf("kt")) + vars.set(Versions.versionMap) + outputs.upToDateWhen { target.get().exists() } +} + +tasks.compileTestKotlin { + dependsOn(preprocessTest) + outputs.upToDateWhen { + preprocessTest.outcomingFiles.files.isEmpty() + } + + doFirst { + kotlin { + sourceSets { + test { + kotlin.setSrcDirs(listOf(preprocessTest.target.get())) + } + } + } + } + + doLast { + kotlin { + sourceSets { + test { + kotlin.setSrcDirs(kotlinTestSources) + } + } + } + } +} + +tasks.withType { + dokkaSourceSets { + create("kotlin-spark-api") { + sourceRoot(preprocessMain.target.get()) + } + } +} + +mavenPublishing { + configure(KotlinJvm(Dokka("dokkaHtml"))) +} + + + + + diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Arities.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Arities.kt similarity index 100% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Arities.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Arities.kt diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Column.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Column.kt similarity index 100% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Column.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Column.kt diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Conversions.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Conversions.kt similarity index 100% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Conversions.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Conversions.kt diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/DataStreamWriter.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/DataStreamWriter.kt similarity index 100% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/DataStreamWriter.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/DataStreamWriter.kt diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt similarity index 99% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt index d44c62dd..fe936e58 100644 --- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt +++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt @@ -371,11 +371,13 @@ fun Dataset>.sortByKey(): Dataset> = sort fun Dataset>.sortByValue(): Dataset> = sort("_2") /** Returns a dataset sorted by the first (`_1`) value of each [Arity2] inside. */ +@Suppress("DEPRECATION") @Deprecated("Use Scala tuples instead.", ReplaceWith("")) @JvmName("sortByArity2Key") fun Dataset>.sortByKey(): Dataset> = sort("_1") /** Returns a dataset sorted by the second (`_2`) value of each [Arity2] inside. */ +@Suppress("DEPRECATION") @Deprecated("Use Scala tuples instead.", ReplaceWith("")) @JvmName("sortByArity2Value") fun Dataset>.sortByValue(): Dataset> = sort("_2") diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt similarity index 96% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt index 8f2e2159..b37674c8 100644 --- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt +++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt @@ -41,10 +41,7 @@ import java.beans.PropertyDescriptor import java.math.BigDecimal import java.sql.Date import java.sql.Timestamp -import java.time.Duration -import java.time.Instant -import java.time.LocalDate -import java.time.Period +import java.time.* import java.util.* import java.util.concurrent.ConcurrentHashMap import kotlin.Any @@ -88,14 +85,24 @@ val ENCODERS: Map, Encoder<*>> = mapOf( String::class to STRING(), BigDecimal::class to DECIMAL(), Date::class to DATE(), - LocalDate::class to LOCALDATE(), // 3.0+ + LocalDate::class to LOCALDATE(), Timestamp::class to TIMESTAMP(), - Instant::class to INSTANT(), // 3.0+ + Instant::class to INSTANT(), ByteArray::class to BINARY(), - Duration::class to DURATION(), // 3.2+ - Period::class to PERIOD(), // 3.2+ + //#if sparkMinor >= 3.2 + Duration::class to DURATION(), + Period::class to PERIOD(), + //#endif ) +private fun checkIfEncoderRequiresNewerVersion(kClass: KClass<*>) { + when (kClass) { + //#if sparkMinor < 3.2 + //$Duration::class, Period::class -> throw IllegalArgumentException("$kClass is supported in Spark 3.2+") + //#endif + } +} + private val knownDataTypes: Map, DataType> = mapOf( Byte::class to DataTypes.ByteType, Short::class to DataTypes.ShortType, @@ -131,11 +138,13 @@ inline fun encoder(): Encoder = generateEncoder(typeOf(), T::c * @see encoder */ @Suppress("UNCHECKED_CAST") -fun generateEncoder(type: KType, cls: KClass<*>): Encoder = - when { +fun generateEncoder(type: KType, cls: KClass<*>): Encoder { + checkIfEncoderRequiresNewerVersion(cls) + return when { isSupportedByKotlinClassEncoder(cls) -> kotlinClassEncoder(schema = memoizedSchema(type), kClass = cls) else -> ENCODERS[cls] as? Encoder? ?: bean(cls.java) } as Encoder +} private fun isSupportedByKotlinClassEncoder(cls: KClass<*>): Boolean = when { diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/GroupState.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/GroupState.kt similarity index 100% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/GroupState.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/GroupState.kt diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Iterators.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Iterators.kt similarity index 100% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Iterators.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Iterators.kt diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/KeyValueGroupedDataset.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/KeyValueGroupedDataset.kt similarity index 100% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/KeyValueGroupedDataset.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/KeyValueGroupedDataset.kt diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt similarity index 99% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt index 94971f43..27a7294c 100644 --- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt +++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/SparkSession.kt @@ -54,6 +54,8 @@ import java.io.Serializable */ class KSparkSession(val spark: SparkSession) { + val sparkVersion = /*$"\""+spark+"\""$*/ /*-*/ "nope" + /** Lazy instance of [JavaSparkContext] wrapper around [sparkContext]. */ val sc: JavaSparkContext by lazy { JavaSparkContext(spark.sparkContext) } diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/StreamingKeyValues.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/StreamingKeyValues.kt similarity index 100% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/StreamingKeyValues.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/StreamingKeyValues.kt diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt similarity index 99% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt index ccbdc01b..bd08d92c 100644 --- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt +++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UDFRegister.kt @@ -22,7 +22,7 @@ * This file contains functions to register UDFs easily from Kotlin. */ -@file:Suppress("DuplicatedCode") +@file:Suppress("DuplicatedCode", "DEPRECATION") package org.jetbrains.kotlinx.spark.api diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedAggregateFunction.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedAggregateFunction.kt similarity index 100% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedAggregateFunction.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedAggregateFunction.kt diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt similarity index 97% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt index d9013334..fe5279e2 100644 --- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt +++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunction.kt @@ -24,7 +24,6 @@ package org.jetbrains.kotlinx.spark.api import org.apache.spark.sql.* import org.apache.spark.sql.types.DataType import scala.collection.Seq -import scala.collection.mutable.WrappedArray import kotlin.reflect.KClass import kotlin.reflect.KProperty import kotlin.reflect.full.isSubclassOf @@ -43,7 +42,11 @@ fun DataType.unWrap(): DataType = */ @PublishedApi internal fun KClass<*>.checkForValidType(parameterName: String) { - if (this == String::class || isSubclassOf(WrappedArray::class) || isSubclassOf(Seq::class)) + if (this == String::class || isSubclassOf(Seq::class) + //#if scalaCompat < 2.13 + //$|| isSubclassOf(scala.collection.mutable.WrappedArray::class) + //#endif + ) return // Most of the time we need strings or WrappedArrays/Seqs if (isSubclassOf(Iterable::class) diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt similarity index 99% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt index cf0c066e..e23aa160 100644 --- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt +++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctionVararg.kt @@ -17,6 +17,8 @@ * limitations under the License. * =LICENSEEND= */ +@file:Suppress("DEPRECATION", "unused", "UNCHECKED_CAST") + package org.jetbrains.kotlinx.spark.api diff --git a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt similarity index 99% rename from kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt rename to kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt index 9b006e35..c5fa749a 100644 --- a/kotlin-spark-api/3.2/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt +++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/UserDefinedFunctions.kt @@ -17,6 +17,7 @@ * limitations under the License. * =LICENSEEND= */ +@file:Suppress("DEPRECATION", "UNCHECKED_CAST", "unused") package org.jetbrains.kotlinx.spark.api import org.apache.spark.sql.* diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt similarity index 97% rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt index e95a65f7..c3362e91 100644 --- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt +++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ApiTest.kt @@ -53,7 +53,7 @@ class ApiTest : ShouldSpec({ } .collectAsList() - expect(result).contains.inOrder.only.values(3.0, 5.0, 7.0, 9.0, 11.0) + expect(result).toContain.inOrder.only.values(3.0, 5.0, 7.0, 9.0, 11.0) } should("Handle JavaConversions in Kotlin") { diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/DatasetFunctionTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/DatasetFunctionTest.kt similarity index 100% rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/DatasetFunctionTest.kt rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/DatasetFunctionTest.kt diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt similarity index 95% rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt index afb87435..1b7dec29 100644 --- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt +++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/EncodingTest.kt @@ -64,11 +64,14 @@ class EncodingTest : ShouldSpec({ dataset.collectAsList() shouldBe timeStamps } + //#if sparkMinor >= 3.2 should("handle Duration Datasets") { val dataset = dsOf(Duration.ZERO) dataset.collectAsList() shouldBe listOf(Duration.ZERO) } + //#endif + //#if sparkMinor >= 3.2 should("handle Period Datasets") { val periods = listOf(Period.ZERO, Period.ofDays(2)) val dataset = periods.toDS() @@ -82,6 +85,7 @@ class EncodingTest : ShouldSpec({ it[1] shouldBe Period.ofDays(0) } } + //#endif should("handle binary datasets") { val byteArray = "Hello there".encodeToByteArray() @@ -272,21 +276,21 @@ class EncodingTest : ShouldSpec({ val ll1 = LonLat(1.0, 2.0) val ll2 = LonLat(3.0, 4.0) val lonlats = dsOf(ll1, ll2).collectAsList() - expect(lonlats).contains.inAnyOrder.only.values(ll1.copy(), ll2.copy()) + expect(lonlats).toContain.inAnyOrder.only.values(ll1.copy(), ll2.copy()) } should("contain all generic primitives with complex schema") { val primitives = t(1, 1.0, 1.toFloat(), 1.toByte(), LocalDate.now(), true) val primitives2 = t(2, 2.0, 2.toFloat(), 2.toByte(), LocalDate.now().plusDays(1), false) val tuples = dsOf(primitives, primitives2).collectAsList() - expect(tuples).contains.inAnyOrder.only.values(primitives, primitives2) + expect(tuples).toContain.inAnyOrder.only.values(primitives, primitives2) } should("contain all generic primitives with complex nullable schema") { val primitives = t(1, 1.0, 1.toFloat(), 1.toByte(), LocalDate.now(), true) val nulls = t(null, null, null, null, null, null) val tuples = dsOf(primitives, nulls).collectAsList() - expect(tuples).contains.inAnyOrder.only.values(primitives, nulls) + expect(tuples).toContain.inAnyOrder.only.values(primitives, nulls) } should("Be able to serialize lists of data classes") { @@ -387,7 +391,7 @@ class EncodingTest : ShouldSpec({ .filterNotNull() .distinct() .collectAsList() - expect(result).contains.inOrder.only.value("y") + expect(result).toContain.inOrder.only.value("y") } should("work with lists of lists") { @@ -399,7 +403,7 @@ class EncodingTest : ShouldSpec({ .map { it.last() } .map { it.first() } .reduceK { a, b -> a + b } - expect(result).toBe(3) + expect(result).toEqual(3) } should("Generate schema correctly with nullalble list and map") { @@ -417,7 +421,7 @@ class EncodingTest : ShouldSpec({ .map { MovieExpanded(it.id, it.genres.split("|").toList()) } .filter { it.genres.contains("Comedy") } .collectAsList() - expect(comedies).contains.inAnyOrder.only.values( + expect(comedies).toContain.inAnyOrder.only.values( MovieExpanded( 1, listOf("Comedy", "Romance") @@ -449,7 +453,7 @@ class EncodingTest : ShouldSpec({ .filter { it.genres.contains("Comedy") } .collectAsList() - expect(comedies).contains.inAnyOrder.only.values( + expect(comedies).toContain.inAnyOrder.only.values( MovieExpanded( 1, arrayOf("Comedy", "Romance") @@ -465,7 +469,7 @@ class EncodingTest : ShouldSpec({ .map { it.id to it.data.firstOrNull { liEl -> liEl.first < 6 } } .map { it.second } .collectAsList() - expect(result).contains.inOrder.only.values(5.1 to 6) + expect(result).toContain.inOrder.only.values(5.1 to 6) } should("handle lists of generics") { @@ -476,7 +480,7 @@ class EncodingTest : ShouldSpec({ .map { it.id to it.data.firstOrNull { liEl -> liEl.first < 6 } } .map { it.second } .collectAsList() - expect(result).contains.inOrder.only.values(5.1 to 6) + expect(result).toContain.inOrder.only.values(5.1 to 6) } should("!handle primitive arrays") { @@ -485,7 +489,7 @@ class EncodingTest : ShouldSpec({ .map { it.map { ai -> ai + 1 } } .collectAsList() .flatten() - expect(result).contains.inOrder.only.values(2, 3, 4, 5) + expect(result).toContain.inOrder.only.values(2, 3, 4, 5) } } } diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/KafkaStreamingTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/KafkaStreamingTest.kt similarity index 100% rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/KafkaStreamingTest.kt rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/KafkaStreamingTest.kt diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ProjectConfig.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ProjectConfig.kt similarity index 100% rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ProjectConfig.kt rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/ProjectConfig.kt diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt similarity index 93% rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt index 9719e8fc..4cc8b9c8 100644 --- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt +++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/StreamingTest.kt @@ -36,6 +36,8 @@ import org.jetbrains.kotlinx.spark.api.tuples.X import org.jetbrains.kotlinx.spark.api.tuples.component1 import org.jetbrains.kotlinx.spark.api.tuples.component2 import org.jetbrains.kotlinx.spark.api.tuples.t +import org.jetbrains.kotlinx.spark.extensions.KSparkExtensions +import org.jetbrains.kotlinx.spark.extensions.`KSparkExtensions$` import scala.Tuple2 import java.io.File import java.io.Serializable @@ -199,13 +201,18 @@ class StreamingTest : ShouldSpec({ } }) -private fun createTempDir() = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark") - .apply { deleteOnExit() } + +private val scalaCompatVersion = `KSparkExtensions$`.`MODULE$`.scalaCompatVersion() +private val sparkVersion = `KSparkExtensions$`.`MODULE$`.sparkVersion() +private fun createTempDir() = Utils.createTempDir( + System.getProperty("java.io.tmpdir"), + "spark_${scalaCompatVersion}_${sparkVersion}" +).apply { deleteOnExit() } private fun createCorruptedCheckpoint(): String { val checkpointDirectory = createTempDir().absolutePath val fakeCheckpointFile = Checkpoint.checkpointFile(checkpointDirectory, Time(1000)) - FileUtils.write(File(fakeCheckpointFile.toString()), "blablabla", StandardCharsets.UTF_8) + FileUtils.write(File(fakeCheckpointFile.toString()), "spark_corrupt_${scalaCompatVersion}_${sparkVersion}", StandardCharsets.UTF_8) assert(Checkpoint.getCheckpointFiles(checkpointDirectory, (null as FileSystem?).toOption()).nonEmpty()) return checkpointDirectory } diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt similarity index 74% rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt index 2d481e79..1eb54ba4 100644 --- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt +++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/TypeInferenceTest.kt @@ -40,15 +40,19 @@ class TypeInferenceTest : ShouldSpec({ val struct = Struct.fromJson(schema(typeOf>>()).prettyJson())!! should("contain correct typings") { - expect(struct.fields).notToBeNull().contains.inAnyOrder.only.entries( + expect(struct.fields).notToEqualNull().toContain.inAnyOrder.only.entries( hasField("first", "string"), - hasStruct("second", + hasStruct( + "second", hasField("vala", "integer"), - hasStruct("tripl1", + hasStruct( + "tripl1", hasField("first", "integer"), - hasStruct("second", + hasStruct( + "second", hasField("vala2", "long"), - hasStruct("para2", + hasStruct( + "para2", hasField("first", "long"), hasField("second", "string") ) @@ -66,17 +70,22 @@ class TypeInferenceTest : ShouldSpec({ val struct = Struct.fromJson(schema(typeOf>>()).prettyJson())!! should("contain correct typings") { - expect(struct.fields).notToBeNull().contains.inAnyOrder.only.entries( + expect(struct.fields).notToEqualNull().toContain.inAnyOrder.only.entries( hasField("first", "string"), - hasStruct("second", + hasStruct( + "second", hasField("vala", "integer"), - hasStruct("tripl1", + hasStruct( + "tripl1", hasField("first", "integer"), - hasStruct("second", + hasStruct( + "second", hasField("vala2", "long"), - hasStruct("para2", + hasStruct( + "para2", hasField("first", "long"), - hasStruct("second", + hasStruct( + "second", hasField("vala3", "double") ) ) @@ -92,7 +101,7 @@ class TypeInferenceTest : ShouldSpec({ val struct = Struct.fromJson(schema(typeOf()).prettyJson())!! should("return correct types too") { - expect(struct.fields).notToBeNull().contains.inAnyOrder.only.entries( + expect(struct.fields).notToEqualNull().toContain.inAnyOrder.only.entries( hasField("a", "string"), hasField("b", "integer"), hasField("c", "double") @@ -104,7 +113,7 @@ class TypeInferenceTest : ShouldSpec({ should("return correct types too") { expect(struct) { isOfType("array") - feature { f(it::elementType) }.toBe(SimpleElement("integer")) + feature { f(it::elementType) }.toEqual(SimpleElement("integer")) } } } @@ -113,12 +122,12 @@ class TypeInferenceTest : ShouldSpec({ should("return correct types too") { expect(struct) { isOfType("array") - feature { f(it::elementType) }.notToBeNull().isA { - feature { f(it.value::fields) }.notToBeNull().contains.inAnyOrder.only.entries( + feature { f(it::elementType) }.notToEqualNull().toBeAnInstanceOf(fun Expect.() { + feature { f(it.value::fields) }.notToEqualNull().toContain.inAnyOrder.only.entries( hasField("first", "integer"), hasField("second", "long") ) - } + }) } } } @@ -129,11 +138,11 @@ class TypeInferenceTest : ShouldSpec({ should("return correct types too") { expect(struct) { isOfType("array") - feature { f(it::elementType) }.notToBeNull().isA { - feature { f(it.value::fields) }.notToBeNull().contains.inAnyOrder.only.entries( + feature { f(it::elementType) }.notToEqualNull().toBeAnInstanceOf(fun Expect.() { + feature { f(it.value::fields) }.notToEqualNull().toContain.inAnyOrder.only.entries( hasField("e", "string") ) - } + }) } } } @@ -142,9 +151,9 @@ class TypeInferenceTest : ShouldSpec({ should("return correct types too") { expect(struct) { isOfType("array") - feature { f(it::elementType) }.notToBeNull().isA { - feature { f(it.value::elementType) }.toBe(SimpleElement("integer")) - } + feature { f(it::elementType) }.notToEqualNull().toBeAnInstanceOf(fun Expect.() { + feature { f(it.value::elementType) }.toEqual(SimpleElement("integer")) + }) } } } @@ -153,8 +162,8 @@ class TypeInferenceTest : ShouldSpec({ should("return correct types too") { expect(struct) { isOfType("array") - feature { f(it::elementType) }.toBe(SimpleElement("integer")) - feature { f(it::containsNull) }.toBe(false) + feature { f(it::elementType) }.toEqual(SimpleElement("integer")) + feature { f(it::containsNull) }.toEqual(false) } } } @@ -163,8 +172,8 @@ class TypeInferenceTest : ShouldSpec({ should("return correct types too") { expect(struct) { isOfType("array") - feature { f(it::elementType) }.toBe(SimpleElement("integer")) - feature { f(it::containsNull) }.toBe(true) + feature { f(it::elementType) }.toEqual(SimpleElement("integer")) + feature { f(it::containsNull) }.toEqual(true) } } } @@ -173,7 +182,7 @@ class TypeInferenceTest : ShouldSpec({ val struct = Struct.fromJson(schema(typeOf()).prettyJson())!! should("Not change order of fields") { - expect(struct.fields).notToBeNull().containsExactly( + expect(struct.fields).notToEqualNull().containsExactly( hasField("lon", "double"), hasField("lat", "double") ) @@ -187,16 +196,16 @@ class TypeInferenceTest : ShouldSpec({ should("show that list is nullable and element is not") { expect(struct) .feature("some", { fields }) { - notToBeNull().contains.inOrder.only.entry { + notToEqualNull().toContain.inOrder.only.entry { this - .feature("field name", { name }) { toBe("optionList") } - .feature("optionList is nullable", { nullable }) { toBe(true) } + .feature("field name", { name }) { toEqual("optionList") } + .feature("optionList is nullable", { nullable }) { toEqual(true) } .feature("optionList", { type }) { this .isA() .feature("element type of optionList", - { value.elementType }) { toBe(SimpleElement("integer")) } - .feature("optionList contains null", { value.containsNull }) { toBe(false) } + { value.elementType }) { toEqual(SimpleElement("integer")) } + .feature("optionList contains null", { value.containsNull }) { toEqual(false) } .feature("optionList type", { value }) { isOfType("array") } } } @@ -207,16 +216,16 @@ class TypeInferenceTest : ShouldSpec({ expect(encoder().schema()) { this .feature("data type", { this.fields()?.toList() }) { - this.notToBeNull().contains.inOrder.only.entry { + this.notToEqualNull().toContain.inOrder.only.entry { this - .feature("element name", { name() }) { toBe("optionList") } + .feature("element name", { name() }) { toEqual("optionList") } .feature("field type", { dataType() }, { this .isA() .feature("element type", { elementType() }) { isA() } - .feature("element nullable", { containsNull() }) { toBe(expected = false) } + .feature("element nullable", { containsNull() }) { toEqual(expected = false) } }) - .feature("optionList nullable", { nullable() }) { toBe(true) } + .feature("optionList nullable", { nullable() }) { toEqual(true) } } } } @@ -226,7 +235,7 @@ class TypeInferenceTest : ShouldSpec({ }) private fun Expect.isOfType(type: String) { - feature { f(it::type) }.toBe(type) + feature { f(it::type) }.toEqual(type) } private fun hasStruct( @@ -235,15 +244,17 @@ private fun hasStruct( vararg expectedFields: Expect.() -> Unit, ): Expect.() -> Unit { return { - feature { f(it::name) }.toBe(name) - feature { f(it::type) }.isA { - feature { f(it.value::fields) }.notToBeNull().contains.inAnyOrder.only.entries(expectedField, - *expectedFields) - } + feature { f(it::name) }.toEqual(name) + feature { f(it::type) }.toBeAnInstanceOf(fun Expect.() { + feature { f(it.value::fields) }.notToEqualNull().toContain.inAnyOrder.only.entries( + expectedField, + *expectedFields + ) + }) } } private fun hasField(name: String, type: String): Expect.() -> Unit = { - feature { f(it::name) }.toBe(name) - feature { f(it::type) }.isA().feature { f(it::value) }.toBe(type) + feature { f(it::name) }.toEqual(name) + feature { f(it::type) }.isA().feature { f(it::value) }.toEqual(type) } diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt similarity index 97% rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt index a4848d40..ad142c00 100644 --- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt +++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt @@ -35,7 +35,6 @@ import org.apache.spark.sql.expressions.Aggregator import org.intellij.lang.annotations.Language import org.junit.jupiter.api.assertThrows import scala.collection.Seq -import scala.collection.mutable.WrappedArray import java.io.Serializable import kotlin.random.Random @@ -204,11 +203,13 @@ class UDFTest : ShouldSpec({ } } - should("succeed when using a WrappedArray") { - udf.register("shouldSucceed") { array: WrappedArray -> - array.asKotlinIterable().joinToString(" ") - } - } + //#if scalaCompat <= 2.12 + //$should("succeed when using a WrappedArray") { + //$ udf.register("shouldSucceed") { array: scala.collection.mutable.WrappedArray -> + //$ array.asKotlinIterable().joinToString(" ") + //$ } + //$} + //#endif should("succeed when using a Seq") { udf.register("shouldSucceed") { array: Seq -> @@ -241,25 +242,28 @@ class UDFTest : ShouldSpec({ context("calling the UDF-Wrapper") { withSpark(logLevel = SparkLogLevel.DEBUG) { - should("succeed in withColumn") { - - val stringArrayMerger = udf { it: WrappedArray -> - it.asKotlinIterable().joinToString(" ") - } - val testData = dsOf(arrayOf("a", "b")) - val newData = testData.withColumn( - "text", - stringArrayMerger( - testData.singleCol().asSeq() - ), - ) - - (newData.select("text").collectAsList() zip newData.select("value").collectAsList()) - .forEach { (text, textArray) -> - assert(text.getString(0) == textArray.getList(0).joinToString(" ")) - } - } + //#if scalaCompat <= 2.12 + //$should("succeed in withColumn with WrappedArray") { + //$ + //$ val stringArrayMerger = udf { it: scala.collection.mutable.WrappedArray -> + //$ it.asKotlinIterable().joinToString(" ") + //$ } + //$ + //$ val testData = dsOf(arrayOf("a", "b")) + //$ val newData = testData.withColumn( + //$ "text", + //$ stringArrayMerger( + //$ testData.singleCol().typed() + //$ ), + //$ ) + //$ + //$ (newData.select("text").collectAsList() zip newData.select("value").collectAsList()) + //$ .forEach { (text, textArray) -> + //$ assert(text.getString(0) == textArray.getList(0).joinToString(" ")) + //$ } + //$} + //#endif should("succeed in withColumn using Seq") { diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt similarity index 98% rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt index 411bee2c..a0edddeb 100644 --- a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt +++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UdtTest.kt @@ -21,7 +21,7 @@ package org.jetbrains.kotlinx.spark.api import io.kotest.core.spec.style.ShouldSpec import io.kotest.matchers.shouldBe -import org.apache.hadoop.shaded.com.google.common.base.MoreObjects +import org.glassfish.jersey.internal.guava.MoreObjects import org.apache.spark.ml.linalg.* import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.GenericInternalRow diff --git a/kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/struct/model/models.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/struct/model/models.kt similarity index 100% rename from kotlin-spark-api/3.2/src/test/kotlin/org/jetbrains/kotlinx/spark/api/struct/model/models.kt rename to kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/struct/model/models.kt diff --git a/mvnw b/mvnw deleted file mode 100755 index 41c0f0c2..00000000 --- a/mvnw +++ /dev/null @@ -1,310 +0,0 @@ -#!/bin/sh -# ---------------------------------------------------------------------------- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---------------------------------------------------------------------------- - -# ---------------------------------------------------------------------------- -# Maven Start Up Batch script -# -# Required ENV vars: -# ------------------ -# JAVA_HOME - location of a JDK home dir -# -# Optional ENV vars -# ----------------- -# M2_HOME - location of maven2's installed home dir -# MAVEN_OPTS - parameters passed to the Java VM when running Maven -# e.g. to debug Maven itself, use -# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -# MAVEN_SKIP_RC - flag to disable loading of mavenrc files -# ---------------------------------------------------------------------------- - -if [ -z "$MAVEN_SKIP_RC" ] ; then - - if [ -f /etc/mavenrc ] ; then - . /etc/mavenrc - fi - - if [ -f "$HOME/.mavenrc" ] ; then - . "$HOME/.mavenrc" - fi - -fi - -# OS specific support. $var _must_ be set to either true or false. -cygwin=false; -darwin=false; -mingw=false -case "`uname`" in - CYGWIN*) cygwin=true ;; - MINGW*) mingw=true;; - Darwin*) darwin=true - # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home - # See https://developer.apple.com/library/mac/qa/qa1170/_index.html - if [ -z "$JAVA_HOME" ]; then - if [ -x "/usr/libexec/java_home" ]; then - export JAVA_HOME="`/usr/libexec/java_home`" - else - export JAVA_HOME="/Library/Java/Home" - fi - fi - ;; -esac - -if [ -z "$JAVA_HOME" ] ; then - if [ -r /etc/gentoo-release ] ; then - JAVA_HOME=`java-config --jre-home` - fi -fi - -if [ -z "$M2_HOME" ] ; then - ## resolve links - $0 may be a link to maven's home - PRG="$0" - - # need this for relative symlinks - while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG="`dirname "$PRG"`/$link" - fi - done - - saveddir=`pwd` - - M2_HOME=`dirname "$PRG"`/.. - - # make it fully qualified - M2_HOME=`cd "$M2_HOME" && pwd` - - cd "$saveddir" - # echo Using m2 at $M2_HOME -fi - -# For Cygwin, ensure paths are in UNIX format before anything is touched -if $cygwin ; then - [ -n "$M2_HOME" ] && - M2_HOME=`cygpath --unix "$M2_HOME"` - [ -n "$JAVA_HOME" ] && - JAVA_HOME=`cygpath --unix "$JAVA_HOME"` - [ -n "$CLASSPATH" ] && - CLASSPATH=`cygpath --path --unix "$CLASSPATH"` -fi - -# For Mingw, ensure paths are in UNIX format before anything is touched -if $mingw ; then - [ -n "$M2_HOME" ] && - M2_HOME="`(cd "$M2_HOME"; pwd)`" - [ -n "$JAVA_HOME" ] && - JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" -fi - -if [ -z "$JAVA_HOME" ]; then - javaExecutable="`which javac`" - if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=`which readlink` - if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then - if $darwin ; then - javaHome="`dirname \"$javaExecutable\"`" - javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" - else - javaExecutable="`readlink -f \"$javaExecutable\"`" - fi - javaHome="`dirname \"$javaExecutable\"`" - javaHome=`expr "$javaHome" : '\(.*\)/bin'` - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "$JAVACMD" ] ; then - if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - else - JAVACMD="`which java`" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." >&2 - echo " We cannot execute $JAVACMD" >&2 - exit 1 -fi - -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi - -CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher - -# traverses directory structure from process work directory to filesystem root -# first directory with .mvn subdirectory is considered project base directory -find_maven_basedir() { - - if [ -z "$1" ] - then - echo "Path not specified to find_maven_basedir" - return 1 - fi - - basedir="$1" - wdir="$1" - while [ "$wdir" != '/' ] ; do - if [ -d "$wdir"/.mvn ] ; then - basedir=$wdir - break - fi - # workaround for JBEAP-8937 (on Solaris 10/Sparc) - if [ -d "${wdir}" ]; then - wdir=`cd "$wdir/.."; pwd` - fi - # end of workaround - done - echo "${basedir}" -} - -# concatenates all lines of a file -concat_lines() { - if [ -f "$1" ]; then - echo "$(tr -s '\n' ' ' < "$1")" - fi -} - -BASE_DIR=`find_maven_basedir "$(pwd)"` -if [ -z "$BASE_DIR" ]; then - exit 1; -fi - -########################################################################################## -# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -# This allows using the maven wrapper in projects that prohibit checking in binary data. -########################################################################################## -if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found .mvn/wrapper/maven-wrapper.jar" - fi -else - if [ "$MVNW_VERBOSE" = true ]; then - echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." - fi - if [ -n "$MVNW_REPOURL" ]; then - jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" - else - jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" - fi - while IFS="=" read key value; do - case "$key" in (wrapperUrl) jarUrl="$value"; break ;; - esac - done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" - if [ "$MVNW_VERBOSE" = true ]; then - echo "Downloading from: $jarUrl" - fi - wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" - if $cygwin; then - wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` - fi - - if command -v wget > /dev/null; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found wget ... using wget" - fi - if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then - wget "$jarUrl" -O "$wrapperJarPath" - else - wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" - fi - elif command -v curl > /dev/null; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found curl ... using curl" - fi - if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then - curl -o "$wrapperJarPath" "$jarUrl" -f - else - curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f - fi - - else - if [ "$MVNW_VERBOSE" = true ]; then - echo "Falling back to using Java to download" - fi - javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" - # For Cygwin, switch paths to Windows format before running javac - if $cygwin; then - javaClass=`cygpath --path --windows "$javaClass"` - fi - if [ -e "$javaClass" ]; then - if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then - if [ "$MVNW_VERBOSE" = true ]; then - echo " - Compiling MavenWrapperDownloader.java ..." - fi - # Compiling the Java class - ("$JAVA_HOME/bin/javac" "$javaClass") - fi - if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then - # Running the downloader - if [ "$MVNW_VERBOSE" = true ]; then - echo " - Running MavenWrapperDownloader.java ..." - fi - ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") - fi - fi - fi -fi -########################################################################################## -# End of extension -########################################################################################## - -export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} -if [ "$MVNW_VERBOSE" = true ]; then - echo $MAVEN_PROJECTBASEDIR -fi -MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" - -# For Cygwin, switch paths to Windows format before running java -if $cygwin; then - [ -n "$M2_HOME" ] && - M2_HOME=`cygpath --path --windows "$M2_HOME"` - [ -n "$JAVA_HOME" ] && - JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` - [ -n "$CLASSPATH" ] && - CLASSPATH=`cygpath --path --windows "$CLASSPATH"` - [ -n "$MAVEN_PROJECTBASEDIR" ] && - MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` -fi - -# Provide a "standardized" way to retrieve the CLI args that will -# work with both Windows and non-Windows executions. -MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" -export MAVEN_CMD_LINE_ARGS - -WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -exec "$JAVACMD" \ - $MAVEN_OPTS \ - -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ - "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ - ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/mvnw.cmd b/mvnw.cmd deleted file mode 100644 index 86115719..00000000 --- a/mvnw.cmd +++ /dev/null @@ -1,182 +0,0 @@ -@REM ---------------------------------------------------------------------------- -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. -@REM ---------------------------------------------------------------------------- - -@REM ---------------------------------------------------------------------------- -@REM Maven Start Up Batch script -@REM -@REM Required ENV vars: -@REM JAVA_HOME - location of a JDK home dir -@REM -@REM Optional ENV vars -@REM M2_HOME - location of maven2's installed home dir -@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands -@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending -@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven -@REM e.g. to debug Maven itself, use -@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files -@REM ---------------------------------------------------------------------------- - -@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' -@echo off -@REM set title of command window -title %0 -@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' -@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% - -@REM set %HOME% to equivalent of $HOME -if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") - -@REM Execute a user defined script before this one -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre -@REM check for pre script, once with legacy .bat ending and once with .cmd ending -if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" -if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" -:skipRcPre - -@setlocal - -set ERROR_CODE=0 - -@REM To isolate internal variables from possible post scripts, we use another setlocal -@setlocal - -@REM ==== START VALIDATION ==== -if not "%JAVA_HOME%" == "" goto OkJHome - -echo. -echo Error: JAVA_HOME not found in your environment. >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -:OkJHome -if exist "%JAVA_HOME%\bin\java.exe" goto init - -echo. -echo Error: JAVA_HOME is set to an invalid directory. >&2 -echo JAVA_HOME = "%JAVA_HOME%" >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -@REM ==== END VALIDATION ==== - -:init - -@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". -@REM Fallback to current working directory if not found. - -set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% -IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir - -set EXEC_DIR=%CD% -set WDIR=%EXEC_DIR% -:findBaseDir -IF EXIST "%WDIR%"\.mvn goto baseDirFound -cd .. -IF "%WDIR%"=="%CD%" goto baseDirNotFound -set WDIR=%CD% -goto findBaseDir - -:baseDirFound -set MAVEN_PROJECTBASEDIR=%WDIR% -cd "%EXEC_DIR%" -goto endDetectBaseDir - -:baseDirNotFound -set MAVEN_PROJECTBASEDIR=%EXEC_DIR% -cd "%EXEC_DIR%" - -:endDetectBaseDir - -IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig - -@setlocal EnableExtensions EnableDelayedExpansion -for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a -@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% - -:endReadAdditionalConfig - -SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" -set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" -set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" - -FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( - IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B -) - -@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -@REM This allows using the maven wrapper in projects that prohibit checking in binary data. -if exist %WRAPPER_JAR% ( - if "%MVNW_VERBOSE%" == "true" ( - echo Found %WRAPPER_JAR% - ) -) else ( - if not "%MVNW_REPOURL%" == "" ( - SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" - ) - if "%MVNW_VERBOSE%" == "true" ( - echo Couldn't find %WRAPPER_JAR%, downloading it ... - echo Downloading from: %DOWNLOAD_URL% - ) - - powershell -Command "&{"^ - "$webclient = new-object System.Net.WebClient;"^ - "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ - "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ - "}"^ - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ - "}" - if "%MVNW_VERBOSE%" == "true" ( - echo Finished downloading %WRAPPER_JAR% - ) -) -@REM End of extension - -@REM Provide a "standardized" way to retrieve the CLI args that will -@REM work with both Windows and non-Windows executions. -set MAVEN_CMD_LINE_ARGS=%* - -%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* -if ERRORLEVEL 1 goto error -goto end - -:error -set ERROR_CODE=1 - -:end -@endlocal & set ERROR_CODE=%ERROR_CODE% - -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost -@REM check for post script, once with legacy .bat ending and once with .cmd ending -if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" -if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" -:skipRcPost - -@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' -if "%MAVEN_BATCH_PAUSE%" == "on" pause - -if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% - -exit /B %ERROR_CODE% diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 6e08e4d2..00000000 --- a/pom.xml +++ /dev/null @@ -1,337 +0,0 @@ - - - 4.0.0 - - Kotlin Spark API: Parent for Spark 3.2+ - Parent project for Kotlin for Apache Spark - org.jetbrains.kotlinx.spark - kotlin-spark-api-parent-3.2 - 1.1.1-SNAPSHOT - pom - - - 0.17.0 - 1.6.10 - 3.1.0 - 3.3.1 - 1.3.1 - 5.3.1 - 0.11.0-95 - 1.7.0 - 0.7.5 - 3.2.1 - - - 0.8.7 - 5.5 - official - 2.0.0 - 3.3.0 - 3.10.1 - 3.0.0-M1 - 3.0.0-M3 - 3.0.1 - 3.2.0 - 3.9.1 - 3.2.1 - 3.0.0-M6 - 1.6.8 - 4.5.6 - - - - dummy - - - - - - org.jetbrains.kotlin - kotlin-stdlib-jdk8 - ${kotlin.version} - - - org.jetbrains.kotlin - kotlin-reflect - ${kotlin.version} - - - - - - - - - org.apache.maven.plugins - maven-release-plugin - 3.0.0-M1 - - - org.sonatype.plugins - nexus-staging-maven-plugin - ${nexus-staging-plugin.version} - - - org.jacoco - jacoco-maven-plugin - ${jacoco-maven-plugin.version} - - - - prepare-agent - - - - report - - report - - prepare-package - - - - - org.jetbrains.kotlin - kotlin-maven-plugin - ${kotlin.version} - - - compile - - compile - - compile - - true - 1.8 - - -Xopt-in=kotlin.RequiresOptIn - -XXLanguage:+InlineClasses - - - - - test-compile - - test-compile - - test-compile - - true - 1.8 - - -Xopt-in=kotlin.RequiresOptIn - -XXLanguage:+InlineClasses - - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - org.apache.maven.plugins - maven-site-plugin - ${maven-site-plugin.version} - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - native - - - - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - - src/main/ - src/test/ - - apache_v2 - =LICENSE= - =LICENSE END= - ---------- - - - - first - - update-file-header - - - - **/*.json - **/*.css - - - process-sources - - - - - org.apache.maven.plugins - maven-source-plugin - ${maven-source-plugin.version} - - - attach-sources - - jar - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - ${maven-enforcer-plugin.version} - - - enforce-maven - - enforce - - - - - 3.5.4 - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - - org.apache.maven.plugins - maven-release-plugin - - true - false - forked-path - scala-2.12,central-deploy - - - - - - - http://maven.apache.org - 2019 - - JetBrains - https://www.jetbrains.com/ - - - - - Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - - - - - - asm0dey - Pasha Finkelshteyn - asm0dey@jetbrains.com - GMT+3 - - - vitaly.khudobakhshov - Vitaly Khudobakhshov - vitaly.khudobakhshov@jetbrains.com - GMT+3 - - - - scm:git:https://github.com/JetBrains/kotlin-spark-api.git - https://github.com/JetBrains/kotlin-spark-api - HEAD - - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - github - GitHub JetBrains Apache Maven Packages - https://maven.pkg.github.com/Kotlin/kotlin-spark-api - - - - - - - - scala-2.12 - - true - - - pom_2.12.xml - - - - central-deploy - - - performRelease - true - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - ${nexus-staging-plugin.version} - true - - ossrh - https://oss.sonatype.org/ - false - 20 - - - - org.apache.maven.plugins - maven-gpg-plugin - ${maven-gpg-plugin.version} - - - sign-artifacts - - sign - - verify - - - - - - - - - diff --git a/pom_2.12.xml b/pom_2.12.xml deleted file mode 100644 index f569bbb7..00000000 --- a/pom_2.12.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - 4.0.0 - - Kotlin Spark API: Parent (Scala 2.12) - Parent project for Kotlin for Apache Spark - kotlin-spark-api-parent-3.2_2.12 - - org.jetbrains.kotlinx.spark - kotlin-spark-api-parent-3.2 - 1.1.1-SNAPSHOT - pom.xml - - pom - - - true - 2.12.15 - 2.12 - - - - core/3.2/pom_2.12.xml - scala-tuples-in-kotlin/pom_2.12.xml - kotlin-spark-api/3.2/pom_2.12.xml - examples/pom-3.2_2.12.xml - jupyter - - - - - - org.jetbrains.kotlinx.spark - core-3.2_${scala.compat.version} - ${project.version} - - - org.jetbrains.kotlinx.spark - scala-tuples-in-kotlin-3.2 - ${project.version} - - - - diff --git a/scala-tuples-in-kotlin/build.gradle.kts b/scala-tuples-in-kotlin/build.gradle.kts new file mode 100644 index 00000000..5209a43f --- /dev/null +++ b/scala-tuples-in-kotlin/build.gradle.kts @@ -0,0 +1,65 @@ +@file:Suppress("UnstableApiUsage") + +import com.vanniktech.maven.publish.JavadocJar.Dokka +import com.vanniktech.maven.publish.KotlinJvm +import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask +import org.jetbrains.dokka.gradle.DokkaTask +import org.jetbrains.dokka.gradle.DokkaTaskPartial + +plugins { + scala + kotlin + dokka + mavenPublishBase +} + +group = Versions.groupID +version = Versions.project + +repositories { + mavenCentral() +} + +tasks.withType().configureEach { + useJUnitPlatform() +} + +dependencies { + with(Dependencies) { + implementation( + kotlinStdLib, + scalaLibrary, + ) + testImplementation( + kotest, + atrium, + kotlinTest, + ) + } +} + +tasks.withType { + dokkaSourceSets { + create("scala-tuples-in-kotlin") { + sourceRoot( + kotlin.sourceSets + .main.get() + .kotlin + .srcDirs + .first { it.path.endsWith("kotlin") } + ) + } + } +} + +mavenPublishing { + configure(KotlinJvm(Dokka("dokkaHtml"))) +} + + +// Publishing of scala-tuples-in-kotlin can be skipped since it's only dependent on the Scala version +val skipScalaTuplesInKotlin = System.getProperty("skipScalaTuplesInKotlin").toBoolean() +tasks + .filter { "publish" in it.name } + .forEach { it.onlyIf { !skipScalaTuplesInKotlin } } + diff --git a/scala-tuples-in-kotlin/pom_2.12.xml b/scala-tuples-in-kotlin/pom_2.12.xml deleted file mode 100644 index a5169fbb..00000000 --- a/scala-tuples-in-kotlin/pom_2.12.xml +++ /dev/null @@ -1,153 +0,0 @@ - - - 4.0.0 - - Kotlin Spark API: Scala Tuples in Kotlin - Scala Tuple helper functions for kotlin - scala-tuples-in-kotlin-3.2 - - org.jetbrains.kotlinx.spark - kotlin-spark-api-parent-3.2_2.12 - 1.1.1-SNAPSHOT - ../pom_2.12.xml - - - - - org.scala-lang - scala-library - ${scala.version} - - - org.jetbrains.kotlin - kotlin-stdlib-jdk8 - ${kotlin.version} - - - - - io.kotest - kotest-runner-junit5-jvm - ${kotest.version} - test - - - com.beust - klaxon - ${klaxon.version} - test - - - ch.tutteli.atrium - atrium-fluent-en_GB - ${atrium.version} - test - - - - org.jetbrains.kotlin - kotlin-test - ${kotlin.version} - test - - - - - src/main/kotlin - src/test/kotlin - target/${scala.compat.version} - - - - org.jetbrains.dokka - dokka-maven-plugin - ${dokka.version} - - 8 - - - - dokka - - dokka - - pre-site - - - javadocjar - - javadocJar - - pre-integration-test - - - - - - org.jetbrains.kotlin - kotlin-maven-plugin - ${kotlin.version} - - - compile - compile - - compile - - - - test-compile - test-compile - - test-compile - - - - - 1.8 - - - - - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - - - jar-with-dependencies - - - - org.jetbrains.spark.api.examples.WordCountKt - - - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - - - - org.apache.maven.plugins - maven-deploy-plugin - - false - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - - false - - - - - - diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 00000000..b94c65ba --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,32 @@ +val spark: String by settings +val scala: String by settings +val skipScalaTuplesInKotlin: String by settings +System.setProperty("spark", spark) +System.setProperty("scala", scala) +System.setProperty("skipScalaTuplesInKotlin", skipScalaTuplesInKotlin) + + +val scalaCompat + get() = scala.substringBeforeLast('.') + +val versions = "${spark}_${scalaCompat}" + +rootProject.name = "kotlin-spark-api-parent_$versions" + +include("core") +include("scala-tuples-in-kotlin") +include("kotlin-spark-api") +include("jupyter") +include("examples") + +project(":core").name = "core_$versions" +project(":scala-tuples-in-kotlin").name = "scala-tuples-in-kotlin_$scalaCompat" +project(":kotlin-spark-api").name = "kotlin-spark-api_$versions" +project(":jupyter").name = "jupyter_$versions" +project(":examples").name = "examples_$versions" + +buildCache { + local { + removeUnusedEntriesAfterDays = 30 + } +} \ No newline at end of file