Add conda dev environments for Python 3.7/3.8, JDK 8/11 (#238)

* Add conda env files

* Add pytest skip for hive tests when Docker images are missing

* Make sure to specify Python version

* Add pip dependencies to dev environments

* Update docs to use dev environment

* First pass at updating CI to use conda envs

* Use environment variable for conda file location in matrix jobs

* Force openjdk reinstall to resolve Windows failures

* Split off env files by JDK version

* Add sasl to dev envs

* Only install sasl on ubuntu

* Skip hive/postgres testing on Windows

* Attempt to get conda cache working

* Remove shell mamba installs

* Skip failing tests on Windows

* xfail Windows tests instead of skipping

* Update references to conda env files

* Don't copy env file in Docker image

* Attempt to set JAVA_HOME explicitly

* Set correct JAVA_HOME only on Windows

* No longer need to xfail JAVA_HOME check

* Remove unnecessary sys import

* Remove env files with unpinned JDK

* Caching isn't working - try the old method of installing?

* Caching seems to slow down runs - try without?

* Remove conda caching from workflows

* Make maven caching key more obvious

* Keep original conda.txt for Docker purposes

Author: charlesbluca

.github/workflows/deploy.yml

@@ -14,22 +14,18 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-v1-${{ hashFiles('**/pom.xml') }}
-      - name: Cache downloaded conda packages
-        uses: actions/cache@v2
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-v3-${{ hashFiles('conda.txt') }}
+          key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
       - name: Set up Python
         uses: conda-incubator/setup-miniconda@v2
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
           python-version: 3.8
+          activate-environment: dask-sql
+          environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml
       - name: Install dependencies
         shell: bash -l {0}
         run: |
-          conda install --file conda.txt -c conda-forge
           pip install setuptools wheel twine
           which python
           pip list

.github/workflows/test.yml

@@ -21,29 +21,18 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-v1-${{ hashFiles('**/pom.xml') }}
-      - name: Cache downloaded conda packages
-        uses: actions/cache@v2
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-v3-${{ hashFiles('conda.txt') }}
+          key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
       - name: Set up Python
         uses: conda-incubator/setup-miniconda@v2
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
           python-version: 3.8
+          activate-environment: dask-sql
+          environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml
       - name: Install dependencies and build the jar
         shell: bash -l {0}
         run: |
-          mamba install --file conda.txt -c conda-forge
-          which python
-          pip list
-          mamba list
-
-          # This needs to happen in the same
-          # shell, because otherwise the JAVA_HOME
-          # will be wrong on windows
           python setup.py java
       - name: Upload the jar
         uses: actions/upload-artifact@v1
@@ -55,6 +44,8 @@ jobs:
     name: "Test (${{ matrix.os }}, java: ${{ matrix.java }}, python: ${{ matrix.python }})"
     needs: build
     runs-on: ${{ matrix.os }}
+    env:
+      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-jdk${{ matrix.java }}-dev.yaml
     strategy:
       matrix:
         java: [8, 11]
@@ -66,58 +57,35 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-v1-${{ matrix.java }}-${{ hashFiles('**/pom.xml') }}
-      - name: Cache downloaded conda packages
-        uses: actions/cache@v2
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-v3-${{ matrix.java }}-${{ matrix.python }}-${{ hashFiles('conda.txt') }}
+          key: ${{ runner.os }}-maven-v1-jdk${{ matrix.java }}-${{ hashFiles('**/pom.xml') }}
       - name: Set up Python
         uses: conda-incubator/setup-miniconda@v2
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
           python-version: ${{ matrix.python }}
+          activate-environment: dask-sql
+          environment-file: ${{ env.CONDA_FILE }}
       - name: Download the pre-build jar
         uses: actions/download-artifact@v1
         with:
           name: jar
           path: dask_sql/jar/
-      - name: Install dependencies
-        shell: bash -l {0}
-        run: |
-          mamba install python=${{ matrix.python }} --file conda.txt -c conda-forge
-      - name: Install sqlalchemy and docker pkg for postgres test
+      - name: Install hive testing dependencies for Linux
         shell: bash -l {0}
         run: |
-          # explicitly install docker, fugue and other packages
-          mamba install \
-            sasl>=0.3.1 \
-            sqlalchemy>=1.4.23 \
-            pyhive>=0.6.4 \
-            psycopg2>=2.9.1 \
-            ciso8601>=2.2.0 \
-            tpot>=0.11.7 \
-            mlflow>=1.19.0 \
-            docker-py>=5.0.0 \
-            -c conda-forge
-          pip install "fugue[sql]>=0.5.3"
+          mamba install -c conda-forge sasl>=0.3.1
           docker pull bde2020/hive:2.3.2-postgresql-metastore
           docker pull bde2020/hive-metastore-postgresql:2.3.0
         if: matrix.os == 'ubuntu-latest'
-      - name: Install Java (again) and test with pytest
+      - name: Set proper JAVA_HOME for Windows
+        shell: bash -l {0}
+        run: |
+          echo "JAVA_HOME=${{ env.CONDA }}\envs\dask-sql\Library" >> $GITHUB_ENV
+        if: matrix.os == 'windows-latest'
+      - name: Test with pytest
         shell: bash -l {0}
         run: |
-          mamba install openjdk=${{ matrix.java }}
-          which python
-          pip list
-          mamba list
-
-          # This needs to happen in the same
-          # shell, because otherwise the JAVA_HOME
-          # will be wrong on windows
-          # The --dist loadfile makes sure, that tests are distributed according to their file
-          # this is especially important to not run the docker-fixtures more than once
           pytest --junitxml=junit/test-results.xml --cov-report=xml -n auto tests --dist loadfile
       - name: Upload pytest test results
         uses: actions/upload-artifact@v1
@@ -141,18 +109,15 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-v1-11-${{ hashFiles('**/pom.xml') }}
-      - name: Cache downloaded conda packages
-        uses: actions/cache@v2
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-v3-11-${{ hashFiles('conda.txt') }}
+          key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
       - name: Set up Python
         uses: conda-incubator/setup-miniconda@v2
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
           python-version: 3.8
+          activate-environment: dask-sql
+          environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml
       - name: Download the pre-build jar
         uses: actions/download-artifact@v1
         with:
@@ -161,7 +126,7 @@ jobs:
       - name: Install dependencies
         shell: bash -l {0}
         run: |
-          mamba install python=3.8 python-blosc lz4 --file conda.txt -c conda-forge
+          mamba install python-blosc lz4 -c conda-forge
 
           which python
           pip list
@@ -192,20 +157,14 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-v1-11-${{ hashFiles('**/pom.xml') }}
-      - name: Cache downloaded conda packages
-        uses: actions/cache@v2
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-v2-11-${{ hashFiles('conda.txt') }}
+          key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
       - name: Set up Python
         uses: conda-incubator/setup-miniconda@v2
         with:
           python-version: 3.8
           mamba-version: "*"
           channels: conda-forge,defaults
           channel-priority: true
-          use-only-tar-bz2: true
       - name: Download the pre-build jar
         uses: actions/download-artifact@v1
         with:

README.md

@@ -113,11 +113,11 @@ If you want to have the newest (unreleased) `dask-sql` version or if you plan to
 
 Create a new conda environment and install the development environment:
 
-    conda create -n dask-sql --file conda.txt -c conda-forge
+    conda env create -f continuous_integration/environment-3.8-jdk11-dev.yaml
 
 It is not recommended to use `pip` instead of `conda` for the environment setup.
 If you however need to, make sure to have Java (jdk >= 8) and maven installed and correctly setup before continuing.
-Have a look into `conda.txt` for the rest of the development environment.
+Have a look into `environment-3.8-jdk11-dev.yaml` for the rest of the development environment.
 
 After that, you can install the package in development mode
 

continuous_integration/environment-3.7-jdk11-dev.yaml

@@ -0,0 +1,42 @@
+name: dask-sql
+channels:
+- conda-forge
+- defaults
+dependencies:
+- adagio>=0.2.3
+- antlr4-python3-runtime>=4.9.2
+- black=19.10b0
+- ciso8601>=2.2.0
+- dask-ml>=1.7.0
+- dask>=2.19.0,!=2021.3.0  # dask 2021.3.0 makes dask-ml fail (see https://github.com/dask/dask-ml/issues/803)
+- fastapi>=0.61.1
+- fs>=2.4.11
+- intake>=0.6.0
+- isort=5.7.0
+- jpype1>=1.0.2
+- lightgbm>=3.2.1
+- maven>=3.6.0
+- mlflow>=1.19.0
+- mock>=4.0.3
+- nest-asyncio>=1.4.3
+- openjdk=11
+- pandas>=1.0.0  # below 1.0, there were no nullable ext. types
+- pip=20.2.4
+- pre-commit>=2.11.1
+- prompt_toolkit>=3.0.8
+- psycopg2>=2.9.1
+- pyarrow>=0.15.1
+- pygments>=2.7.1
+- pyhive>=0.6.4
+- pytest-cov>=2.10.1
+- pytest-xdist
+- pytest>=6.0.1
+- python=3.7
+- scikit-learn>=0.24.2
+- sphinx>=3.2.1
+- tpot>=0.11.7
+- triad>=0.5.4
+- tzlocal>=2.1
+- uvicorn>=0.11.3
+- pip:
+  - fugue[sql]>=0.5.3

continuous_integration/environment-3.7-jdk8-dev.yaml

@@ -0,0 +1,42 @@
+name: dask-sql
+channels:
+- conda-forge
+- defaults
+dependencies:
+- adagio>=0.2.3
+- antlr4-python3-runtime>=4.9.2
+- black=19.10b0
+- ciso8601>=2.2.0
+- dask-ml>=1.7.0
+- dask>=2.19.0,!=2021.3.0  # dask 2021.3.0 makes dask-ml fail (see https://github.com/dask/dask-ml/issues/803)
+- fastapi>=0.61.1
+- fs>=2.4.11
+- intake>=0.6.0
+- isort=5.7.0
+- jpype1>=1.0.2
+- lightgbm>=3.2.1
+- maven>=3.6.0
+- mlflow>=1.19.0
+- mock>=4.0.3
+- nest-asyncio>=1.4.3
+- openjdk=8
+- pandas>=1.0.0  # below 1.0, there were no nullable ext. types
+- pip=20.2.4
+- pre-commit>=2.11.1
+- prompt_toolkit>=3.0.8
+- psycopg2>=2.9.1
+- pyarrow>=0.15.1
+- pygments>=2.7.1
+- pyhive>=0.6.4
+- pytest-cov>=2.10.1
+- pytest-xdist
+- pytest>=6.0.1
+- python=3.7
+- scikit-learn>=0.24.2
+- sphinx>=3.2.1
+- tpot>=0.11.7
+- triad>=0.5.4
+- tzlocal>=2.1
+- uvicorn>=0.11.3
+- pip:
+  - fugue[sql]>=0.5.3

continuous_integration/environment-3.8-jdk11-dev.yaml

@@ -0,0 +1,42 @@
+name: dask-sql
+channels:
+- conda-forge
+- defaults
+dependencies:
+- adagio>=0.2.3
+- antlr4-python3-runtime>=4.9.2
+- black=19.10b0
+- ciso8601>=2.2.0
+- dask-ml>=1.7.0
+- dask>=2.19.0,!=2021.3.0  # dask 2021.3.0 makes dask-ml fail (see https://github.com/dask/dask-ml/issues/803)
+- fastapi>=0.61.1
+- fs>=2.4.11
+- intake>=0.6.0
+- isort=5.7.0
+- jpype1>=1.0.2
+- lightgbm>=3.2.1
+- maven>=3.6.0
+- mlflow>=1.19.0
+- mock>=4.0.3
+- nest-asyncio>=1.4.3
+- openjdk=11
+- pandas>=1.0.0  # below 1.0, there were no nullable ext. types
+- pip=20.2.4
+- pre-commit>=2.11.1
+- prompt_toolkit>=3.0.8
+- psycopg2>=2.9.1
+- pyarrow>=0.15.1
+- pygments>=2.7.1
+- pyhive>=0.6.4
+- pytest-cov>=2.10.1
+- pytest-xdist
+- pytest>=6.0.1
+- python=3.8
+- scikit-learn>=0.24.2
+- sphinx>=3.2.1
+- tpot>=0.11.7
+- triad>=0.5.4
+- tzlocal>=2.1
+- uvicorn>=0.11.3
+- pip:
+  - fugue[sql]>=0.5.3

continuous_integration/environment-3.8-jdk8-dev.yaml

@@ -0,0 +1,42 @@
+name: dask-sql
+channels:
+- conda-forge
+- defaults
+dependencies:
+- adagio>=0.2.3
+- antlr4-python3-runtime>=4.9.2
+- black=19.10b0
+- ciso8601>=2.2.0
+- dask-ml>=1.7.0
+- dask>=2.19.0,!=2021.3.0  # dask 2021.3.0 makes dask-ml fail (see https://github.com/dask/dask-ml/issues/803)
+- fastapi>=0.61.1
+- fs>=2.4.11
+- intake>=0.6.0
+- isort=5.7.0
+- jpype1>=1.0.2
+- lightgbm>=3.2.1
+- maven>=3.6.0
+- mlflow>=1.19.0
+- mock>=4.0.3
+- nest-asyncio>=1.4.3
+- openjdk=8
+- pandas>=1.0.0  # below 1.0, there were no nullable ext. types
+- pip=20.2.4
+- pre-commit>=2.11.1
+- prompt_toolkit>=3.0.8
+- psycopg2>=2.9.1
+- pyarrow>=0.15.1
+- pygments>=2.7.1
+- pyhive>=0.6.4
+- pytest-cov>=2.10.1
+- pytest-xdist
+- pytest>=6.0.1
+- python=3.8
+- scikit-learn>=0.24.2
+- sphinx>=3.2.1
+- tpot>=0.11.7
+- triad>=0.5.4
+- tzlocal>=2.1
+- uvicorn>=0.11.3
+- pip:
+  - fugue[sql]>=0.5.3

conda.txt renamed to docker/conda.txt

@@ -4,7 +4,7 @@ FROM daskdev/dask:latest
 LABEL author "Nils Braun <[email protected]>"
 
 # Install dependencies for dask-sql
-COPY conda.txt /opt/dask_sql/
+COPY docker/conda.txt /opt/dask_sql/
 RUN conda config --add channels conda-forge \
     && /opt/conda/bin/conda install --freeze-installed \
     "jpype1>=1.0.2" \

docker/main.dockerfile

@@ -57,11 +57,11 @@ Create a new conda environment and install the development environment:
 
 .. code-block:: bash
 
-    conda create -n dask-sql --file conda.txt -c conda-forge
+    conda env create -f continuous_integration/environment-3.8-jdk11-dev.yaml
 
 It is not recommended to use ``pip`` instead of ``conda``.
 If you however need to, make sure to have Java (jdk >= 8) and maven installed and correctly setup before continuing.
-Have a look into ``conda.txt`` for the rest of the development environment.
+Have a look into ``environment-3.8-jdk11-dev.yaml`` for the rest of the development environment.
 
 After that, you can install the package in development mode