From f09761c9aec8d1578af6a03dba1e4f595c69f0fc Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 4 Jun 2019 16:19:18 +0100 Subject: [PATCH 1/9] CI/DOC: Building the documentation with azure-pipelines --- azure-pipelines.yml | 61 +++++++++++++++++++++++++++++++++++++++++++++ environment.yml | 7 ++++++ 2 files changed, 68 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 85325c52e7e6d..f94059aceec9e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -116,3 +116,64 @@ jobs: fi displayName: 'Running benchmarks' condition: true + +- job: 'Docs' + pool: + vmImage: ubuntu-16.04 + timeoutInMinutes: 90 + steps: + - script: | + echo '##vso[task.setvariable variable=CONDA_ENV]pandas-dev' + echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' + displayName: 'Setting environment variables' + + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + sudo apt-get install -y libc6-dev-i386 + ci/setup_env.sh + displayName: 'Setup environment and build pandas' + + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev + doc/make.py + displayName: 'Build documentation' + + - script: | + cd doc/build/html + git init + touch .nojekyll + echo "dev.pandas.io" > CNAME + git add --all . + git config user.email "pandas-dev@python.org" + git config user.name "pandas-docs-bot" + git commit -m "pandas documentation in master" + displayName: 'Create git repo for docs build' + condition : | + and(not(eq(variables['Build.Reason'], 'PullRequest')), + eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + # This task to work requires next steps: + # 1. Got to "Library > Secure files" in the azure-pipelines dashboard: https://dev.azure.com/pandas-dev/pandas/_library?itemType=SecureFiles + # 2. Click on "+ Secure file" + # 3. Upload the private key (the name of the file must match with the specified in "sshKeySecureFile" input below, "pandas_docs_key") + # 4. Click on file name after it is created, tick the box "Authorize for use in all pipelines" and save + # 5. The public key specified in "sshPublicKey" is the pair of the uploaded private key, and needs to be specified as a deploy key of the repo where the docs will be pushed: https://github.com/pandas-dev/pandas-dev.github.io/settings/keys + - task: InstallSSHKey@0 + inputs: + hostName: 'github.com,192.30.252.128 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' + sshPublicKey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDHmz3l/EdqrgNxEUKkwDUuUcLv91unig03pYFGO/DMIgCmPdMG96zAgfnESd837Rm0wSSqylwSzkRJt5MV/TpFlcVifDLDQmUhqCeO8Z6dLl/oe35UKmyYICVwcvQTAaHNnYRpKC5IUlTh0JEtw9fGlnp1Ta7U1ENBLbKdpywczElhZu+hOQ892zqOj3CwA+U2329/d6cd7YnqIKoFN9DWT3kS5K6JE4IoBfQEVekIOs23bKjNLvPoOmi6CroAhu/K8j+NCWQjge5eJf2x/yTnIIP1PlEcXoHIr8io517posIx3TBup+CN8bNS1PpDW3jyD3ttl1uoBudjOQrobNnJeR6Rn67DRkG6IhSwr3BWj8alwUG5mTdZzwV5Pa9KZFdIiqX7NoDGg+itsR39QCn0thK8lGRNSR8KrWC1PSjecwelKBO7uQ7rnk/rkrZdBWR4oEA8YgNH8tirUw5WfOr5a0AIaJicKxGKNdMxZt+zmC+bS7F4YCOGIm9KHa43RrKhoGRhRf9fHHHKUPwFGqtWG4ykcUgoamDOURJyepesBAO3FiRE9rLU6ILbB3yEqqoekborHmAJD5vf7PWItW3Q/YQKuk3kkqRcKnexPyzyyq5lUgTi8CxxZdaASIOu294wjBhhdyHlXEkVTNJ9JKkj/obF+XiIIp0cBDsOXY9hDQ== pandas-dev@python.org' + sshKeySecureFile: 'pandas_docs_key' + displayName: 'Install GitHub ssh deployment key' + condition : | + and(not(eq(variables['Build.Reason'], 'PullRequest')), + eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + - script: | + cd doc/build/html + git remote add origin git@github.com:pandas-dev/pandas-dev.github.io.git + git push origin master -f + displayName: 'Publish docs to GitHub pages' + condition : | + and(not(eq(variables['Build.Reason'], 'PullRequest')), + eq(variables['Build.SourceBranch'], 'refs/heads/master')) diff --git a/environment.yml b/environment.yml index cf17dc1281ec9..91ea26eef4b61 100644 --- a/environment.yml +++ b/environment.yml @@ -17,10 +17,17 @@ dependencies: - flake8-rst>=0.6.0,<=0.7.0 - gitpython - hypothesis>=3.82 + - ipywidgets - isort - moto - mypy + - nbconvert>=5.4.1 + - nbformat + - notebook>=5.7.5 + - pandoc - pycodestyle + - pyqt + - python-snappy - pytest>=4.0.2 - pytest-mock - sphinx From c959c2cadfbdd3a382fd161b45d1ede83149db99 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 4 Jun 2019 17:18:14 +0100 Subject: [PATCH 2/9] Synchronizing requirements-dev.txt --- requirements-dev.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 115a93495c95b..e6085920a9999 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,10 +8,17 @@ flake8-comprehensions flake8-rst>=0.6.0,<=0.7.0 gitpython hypothesis>=3.82 +ipywidgets isort moto mypy +nbconvert>=5.4.1 +nbformat +notebook>=5.7.5 +pandoc pycodestyle +pyqt +python-snappy pytest>=4.0.2 pytest-mock sphinx From bc8ff7ea6cf73f0ada8e6df79cfc0e3aaea1943b Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 5 Jun 2019 11:10:40 +0100 Subject: [PATCH 3/9] Changing dependencies of the doc build in azure --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f94059aceec9e..4e2edde758d55 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -124,7 +124,7 @@ jobs: steps: - script: | echo '##vso[task.setvariable variable=CONDA_ENV]pandas-dev' - echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' + echo '##vso[task.setvariable variable=ENV_FILE]ci/deps/travis-36-doc.yaml' displayName: 'Setting environment variables' - script: | From 861e5b12738ef3e7b40a7acd4e6424e8525337a5 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 5 Jun 2019 12:01:57 +0100 Subject: [PATCH 4/9] Reverting changes to dependencies --- environment.yml | 7 ------- requirements-dev.txt | 7 ------- 2 files changed, 14 deletions(-) diff --git a/environment.yml b/environment.yml index 91ea26eef4b61..cf17dc1281ec9 100644 --- a/environment.yml +++ b/environment.yml @@ -17,17 +17,10 @@ dependencies: - flake8-rst>=0.6.0,<=0.7.0 - gitpython - hypothesis>=3.82 - - ipywidgets - isort - moto - mypy - - nbconvert>=5.4.1 - - nbformat - - notebook>=5.7.5 - - pandoc - pycodestyle - - pyqt - - python-snappy - pytest>=4.0.2 - pytest-mock - sphinx diff --git a/requirements-dev.txt b/requirements-dev.txt index e6085920a9999..115a93495c95b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,17 +8,10 @@ flake8-comprehensions flake8-rst>=0.6.0,<=0.7.0 gitpython hypothesis>=3.82 -ipywidgets isort moto mypy -nbconvert>=5.4.1 -nbformat -notebook>=5.7.5 -pandoc pycodestyle -pyqt -python-snappy pytest>=4.0.2 pytest-mock sphinx From 0a6f5236af5a36000f11b4c5868485e5b26d8dc3 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 5 Jun 2019 15:56:03 +0100 Subject: [PATCH 5/9] Fixing typos, and improving instructions about private/public keys --- azure-pipelines.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4e2edde758d55..ec73a5f41708c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -153,12 +153,13 @@ jobs: and(not(eq(variables['Build.Reason'], 'PullRequest')), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - # This task to work requires next steps: - # 1. Got to "Library > Secure files" in the azure-pipelines dashboard: https://dev.azure.com/pandas-dev/pandas/_library?itemType=SecureFiles - # 2. Click on "+ Secure file" - # 3. Upload the private key (the name of the file must match with the specified in "sshKeySecureFile" input below, "pandas_docs_key") - # 4. Click on file name after it is created, tick the box "Authorize for use in all pipelines" and save - # 5. The public key specified in "sshPublicKey" is the pair of the uploaded private key, and needs to be specified as a deploy key of the repo where the docs will be pushed: https://github.com/pandas-dev/pandas-dev.github.io/settings/keys + # For `InstallSSHKey@0` to work, next steps are required: + # 1. Generate a pair of private/public keys (i.e. `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"` + # 2. Go to "Library > Secure files" in the Azure Pipelines dashboard: https://dev.azure.com/pandas-dev/pandas/_library?itemType=SecureFiles + # 3. Click on "+ Secure file" + # 4. Upload the private key (the name of the file must match with the specified in "sshKeySecureFile" input below, "pandas_docs_key") + # 5. Click on file name after it is created, tick the box "Authorize for use in all pipelines" and save + # 6. The public key specified in "sshPublicKey" is the pair of the uploaded private key, and needs to be set as a deploy key of the repo where the docs will be pushed (with write access): https://github.com/pandas-dev/pandas-dev.github.io/settings/keys - task: InstallSSHKey@0 inputs: hostName: 'github.com,192.30.252.128 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' From aa079d524c7c76c4057192ba84422a6a6cd0bb1f Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 5 Jun 2019 16:43:00 +0100 Subject: [PATCH 6/9] Adding missing parenthesis in comment --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ec73a5f41708c..b4eea66ce302a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -154,7 +154,7 @@ jobs: eq(variables['Build.SourceBranch'], 'refs/heads/master')) # For `InstallSSHKey@0` to work, next steps are required: - # 1. Generate a pair of private/public keys (i.e. `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"` + # 1. Generate a pair of private/public keys (i.e. `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"`) # 2. Go to "Library > Secure files" in the Azure Pipelines dashboard: https://dev.azure.com/pandas-dev/pandas/_library?itemType=SecureFiles # 3. Click on "+ Secure file" # 4. Upload the private key (the name of the file must match with the specified in "sshKeySecureFile" input below, "pandas_docs_key") From e51c63436ebcb337a19ac757fec9f9689123ec45 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 10 Jun 2019 11:47:43 +0100 Subject: [PATCH 7/9] Not failing the build if the push of the docs fails (making the push fail to test if it works) --- azure-pipelines.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b4eea66ce302a..60023164bd6f2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -173,7 +173,10 @@ jobs: - script: | cd doc/build/html git remote add origin git@github.com:pandas-dev/pandas-dev.github.io.git - git push origin master -f + # FIXME not forcing the push will fail, testing if the exit 0 makes the build green anyway + # git push -f origin master + git push origin master + exit 0 displayName: 'Publish docs to GitHub pages' condition : | and(not(eq(variables['Build.Reason'], 'PullRequest')), From f31ae5002e5b8635365290b0a89482c947b557dc Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 10 Jun 2019 13:18:17 +0100 Subject: [PATCH 8/9] Leaving the doc build green even if the push fails (removing the failure of the build for testing) --- azure-pipelines.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 60023164bd6f2..e8b0aa414aebb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -173,10 +173,8 @@ jobs: - script: | cd doc/build/html git remote add origin git@github.com:pandas-dev/pandas-dev.github.io.git - # FIXME not forcing the push will fail, testing if the exit 0 makes the build green anyway - # git push -f origin master - git push origin master - exit 0 + git push -f origin master + exit 0 # FIXME this will leave the build green even if the step fails. To be removed when we are confident with this. displayName: 'Publish docs to GitHub pages' condition : | and(not(eq(variables['Build.Reason'], 'PullRequest')), From 9a0523af1ec74a86732d0b46c389ed9fbfa08cc3 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 12 Jun 2019 10:02:28 +0100 Subject: [PATCH 9/9] Removing unused CONDA_ENV variable --- azure-pipelines.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b4eea66ce302a..f4d1078895372 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,7 +24,6 @@ jobs: # XXX next command should avoid redefining the path in every step, but # made the process crash as it couldn't find deactivate #echo '##vso[task.prependpath]$HOME/miniconda3/bin' - echo '##vso[task.setvariable variable=CONDA_ENV]pandas-dev' echo '##vso[task.setvariable variable=ENV_FILE]environment.yml' echo '##vso[task.setvariable variable=AZURE]true' displayName: 'Setting environment variables' @@ -123,7 +122,6 @@ jobs: timeoutInMinutes: 90 steps: - script: | - echo '##vso[task.setvariable variable=CONDA_ENV]pandas-dev' echo '##vso[task.setvariable variable=ENV_FILE]ci/deps/travis-36-doc.yaml' displayName: 'Setting environment variables'