Skip to content

Commit 5fa0ea5

Browse files
soumithsoumith
authored
Merge pull request #61 from pjh5/nightly_condas
Conda changes for easier CI integration
2 parents adcb783 + c486bdc commit 5fa0ea5

File tree

3 files changed

+79
-49
lines changed

3 files changed

+79
-49
lines changed

‎conda/build_pytorch.sh

Lines changed: 75 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,73 +1,101 @@
11
#!/usr/bin/env bash
2-
. /remote/anaconda_token || true
2+
if [[ -x "/remote/anaconda_token" ]]; then
3+
. /remote/anaconda_token || true
4+
fi
5+
6+
set -ex
37

4-
set -e
8+
# Defined a portable sed that should work on both mac and linux
9+
if [ "$(uname)" == 'Darwin' ]; then
10+
portable_sed="sed -E -i ''"
11+
else
12+
portable_sed='sed --regexp-extended -i'
13+
fi
514

615
echo "Building cuda version $1 and pytorch version: $2 build_number: $3"
16+
desired_cuda="$1"
17+
build_version="$2"
18+
build_number="$3"
719

20+
# setup.py is hardcoded to use these variables
21+
export PYTORCH_BUILD_VERSION=$build_version
22+
export PYTORCH_BUILD_NUMBER=$build_number
23+
24+
if [[ "$build_version" == "nightly" ]]; then
25+
export PYTORCH_BUILD_VERSION="$(date +"%Y.%m.%d")"
26+
fi
27+
28+
29+
# This is the channel that finished packages will be uploaded to
30+
if [[ -z "$ANACONDA_USER" ]]; then
31+
ANACONDA_USER=soumith
32+
fi
33+
34+
# Token needed to upload to the conda channel above
835
if [ -z "$ANACONDA_TOKEN" ]; then
936
echo "ANACONDA_TOKEN is unset. Please set it in your environment before running this script";
10-
exit 1
1137
fi
1238

13-
ANACONDA_USER=soumith
39+
# Don't upload the packages until we've verified that they're correct
40+
conda config --set anaconda_upload no
1441

15-
BUILD_VERSION="$2"
16-
BUILD_NUMBER="$3"
42+
# Build for a specified Python, or if none were given then all of them
43+
if [[ -z "$DESIRED_PYTHON" ]]; then
44+
DESIRED_PYTHON=('2.7' '3.5' '3.6' '3.7')
45+
fi
46+
echo "Will build for all Pythons: ${DESIRED_PYTHON[@]}"
47+
echo "Will build for all CUDA versions: ${desired_cuda[@]}"
1748

18-
export PYTORCH_BUILD_VERSION=$BUILD_VERSION
19-
export PYTORCH_BUILD_NUMBER=$BUILD_NUMBER
2049

21-
if [[ "$BUILD_VERSION" == "nightly" ]]; then
22-
export PYTORCH_BUILD_VERSION="$(date +"%Y.%m.%d")"
50+
# Determine which build folder to use, if not given it directly
51+
if [[ -n "$TORCH_CONDA_BUILD_FOLDER" ]]; then
52+
build_folder="$TORCH_CONDA_BUILD_FOLDER"
53+
else
54+
if [[ "$OSTYPE" == 'darwin'* || "$desired_cuda" == '90' ]]; then
55+
build_folder='pytorch'
56+
elif [[ "$desired_cuda" == 'cpu' ]]; then
57+
build_folder='pytorch-cpu'
58+
else
59+
build_folder="pytorch-$desired_cuda"
60+
fi
61+
build_folder="$build_folder-$build_version"
2362
fi
63+
echo "Using conda-build folder $build_folder"
2464

25-
conda config --set anaconda_upload no
26-
65+
# Switch the CUDA version that /usr/local/cuda points to
2766
if [[ "$OSTYPE" == "darwin"* ]]; then
2867
export CUDA_VERSION="0.0"
2968
export CUDNN_VERSION="0.0"
30-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 pytorch-$BUILD_VERSION
31-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 pytorch-$BUILD_VERSION
32-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 pytorch-$BUILD_VERSION
33-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 pytorch-$BUILD_VERSION
34-
else
35-
if [[ "$1" == "80" ]]; then
36-
. ./switch_cuda_version.sh 8.0
37-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 pytorch-cuda80-$BUILD_VERSION
38-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 pytorch-cuda80-$BUILD_VERSION
39-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 pytorch-cuda80-$BUILD_VERSION
40-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 pytorch-cuda80-$BUILD_VERSION
41-
elif [[ "$1" == "90" ]]; then
42-
. ./switch_cuda_version.sh 9.0
43-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 pytorch-$BUILD_VERSION
44-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 pytorch-$BUILD_VERSION
45-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 pytorch-$BUILD_VERSION
46-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 pytorch-$BUILD_VERSION
47-
elif [[ "$1" == "92" ]]; then
48-
. ./switch_cuda_version.sh 9.2
49-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 pytorch-cuda92-$BUILD_VERSION
50-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 pytorch-cuda92-$BUILD_VERSION
51-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 pytorch-cuda92-$BUILD_VERSION
52-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 pytorch-cuda92-$BUILD_VERSION
53-
elif [[ "$1" == "cpu" ]]; then
54-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 pytorch-cpu-$BUILD_VERSION
55-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 pytorch-cpu-$BUILD_VERSION
56-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 pytorch-cpu-$BUILD_VERSION
57-
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 pytorch-cpu-$BUILD_VERSION
58-
else
59-
echo "Error, unknown argument $1"
60-
exit 1
61-
fi
69+
elif [[ "$desired_cuda" != 'cpu' ]]; then
70+
echo "Switching to CUDA version ${desired_cuda:0:1}.${desired_cuda:1:1}"
71+
. ./switch_cuda_version.sh "${desired_cuda:0:1}.${desired_cuda:1:1}"
72+
$portable_sed "s/cudatoolkit =[0-9]/cudatoolkit =${desired_cuda:0:1}/g" "$build_folder/meta.yaml"
73+
fi
74+
75+
# Alter the meta.yaml to use passed in Github repo/branch
76+
if [[ -n "$GITHUB_ORG" ]]; then
77+
$portable_sed "s#git_url:.*#git_url: https://github.com/$GITHUB_ORG/pytorch#g" "$build_folder/meta.yaml"
78+
fi
79+
if [[ -n "$PYTORCH_BRANCH" ]]; then
80+
$portable_sed "s#git_rev:.*#git_rev: $PYTORCH_BRANCH#g" "$build_folder/meta.yaml"
6281
fi
6382

83+
# Loop through all Python versions to build a package for each
84+
for py_ver in "${DESIRED_PYTHON[@]}"; do
85+
echo "Build $build_folder for Python version $py_ver"
86+
time conda build -c "$ANACONDA_USER" \
87+
--no-anaconda-upload \
88+
--python "$py_ver" \
89+
"$build_folder"
90+
done
91+
6492
echo "All builds succeeded, uploading binaries"
6593

6694
set +e
6795

68-
# anaconda -t $ANACONDA_TOKEN upload --user $ANACONDA_USER $(conda build -c $ANACONDA_USER --python 2.7 pytorch-$BUILD_VERSION --output)
69-
# anaconda -t $ANACONDA_TOKEN upload --user $ANACONDA_USER $(conda build -c $ANACONDA_USER --python 3.5 pytorch-$BUILD_VERSION --output)
70-
# anaconda -t $ANACONDA_TOKEN upload --user $ANACONDA_USER $(conda build -c $ANACONDA_USER --python 3.6 pytorch-$BUILD_VERSION --output)
96+
# anaconda -t $ANACONDA_TOKEN upload --user $ANACONDA_USER $(conda build -c $ANACONDA_USER --python 2.7 pytorch-$build_version --output)
97+
# anaconda -t $ANACONDA_TOKEN upload --user $ANACONDA_USER $(conda build -c $ANACONDA_USER --python 3.5 pytorch-$build_version --output)
98+
# anaconda -t $ANACONDA_TOKEN upload --user $ANACONDA_USER $(conda build -c $ANACONDA_USER --python 3.6 pytorch-$build_version --output)
7199

72100
unset PYTORCH_BUILD_VERSION
73101
unset PYTORCH_BUILD_NUMBER

‎conda/pytorch-nightly/build.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#!/usr/bin/env bash
2+
set -ex
23

34
export CMAKE_LIBRARY_PATH=$PREFIX/lib:$PREFIX/include:$CMAKE_LIBRARY_PATH
45
export CMAKE_PREFIX_PATH=$PREFIX
@@ -12,6 +13,7 @@ export NCCL_ROOT_DIR=/usr/local/cuda
1213
export USE_STATIC_CUDNN=1
1314
export USE_STATIC_NCCL=1
1415
export ATEN_STATIC_CUDA=1
16+
export USE_CUDA_STATIC_LINK=1
1517

1618
fname_with_sha256() {
1719
HASH=$(sha256sum $1 | cut -c1-8)

‎conda/pytorch-nightly/meta.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ package:
44

55
source:
66
git_url: https://github.com/pytorch/pytorch
7-
git_rev: "nightly"
7+
git_rev: "master"
88

99
requirements:
1010
build:
@@ -47,4 +47,4 @@ about:
4747
license: BSD 3-Clause
4848
license_family: BSD
4949
license_file: LICENSE
50-
summary: PyTorch is an optimized tensor library for deep learning using GPUs and CPUs.
50+
summary: PyTorch is an optimized tensor library for deep learning using GPUs and CPUs.

0 commit comments

Comments
 (0)