From d384dd68ae1b94e4662c5cf336736ddc1b6f2e89 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 11 Dec 2019 09:16:25 -0800 Subject: [PATCH 1/6] Improved performanc of setup; add parallel build support --- setup.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/setup.py b/setup.py index e6a95d4e7afd8..4d6b67ae978d1 100755 --- a/setup.py +++ b/setup.py @@ -129,12 +129,7 @@ def build_extensions(self): if cython: self.render_templates(_pxifiles) - numpy_incl = pkg_resources.resource_filename("numpy", "core/include") - - for ext in self.extensions: - if hasattr(ext, "include_dirs") and numpy_incl not in ext.include_dirs: - ext.include_dirs.append(numpy_incl) - _build_ext.build_extensions(self) + super().build_extensions() DESCRIPTION = "Powerful data structures for data analysis, time series, and statistics" From 581cb5544678502766b18184d61e7703ea1ca9b4 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 11 Dec 2019 11:20:01 -0800 Subject: [PATCH 2/6] Parallel cythonize calls --- setup.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4d6b67ae978d1..3036c03155586 100755 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ BSD license. Parts are from lxml (https://github.com/lxml/lxml) """ +import argparse from distutils.sysconfig import get_config_vars from distutils.version import LooseVersion import os @@ -525,6 +526,19 @@ def maybe_cythonize(extensions, *args, **kwargs): if hasattr(ext, "include_dirs") and numpy_incl not in ext.include_dirs: ext.include_dirs.append(numpy_incl) + # reuse any parallel arguments provided for compliation to cythonize + parser = argparse.ArgumentParser() + parser.add_argument("-j", type=int) + parser.add_argument("--parallel", type=int) + parsed, _ = parser.parse_known_args() + + nthreads = 0 + if "parallel" in parsed: + nthreads = parsed.parallel + elif "j" is parsed: + nthreads = parsed.j + + kwargs["nthreads"] = nthreads build_ext.render_templates(_pxifiles) return cythonize(extensions, *args, **kwargs) @@ -681,7 +695,7 @@ def srcpath(name=None, suffix=".pyx", subdir="src"): "_libs.window.aggregations": { "pyxfile": "_libs/window/aggregations", "language": "c++", - "suffix": ".cpp" + "suffix": ".cpp", }, "_libs.window.indexers": {"pyxfile": "_libs/window/indexers"}, "_libs.writers": {"pyxfile": "_libs/writers"}, From 38f0dbaae2deb4ecd12a0da61b2ffaba07b5e44d Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 11 Dec 2019 11:28:17 -0800 Subject: [PATCH 3/6] typo fixes --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3036c03155586..1d72cb63fe06e 100755 --- a/setup.py +++ b/setup.py @@ -533,9 +533,9 @@ def maybe_cythonize(extensions, *args, **kwargs): parsed, _ = parser.parse_known_args() nthreads = 0 - if "parallel" in parsed: + if "parallel" in parsed and parsed.parallel is not None: nthreads = parsed.parallel - elif "j" is parsed: + elif "j" in parsed and parsed.j is not None: nthreads = parsed.j kwargs["nthreads"] = nthreads From b3e0533d327587f44d829050567fa3d90d4f28fa Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 11 Dec 2019 11:32:22 -0800 Subject: [PATCH 4/6] simplified --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1d72cb63fe06e..f7eb467cca8bc 100755 --- a/setup.py +++ b/setup.py @@ -533,9 +533,9 @@ def maybe_cythonize(extensions, *args, **kwargs): parsed, _ = parser.parse_known_args() nthreads = 0 - if "parallel" in parsed and parsed.parallel is not None: + if parsed.parallel: nthreads = parsed.parallel - elif "j" in parsed and parsed.j is not None: + elif parsed.j: nthreads = parsed.j kwargs["nthreads"] = nthreads From 41c69174ea8459390f79e1984a176e691f18c3d6 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 11 Dec 2019 11:37:06 -0800 Subject: [PATCH 5/6] Add flags to CI --- ci/incremental/build.cmd | 2 +- ci/setup_env.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/incremental/build.cmd b/ci/incremental/build.cmd index b61b59e287299..cf5942abe92e7 100644 --- a/ci/incremental/build.cmd +++ b/ci/incremental/build.cmd @@ -1,7 +1,7 @@ @rem https://github.com/numba/numba/blob/master/buildscripts/incremental/build.cmd @rem Build extensions -python setup.py build_ext -q -i +python setup.py build_ext -q -i -j4 @rem Install pandas python -m pip install --no-build-isolation -e . diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 3d79c0cfd7000..a58dbe09dec01 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -121,7 +121,7 @@ conda list pandas # Make sure any error below is reported as such echo "[Build extensions]" -python setup.py build_ext -q -i +python setup.py build_ext -q -i -j4 # XXX: Some of our environments end up with old versions of pip (10.x) # Adding a new enough version of pip to the requirements explodes the From de87d56852d4786528c18d7fa5d78488b1130d11 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 11 Dec 2019 12:08:20 -0800 Subject: [PATCH 6/6] Remove windows --- ci/incremental/build.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/incremental/build.cmd b/ci/incremental/build.cmd index cf5942abe92e7..b61b59e287299 100644 --- a/ci/incremental/build.cmd +++ b/ci/incremental/build.cmd @@ -1,7 +1,7 @@ @rem https://github.com/numba/numba/blob/master/buildscripts/incremental/build.cmd @rem Build extensions -python setup.py build_ext -q -i -j4 +python setup.py build_ext -q -i @rem Install pandas python -m pip install --no-build-isolation -e .