From 25c5dd09d81403a0921e92aedbaaa3d969df7af0 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 29 Jan 2024 15:04:12 -0500 Subject: [PATCH 1/9] Use the same number of loops --- pyperformance/cli.py | 4 ++++ pyperformance/run.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/pyperformance/cli.py b/pyperformance/cli.py index 34544a11..3d83772b 100644 --- a/pyperformance/cli.py +++ b/pyperformance/cli.py @@ -75,6 +75,10 @@ def parse_args(): cmd.add_argument("--min-time", metavar="MIN_TIME", help="Minimum duration in seconds of a single " "value, used to calibrate the number of loops") + cmd.add_argument("--same-loops", + help="Use the same number of loops as a previous run " + "(i.e., don't recalibrate). Should be a path to a " + ".json file from a previous run.") filter_opts(cmd) # show diff --git a/pyperformance/run.py b/pyperformance/run.py index aa2b3744..2d9bf2b6 100644 --- a/pyperformance/run.py +++ b/pyperformance/run.py @@ -1,5 +1,6 @@ from collections import namedtuple import hashlib +import json import sys import time import traceback @@ -50,7 +51,28 @@ def get_run_id(python, bench=None): return RunID(py_id, compat_id, bench, ts) +def get_loops_from_file(filename): + with open(filename) as fd: + data = json.load(fd) + + loops = {} + for benchmark in data["benchmarks"]: + metadata = benchmark.get("metadata", data["metadata"]) + name = metadata["name"] + if name.endswith("_none"): + name = name[:-len("_none")] + if "loops" in metadata: + loops[name] = metadata["loops"] + + return loops + + def run_benchmarks(should_run, python, options): + if hasattr(options, "same_loops"): + loops = get_loops_from_file(options.same_loops) + else: + loops = {} + to_run = sorted(should_run) info = _pythoninfo.get_info(python) @@ -136,6 +158,10 @@ def add_bench(dest_suite, obj): return dest_suite + if name in loops: + pyperf_opts.append(f"--loops={loops[name]}") + pyperf_opts.append("--warmups=1") + bench_venv, bench_runid = benchmarks.get(bench) if bench_venv is None: print("ERROR: Benchmark %s failed: could not install requirements" % name) From aa39cdeef7441388863f3c2132cb7a262e3965fc Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 30 Jan 2024 11:28:20 -0500 Subject: [PATCH 2/9] Change how sqlglot is multiple benchmarks --- .../bm_sqlglot/bm_sqlglot_optimize.toml | 3 +++ .../bm_sqlglot/bm_sqlglot_parse.toml | 3 +++ .../bm_sqlglot/bm_sqlglot_transpile.toml | 3 +++ .../benchmarks/bm_sqlglot/pyproject.toml | 3 ++- .../benchmarks/bm_sqlglot/run_benchmark.py | 25 ++++++++++++++++--- 5 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_optimize.toml create mode 100644 pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_parse.toml create mode 100644 pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_transpile.toml diff --git a/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_optimize.toml b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_optimize.toml new file mode 100644 index 00000000..5c1bd792 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_optimize.toml @@ -0,0 +1,3 @@ +[tool.pyperformance] +name = "sqlglot_optimize" +extra_opts = ["optimize"] \ No newline at end of file diff --git a/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_parse.toml b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_parse.toml new file mode 100644 index 00000000..d0b06e03 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_parse.toml @@ -0,0 +1,3 @@ +[tool.pyperformance] +name = "sqlglot_parse" +extra_opts = ["parse"] \ No newline at end of file diff --git a/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_transpile.toml b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_transpile.toml new file mode 100644 index 00000000..babb81b4 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_transpile.toml @@ -0,0 +1,3 @@ +[tool.pyperformance] +name = "sqlglot_transpile" +extra_opts = ["transpile"] \ No newline at end of file diff --git a/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml b/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml index 6e59a668..bfa6d8e2 100644 --- a/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml +++ b/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml @@ -9,4 +9,5 @@ urls = {repository = "https://github.com/python/pyperformance"} dynamic = ["version"] [tool.pyperformance] -name = "sqlglot" +name = "sqlglot_normalize" +extra_args = ["normalize"] diff --git a/pyperformance/data-files/benchmarks/bm_sqlglot/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_sqlglot/run_benchmark.py index fa7d9efb..653df423 100644 --- a/pyperformance/data-files/benchmarks/bm_sqlglot/run_benchmark.py +++ b/pyperformance/data-files/benchmarks/bm_sqlglot/run_benchmark.py @@ -164,10 +164,27 @@ def bench_normalize(loops): return elapsed +BENCHMARKS = { + "parse": bench_parse, + "transpile": bench_transpile, + "optimize": bench_optimize, + "normalize": bench_normalize +} + + +def add_parser_args(parser): + parser.add_argument( + "benchmark", + choices=BENCHMARKS, + help="Which benchmark to run." + ) + + if __name__ == "__main__": runner = pyperf.Runner() runner.metadata['description'] = "SQLGlot benchmark" - runner.bench_time_func("sqlglot_parse", bench_parse) - runner.bench_time_func("sqlglot_transpile", bench_transpile) - runner.bench_time_func("sqlglot_optimize", bench_optimize) - runner.bench_time_func("sqlglot_normalize", bench_normalize) + add_parser_args(runner.argparser) + args = runner.parse_args() + benchmark = args.benchmark + + runner.bench_time_func(f"sqlglot_{benchmark}", BENCHMARKS[benchmark]) From 9cee0601f8a502176fbb7297b9c7aa7a6cab005c Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 30 Jan 2024 15:12:57 -0500 Subject: [PATCH 3/9] Add docs --- doc/changelog.rst | 3 +++ doc/usage.rst | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index de5c0126..070d2bb5 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,9 @@ Changelog ========= +* Add a --same-loops option to the run command to use the exact same number of + loops as a previous run (without recalibrating). + Version 1.10.0 (2023-10-22) -------------- * Add benchmark for asyncio_webockets diff --git a/doc/usage.rst b/doc/usage.rst index c7336407..34706144 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -140,6 +140,10 @@ options:: -p PYTHON, --python PYTHON Python executable (default: use running Python) + --same-loops SAME_LOOPS + Use the same number of loops as a previous run + (i.e., don't recalibrate). Should be a path to a + .json file from a previous run. show ---- From 8a1966c9aa2688958a31826854447a37e0112bfd Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 30 Jan 2024 15:41:04 -0500 Subject: [PATCH 4/9] Fix sqlglot configuration --- pyperformance/data-files/benchmarks/MANIFEST | 4 ++++ .../data-files/benchmarks/bm_sqlglot/pyproject.toml | 2 +- .../data-files/benchmarks/bm_sqlglot/run_benchmark.py | 6 +++++- pyperformance/run.py | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pyperformance/data-files/benchmarks/MANIFEST b/pyperformance/data-files/benchmarks/MANIFEST index b4d22f7b..d9b1d8a7 100644 --- a/pyperformance/data-files/benchmarks/MANIFEST +++ b/pyperformance/data-files/benchmarks/MANIFEST @@ -77,6 +77,10 @@ spectral_norm sqlalchemy_declarative sqlalchemy_imperative sqlglot +sqlglot_parse +sqlglot_transpile +sqlglot_optimize +sqlglot_normalize sqlite_synth sympy telco diff --git a/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml b/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml index bfa6d8e2..67be5130 100644 --- a/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml +++ b/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml @@ -10,4 +10,4 @@ dynamic = ["version"] [tool.pyperformance] name = "sqlglot_normalize" -extra_args = ["normalize"] +extra_opts = ["normalize"] diff --git a/pyperformance/data-files/benchmarks/bm_sqlglot/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_sqlglot/run_benchmark.py index 653df423..f8fbb79a 100644 --- a/pyperformance/data-files/benchmarks/bm_sqlglot/run_benchmark.py +++ b/pyperformance/data-files/benchmarks/bm_sqlglot/run_benchmark.py @@ -172,6 +172,10 @@ def bench_normalize(loops): } +def add_cmdline_args(cmd, args): + cmd.append(args.benchmark) + + def add_parser_args(parser): parser.add_argument( "benchmark", @@ -181,7 +185,7 @@ def add_parser_args(parser): if __name__ == "__main__": - runner = pyperf.Runner() + runner = pyperf.Runner(add_cmdline_args=add_cmdline_args) runner.metadata['description'] = "SQLGlot benchmark" add_parser_args(runner.argparser) args = runner.parse_args() diff --git a/pyperformance/run.py b/pyperformance/run.py index 2d9bf2b6..622604a5 100644 --- a/pyperformance/run.py +++ b/pyperformance/run.py @@ -68,7 +68,7 @@ def get_loops_from_file(filename): def run_benchmarks(should_run, python, options): - if hasattr(options, "same_loops"): + if options.same_loops is not None: loops = get_loops_from_file(options.same_loops) else: loops = {} From 50b93bd93680370a5b428188b24b977b20fdff79 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 30 Jan 2024 15:51:10 -0500 Subject: [PATCH 5/9] Fix sqlglot --- pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml b/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml index 67be5130..cb8656a2 100644 --- a/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml +++ b/pyperformance/data-files/benchmarks/bm_sqlglot/pyproject.toml @@ -9,5 +9,5 @@ urls = {repository = "https://github.com/python/pyperformance"} dynamic = ["version"] [tool.pyperformance] -name = "sqlglot_normalize" +name = "sqlglot" extra_opts = ["normalize"] From 9e131179a13cc64c880236b9d59e34b29f20905a Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 30 Jan 2024 15:52:58 -0500 Subject: [PATCH 6/9] Fix MANIFEST --- pyperformance/data-files/benchmarks/MANIFEST | 1 - 1 file changed, 1 deletion(-) diff --git a/pyperformance/data-files/benchmarks/MANIFEST b/pyperformance/data-files/benchmarks/MANIFEST index d9b1d8a7..3210b97f 100644 --- a/pyperformance/data-files/benchmarks/MANIFEST +++ b/pyperformance/data-files/benchmarks/MANIFEST @@ -80,7 +80,6 @@ sqlglot sqlglot_parse sqlglot_transpile sqlglot_optimize -sqlglot_normalize sqlite_synth sympy telco From 3377159955e38d3cfb69beab7938553ce998e09f Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 31 Jan 2024 08:34:28 -0500 Subject: [PATCH 7/9] Add newlines --- .../data-files/benchmarks/bm_sqlglot/bm_sqlglot_optimize.toml | 2 +- .../data-files/benchmarks/bm_sqlglot/bm_sqlglot_parse.toml | 2 +- .../data-files/benchmarks/bm_sqlglot/bm_sqlglot_transpile.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_optimize.toml b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_optimize.toml index 5c1bd792..7f59f0b8 100644 --- a/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_optimize.toml +++ b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_optimize.toml @@ -1,3 +1,3 @@ [tool.pyperformance] name = "sqlglot_optimize" -extra_opts = ["optimize"] \ No newline at end of file +extra_opts = ["optimize"] diff --git a/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_parse.toml b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_parse.toml index d0b06e03..b886688a 100644 --- a/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_parse.toml +++ b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_parse.toml @@ -1,3 +1,3 @@ [tool.pyperformance] name = "sqlglot_parse" -extra_opts = ["parse"] \ No newline at end of file +extra_opts = ["parse"] diff --git a/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_transpile.toml b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_transpile.toml index babb81b4..25a26a3f 100644 --- a/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_transpile.toml +++ b/pyperformance/data-files/benchmarks/bm_sqlglot/bm_sqlglot_transpile.toml @@ -1,3 +1,3 @@ [tool.pyperformance] name = "sqlglot_transpile" -extra_opts = ["transpile"] \ No newline at end of file +extra_opts = ["transpile"] From b2a3e7a172167ef4a33a74ad4cbeefc6d974408e Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 1 Feb 2024 11:47:14 -0500 Subject: [PATCH 8/9] Add same_loops to config --- pyperformance/compile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyperformance/compile.py b/pyperformance/compile.py index 0a7c8332..8f26aded 100644 --- a/pyperformance/compile.py +++ b/pyperformance/compile.py @@ -543,6 +543,8 @@ def run_benchmark(self, python=None): cmd.extend(('--affinity', self.conf.affinity)) if self.conf.debug: cmd.append('--debug-single-value') + if self.conf.same_loops: + cmd.append('--same_loops=%s' % self.conf.same_loops) exitcode = self.run_nocheck(*cmd) if os.path.exists(self.filename): @@ -812,6 +814,7 @@ def getint(section, key, default=None): conf.benchmarks = getstr('run_benchmark', 'benchmarks', default='') conf.affinity = getstr('run_benchmark', 'affinity', default='') conf.upload = getboolean('run_benchmark', 'upload', False) + conf.same_loops = getfile('run_benchmark', 'same_loops', default='') # paths conf.build_dir = os.path.join(conf.directory, 'build') From 14cd648f33db50539dce9ccf8bcde8c4b02ca39a Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 1 Feb 2024 18:07:27 -0500 Subject: [PATCH 9/9] Don't pass hardcoded warmups --- pyperformance/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyperformance/run.py b/pyperformance/run.py index 622604a5..f572181c 100644 --- a/pyperformance/run.py +++ b/pyperformance/run.py @@ -160,7 +160,6 @@ def add_bench(dest_suite, obj): if name in loops: pyperf_opts.append(f"--loops={loops[name]}") - pyperf_opts.append("--warmups=1") bench_venv, bench_runid = benchmarks.get(bench) if bench_venv is None: