diff --git a/ci.jsonnet b/ci.jsonnet index 720e4d114f..cc9cf3fd93 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -475,6 +475,7 @@ [bench]: bench_task(bench, PY_BENCHMARKS) + platform_spec(no_jobs) + raw_results + bench_variants({ "vm_name:graalpython_enterprise" : {"linux:amd64:jdk-latest" : weekly + t("08:00:00")}, "vm_name:graalvm_ee_default" : {"linux:amd64:jdk-latest" : weekly + t("08:00:00") + need_pgo}, + "vm_name:graalvm_ee_default_interpreter" : {"linux:amd64:jdk-latest" : weekly + t("08:00:00") + need_pgo}, "vm_name:cpython_launcher" : {"linux:amd64:jdk-latest" : monthly + t("08:00:00")}, "vm_name:pypy_launcher" : {"linux:amd64:jdk-latest" : on_demand + t("08:00:00")}, }), diff --git a/mx.graalpython/mx_graalpython_benchmark.py b/mx.graalpython/mx_graalpython_benchmark.py index bf815c23f6..ab1eddaec1 100644 --- a/mx.graalpython/mx_graalpython_benchmark.py +++ b/mx.graalpython/mx_graalpython_benchmark.py @@ -160,6 +160,10 @@ def interpreter(self): def post_process_command_line_args(self, args): return args + @contextmanager + def run_environment(self): + yield + def run_vm(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False, env=None): cmd = [self.interpreter] + args cmd = mx.apply_command_mapper_hooks(cmd, self.command_mapper_hooks) @@ -339,6 +343,20 @@ def run(self, *args, **kwargs): mx.abort(f"ERROR: benchmark config '{CONFIGURATION_UNCACHED}' not consistent with what runtime reported.") return code, out, dims + @contextmanager + def run_environment(self): + with super().run_environment(): + from mx_graalpython import set_env + + existing = os.environ.get("GRAAL_PYTHON_VM_ARGS") + if existing: + existing_args = existing[1:].split("\v") if existing.startswith("\v") else shlex.split(existing) + else: + existing_args = [] + value = "\v" + "\v".join(existing_args + list(self.get_extra_polyglot_args())) + with set_env(GRAAL_PYTHON_VM_ARGS=value): + yield + def get_extra_polyglot_args(self): return ["--experimental-options", "-snapshot-startup", "--python.MaxNativeMemory=%s" % (2**34), *self._extra_polyglot_args] diff --git a/mx.graalpython/mx_graalpython_python_benchmarks.py b/mx.graalpython/mx_graalpython_python_benchmarks.py index ab08ff8e5d..212b2650af 100644 --- a/mx.graalpython/mx_graalpython_python_benchmarks.py +++ b/mx.graalpython/mx_graalpython_python_benchmarks.py @@ -534,39 +534,40 @@ def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs): bms = ["-b", ",".join(benchmarks)] else: bms = ["-b", ",".join(DEFAULT_PYPERFORMANCE_BENCHMARKS)] - json_file = f"{vm_venv}.json" - retcode = mx.run( - [ - join(vm_venv, "bin", "pyperformance"), - "run", - "--inherit-environ", - "PIP_INDEX_URL,PIP_EXTRA_INDEX_URL,PIP_TRUSTED_HOST,PIP_TIMEOUT,PIP_RETRIES,LD_LIBRARY_PATH,LIBRARY_PATH,CPATH,PATH,PYPY_GC_MAX,JAVA_OPTS,GRAAL_PYTHON_ARGS,GRAAL_PYTHON_VM_ARGS", - "-o", - json_file, - *bms, - ], - cwd=workdir, - nonZeroIsFatal=False, - ) - mx.log(f"Return code of benchmark harness: {retcode}") - # run again in single shot mode for memory measurements - json_file_memory = f"{vm_venv}_memory.json" - retcode = mx.run( - [ - join(vm_venv, "bin", "pyperformance"), - "run", - "--debug-single-value", - "--track-memory", - "--inherit-environ", - "PIP_INDEX_URL,PIP_EXTRA_INDEX_URL,PIP_TRUSTED_HOST,PIP_TIMEOUT,PIP_RETRIES,LD_LIBRARY_PATH,LIBRARY_PATH,CPATH,PATH,PYPY_GC_MAX,JAVA_OPTS,GRAAL_PYTHON_ARGS,GRAAL_PYTHON_VM_ARGS", - "-o", - json_file_memory, - *bms, - ], - cwd=workdir, - nonZeroIsFatal=False, - ) - mx.log(f"Return code of benchmark harness: {retcode}") + with vm.run_environment(): + json_file = f"{vm_venv}.json" + retcode = mx.run( + [ + join(vm_venv, "bin", "pyperformance"), + "run", + "--inherit-environ", + "PIP_INDEX_URL,PIP_EXTRA_INDEX_URL,PIP_TRUSTED_HOST,PIP_TIMEOUT,PIP_RETRIES,LD_LIBRARY_PATH,LIBRARY_PATH,CPATH,PATH,PYPY_GC_MAX,JAVA_OPTS,GRAAL_PYTHON_ARGS,GRAAL_PYTHON_VM_ARGS", + "-o", + json_file, + *bms, + ], + cwd=workdir, + nonZeroIsFatal=False, + ) + mx.log(f"Return code of benchmark harness: {retcode}") + # run again in single shot mode for memory measurements + json_file_memory = f"{vm_venv}_memory.json" + retcode = mx.run( + [ + join(vm_venv, "bin", "pyperformance"), + "run", + "--debug-single-value", + "--track-memory", + "--inherit-environ", + "PIP_INDEX_URL,PIP_EXTRA_INDEX_URL,PIP_TRUSTED_HOST,PIP_TIMEOUT,PIP_RETRIES,LD_LIBRARY_PATH,LIBRARY_PATH,CPATH,PATH,PYPY_GC_MAX,JAVA_OPTS,GRAAL_PYTHON_ARGS,GRAAL_PYTHON_VM_ARGS", + "-o", + json_file_memory, + *bms, + ], + cwd=workdir, + nonZeroIsFatal=False, + ) + mx.log(f"Return code of benchmark harness: {retcode}") shutil.copy(join(workdir, json_file), join(SUITE.dir, "raw_results.json")) shutil.copy(join(workdir, json_file_memory), join(SUITE.dir, "raw_results_memory.json")) return retcode, ",".join([join(workdir, json_file), join(workdir, json_file_memory)]), vm_dims @@ -637,18 +638,19 @@ def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs): bms = ["-b", ",".join(benchmarks)] else: bms = ["-b", ",".join(DEFAULT_PYPY_BENCHMARKS)] - retcode = mx.run( - [ - sys.executable, - join(workdir, "benchmarks", "run_local.py"), - f"{vm_venv}/bin/python", - "-o", - join(workdir, json_file), - *bms, - ], - cwd=workdir, - nonZeroIsFatal=False, - ) + with vm.run_environment(): + retcode = mx.run( + [ + sys.executable, + join(workdir, "benchmarks", "run_local.py"), + f"{vm_venv}/bin/python", + "-o", + join(workdir, json_file), + *bms, + ], + cwd=workdir, + nonZeroIsFatal=False, + ) shutil.copy(join(workdir, json_file), join(SUITE.dir, "raw_results.json")) mx.log(f"Return code of benchmark harness: {retcode}") return retcode, join(workdir, json_file), vm_dims @@ -739,23 +741,24 @@ def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs): if not benchmarks: benchmarks = DEFAULT_NUMPY_BENCHMARKS - retcode = mx.run( - [ - join(workdir, vm_venv, "bin", "asv"), - "run", - "--strict", - "--record-samples", - "-e", - "--python=same", - "--set-commit-hash", - f"v{self.VERSION}", - "-b", create_asv_benchmark_selection( - benchmarks, skipped=SKIPPED_NUMPY_BENCHMARKS, skipped_patterns=SKIPPED_NUMPY_BENCHMARK_PATTERNS - ), - ], - cwd=benchdir, - nonZeroIsFatal=False, - ) + with vm.run_environment(): + retcode = mx.run( + [ + join(workdir, vm_venv, "bin", "asv"), + "run", + "--strict", + "--record-samples", + "-e", + "--python=same", + "--set-commit-hash", + f"v{self.VERSION}", + "-b", create_asv_benchmark_selection( + benchmarks, skipped=SKIPPED_NUMPY_BENCHMARKS, skipped_patterns=SKIPPED_NUMPY_BENCHMARK_PATTERNS + ), + ], + cwd=benchdir, + nonZeroIsFatal=False, + ) json_file = glob.glob(join(benchdir, "results", "*", "*numpy*.json")) mx.log(f"Return code of benchmark harness: {retcode}") @@ -885,21 +888,22 @@ def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs): if not benchmarks: benchmarks = DEFAULT_PANDAS_BENCHMARKS - retcode = mx.run( - [ - join(workdir, vm_venv, "bin", "asv"), - "run", - "--strict", - "--record-samples", - "-e", - "--python=same", - "--set-commit-hash", - self.VERSION_TAG, - "-b", create_asv_benchmark_selection(benchmarks, skipped=SKIPPED_PANDAS_BENCHMARKS), - ], - cwd=benchdir, - nonZeroIsFatal=False, - ) + with vm.run_environment(): + retcode = mx.run( + [ + join(workdir, vm_venv, "bin", "asv"), + "run", + "--strict", + "--record-samples", + "-e", + "--python=same", + "--set-commit-hash", + self.VERSION_TAG, + "-b", create_asv_benchmark_selection(benchmarks, skipped=SKIPPED_PANDAS_BENCHMARKS), + ], + cwd=benchdir, + nonZeroIsFatal=False, + ) json_file = glob.glob(join(benchdir, "results", "*", "*pandas*.json")) mx.log(f"Return code of benchmark harness: {retcode}")