Skip to content

[GR-79899] Expand interpreter benchmarks to pyperformance #1171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
graalvmbot merged 1 commit into from
Sep 25, 2026
+100 −77
Merged

[GR-79899] Expand interpreter benchmarks to pyperformance #1171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ci.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,7 @@
[bench]: bench_task(bench, PY_BENCHMARKS) + platform_spec(no_jobs) + raw_results + bench_variants({
"vm_name:graalpython_enterprise" : {"linux:amd64:jdk-latest" : weekly + t("08:00:00")},
"vm_name:graalvm_ee_default" : {"linux:amd64:jdk-latest" : weekly + t("08:00:00") + need_pgo},
"vm_name:graalvm_ee_default_interpreter" : {"linux:amd64:jdk-latest" : weekly + t("08:00:00") + need_pgo},
"vm_name:cpython_launcher" : {"linux:amd64:jdk-latest" : monthly + t("08:00:00")},
"vm_name:pypy_launcher" : {"linux:amd64:jdk-latest" : on_demand + t("08:00:00")},
}),
Expand Down
18 changes: 18 additions & 0 deletions mx.graalpython/mx_graalpython_benchmark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,10 @@ def interpreter(self):
def post_process_command_line_args(self, args):
return args

@contextmanager
def run_environment(self):
yield

def run_vm(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False, env=None):
cmd = [self.interpreter] + args
cmd = mx.apply_command_mapper_hooks(cmd, self.command_mapper_hooks)
Expand Down Expand Up @@ -339,6 +343,20 @@ def run(self, *args, **kwargs):
mx.abort(f"ERROR: benchmark config '{CONFIGURATION_UNCACHED}' not consistent with what runtime reported.")
return code, out, dims

@contextmanager
def run_environment(self):
with super().run_environment():
from mx_graalpython import set_env

existing = os.environ.get("GRAAL_PYTHON_VM_ARGS")
if existing:
existing_args = existing[1:].split("\v") if existing.startswith("\v") else shlex.split(existing)
else:
existing_args = []
value = "\v" + "\v".join(existing_args + list(self.get_extra_polyglot_args()))
with set_env(GRAAL_PYTHON_VM_ARGS=value):
yield

def get_extra_polyglot_args(self):
return ["--experimental-options", "-snapshot-startup", "--python.MaxNativeMemory=%s" % (2**34), *self._extra_polyglot_args]

Expand Down
158 changes: 81 additions & 77 deletions mx.graalpython/mx_graalpython_python_benchmarks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,39 +534,40 @@ def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs):
bms = ["-b", ",".join(benchmarks)]
else:
bms = ["-b", ",".join(DEFAULT_PYPERFORMANCE_BENCHMARKS)]
json_file = f"{vm_venv}.json"
retcode = mx.run(
[
join(vm_venv, "bin", "pyperformance"),
"run",
"--inherit-environ",
"PIP_INDEX_URL,PIP_EXTRA_INDEX_URL,PIP_TRUSTED_HOST,PIP_TIMEOUT,PIP_RETRIES,LD_LIBRARY_PATH,LIBRARY_PATH,CPATH,PATH,PYPY_GC_MAX,JAVA_OPTS,GRAAL_PYTHON_ARGS,GRAAL_PYTHON_VM_ARGS",
"-o",
json_file,
*bms,
],
cwd=workdir,
nonZeroIsFatal=False,
)
mx.log(f"Return code of benchmark harness: {retcode}")
# run again in single shot mode for memory measurements
json_file_memory = f"{vm_venv}_memory.json"
retcode = mx.run(
[
join(vm_venv, "bin", "pyperformance"),
"run",
"--debug-single-value",
"--track-memory",
"--inherit-environ",
"PIP_INDEX_URL,PIP_EXTRA_INDEX_URL,PIP_TRUSTED_HOST,PIP_TIMEOUT,PIP_RETRIES,LD_LIBRARY_PATH,LIBRARY_PATH,CPATH,PATH,PYPY_GC_MAX,JAVA_OPTS,GRAAL_PYTHON_ARGS,GRAAL_PYTHON_VM_ARGS",
"-o",
json_file_memory,
*bms,
],
cwd=workdir,
nonZeroIsFatal=False,
)
mx.log(f"Return code of benchmark harness: {retcode}")
with vm.run_environment():
json_file = f"{vm_venv}.json"
retcode = mx.run(
[
join(vm_venv, "bin", "pyperformance"),
"run",
"--inherit-environ",
"PIP_INDEX_URL,PIP_EXTRA_INDEX_URL,PIP_TRUSTED_HOST,PIP_TIMEOUT,PIP_RETRIES,LD_LIBRARY_PATH,LIBRARY_PATH,CPATH,PATH,PYPY_GC_MAX,JAVA_OPTS,GRAAL_PYTHON_ARGS,GRAAL_PYTHON_VM_ARGS",
"-o",
json_file,
*bms,
],
cwd=workdir,
nonZeroIsFatal=False,
)
mx.log(f"Return code of benchmark harness: {retcode}")
# run again in single shot mode for memory measurements
json_file_memory = f"{vm_venv}_memory.json"
retcode = mx.run(
[
join(vm_venv, "bin", "pyperformance"),
"run",
"--debug-single-value",
"--track-memory",
"--inherit-environ",
"PIP_INDEX_URL,PIP_EXTRA_INDEX_URL,PIP_TRUSTED_HOST,PIP_TIMEOUT,PIP_RETRIES,LD_LIBRARY_PATH,LIBRARY_PATH,CPATH,PATH,PYPY_GC_MAX,JAVA_OPTS,GRAAL_PYTHON_ARGS,GRAAL_PYTHON_VM_ARGS",
"-o",
json_file_memory,
*bms,
],
cwd=workdir,
nonZeroIsFatal=False,
)
mx.log(f"Return code of benchmark harness: {retcode}")
shutil.copy(join(workdir, json_file), join(SUITE.dir, "raw_results.json"))
shutil.copy(join(workdir, json_file_memory), join(SUITE.dir, "raw_results_memory.json"))
return retcode, ",".join([join(workdir, json_file), join(workdir, json_file_memory)]), vm_dims
Expand Down Expand Up @@ -637,18 +638,19 @@ def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs):
bms = ["-b", ",".join(benchmarks)]
else:
bms = ["-b", ",".join(DEFAULT_PYPY_BENCHMARKS)]
retcode = mx.run(
[
sys.executable,
join(workdir, "benchmarks", "run_local.py"),
f"{vm_venv}/bin/python",
"-o",
join(workdir, json_file),
*bms,
],
cwd=workdir,
nonZeroIsFatal=False,
)
with vm.run_environment():
retcode = mx.run(
[
sys.executable,
join(workdir, "benchmarks", "run_local.py"),
f"{vm_venv}/bin/python",
"-o",
join(workdir, json_file),
*bms,
],
cwd=workdir,
nonZeroIsFatal=False,
)
shutil.copy(join(workdir, json_file), join(SUITE.dir, "raw_results.json"))
mx.log(f"Return code of benchmark harness: {retcode}")
return retcode, join(workdir, json_file), vm_dims
Expand Down Expand Up @@ -739,23 +741,24 @@ def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs):

if not benchmarks:
benchmarks = DEFAULT_NUMPY_BENCHMARKS
retcode = mx.run(
[
join(workdir, vm_venv, "bin", "asv"),
"run",
"--strict",
"--record-samples",
"-e",
"--python=same",
"--set-commit-hash",
f"v{self.VERSION}",
"-b", create_asv_benchmark_selection(
benchmarks, skipped=SKIPPED_NUMPY_BENCHMARKS, skipped_patterns=SKIPPED_NUMPY_BENCHMARK_PATTERNS
),
],
cwd=benchdir,
nonZeroIsFatal=False,
)
with vm.run_environment():
retcode = mx.run(
[
join(workdir, vm_venv, "bin", "asv"),
"run",
"--strict",
"--record-samples",
"-e",
"--python=same",
"--set-commit-hash",
f"v{self.VERSION}",
"-b", create_asv_benchmark_selection(
benchmarks, skipped=SKIPPED_NUMPY_BENCHMARKS, skipped_patterns=SKIPPED_NUMPY_BENCHMARK_PATTERNS
),
],
cwd=benchdir,
nonZeroIsFatal=False,
)

json_file = glob.glob(join(benchdir, "results", "*", "*numpy*.json"))
mx.log(f"Return code of benchmark harness: {retcode}")
Expand Down Expand Up @@ -885,21 +888,22 @@ def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs):

if not benchmarks:
benchmarks = DEFAULT_PANDAS_BENCHMARKS
retcode = mx.run(
[
join(workdir, vm_venv, "bin", "asv"),
"run",
"--strict",
"--record-samples",
"-e",
"--python=same",
"--set-commit-hash",
self.VERSION_TAG,
"-b", create_asv_benchmark_selection(benchmarks, skipped=SKIPPED_PANDAS_BENCHMARKS),
],
cwd=benchdir,
nonZeroIsFatal=False,
)
with vm.run_environment():
retcode = mx.run(
[
join(workdir, vm_venv, "bin", "asv"),
"run",
"--strict",
"--record-samples",
"-e",
"--python=same",
"--set-commit-hash",
self.VERSION_TAG,
"-b", create_asv_benchmark_selection(benchmarks, skipped=SKIPPED_PANDAS_BENCHMARKS),
],
cwd=benchdir,
nonZeroIsFatal=False,
)

json_file = glob.glob(join(benchdir, "results", "*", "*pandas*.json"))
mx.log(f"Return code of benchmark harness: {retcode}")
Expand Down
Loading