diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0c4782f..7ea1dc6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: matrix: # Test all supported versions on Ubuntu: os: [ubuntu-latest] - python: ["3.9", "3.10", "3.11", "pypy-3.11"] + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "pypy-3.11"] experimental: [false] # include: # - os: macos-latest diff --git a/setup.py b/setup.py index 40a40b6..a08d79c 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,38 @@ from setuptools import setup, find_packages, Extension -from distutils.command.build_py import build_py +from setuptools.command.build_py import build_py +from setuptools.command.build_ext import build_ext import os, sys import subprocess -import platform IS_PYPY = '__pypy__' in sys.builtin_module_names +BASEDIR = os.path.dirname(os.path.abspath(__file__)) + class vmprof_build(build_py, object): def run(self): super(vmprof_build, self).run() -BASEDIR = os.path.dirname(os.path.abspath(__file__)) +class vmprof_build_ext(build_ext, object): + """build_ext that runs libbacktrace configure before building. + This is needed because libbacktrace does not have a pre-built library for all platforms. + """ + def run(self): + # configure libbacktrace on Unix systems (not Windows/macOS) + if sys.platform.startswith('linux') or sys.platform.startswith('freebsd'): + libbacktrace_dir = os.path.join(BASEDIR, "src", "libbacktrace") + config_h = os.path.join(libbacktrace_dir, "config.h") + # only run configure if config.h doesn't exist + if not os.path.exists(config_h): + orig_dir = os.getcwd() + os.chdir(libbacktrace_dir) + try: + # generate configure script if it doesn't exist + if not os.path.exists("configure"): + subprocess.check_call(["autoreconf", "-i"]) + subprocess.check_call(["./configure"]) + finally: + os.chdir(orig_dir) + super(vmprof_build_ext, self).run() def _supported_unix(): if sys.platform.startswith('linux'): @@ -65,20 +87,13 @@ def _supported_unix(): 'src/libbacktrace/posix.c', 'src/libbacktrace/sort.c', ] - # configure libbacktrace!! - class vmprof_build(build_py, object): - def run(self): - orig_dir = os.getcwd() - os.chdir(os.path.join(BASEDIR, "src", "libbacktrace")) - subprocess.check_call(["./configure"]) - os.chdir(orig_dir) - super(vmprof_build, self).run() else: raise NotImplementedError("platform '%s' is not supported!" % sys.platform) - extra_compile_args.append('-I src/') - extra_compile_args.append('-I src/libbacktrace') - if sys.version_info[:2] == (3,11): + # use absolute paths for include directories so compilation works from any directory + extra_compile_args.append('-I' + os.path.join(BASEDIR, 'src')) + extra_compile_args.append('-I' + os.path.join(BASEDIR, 'src', 'libbacktrace')) + if sys.version_info[:2] >= (3,11): extra_source_files += ['src/populate_frames.c'] ext_modules = [Extension('_vmprof', sources=[ @@ -116,14 +131,14 @@ def run(self): description="Python's vmprof client", long_description='See https://vmprof.readthedocs.org/', url='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/vmprof/vmprof-python', - cmdclass={'build_py': vmprof_build}, + cmdclass={'build_py': vmprof_build, 'build_ext': vmprof_build_ext}, install_requires=[ 'requests', 'six', 'pytz', 'colorama', ] + extra_install_requires, - python_requires='<3.12', + python_requires='<3.15', tests_require=['pytest','cffi','hypothesis'], entry_points = { 'console_scripts': [ @@ -140,4 +155,4 @@ def run(self): zip_safe=False, include_package_data=True, ext_modules=ext_modules, -) +) \ No newline at end of file diff --git a/src/_vmprof.c b/src/_vmprof.c index 94a7178..dcd9546 100644 --- a/src/_vmprof.c +++ b/src/_vmprof.c @@ -14,7 +14,6 @@ #ifndef RPYTHON_VMPROF #if PY_VERSION_HEX >= 0x030b00f0 /* >= 3.11 */ - #include "internal/pycore_frame.h" #include "populate_frames.h" #endif #endif @@ -140,7 +139,11 @@ void emit_all_code_objects(PyObject * seen_code_ids) Py_ssize_t i, size; void * param[2]; +#if PY_VERSION_HEX >= 0x030D0000 /* >= 3.13 */ + gc_module = PyImport_ImportModule("gc"); +#else gc_module = PyImport_ImportModuleNoBlock("gc"); +#endif if (gc_module == NULL) goto error; diff --git a/src/populate_frames.c b/src/populate_frames.c index e9d2bfa..0b1c74b 100644 --- a/src/populate_frames.c +++ b/src/populate_frames.c @@ -7,6 +7,11 @@ // 0x030B0000 is 3.11. #define PY_311 0x030B0000 +// 0x030D0000 is 3.13. +#define PY_313 0x030D0000 +// 0x030E0000 is 3.14. +#define PY_314 0x030E0000 + #if PY_VERSION_HEX >= PY_311 /** @@ -22,7 +27,12 @@ */ #define Py_BUILD_CORE +#if PY_VERSION_HEX >= PY_314 +// Python 3.14 moved frame internals to pycore_interpframe.h +#include "internal/pycore_interpframe.h" +#else #include "internal/pycore_frame.h" +#endif #undef Py_BUILD_CORE // Modified from @@ -30,7 +40,13 @@ _PyInterpreterFrame *unsafe_PyThreadState_GetInterpreterFrame( PyThreadState *tstate) { assert(tstate != NULL); +#if PY_VERSION_HEX >= PY_313 + // In Python 3.13+, cframe was removed and current_frame is directly on tstate + _PyInterpreterFrame *f = tstate->current_frame; +#else + // Python 3.11 and 3.12 use cframe->current_frame _PyInterpreterFrame *f = tstate->cframe->current_frame; +#endif while (f && _PyFrame_IsIncomplete(f)) { f = f->previous; } @@ -47,7 +63,13 @@ PyCodeObject *unsafe_PyInterpreterFrame_GetCode( _PyInterpreterFrame *frame) { assert(frame != NULL); assert(!_PyFrame_IsIncomplete(frame)); +#if PY_VERSION_HEX >= PY_313 + // In Python 3.13+, use the _PyFrame_GetCode inline function + // f_code was renamed to f_executable + PyCodeObject *code = _PyFrame_GetCode(frame); +#else PyCodeObject *code = frame->f_code; +#endif assert(code != NULL); return code; } @@ -71,6 +93,10 @@ _PyInterpreterFrame *unsafe_PyInterpreterFrame_GetBack( // this function is not available in libpython int _PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame) { int addr = _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); +#if PY_VERSION_HEX >= PY_313 + return PyCode_Addr2Line(_PyFrame_GetCode(frame), addr); +#else return PyCode_Addr2Line(frame->f_code, addr); +#endif } -#endif // PY_VERSION_HEX >= PY_311 \ No newline at end of file +#endif // PY_VERSION_HEX >= PY_311 diff --git a/src/populate_frames.h b/src/populate_frames.h index 509b997..7ff6dfe 100644 --- a/src/populate_frames.h +++ b/src/populate_frames.h @@ -7,8 +7,16 @@ #include +// 0x030E0000 is 3.14. +#define PY_314 0x030E0000 + #define Py_BUILD_CORE +#if PY_VERSION_HEX >= PY_314 +// Python 3.14 moved frame internals to pycore_interpframe.h +#include "internal/pycore_interpframe.h" +#else #include "internal/pycore_frame.h" +#endif #undef Py_BUILD_CORE _PyInterpreterFrame *unsafe_PyThreadState_GetInterpreterFrame(PyThreadState *tstate); @@ -19,4 +27,4 @@ _PyInterpreterFrame *unsafe_PyInterpreterFrame_GetBack(_PyInterpreterFrame *fram int _PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame); -#endif \ No newline at end of file +#endif diff --git a/src/vmp_stack.h b/src/vmp_stack.h index c1f76d4..a0e5d8d 100644 --- a/src/vmp_stack.h +++ b/src/vmp_stack.h @@ -4,7 +4,13 @@ #ifndef RPYTHON_VMPROF #if PY_VERSION_HEX >= 0x030b00f0 /* >= 3.11 */ + #define Py_BUILD_CORE + #if PY_VERSION_HEX >= 0x030E0000 /* >= 3.14 */ + #include "internal/pycore_interpframe.h" + #else #include "internal/pycore_frame.h" + #endif + #undef Py_BUILD_CORE #include "populate_frames.h" #endif #endif diff --git a/src/vmprof_win.c b/src/vmprof_win.c index 94eac4c..3511449 100644 --- a/src/vmprof_win.c +++ b/src/vmprof_win.c @@ -2,13 +2,15 @@ #ifndef RPYTHON_VMPROF #if PY_VERSION_HEX >= 0x030b00f0 /* >= 3.11 */ - #include "internal/pycore_frame.h" #include "populate_frames.h" #endif #endif volatile int thread_started = 0; volatile int enabled = 0; +#ifndef RPYTHON_VMPROF +static PY_WIN_THREAD_STATE *target_tstate = NULL; +#endif HANDLE write_mutex; @@ -106,18 +108,39 @@ int vmprof_snapshot_thread(DWORD thread_id, PY_WIN_THREAD_STATE *tstate, prof_st return depth; #else -#if PY_VERSION_HEX >= 0x030B0000 /* < 3.11 */ - _PyInterpreterFrame * frame = unsafe_PyThreadState_GetInterpreterFrame(tstate); +#if PY_VERSION_HEX >= 0x030B0000 /* >= 3.11 */ + _PyInterpreterFrame * frame = NULL; #else - PY_STACK_FRAME_T * frame = PyThreadState_GetFrame(tstate); + PY_STACK_FRAME_T * frame = NULL; #endif - depth = vmp_walk_and_record_stack(frame, stack->stack, - MAX_STACK_DEPTH, 0, 0); + /* The target thread was suspended at an arbitrary instruction, possibly + in the middle of pushing or popping an interpreter frame, so the frame + chain we are about to read may be inconsistent. Under MSVC, catch the + resulting access violation and drop the sample instead of crashing. */ +#ifdef _MSC_VER + __try { +#endif +#if PY_VERSION_HEX >= 0x030B0000 /* >= 3.11 */ + frame = unsafe_PyThreadState_GetInterpreterFrame(tstate); +#else + frame = PyThreadState_GetFrame(tstate); +#endif + depth = vmp_walk_and_record_stack(frame, stack->stack, + MAX_STACK_DEPTH, 0, 0); +#ifdef _MSC_VER + } __except (EXCEPTION_EXECUTE_HANDLER) { + depth = -1; + } +#endif #if PY_VERSION_HEX < 0x030B0000 /* < 3.11 */ Py_XDECREF(frame); #endif + if (depth < 0) { + ResumeThread(hThread); + return -1; + } stack->depth = depth; stack->stack[depth++] = (void*)((ULONG_PTR)thread_id); stack->count = 1; @@ -174,6 +197,8 @@ long __stdcall vmprof_mainloop(void *arg) continue; } tstate = get_current_thread_state(); + if (!tstate) + tstate = target_tstate; if (!tstate) continue; depth = vmprof_snapshot_thread(tstate->thread_id, tstate, stack); @@ -221,6 +246,9 @@ int vmprof_enable(int memory, int native, int real_time) thread_started = 1; } enabled = 1; +#ifndef RPYTHON_VMPROF + target_tstate = PyThreadState_Get(); +#endif return 0; } @@ -231,6 +259,9 @@ int vmprof_disable(void) (void)vmp_write_time_now(MARKER_TRAILER); enabled = 0; +#ifndef RPYTHON_VMPROF + target_tstate = NULL; +#endif vmp_set_profile_fileno(-1); return 0; } diff --git a/test_requirements.txt b/test_requirements.txt index 1cc9ec8..49a9055 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -3,6 +3,7 @@ requests urllib3==1.26.6 ; python_version < '3.9' backports.shutil_which cffi +setuptools pytest hypothesis colorama diff --git a/vmprof/cli.py b/vmprof/cli.py index b65e6a4..5f067a1 100644 --- a/vmprof/cli.py +++ b/vmprof/cli.py @@ -118,7 +118,7 @@ class IniParser(object): def __init__(self, f): self.ini_parser = configparser.ConfigParser() - self.ini_parser.readfp(f) + self.ini_parser.read_file(f) def get_option(self, name, type, default=None): if type == float: diff --git a/vmprof/test/test_c_source.py b/vmprof/test/test_c_source.py index 0d609cb..1660bd0 100644 --- a/vmprof/test/test_c_source.py +++ b/vmprof/test/test_c_source.py @@ -31,8 +31,8 @@ def setup_class(cls): libs.append('unwind-x86_64') # trick: compile with _CFFI_USE_EMBEDDING=1 which will not define Py_LIMITED_API sources = [] - if sys.version_info[:2] == (3,11): - sources += ['src/populate_frames.c']# needed for cp311 but must not be included in py < 3.11 + if sys.version_info[:2] >= (3, 11): + sources += ['src/populate_frames.c']# needed for py 3.11+ but must not be included in py < 3.11 stack_ffi.set_source("vmprof.test._test_stack", source, include_dirs=['src'], define_macros=[('_CFFI_USE_EMBEDDING',1), ('PY_TEST',1), ('VMP_SUPPORTS_NATIVE_PROFILING',1)], diff --git a/vmprof/test/test_run.py b/vmprof/test/test_run.py index 5d24b2e..78a4e50 100644 --- a/vmprof/test/test_run.py +++ b/vmprof/test/test_run.py @@ -199,8 +199,13 @@ def test_nested_call(): assert len(t.children) == 1 assert 'function_foo' in t[''].name if PY3K: - assert len(t[''].children) == 1 - assert '' in t[''][''].name + # In Python 3.12+, list comprehensions are inlined and don't create + # a separate stack frame (PEP 709), so won't appear + if sys.version_info >= (3, 12): + assert len(t[''].children) == 0 + else: + assert len(t[''].children) == 1 + assert '' in t[''][''].name else: assert len(t[''].children) == 0