diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26225103..daeb5df9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,9 +91,76 @@ jobs: cmake -S . -B build-external -DLLVM_DIR="$LLVM_DIR" -DCppInterOp_DIR="$RUNNER_TEMP/cppinterop-build/lib/cmake/CppInterOp" cmake -S . -B build-external -DLLVM_DIR="$LLVM_DIR" -DCppInterOp_DIR="$RUNNER_TEMP/cppinterop-build/lib/cmake/CppInterOp" + # The wheel toolchain on the manylinux floor and on the current release: + # AlmaLinux 8 builds with gcc-toolset-14 and JIT-compiles against the GCC 8 + # headers and runtime, AlmaLinux 10 uses its system compiler throughout. + build-test-almalinux: + name: almalinux:${{ matrix.release }}/llvm21/py3.12/c++${{ matrix.cxx }} + strategy: + fail-fast: false + matrix: + include: + - { release: '8', cxx: '17', python: python3.12, packages: gcc-toolset-14-gcc-c++, bin: /opt/rh/gcc-toolset-14/root/usr/bin } + - { release: '10', cxx: '20', python: python3, packages: eigen3-devel, bin: /usr/bin } + runs-on: ubuntu-24.04 + container: almalinux:${{ matrix.release }} + steps: + - name: Install the build and test dependencies + env: + PACKAGES: ${{ matrix.packages }} + PYTHON: ${{ matrix.python }} + run: dnf -y install gcc-c++ make git zstd zlib-devel boost-devel "$PYTHON-devel" "$PYTHON-pip" "$PACKAGES" + + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Put the distribution's python on PATH + env: + PYTHON: ${{ matrix.python }} + run: | + "$PYTHON" -m venv "$RUNNER_TEMP/venv" + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" + + - uses: compiler-research/ci-workflows/actions/setup-recipe@main + id: llvm + with: + recipe: llvm-wheel + version: '21.1.8' + os: ubuntu-24.04 + arch: x86_64 + ref: 02f380b2e46746cd1c948c085894b8594a867fba + + - name: Build and install cppjit with the test requirements + env: + CC: ${{ matrix.bin }}/gcc + CXX: ${{ matrix.bin }}/g++ + LLVM_DIR: ${{ steps.llvm.outputs.path }}/lib/cmake/llvm + run: python -m pip install . -v --config-settings=cmake.define.LLVM_DIR="$LLVM_DIR" -r requirements.txt + + - name: Run the test suite + env: + CPPINTEROP_EXTRA_INTERPRETER_ARGS: -std=c++${{ matrix.cxx }} + run: | + set -o pipefail + cd test + make -j"$(nproc)" PYTHON=python + python -m pytest -ra --tb=short -p no:cacheprovider --durations=25 \ + -n auto --max-worker-restart 2 | tee "$RUNNER_TEMP/pytest.log" + + - name: Capture summary + if: always() + run: tail -n1 "$RUNNER_TEMP/pytest.log" > pytest-summary.txt 2>/dev/null || echo 'no summary (build or setup failed)' > pytest-summary.txt + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: result-almalinux-${{ matrix.release }}-llvm21-py3.12-cxx${{ matrix.cxx }} + path: pytest-summary.txt + report: if: ${{ always() && github.event_name == 'pull_request' }} - needs: build-test + needs: [build-test, build-test-almalinux] runs-on: ubuntu-latest permissions: contents: read diff --git a/CMakeLists.txt b/CMakeLists.txt index 79828f6e..082b1f80 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,6 +97,12 @@ if(CMAKE_COMPILER_IS_GNUCXX) target_link_options(cppjit PRIVATE -Wl,-Bsymbolic-functions) endif() +# Keep the statically linked toolchain runtime (libstdc++_nonshared, libgcc) +# out of the dynamic symbol table, or the JIT resolves against it. +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(cppjit PRIVATE -Wl,--exclude-libs,ALL) +endif() + # macOS needs to allow undefined Python symbols if(APPLE) set_target_properties(cppjit PROPERTIES diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index 56954755..5782aedf 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -28,7 +28,16 @@ This release supports Python 3.12-3.14 and LLVM 21-22. ## Other changes -- ... +- `import cppjit` raises a `RuntimeError` when the C++ standard headers do + not parse, for example on a host without a C++ toolchain. +- `libcppjit` no longer exports the standard-library symbols it links + statically. A feature the host `libstdc++` lacks, such as + `std::filesystem` on a GCC 8 runtime, raises instead of crashing the + process. +- `cppdef`, `cppexec`, `include` and `c_include` errors carry the + compiler's diagnostic text. +- A call whose JIT wrapper cannot be compiled raises a `RuntimeError` + carrying the JIT's report, whatever the return type. ## Contributors diff --git a/python/cppjit/_cpython_cppjit.py b/python/cppjit/_cpython_cppjit.py index 50a32f37..dcca7aaf 100644 --- a/python/cppjit/_cpython_cppjit.py +++ b/python/cppjit/_cpython_cppjit.py @@ -26,7 +26,10 @@ def _preload_backend_library(): raise ImportError("cannot locate the cppjit.libcppjit extension module") lib = ctypes.CDLL(spec.origin, ctypes.RTLD_GLOBAL) if not lib.LoadCppInterOp(): - raise RuntimeError("failed to load CppInterOp (LoadCppInterOp returned 0)") + raise RuntimeError( + "cppjit could not initialize its C++ interpreter, see the diagnostic " + "above (LoadCppInterOp failed)" + ) return lib diff --git a/src/cpyrt/CPPInstance.cxx b/src/cpyrt/CPPInstance.cxx index 04566692..f64f9cc7 100644 --- a/src/cpyrt/CPPInstance.cxx +++ b/src/cpyrt/CPPInstance.cxx @@ -718,13 +718,12 @@ static Py_hash_t op_hash(CPPInstance* self) { // Try to locate an std::hash for this type and use that if it exists CPPClass* klass = (CPPClass*)Py_TYPE(self); if (klass->fOperators && klass->fOperators->fHash) { - Py_hash_t h = 0; PyObject* hashval = PyObject_CallFunctionObjArgs(klass->fOperators->fHash, (PyObject*)self, nullptr); - if (hashval) { - h = cpyrt_PyLong_AsHash_t(hashval); - Py_DECREF(hashval); - } + if (!hashval) + return -1; + Py_hash_t h = cpyrt_PyLong_AsHash_t(hashval); + Py_DECREF(hashval); return h; } @@ -737,18 +736,19 @@ static Py_hash_t op_hash(CPPInstance* self) { Py_DECREF(dct); if (isValid) { PyObject* hashobj = PyObject_CallObject(hashcls, nullptr); + Py_DECREF(hashcls); + if (!hashobj) + return -1; if (!klass->fOperators) klass->fOperators = new Utility::PyOperators{}; klass->fOperators->fHash = hashobj; - Py_DECREF(hashcls); - Py_hash_t h = 0; PyObject* hashval = PyObject_CallFunctionObjArgs(hashobj, (PyObject*)self, nullptr); - if (hashval) { - h = cpyrt_PyLong_AsHash_t(hashval); - Py_DECREF(hashval); - } + if (!hashval) + return -1; + Py_hash_t h = cpyrt_PyLong_AsHash_t(hashval); + Py_DECREF(hashval); return h; } Py_DECREF(hashcls); diff --git a/src/cpyrt/Executors.cxx b/src/cpyrt/Executors.cxx index 5465a1e9..fbfae1ee 100644 --- a/src/cpyrt/Executors.cxx +++ b/src/cpyrt/Executors.cxx @@ -11,6 +11,8 @@ using namespace cppjit; #include "Utility.h" #include "cppjit_interop.h" +#include "cpyrt/PyException.h" + // Standard #include #include @@ -33,43 +35,67 @@ extern std::set gIteratorTypes; //- helpers ------------------------------------------------------------------ namespace { -#ifdef WITH_THREAD -class GILControl { +// releases the GIL around a call when the call context asks for it +class GILRelease { public: - GILControl() : fSave(PyEval_SaveThread()) {} - ~GILControl() { PyEval_RestoreThread(fSave); } + GILRelease(cpyrt::CallContext* ctxt) : fSave(nullptr) { +#ifdef WITH_THREAD + if (ReleasesGIL(ctxt)) + fSave = PyEval_SaveThread(); +#endif + } + ~GILRelease() { + if (fSave) + PyEval_RestoreThread(fSave); + } private: PyThreadState* fSave; }; -#endif + +// raises when the JIT could not compile the call wrapper. An unsizable +// return type is reported by the executor that expected the temporary. +inline void RaiseOnCallFailure(interop::TCppMethod_t method) { + if (interop::LastCallFailure() != interop::kWrapperJitFailed) + return; + std::string proto = interop::GetName(interop::TCppScope_t(method.data)) + + interop::GetMethodSignature(method, true); + if (!interop::IsConstructor(method)) + proto = interop::GetMethodReturnTypeAsString(method) + " " + proto; + std::string report = interop::LastCallFailureReport(); + PyErr_Format(PyExc_RuntimeError, + "failed to JIT-compile the call wrapper for '%s'%s%s", + proto.c_str(), report.empty() ? "" : ":\n", report.c_str()); + throw cpyrt::PyException(); +} } // unnamed namespace -#ifdef WITH_THREAD #define CPPJIT_IMPL_GILCALL(rtype, tcode) \ static inline rtype GILCall##tcode(interop::TCppMethod_t method, \ interop::TCppObject_t self, \ cpyrt::CallContext* ctxt) { \ - if (!ReleasesGIL(ctxt)) \ - return interop::Call##tcode(method, self, ctxt->GetEncodedSize(), \ - ctxt->GetArgs()); \ - GILControl gc{}; \ - return interop::Call##tcode(method, self, ctxt->GetEncodedSize(), \ - ctxt->GetArgs()); \ + rtype result; \ + { \ + GILRelease gil{ctxt}; \ + result = interop::Call##tcode(method, self, ctxt->GetEncodedSize(), \ + ctxt->GetArgs()); \ + } \ + RaiseOnCallFailure(method); \ + return result; \ } -#else -#define CPPJIT_IMPL_GILCALL(rtype, tcode) \ - static inline rtype GILCall##tcode(interop::TCppMethod_t method, \ - interop::TCppObject_t self, \ - cpyrt::CallContext* ctxt) { \ - return interop::Call##tcode(method, self, ctxt->GetEncodedSize(), \ - ctxt->GetArgs()); \ + +static inline void GILCallV(interop::TCppMethod_t method, + interop::TCppObject_t self, + cpyrt::CallContext* ctxt) { + { + GILRelease gil{ctxt}; + interop::CallV(method, self, ctxt->GetEncodedSize(), ctxt->GetArgs()); } -#endif + RaiseOnCallFailure(method); +} // clang-format off -CPPJIT_IMPL_GILCALL(void, V) CPPJIT_IMPL_GILCALL(unsigned char, B) CPPJIT_IMPL_GILCALL(char, C) CPPJIT_IMPL_GILCALL(short, H) @@ -87,31 +113,27 @@ static inline interop::TCppObject_t GILCallO(interop::TCppMethod_t method, cpyrt::CallContext* ctxt, interop::TCppScope_t klass) { interop::TCppType_t klass_ty = interop::GetTypeFromScope(klass); -#ifdef WITH_THREAD - if (!ReleasesGIL(ctxt)) -#endif - return interop::CallO(method, self, ctxt->GetEncodedSize(), ctxt->GetArgs(), - klass_ty); -#ifdef WITH_THREAD - GILControl gc{}; - return interop::CallO(method, self, ctxt->GetEncodedSize(), ctxt->GetArgs(), - klass_ty); -#endif + interop::TCppObject_t result; + { + GILRelease gil{ctxt}; + result = interop::CallO(method, self, ctxt->GetEncodedSize(), + ctxt->GetArgs(), klass_ty); + } + RaiseOnCallFailure(method); + return result; } static inline interop::TCppObject_t GILCallConstructor(interop::TCppMethod_t method, interop::TCppScope_t klass, cpyrt::CallContext* ctxt) { -#ifdef WITH_THREAD - if (!ReleasesGIL(ctxt)) -#endif - return interop::CallConstructor(method, klass, ctxt->GetEncodedSize(), - ctxt->GetArgs()); -#ifdef WITH_THREAD - GILControl gc{}; - return interop::CallConstructor(method, klass, ctxt->GetEncodedSize(), - ctxt->GetArgs()); -#endif + interop::TCppObject_t result; + { + GILRelease gil{ctxt}; + result = interop::CallConstructor(method, klass, ctxt->GetEncodedSize(), + ctxt->GetArgs()); + } + RaiseOnCallFailure(method); + return result; } static inline PyObject* cpyrt_PyText_FromLong(long cl) { @@ -368,7 +390,7 @@ bool cpyrt::RefExecutor::SetAssignable(PyObject* pyobject) { interop::TCppObject_t self, \ CallContext* ctxt) { \ type* ref = (type*)GILCallR(method, self, ctxt); \ - if (!ref) { /* can happen if wrapper compilation fails */ \ + if (!ref) { /* a reference bound to a null address */ \ PyErr_SetString(PyExc_ReferenceError, \ "attempt to access a null-pointer"); \ return nullptr; \ @@ -429,6 +451,10 @@ PyObject* cpyrt::STLStringRefExecutor::Execute(interop::TCppMethod_t method, interop::GetFullScope("std::string"); std::string* result = (std::string*)GILCallR(method, self, ctxt); + if (!result) { // a reference bound to a null address + PyErr_SetString(PyExc_ReferenceError, "attempt to access a null-pointer"); + return nullptr; + } if (!fAssignable) { std::string* rescp = new std::string{*result}; return BindCppObjectNoCast((void*)rescp, sSTLStringScope, @@ -622,9 +648,13 @@ PyObject* cpyrt::STLStringExecutor::Execute(interop::TCppMethod_t method, interop::GetFullScope("std::string"); std::string* result = (std::string*)GILCallO(method, self, ctxt, sSTLStringScope).data; - if (!result) - result = new std::string{}; - else if (PyErr_Occurred()) { + if (!result) { + if (!PyErr_Occurred()) // the callee may have set a python error itself + PyErr_SetString(PyExc_ValueError, + "nullptr result where temporary expected"); + return nullptr; + } + if (PyErr_Occurred()) { delete result; return nullptr; } @@ -644,8 +674,10 @@ PyObject* cpyrt::STLWStringExecutor::Execute(interop::TCppMethod_t method, std::wstring* result = (std::wstring*)GILCallO(method, self, ctxt, sSTLWStringScope).data; if (!result) { - wchar_t w = L'\0'; - return PyUnicode_FromWideChar(&w, 0); + if (!PyErr_Occurred()) // the callee may have set a python error itself + PyErr_SetString(PyExc_ValueError, + "nullptr result where temporary expected"); + return nullptr; } PyObject* pyresult = PyUnicode_FromWideChar(result->c_str(), result->size()); @@ -773,6 +805,10 @@ PyObject* cpyrt::InstancePtrPtrExecutor::Execute(interop::TCppMethod_t method, return BindCppObject((void*)result, fClass, CPPInstance::kIsPtrPtr | CPPInstance::kIsReference); + if (!result) { // nothing to assign to + PyErr_SetString(PyExc_ReferenceError, "attempt to access a null-pointer"); + return nullptr; + } CPPInstance* cppinst = (CPPInstance*)fAssignable; *result = cppinst->GetObject(); @@ -792,6 +828,10 @@ PyObject* cpyrt::InstancePtrRefExecutor::Execute(interop::TCppMethod_t method, return SetInstanceCheckError(fAssignable); void** result = (void**)GILCallR(method, self, ctxt); + if (!result) { // a reference bound to a null address + PyErr_SetString(PyExc_ReferenceError, "attempt to access a null-pointer"); + return nullptr; + } if (!fAssignable) return BindCppObject(*result, fClass); diff --git a/src/cpyrt/Pythonize.cxx b/src/cpyrt/Pythonize.cxx index 45fc0d26..4347d0bb 100644 --- a/src/cpyrt/Pythonize.cxx +++ b/src/cpyrt/Pythonize.cxx @@ -106,6 +106,8 @@ PyObject* PyStyleIndex(PyObject* self, PyObject* index) { return nullptr; Py_ssize_t size = PySequence_Size(self); + if (size == -1 && PyErr_Occurred()) + return nullptr; if (idx >= size || (idx < 0 && idx < -size)) { PyErr_SetString(PyExc_IndexError, "index out of range"); return nullptr; @@ -329,6 +331,8 @@ static bool FillVector(PyObject* vecin, PyObject* args, ItemGetter* getter) { if (0 < sz) { PyObject* res = PyObject_CallMethod(vecin, (char*)"reserve", (char*)"n", sz); + if (!res) + return false; Py_DECREF(res); } else // i.e. sz == 0, so empty container: done return true; @@ -551,7 +555,13 @@ PyObject* VectorData(PyObject* self, PyObject*) { //--------------------------------------------------------------------------- PyObject* VectorArray(PyObject* self, PyObject* args, PyObject* kwargs) { PyObject* pydata = VectorData(self, nullptr); + if (!pydata) + return nullptr; PyObject* arrcall = PyObject_GetAttr(pydata, PyStrings::gArray); + if (!arrcall) { + Py_DECREF(pydata); + return nullptr; + } PyObject* newarr = PyObject_Call(arrcall, args, kwargs); Py_DECREF(arrcall); Py_DECREF(pydata); @@ -634,11 +644,15 @@ static PyObject* vector_iter(PyObject* v) { } PyObject* pydata = CallPyObjMethod(v, "__real_data"); - if (!pydata || Utility::GetBuffer(pydata, '*', 1, vi->vi_data, false) == 0) + if (!pydata) { + Py_DECREF((PyObject*)vi); + return nullptr; + } + if (Utility::GetBuffer(pydata, '*', 1, vi->vi_data, false) == 0) vi->vi_data = CPPInstance_Check(pydata) ? ((CPPInstance*)pydata)->GetObjectRaw() : nullptr; - Py_XDECREF(pydata); + Py_DECREF(pydata); } else { PyErr_Clear(); @@ -653,6 +667,10 @@ static PyObject* vector_iter(PyObject* v) { vi->ii_pos = 0; vi->ii_len = PySequence_Size(v); + if (vi->ii_len == -1 && PyErr_Occurred()) { + Py_DECREF((PyObject*)vi); + return nullptr; + } PyObject_GC_Track(vi); return (PyObject*)vi; @@ -668,12 +686,18 @@ PyObject* VectorGetItem(CPPInstance* self, PySliceObject* index) { PyObject* pyclass = (PyObject*)Py_TYPE((PyObject*)self); PyObject* nseq = PyObject_CallObject(pyclass, nullptr); + if (!nseq) + return nullptr; Py_ssize_t start, stop, step; PySlice_GetIndices((cpyrt_PySliceCast)index, PyObject_Length((PyObject*)self), &start, &stop, &step); const Py_ssize_t nlen = PySequence_Size((PyObject*)self); + if (nlen == -1 && PyErr_Occurred()) { + Py_DECREF(nseq); + return nullptr; + } if (!AdjustSlice(nlen, start, stop, step)) return nseq; @@ -682,9 +706,18 @@ PyObject* VectorGetItem(CPPInstance* self, PySliceObject* index) { PyObject* pyidx = PyInt_FromSsize_t(i); PyObject* item = PyObject_CallMethodOneArg((PyObject*)self, PyStrings::gGetNoCheck, pyidx); - CallPyObjMethod(nseq, "push_back", item); - Py_DECREF(item); Py_DECREF(pyidx); + if (!item) { + Py_DECREF(nseq); + return nullptr; + } + PyObject* pbres = CallPyObjMethod(nseq, "push_back", item); + Py_DECREF(item); + if (!pbres) { + Py_DECREF(nseq); + return nullptr; + } + Py_DECREF(pbres); } return nseq; @@ -713,11 +746,17 @@ PyObject* VectorBoolGetItem(CPPInstance* self, PyObject* idx) { if (PySlice_Check(idx)) { PyObject* pyclass = (PyObject*)Py_TYPE((PyObject*)self); PyObject* nseq = PyObject_CallObject(pyclass, nullptr); + if (!nseq) + return nullptr; Py_ssize_t start, stop, step; PySlice_GetIndices((cpyrt_PySliceCast)idx, PyObject_Length((PyObject*)self), &start, &stop, &step); const Py_ssize_t nlen = PySequence_Size((PyObject*)self); + if (nlen == -1 && PyErr_Occurred()) { + Py_DECREF(nseq); + return nullptr; + } if (!AdjustSlice(nlen, start, stop, step)) return nseq; @@ -726,9 +765,18 @@ PyObject* VectorBoolGetItem(CPPInstance* self, PyObject* idx) { PyObject* pyidx = PyInt_FromSsize_t(i); PyObject* item = PyObject_CallMethodOneArg((PyObject*)self, PyStrings::gGetItem, pyidx); - CallPyObjMethod(nseq, "push_back", item); - Py_DECREF(item); Py_DECREF(pyidx); + if (!item) { + Py_DECREF(nseq); + return nullptr; + } + PyObject* pbres = CallPyObjMethod(nseq, "push_back", item); + Py_DECREF(item); + if (!pbres) { + Py_DECREF(nseq); + return nullptr; + } + Py_DECREF(pbres); } return nseq; @@ -803,15 +851,24 @@ PyObject* ArrayInit(PyObject* self, PyObject* args, PyObject* /* kwds */) { PyObject* items = PyTuple_GET_ITEM(args, 0); Py_ssize_t fillsz = PySequence_Size(items); - if (PySequence_Size(self) != fillsz) { + Py_ssize_t selfsz = PySequence_Size(self); + if (selfsz == -1 && PyErr_Occurred()) { + Py_DECREF(result); + return nullptr; + } + if (selfsz != fillsz) { PyErr_Format(PyExc_ValueError, "received sequence of size %zd where %zd expected", fillsz, - PySequence_Size(self)); + selfsz); Py_DECREF(result); return nullptr; } PyObject* si_call = PyObject_GetAttr(self, PyStrings::gSetItem); + if (!si_call) { + Py_DECREF(result); + return nullptr; + } for (Py_ssize_t i = 0; i < fillsz; ++i) { PyObject* item = PySequence_GetItem(items, i); PyObject* index = PyInt_FromSsize_t(i); @@ -851,6 +908,10 @@ static PyObject* MapFromPairs(PyObject* self, PyObject* pairs) { return nullptr; PyObject* si_call = PyObject_GetAttr(self, PyStrings::gSetItem); + if (!si_call) { + Py_DECREF(result); + return nullptr; + } for (Py_ssize_t i = 0; i < PySequence_Size(pairs); ++i) { PyObject* pair = PySequence_GetItem(pairs, i); PyObject* sires = nullptr; @@ -1095,6 +1156,10 @@ static PyObject* index_iter(PyObject* c) { ii->ii_container = c; ii->ii_pos = 0; ii->ii_len = PySequence_Size(c); + if (ii->ii_len == -1 && PyErr_Occurred()) { + Py_DECREF((PyObject*)ii); + return nullptr; + } PyObject_GC_Track(ii); return (PyObject*)ii; @@ -1429,6 +1494,8 @@ Py_hash_t STLStringHash(PyObject* self) { // std::string objects hash to the same values as Python strings to allow // matches in dictionaries etc. PyObject* data = STLStringGetData(self, false); + if (!data) + return -1; Py_hash_t h = cpyrt_PyText_Type.tp_hash(data); Py_DECREF(data); return h; diff --git a/src/cpyrt/cpyrtModule.cxx b/src/cpyrt/cpyrtModule.cxx index dd54099f..0ac01ae9 100644 --- a/src/cpyrt/cpyrtModule.cxx +++ b/src/cpyrt/cpyrtModule.cxx @@ -14,6 +14,7 @@ using namespace cppjit; #include "MemoryRegulator.h" #include "ProxyWrappers.h" #include "PyStrings.h" +#include "StderrCapture.h" #include "TemplateProxy.h" #include "TupleOfInstances.h" #include "Utility.h" @@ -237,8 +238,6 @@ PyObject* gSegvException = nullptr; PyObject* gIllException = nullptr; PyObject* gAbrtException = nullptr; std::unordered_set gPinnedTypes; -std::ostringstream gCapturedError; -std::streambuf* gOldErrorBuffer = nullptr; std::unordered_map>& pythonizations() { static std::unordered_map> pyzMap; @@ -813,25 +812,17 @@ static PyObject* AddSmartPtrType(PyObject*, PyObject* args) { } //---------------------------------------------------------------------------- -static PyObject* BeginCaptureStderr(PyObject*, PyObject*) { - gOldErrorBuffer = std::cerr.rdbuf(); - std::cerr.rdbuf(gCapturedError.rdbuf()); +static interop::StderrCapture gStderrCapture; +static PyObject* BeginCaptureStderr(PyObject*, PyObject*) { + gStderrCapture.Begin(); Py_RETURN_NONE; } //---------------------------------------------------------------------------- static PyObject* EndCaptureStderr(PyObject*, PyObject*) { - // restore old rdbuf and return captured result - std::cerr.rdbuf(gOldErrorBuffer); - gOldErrorBuffer = nullptr; - - std::string capturedError = std::move(gCapturedError).str(); - - gCapturedError.str(""); - gCapturedError.clear(); - - return Py_BuildValue("s", capturedError.c_str()); + std::string text = gStderrCapture.Stop(); + return PyUnicode_DecodeUTF8(text.data(), (Py_ssize_t)text.size(), "replace"); } } // unnamed namespace diff --git a/src/interop/StderrCapture.h b/src/interop/StderrCapture.h new file mode 100644 index 00000000..727c4ced --- /dev/null +++ b/src/interop/StderrCapture.h @@ -0,0 +1,61 @@ +#ifndef CPPJIT_INTEROP_STDERRCAPTURE_H +#define CPPJIT_INTEROP_STDERRCAPTURE_H + +#include +#include +#include +#ifdef __linux__ +#include +#endif + +namespace cppjit::interop { + +// Redirects descriptor 2 into an anonymous file between Begin and Stop. clang +// and the JIT report on the descriptor, which a std::cerr buffer swap misses. +class StderrCapture { +public: + ~StderrCapture() { Stop(); } + + void Begin() { + if (fFd != -1) + return; + fflush(stderr); +#ifdef __linux__ + fFd = memfd_create("cppjit-stderr", MFD_CLOEXEC); +#else + if (FILE* f = tmpfile()) { + fFd = dup(fileno(f)); + fclose(f); + } +#endif + if (fFd == -1 || (fSaved = dup(2)) == -1 || dup2(fFd, 2) == -1) + Stop(); // the output stays on the descriptor + } + + // restores the descriptor and returns the text written meanwhile + std::string Stop() { + std::string text; + if (fFd == -1) + return text; + fflush(stderr); + if (fSaved != -1) { + dup2(fSaved, 2); + close(fSaved); + } + lseek(fFd, 0, SEEK_SET); + char buf[4096]; + for (ssize_t n; (n = read(fFd, buf, sizeof buf)) > 0;) + text.append(buf, n); + close(fFd); + fFd = fSaved = -1; + return text; + } + +private: + int fFd = -1; + int fSaved = -1; +}; + +} // namespace cppjit::interop + +#endif // !CPPJIT_INTEROP_STDERRCAPTURE_H diff --git a/src/interop/cppjit_interop.h b/src/interop/cppjit_interop.h index 883f0a38..4e1d48a0 100644 --- a/src/interop/cppjit_interop.h +++ b/src/interop/cppjit_interop.h @@ -173,6 +173,14 @@ RPY_EXPORTED TCppObject_t CallO(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args, TCppType_t result_type); +// why the most recent call on this thread returned no result +enum ECallFailure { kCallCompleted = 0, kUnsizableReturn, kWrapperJitFailed }; +RPY_EXPORTED +ECallFailure LastCallFailure(); +// the JIT's report behind kWrapperJitFailed, with symbol names demangled +RPY_EXPORTED +std::string LastCallFailureReport(); + RPY_EXPORTED TCppFuncAddr_t GetFunctionAddress(TCppMethod_t method, bool check_enabled = true); diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index 735e207c..6b5ff2e2 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -8,6 +8,7 @@ #include "precommondefs.h" // This defines several system feature macros and should be included before any system header. // Bindings +#include "StderrCapture.h" #include "cppjit_interop.h" using namespace cppjit; @@ -24,6 +25,7 @@ static inline size_t CALL_NARGS(size_t nargs) { return nargs & ~DIRECT_CALL; } // Standard #include // for std::count, std::remove #include +#include #include #include // for getenv #include @@ -37,6 +39,7 @@ static inline size_t CALL_NARGS(size_t nargs) { return nargs & ~DIRECT_CALL; } #include #include #include +#include #include std::recursive_mutex InterOpMutex; @@ -184,7 +187,7 @@ static void configureInterpreter(const InterOpPaths& Paths) { Cpp::LoadLibrary("libstdc++", /* lookup= */ true); } -static void preloadHeaders() { +static bool preloadHeaders() { const char* code = "#include \n" "#include \n" "#include \n" @@ -209,7 +212,7 @@ static void preloadHeaders() { "#include \n" "#endif\n" "#include \n"; - Cpp::Process(code); + return Cpp::Process(code) == 0; } static void defineRuntimeHelpers() { @@ -246,9 +249,18 @@ extern "C" int LoadCppInterOp() { if (!loadDispatchAPI(Paths)) return; - acquireOrCreateInterpreter(Paths); + if (!acquireOrCreateInterpreter(Paths)) { + std::cerr << "[cppjit] Failed to create the C++ interpreter" << std::endl; + return; + } configureInterpreter(Paths); - preloadHeaders(); + if (!preloadHeaders()) { + std::cerr << "[cppjit] The C++ standard headers do not parse, see the " + "diagnostic above. Install a C++ toolchain such as g++ or " + "the conda package cxx-compiler." + << std::endl; + return; + } defineRuntimeHelpers(); Loaded = 1; @@ -863,49 +875,113 @@ static inline void release_args(Parameter* args, size_t nargs) { } } +// call failure record -------------------------------------------------------- +namespace { + +struct CallFailure { + interop::ECallFailure kind = interop::kCallCompleted; + std::string report; +}; + +// The outcome of the most recent call on this thread, recorded after the +// wrapper returns so a nested call cannot overwrite its caller's record. +thread_local CallFailure gCallFailure; + +// methods whose call wrapper compiled once, guarded by InterOpMutex +std::unordered_set gCompiledWrappers; + +// the JIT report with its mangled names demangled and no trailing newlines +std::string demangle_report(const std::string& text) { + auto is_name_char = [](char c) { + return std::isalnum((unsigned char)c) || c == '_' || c == '.' || c == '$'; + }; + std::string out; + size_t pos = 0; + for (size_t start = text.find("_Z"); start != std::string::npos; + start = text.find("_Z", pos)) { + size_t end = start + 2; + while (end < text.size() && is_name_char(text[end])) + ++end; + out.append(text, pos, start - pos); + std::string token = text.substr(start, end - start); + bool own_word = start == 0 || !is_name_char(text[start - 1]); + std::string demangled = own_word ? Cpp::Demangle(token) : std::string{}; + out += demangled.empty() ? token : demangled; + pos = end; + } + out.append(text, pos, std::string::npos); + while (!out.empty() && out.back() == '\n') + out.pop_back(); + return out; +} + +// The first call compiles the wrapper under a capture, as the JIT reports a +// failure on the standard error descriptor only. Caller holds InterOpMutex. +Cpp::JitCall make_callable(interop::TCppMethod_t method, std::string& report) { + if (gCompiledWrappers.count(method.data)) + return Cpp::MakeFunctionCallable(method); + interop::StderrCapture capture; + capture.Begin(); + Cpp::JitCall JC = Cpp::MakeFunctionCallable(method); + report = capture.Stop(); + if (JC) + gCompiledWrappers.insert(method.data); + return JC; +} + +} // unnamed namespace + +interop::ECallFailure interop::LastCallFailure() { return gCallFailure.kind; } + +std::string interop::LastCallFailureReport() { return gCallFailure.report; } + static inline bool WrapperCall(interop::TCppMethod_t method, size_t nargs, void* args_, void* self, void* result) { Parameter* args = (Parameter*)args_; // bool is_direct = nargs & DIRECT_CALL; nargs = CALL_NARGS(nargs); - // if (!is_ready(wrap, is_direct)) - // return false; // happens with compilation error + std::string report; InterOpMutex.lock(); - if (Cpp::JitCall JC = Cpp::MakeFunctionCallable(method)) { - InterOpMutex.unlock(); - bool runRelease = false; - // const auto& fgen = /* is_direct ? faceptr.fDirect : */ faceptr; - if (nargs <= cpyrt::SMALL_ARGS_N) { - void* smallbuf[cpyrt::SMALL_ARGS_N]; - if (nargs) - runRelease = copy_args(args, nargs, smallbuf); - // CLING_CATCH_UNCAUGHT_ - JC.Invoke(result, {smallbuf, nargs}, self); - // _CLING_CATCH_UNCAUGHT - } else { - std::vector buf(nargs); - runRelease = copy_args(args, nargs, buf.data()); - // CLING_CATCH_UNCAUGHT_ - JC.Invoke(result, {buf.data(), nargs}, self); - // _CLING_CATCH_UNCAUGHT - } - if (runRelease) - release_args(args, nargs); - return true; - } + Cpp::JitCall JC = make_callable(method, report); InterOpMutex.unlock(); - return false; + if (!JC) { + gCallFailure = {interop::kWrapperJitFailed, demangle_report(report)}; + return false; + } + // a compiled wrapper's warnings stay visible + if (!report.empty()) + std::cerr << report; + + bool runRelease = false; + // const auto& fgen = /* is_direct ? faceptr.fDirect : */ faceptr; + if (nargs <= cpyrt::SMALL_ARGS_N) { + void* smallbuf[cpyrt::SMALL_ARGS_N]; + if (nargs) + runRelease = copy_args(args, nargs, smallbuf); + // CLING_CATCH_UNCAUGHT_ + JC.Invoke(result, {smallbuf, nargs}, self); + // _CLING_CATCH_UNCAUGHT + } else { + std::vector buf(nargs); + runRelease = copy_args(args, nargs, buf.data()); + // CLING_CATCH_UNCAUGHT_ + JC.Invoke(result, {buf.data(), nargs}, self); + // _CLING_CATCH_UNCAUGHT + } + if (runRelease) + release_args(args, nargs); + gCallFailure = {}; + return true; } template static inline T CallT(interop::TCppMethod_t method, interop::TCppObject_t self, size_t nargs, void* args) { + // a failed call returns T{} and records its reason T t{}; - if (WrapperCall(method, nargs, args, self.data, &t)) - return t; - throw std::runtime_error("failed to resolve function"); - return (T)-1; + WrapperCall(method, nargs, args, self.data, &t); + return t; } #ifdef PRINT_DEBUG @@ -923,8 +999,7 @@ static inline T CallT(interop::TCppMethod_t method, interop::TCppObject_t self, void interop::CallV(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args) { - if (!WrapperCall(method, nargs, args, self.data, nullptr)) - return /* TODO ... report error */; + WrapperCall(method, nargs, args, self.data, nullptr); } // clang-format off @@ -942,9 +1017,8 @@ CPPJIT_IMP_CALL(LD, long double ) void* interop::CallR(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args) { void* r = nullptr; - if (WrapperCall(method, nargs, args, self.data, &r)) - return r; - return nullptr; + WrapperCall(method, nargs, args, self.data, &r); + return r; } char* interop::CallS(TCppMethod_t method, TCppObject_t self, size_t nargs, @@ -980,8 +1054,10 @@ interop::TCppObject_t interop::CallO(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args, TCppType_t result_type) { size_t size = interop::SizeOfType(result_type); - if (size == 0) + if (size == 0) { + gCallFailure = {interop::kUnsizableReturn, {}}; return TCppObject_t{}; // unsizable return type; the caller reports + } void* obj = ::operator new(size); if (WrapperCall(method, nargs, args, self.data, obj)) return (TCppObject_t)obj; diff --git a/test/support.py b/test/support.py index 667394e2..018398f5 100644 --- a/test/support.py +++ b/test/support.py @@ -137,11 +137,22 @@ def setup_make(targetname, optional=False): ) +def run_child(code, *args): + # a failed JIT materialization poisons the session, so a call meant to + # fail runs in a process of its own + return subprocess.run( + [sys.executable, "-c", code, *args], + capture_output=True, + encoding="utf-8", + errors="replace", + timeout=300, + ) + + def _jit_resolves_std_filesystem(): # The JIT resolves std::filesystem against the loaded libstdc++.so, not - # the compile headers -- a manylinux image (gcc-toolset headers over a - # GCC-8 base runtime) compiles the include yet dies resolving the - # symbols. That failure is a fatal JIT error, so ask in a child process. + # the headers, so a GCC 8 runtime under newer headers fails at the call. + # A child process keeps that failure out of the test process. probe = ( "import cppjit\n" 'cppjit.cppdef("""#include \n' @@ -150,12 +161,7 @@ def _jit_resolves_std_filesystem(): "assert cppjit.gbl.fs_probe() > 0\n" ) try: - return ( - subprocess.run( - [sys.executable, "-c", probe], capture_output=True, timeout=300 - ).returncode - == 0 - ) + return run_child(probe).returncode == 0 except subprocess.TimeoutExpired: return False diff --git a/test/test_api.py b/test/test_api.py index f6f49262..4a6c656d 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -1,5 +1,5 @@ -from pytest import mark, skip -from support import IS_LINUX_ARM, IS_MAC, ispypy +from pytest import mark, raises, skip +from support import CAN_JIT_STD_FILESYSTEM, IS_LINUX_ARM, IS_MAC, ispypy, run_child class TestAPI: @@ -229,3 +229,97 @@ class MyArray { assert Sequence_Check(tuple()) assert Sequence_Check(cppjit.gbl.std.vector[ns.MyClass]()) assert not Sequence_Check(cppjit.gbl.std.list[ns.MyClass]()) + + +class TestJITERRORS: + def test01_diagnostics_stay_out_of_stderr(self, capfd): + """A compile error's text lands in the exception, not on fd 2""" + + import cppjit + + with raises(SyntaxError) as e: + cppjit.cppdef("1aap = 42;") + assert "invalid digit" in str(e.value) + assert "invalid digit" not in capfd.readouterr().err + + def test02_missing_runtime_symbol_raises(self): + """A missing runtime symbol raises at the call and the process survives""" + + proc = run_child( + """\ +import sys +import cppjit +from pytest import raises +cppjit.cppdef('''#include +int fs_probe() { return (int)std::filesystem::path("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/a/b.txt").filename().string().size(); }''') +if sys.argv[1] == "1": + assert cppjit.gbl.fs_probe() == 5 +else: + with raises(RuntimeError, match="filesystem"): + cppjit.gbl.fs_probe() +cppjit.cppdef("int after_probe() { return 7; }") +assert cppjit.gbl.after_probe() == 7 +""", + "1" if CAN_JIT_STD_FILESYSTEM else "0", + ) + assert proc.returncode == 0, proc.stderr + assert "CRASH DETECTED" not in proc.stderr + + def test03_failed_wrapper_raises_for_every_return_type(self): + """A call wrapper the JIT cannot compile raises whatever the return type""" + + # one cppdef per function keeps each failure in a module of its own + proc = run_child( + """\ +import cppjit +from pytest import raises +cppjit.cppdef("extern int undefined_fn_x();") +cppjit.cppdef("int f_i() { return undefined_fn_x(); }") +cppjit.cppdef("void f_v() { undefined_fn_x(); }") +cppjit.cppdef('std::string f_s() { undefined_fn_x(); return ""; }') +cppjit.cppdef("struct Opaque; Opaque make_opaque();") +for name in ("f_i", "f_v", "f_s"): + with raises(RuntimeError, match="undefined_fn_x"): + getattr(cppjit.gbl, name)() +with raises(ValueError, match="size of its return type is unknown"): + cppjit.gbl.make_opaque() +""" + ) + assert proc.returncode == 0, proc.stderr + + @mark.xfail(strict=True, reason="a failed module poisons later linkonce_odr code") + def test04_failed_materialization_keeps_later_std_code_working(self): + """Unrelated std::string code still JIT-compiles after a failed call""" + + proc = run_child( + """\ +import cppjit +from pytest import raises +cppjit.cppdef('extern int undefined_fn_p(); std::string first_s() { undefined_fn_p(); return ""; }') +with raises(RuntimeError): + cppjit.gbl.first_s() +cppjit.cppdef('std::string second_s() { return "ok"; }') +assert cppjit.gbl.second_s() == "ok" +""" + ) + assert proc.returncode == 0, proc.stderr + + def test05_failed_wrapper_inside_a_pythonization_raises(self): + """A pythonized method whose C++ call cannot be JIT-compiled raises""" + + # construct first: the poisoned module then holds reserve()'s first definition + proc = run_child( + """\ +import cppjit +from pytest import raises +cppjit.gbl.std.vector["int"]() +cppjit.cppdef('''extern int undefined_fn_q(); +void first_r() { std::vector v; undefined_fn_q(); v.reserve(4); }''') +with raises(RuntimeError): + cppjit.gbl.first_r() +with raises(RuntimeError, match="reserve"): + cppjit.gbl.std.vector["int"](range(10)) +""" + ) + assert proc.returncode == 0, proc.stderr + assert "CRASH DETECTED" not in proc.stderr diff --git a/test/test_boost.py b/test/test_boost.py index 157c066f..03e1421f 100644 --- a/test/test_boost.py +++ b/test/test_boost.py @@ -29,6 +29,17 @@ def add_boost_include_path(): cppjit.add_include_path(boost_extra_inc) +def boost_version(): + import cppjit + + add_boost_include_path() + try: + cppjit.include("boost/version.hpp") + except Exception: + return 0 + return cppjit.evaluate("BOOST_VERSION") + + @mark.skipif(noboost == True, reason="boost not found") class TestBOOSTANY: def setup_class(cls): @@ -178,6 +189,10 @@ class C { }; } """) @mark.skipif(((noboost == True) or IS_MAC_ARM or IS_MAC_X86), reason="boost not found") +@mark.skipif( + not noboost and boost_version() < 106700, + reason="is_placeholder rejects an incomplete concept type before Boost 1.67", +) class TestBOOSTERASURE: def setup_class(cls): import cppjit diff --git a/test/test_fragile.py b/test/test_fragile.py index b72a5b3b..13b2ab54 100644 --- a/test/test_fragile.py +++ b/test/test_fragile.py @@ -526,7 +526,6 @@ def test20_capture_output(self): assert capture.str() == "Hello, World\n" - @mark.xfail(condition=IS_CLANG_REPL, reason="Fails with ClangRepl") def test21_failing_cppcode(self): """Check error behavior of failing C++ code"""