From cabb466057c53d1ff065384ff86c8a98719f3bc2 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Mon, 13 Oct 2025 20:59:57 -0700 Subject: [PATCH 1/7] Use strong reference APIs. For the free-threaded build, it is not safe use borrowed references. Another thread could deallocate the object and cause the reference to become invalid. Replace API calls that borrow references with strong reference APIs. --- src/dict.cpp | 8 ++++++++ src/object/function_doc_signature.cpp | 15 +++++++++++++-- src/wrapper.cpp | 20 ++++++++++++++------ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/dict.cpp b/src/dict.cpp index 77d840d455..296bc21e9c 100644 --- a/src/dict.cpp +++ b/src/dict.cpp @@ -68,8 +68,16 @@ object dict_base::get(object_cref k) const { if (check_exact(this)) { +#ifdef Py_GIL_DISABLED + PyObject* result; + if (PyDict_GetItemRef(this->ptr(),k.ptr(),&result) < 0) { + throw_error_already_set(); + } + return object(detail::new_reference(result ? result : Py_None)); +#else PyObject* result = PyDict_GetItem(this->ptr(),k.ptr()); return object(detail::borrowed_reference(result ? result : Py_None)); +#endif } else { diff --git a/src/object/function_doc_signature.cpp b/src/object/function_doc_signature.cpp index 18d458698d..76b620dcb9 100644 --- a/src/object/function_doc_signature.cpp +++ b/src/object/function_doc_signature.cpp @@ -135,7 +135,15 @@ namespace boost { namespace python { namespace objects { str name(get_qualname(py_type)); if ( py_type->tp_flags & Py_TPFLAGS_HEAPTYPE ) { // Qualify the type name if it is defined in a different module. - PyObject *type_module_name = PyDict_GetItemString(py_type->tp_dict, "__module__"); + PyObject *type_module_name; +#if PY_VERSION_HEX >= 0x030D0000 + if (PyDict_GetItemStringRef(py_type->tp_dict, "__module__", &type_module_name) < 0) { + throw_error_already_set(); + } +#else + type_module_name = PyDict_GetItemString(py_type->tp_dict, "__module__"); + Py_XINCREF(type_module_name); +#endif if ( type_module_name && PyObject_RichCompareBool( @@ -144,8 +152,11 @@ namespace boost { namespace python { namespace objects { Py_NE ) != 0 ) { - return str("%s.%s" % make_tuple(handle<>(borrowed(type_module_name)), name)); + str result = str("%s.%s" % make_tuple(handle<>(type_module_name), name)); + return result; } + // Clean up the strong reference if we didn't use it + Py_XDECREF(type_module_name); } return name; } else { diff --git a/src/wrapper.cpp b/src/wrapper.cpp index 8b1b884769..2b053d8311 100644 --- a/src/wrapper.cpp +++ b/src/wrapper.cpp @@ -21,20 +21,28 @@ namespace detail this->m_self, const_cast(name)))) ) { - PyObject* borrowed_f = 0; - + PyObject* class_f = 0; + if ( PyMethod_Check(m.get()) && PyMethod_GET_SELF(m.get()) == this->m_self && class_object->tp_dict != 0 ) { - borrowed_f = ::PyDict_GetItemString( +#if PY_VERSION_HEX >= 0x030D0000 + if (::PyDict_GetItemStringRef( + class_object->tp_dict, const_cast(name), &class_f) < 0) { + throw_error_already_set(); + } +#else + class_f = ::PyDict_GetItemString( class_object->tp_dict, const_cast(name)); - - + Py_XINCREF(class_f); +#endif } - if (borrowed_f != PyMethod_GET_FUNCTION(m.get())) + bool is_override = (class_f != PyMethod_GET_FUNCTION(m.get())); + Py_XDECREF(class_f); + if (is_override) return override(m); } } From 6f5f3b66074b27b44ddc5f1198003d0b01d31a52 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 30 Oct 2025 14:23:37 -0700 Subject: [PATCH 2/7] Add work-around to crash in ~object_base(). For the free-threaded build (and possibly the debug build), it is not safe to call Py_DECREF() if there is no valid Python thread-state. --- include/boost/python/object_core.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/boost/python/object_core.hpp b/include/boost/python/object_core.hpp index 16480d0d89..074360d415 100644 --- a/include/boost/python/object_core.hpp +++ b/include/boost/python/object_core.hpp @@ -419,6 +419,16 @@ inline api::object_base& api::object_base::operator=(api::object_base const& rhs inline api::object_base::~object_base() { +#ifdef Py_GIL_DISABLED + // This is a not very elegant fix for a problem that occurs with the + // free-threaded build of Python. If this is called when the interpreter + // has already been finalized, the thread-state can be null. Unlike the + // GIL-enabled build, Py_DECREF() requires a valid thread-state. This + // causes a memory leak, rather than crash, which seems preferable. + if (PyThreadState_GetUnchecked() == NULL) { + return; + } +#endif assert( Py_REFCNT(m_ptr) > 0 ); Py_DECREF(m_ptr); } From cfbefe893ca6254fd2377c4f6fcdacfd3b851680 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Mon, 13 Oct 2025 22:42:42 -0700 Subject: [PATCH 3/7] Use re-entrant mutex to protect global state. Add pymutex.hpp which implements a re-entrant mutex on top of Python's PyMutex. Add BOOST_PYTHON_LOCK_STATE() macro that uses RAII to lock mutable global state as required. --- include/boost/python/detail/pymutex.hpp | 103 ++++++++++++++++++++++++ src/converter/from_python.cpp | 7 +- src/converter/registry.cpp | 9 ++- src/converter/type_id.cpp | 6 +- src/errors.cpp | 22 ++++- src/object/inheritance.cpp | 6 ++ 6 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 include/boost/python/detail/pymutex.hpp diff --git a/include/boost/python/detail/pymutex.hpp b/include/boost/python/detail/pymutex.hpp new file mode 100644 index 0000000000..2d2e2d6266 --- /dev/null +++ b/include/boost/python/detail/pymutex.hpp @@ -0,0 +1,103 @@ +// Copyright 2025 Boost.Python Contributors +// Distributed under the Boost Software License, Version 1.0. (See +// accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#ifndef BOOST_PYTHON_DETAIL_PYMUTEX_HPP +#define BOOST_PYTHON_DETAIL_PYMUTEX_HPP + +#include +#ifdef Py_GIL_DISABLED +// needed for pymutex wrapper +#include +#include +#endif + +namespace boost { namespace python { namespace detail { + +#ifdef Py_GIL_DISABLED + +// Re-entrant wrapper around PyMutex for free-threaded Python +// Similar to _PyRecursiveMutex or threading.RLock +class pymutex { + PyMutex m_mutex; + std::atomic m_owner; + std::size_t m_level; + +public: + pymutex() : m_mutex({}), m_owner(0), m_level(0) {} + + // Non-copyable, non-movable + pymutex(const pymutex&) = delete; + pymutex& operator=(const pymutex&) = delete; + + void lock() { + unsigned long thread = PyThread_get_thread_ident(); + if (m_owner.load(std::memory_order_relaxed) == thread) { + m_level++; + return; + } + PyMutex_Lock(&m_mutex); + m_owner.store(thread, std::memory_order_relaxed); + // m_level should be 0 when we acquire the lock + } + + void unlock() { + unsigned long thread = PyThread_get_thread_ident(); + // Verify current thread owns the lock + if (m_owner.load(std::memory_order_relaxed) != thread) { + // This should never happen - programming error + return; + } + if (m_level > 0) { + m_level--; + return; + } + m_owner.store(0, std::memory_order_relaxed); + PyMutex_Unlock(&m_mutex); + } + + bool is_locked_by_current_thread() const { + unsigned long thread = PyThread_get_thread_ident(); + return m_owner.load(std::memory_order_relaxed) == thread; + } +}; + + +// RAII lock guard for pymutex +class pymutex_guard { + pymutex& m_mutex; + +public: + explicit pymutex_guard(pymutex& mutex) : m_mutex(mutex) { + m_mutex.lock(); + } + + ~pymutex_guard() { + m_mutex.unlock(); + } + + // Non-copyable, non-movable + pymutex_guard(const pymutex_guard&) = delete; + pymutex_guard& operator=(const pymutex_guard&) = delete; +}; + +// Global mutex for protecting all Boost.Python internal state +// Similar to pybind11's internals.mutex +BOOST_PYTHON_DECL pymutex& get_global_mutex(); + +// Macro for acquiring the global lock +// Similar to pybind11's PYBIND11_LOCK_INTERNALS +#define BOOST_PYTHON_LOCK_STATE() \ + ::boost::python::detail::pymutex_guard lock(::boost::python::detail::get_global_mutex()) + +#else + +// No-op macro when not in free-threaded mode +#define BOOST_PYTHON_LOCK_STATE() + +#endif // Py_GIL_DISABLED + +}}} // namespace boost::python::detail + +#endif // BOOST_PYTHON_DETAIL_PYMUTEX_HPP diff --git a/src/converter/from_python.cpp b/src/converter/from_python.cpp index f3989ba77f..53a149fa72 100644 --- a/src/converter/from_python.cpp +++ b/src/converter/from_python.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -145,6 +146,8 @@ namespace inline bool visit(rvalue_from_python_chain const* chain) { + BOOST_PYTHON_LOCK_STATE(); + visited_t::iterator const p = std::lower_bound(visited.begin(), visited.end(), chain); if (p != visited.end() && *p == chain) return false; @@ -157,9 +160,11 @@ namespace { unvisit(rvalue_from_python_chain const* chain) : chain(chain) {} - + ~unvisit() { + BOOST_PYTHON_LOCK_STATE(); + visited_t::iterator const p = std::lower_bound(visited.begin(), visited.end(), chain); assert(p != visited.end()); visited.erase(p); diff --git a/src/converter/registry.cpp b/src/converter/registry.cpp index aa20c3f685..1b23dbef48 100644 --- a/src/converter/registry.cpp +++ b/src/converter/registry.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -112,9 +113,9 @@ registration::~registration() namespace // { typedef registration entry; - + typedef std::set registry_t; - + #ifndef BOOST_PYTHON_CONVERTER_REGISTRY_APPLE_MACH_WORKAROUND registry_t& entries() { @@ -181,6 +182,8 @@ namespace // entry* get(type_info type, bool is_shared_ptr = false) { + BOOST_PYTHON_LOCK_STATE(); + # ifdef BOOST_PYTHON_TRACE_REGISTRY registry_t::iterator p = entries().find(entry(type)); @@ -293,6 +296,8 @@ namespace registry registration const* query(type_info type) { + BOOST_PYTHON_LOCK_STATE(); + registry_t::iterator p = entries().find(entry(type)); # ifdef BOOST_PYTHON_TRACE_REGISTRY std::cout << "querying " << type diff --git a/src/converter/type_id.cpp b/src/converter/type_id.cpp index c6a8bf7a04..fafb13619c 100644 --- a/src/converter/type_id.cpp +++ b/src/converter/type_id.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -81,7 +82,7 @@ namespace { free_mem(char*p) : p(p) {} - + ~free_mem() { std::free(p); @@ -92,6 +93,7 @@ namespace bool cxxabi_cxa_demangle_is_broken() { + BOOST_PYTHON_LOCK_STATE(); static bool was_tested = false; static bool is_broken = false; if (!was_tested) { @@ -109,6 +111,8 @@ namespace detail { BOOST_PYTHON_DECL char const* gcc_demangle(char const* mangled) { + BOOST_PYTHON_LOCK_STATE(); + typedef std::vector< std::pair > mangling_map; diff --git a/src/errors.cpp b/src/errors.cpp index 34ea22f43e..7f6b1880d5 100644 --- a/src/errors.cpp +++ b/src/errors.cpp @@ -10,9 +10,21 @@ #include #include #include +#include namespace boost { namespace python { +#ifdef Py_GIL_DISABLED +namespace detail { + // Global mutex for protecting all Boost.Python internal state + pymutex& get_global_mutex() + { + static pymutex mutex; + return mutex; + } +} +#endif + error_already_set::~error_already_set() {} // IMPORTANT: this function may only be called from within a catch block! @@ -20,8 +32,13 @@ BOOST_PYTHON_DECL bool handle_exception_impl(function0 f) { try { - if (detail::exception_handler::chain) - return detail::exception_handler::chain->handle(f); + detail::exception_handler* handler_chain = nullptr; + { + BOOST_PYTHON_LOCK_STATE(); + handler_chain = detail::exception_handler::chain; + } + if (handler_chain) + return handler_chain->handle(f); f(); return false; } @@ -80,6 +97,7 @@ exception_handler::exception_handler(handler_function const& impl) : m_impl(impl) , m_next(0) { + BOOST_PYTHON_LOCK_STATE(); if (chain != 0) tail->m_next = this; else diff --git a/src/object/inheritance.cpp b/src/object/inheritance.cpp index a7b3156e41..44062875a4 100644 --- a/src/object/inheritance.cpp +++ b/src/object/inheritance.cpp @@ -4,6 +4,7 @@ // http://www.boost.org/LICENSE_1_0.txt) #include #include +#include #include #if _MSC_FULL_VER >= 13102171 && _MSC_FULL_VER <= 13102179 # include @@ -390,6 +391,8 @@ namespace inline void* convert_type(void* const p, class_id src_t, class_id dst_t, bool polymorphic) { + BOOST_PYTHON_LOCK_STATE(); + // Quickly rule out unregistered types index_entry* src_p = seek_type(src_t); if (src_p == 0) @@ -452,6 +455,8 @@ BOOST_PYTHON_DECL void* find_static_type(void* p, class_id src_t, class_id dst_t BOOST_PYTHON_DECL void add_cast( class_id src_t, class_id dst_t, cast_function cast, bool is_downcast) { + BOOST_PYTHON_LOCK_STATE(); + // adding an edge will invalidate any record of unreachability in // the cache. static std::size_t expected_cache_len = 0; @@ -490,6 +495,7 @@ BOOST_PYTHON_DECL void add_cast( BOOST_PYTHON_DECL void register_dynamic_id_aux( class_id static_id, dynamic_id_function get_dynamic_id) { + BOOST_PYTHON_LOCK_STATE(); tuples::get(*demand_type(static_id)) = get_dynamic_id; } From fc68878e02bb2cb9a0e107a24daf890b5ffc4477 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Mon, 13 Oct 2025 20:59:37 -0700 Subject: [PATCH 4/7] Set the Py_MOD_GIL_NOT_USED flag on modules. This indicates that the free-threaded build of Python can keep the GIL disabled when the module is loaded. Without this module flag, importing the module will cause the GIL to be re-enabled. A warning is emitted if this happens. --- src/module.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/module.cpp b/src/module.cpp index 57675fa2df..707e433941 100644 --- a/src/module.cpp +++ b/src/module.cpp @@ -40,8 +40,14 @@ BOOST_PYTHON_DECL void scope_setattr_doc(char const* name, object const& x, char BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef& moduledef, void(*init_function)()) { + PyObject *mod = PyModule_Create(&moduledef); +#ifdef Py_GIL_DISABLED + if (mod != NULL) { + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + } +#endif return init_module_in_scope( - PyModule_Create(&moduledef), + mod, init_function); } From 501356431631b3bcb10dd0a9336a5a05d2171889 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 6 Nov 2025 15:45:22 -0800 Subject: [PATCH 5/7] Add "nogil" option for BOOST_PYTHON_MODULE_INIT. Implement optional arguments for BOOST_PYTHON_MODULE_INIT and allow the boost::python::mod_gil_not_used() option. This sets the Py_MOD_GIL_NOT_USED flag for the extension module. To define a module that supports free-threaded Python, define it like this: BOOST_PYTHON_MODULE(my_module, boost::python::mod_gil_not_used()) { ... } --- include/boost/python/module_init.hpp | 75 ++++++++++++++++++++++++++-- src/module.cpp | 5 +- test/fabscript | 1 + test/module_nogil.cpp | 25 ++++++++++ test/module_nogil.py | 29 +++++++++++ 5 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 test/module_nogil.cpp create mode 100644 test/module_nogil.py diff --git a/include/boost/python/module_init.hpp b/include/boost/python/module_init.hpp index 7fe5a1c8a2..390db82cf4 100644 --- a/include/boost/python/module_init.hpp +++ b/include/boost/python/module_init.hpp @@ -11,11 +11,41 @@ # ifndef BOOST_PYTHON_MODULE_INIT -namespace boost { namespace python { namespace detail { +namespace boost { namespace python { + +#ifdef HAS_CXX11 +// Use to activate the Py_MOD_GIL_NOT_USED flag. +class mod_gil_not_used { +public: + explicit mod_gil_not_used(bool flag = true) : flag_(flag) {} + bool flag() const { return flag_; } + +private: + bool flag_; +}; + +namespace detail { + +inline bool gil_not_used_option() { return false; } +template +bool gil_not_used_option(F &&, O &&...o); +template +inline bool gil_not_used_option(mod_gil_not_used f, O &&...o) { + return f.flag() || gil_not_used_option(o...); +} +template +inline bool gil_not_used_option(F &&, O &&...o) { + return gil_not_used_option(o...); +} + +} +#endif // HAS_CXX11 + +namespace detail { # if PY_VERSION_HEX >= 0x03000000 -BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef&, void(*)()); +BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef&, void(*)(), bool gil_not_used = false); #else @@ -27,7 +57,37 @@ BOOST_PYTHON_DECL PyObject* init_module(char const* name, void(*)()); # if PY_VERSION_HEX >= 0x03000000 -# define _BOOST_PYTHON_MODULE_INIT(name) \ +# ifdef HAS_CXX11 +# define _BOOST_PYTHON_MODULE_INIT(name, ...) \ + PyObject* BOOST_PP_CAT(PyInit_, name)() \ + { \ + static PyModuleDef_Base initial_m_base = { \ + PyObject_HEAD_INIT(NULL) \ + 0, /* m_init */ \ + 0, /* m_index */ \ + 0 /* m_copy */ }; \ + static PyMethodDef initial_methods[] = { { 0, 0, 0, 0 } }; \ + \ + static struct PyModuleDef moduledef = { \ + initial_m_base, \ + BOOST_PP_STRINGIZE(name), \ + 0, /* m_doc */ \ + -1, /* m_size */ \ + initial_methods, \ + 0, /* m_reload */ \ + 0, /* m_traverse */ \ + 0, /* m_clear */ \ + 0, /* m_free */ \ + }; \ + \ + return boost::python::detail::init_module( \ + moduledef, BOOST_PP_CAT(init_module_, name), \ + boost::python::detail::gil_not_used_option(__VA_ARGS__) ); \ + } \ + void BOOST_PP_CAT(init_module_, name)() + +# else // !HAS_CXX11 +# define _BOOST_PYTHON_MODULE_INIT(name) \ PyObject* BOOST_PP_CAT(PyInit_, name)() \ { \ static PyModuleDef_Base initial_m_base = { \ @@ -53,6 +113,7 @@ BOOST_PYTHON_DECL PyObject* init_module(char const* name, void(*)()); moduledef, BOOST_PP_CAT(init_module_, name) ); \ } \ void BOOST_PP_CAT(init_module_, name)() +# endif // HAS_CXX11 # else @@ -66,9 +127,15 @@ BOOST_PYTHON_DECL PyObject* init_module(char const* name, void(*)()); # endif -# define BOOST_PYTHON_MODULE_INIT(name) \ +# if defined(HAS_CXX11) && (PY_VERSION_HEX >= 0x03000000) +# define BOOST_PYTHON_MODULE_INIT(name, ...) \ + void BOOST_PP_CAT(init_module_,name)(); \ +extern "C" BOOST_SYMBOL_EXPORT _BOOST_PYTHON_MODULE_INIT(name, __VA_ARGS__) +# else +# define BOOST_PYTHON_MODULE_INIT(name) \ void BOOST_PP_CAT(init_module_,name)(); \ extern "C" BOOST_SYMBOL_EXPORT _BOOST_PYTHON_MODULE_INIT(name) +# endif // HAS_CXX11 && Python 3 # endif diff --git a/src/module.cpp b/src/module.cpp index 707e433941..c32f4187bc 100644 --- a/src/module.cpp +++ b/src/module.cpp @@ -38,11 +38,12 @@ BOOST_PYTHON_DECL void scope_setattr_doc(char const* name, object const& x, char #if PY_VERSION_HEX >= 0x03000000 -BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef& moduledef, void(*init_function)()) +BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef& moduledef, + void(*init_function)(), bool gil_not_used) { PyObject *mod = PyModule_Create(&moduledef); #ifdef Py_GIL_DISABLED - if (mod != NULL) { + if (mod != NULL && gil_not_used) { PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); } #endif diff --git a/test/fabscript b/test/fabscript index a002fb2bf0..7cf22f9c09 100644 --- a/test/fabscript +++ b/test/fabscript @@ -68,6 +68,7 @@ for t in [('injected',), ('raw_ctor',), ('exception_translator',), ('module_init_exception',), + ('module_nogil',), ('test_enum', ['enum_ext']), ('test_cltree', ['cltree']), ('newtest', ['m1', 'm2']), diff --git a/test/module_nogil.cpp b/test/module_nogil.cpp new file mode 100644 index 0000000000..331a73cf31 --- /dev/null +++ b/test/module_nogil.cpp @@ -0,0 +1,25 @@ +// Test for BOOST_PYTHON_MODULE with optional mod_gil_not_used argument + +#include +#include + +// Simple function to export +int get_value() { + return 1234; +} + +#if defined(HAS_CXX11) && (PY_VERSION_HEX >= 0x03000000) +// C++11 build with Python 3: test with mod_gil_not_used option +BOOST_PYTHON_MODULE(module_nogil_ext, boost::python::mod_gil_not_used()) +{ + using namespace boost::python; + def("get_value", get_value); +} +#else +// C++98 build or Python 2: test without optional arguments +BOOST_PYTHON_MODULE(module_nogil_ext) +{ + using namespace boost::python; + def("get_value", get_value); +} +#endif diff --git a/test/module_nogil.py b/test/module_nogil.py new file mode 100644 index 0000000000..c035436014 --- /dev/null +++ b/test/module_nogil.py @@ -0,0 +1,29 @@ +""" +>>> from module_nogil_ext import * +>>> get_value() +1234 +>>> import sys, sysconfig +>>> Py_GIL_DISABLED = bool(sysconfig.get_config_var('Py_GIL_DISABLED')) +>>> if Py_GIL_DISABLED and sys._is_gil_enabled(): +... print('GIL is enabled and should not be') +... else: +... print('okay') +okay +""" + +from __future__ import print_function + +def run(args = None): + import sys + import doctest + + if args is not None: + sys.argv = args + return doctest.testmod(sys.modules.get(__name__)) + +if __name__ == '__main__': + print("running...") + import sys + status = run()[0] + if (status == 0): print("Done.") + sys.exit(status) From e89f86b74f7cdd5682c1ea4e45a809471a11d7f0 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Tue, 11 Nov 2025 10:04:47 -0800 Subject: [PATCH 6/7] Update Linux CI scripts, more Python versions. Update scripts to use actions/setup-python to install different Python versions. Add run-faber.sh and get-py-env.py scripts. Add test-ubuntu-py-ver.yml CI script to test with different Python versions. --- .github/get-py-env.py | 65 +++++++++++++++++++++++++++ .github/run-faber.sh | 46 +++++++++++++++++++ .github/workflows/test-ubuntu-py2.yml | 58 ++++++++++++++++++++++++ .github/workflows/test-ubuntu.yml | 58 +++++++++++------------- 4 files changed, 194 insertions(+), 33 deletions(-) create mode 100755 .github/get-py-env.py create mode 100755 .github/run-faber.sh create mode 100644 .github/workflows/test-ubuntu-py2.yml diff --git a/.github/get-py-env.py b/.github/get-py-env.py new file mode 100755 index 0000000000..a6c41460d8 --- /dev/null +++ b/.github/get-py-env.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# +# Determine info about the Python install and write shell code to stdout, to +# set env variables. This will set the variables PY_LDFLAGS, PY_CFLAGS and +# PY_INC_PATH. +# +# The python3-config tool is used as the source of this info. In theory we +# could use sysconfig as well but the setup-python action from github appears +# to patch python3-config but not patch the sysconfig info. +# +# Usage: +# eval $(python3 get-py-env.py) + +import os +import re +import subprocess + + +def get_output(cmd): + rv = subprocess.run( + cmd, + capture_output=True, # Capture stdout and stderr + text=True, # Decode output as text (UTF-8) + check=True, # Raise an error if the command fails + ) + return rv.stdout + + +def extract_flags(cmd, prefix): + flags = [] + for part in get_output(cmd).split(): + part = part.strip() + if part.startswith(prefix): + flags.append(part) + return ' '.join(flags) + + +def find_python_h(): + """Find the include path that has Python.h contained inside. + We could use INCLUDEPY from sysconfig but github patches + python3-config but not the sysconfig info (after moving the + install). + """ + c_flags = extract_flags(['python3-config', '--cflags'], '-I') + for part in c_flags.split(): + m = re.search(r'-I(\S+)', part) + if not m: + continue + inc_path = m.group(1) + if os.path.exists(os.path.join(inc_path, 'Python.h')): + return inc_path + raise SystemExit('cannot find Python.h') + + +def main(): + ld_flags = extract_flags(['python3-config', '--ldflags'], '-L') + c_flags = extract_flags(['python3-config', '--cflags'], '-I') + include_path = find_python_h() + print(f'PY_LDFLAGS="{ld_flags}"') + print(f'PY_CFLAGS="{c_flags}"') + print(f'PY_INC_PATH="{include_path}"') + + +if __name__ == '__main__': + main() diff --git a/.github/run-faber.sh b/.github/run-faber.sh new file mode 100755 index 0000000000..5cb78be6dd --- /dev/null +++ b/.github/run-faber.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +set -eu + +echo "cxx version: $CXX $($CXX --version)" +echo "cxx std: $CXX_STD" +echo "python3 path: $(which python3)" +echo "python3 version: $(python3 --version)" + +if ! which faber > /dev/null; then + echo "Installing faber..." + python3 -m pip install --upgrade pip + python3 -m pip install -U faber +fi +echo "faber version: $(faber -v)" + +# find and set PY_LDFLAGS and PY_INC_PATH +eval $(python3 .github/get-py-env.py) + +echo "PY_INC_PATH=$PY_INC_PATH" +echo "PY_LDFLAGS=$PY_LDFLAGS" + +case $(python3-config --abiflags) in + *t*) + # When running with free-threaded, we always want to disable the GIL + # even for extensions without the mod_gil_not_used() flag + export PYTHON_GIL=0 + ;; +esac + +# this could be set by LD_LIBRARY_PATH but faber overrides it +prefix=$(python3-config --prefix) +echo "${prefix}/lib" > /etc/ld.so.conf.d/boost-ci.conf && ldconfig + +sed -e "s/\$PYTHON/python3/g" .ci/faber > $HOME/.faber + +faber \ + --with-boost-include=${BOOST_PY_DEPS} \ + --builddir=build \ + cxx.name="${CXX}" \ + cxxflags="-std=${CXX_STD}" \ + cppflags="-std=${CXX_STD}" \ + include="${PY_INC_PATH}" \ + ldflags="${PY_LDFLAGS}" \ + -j`nproc` \ + "$@" diff --git a/.github/workflows/test-ubuntu-py2.yml b/.github/workflows/test-ubuntu-py2.yml new file mode 100644 index 0000000000..801664c105 --- /dev/null +++ b/.github/workflows/test-ubuntu-py2.yml @@ -0,0 +1,58 @@ +name: Test Ubuntu, Python 2.x + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python: [python] + cxx: [g++] + std: [c++11] + include: + # Add the appropriate docker image for each compiler. + # The images from teeks99/boost-python-test already have boost::python + # pre-reqs installed, see: + # https://github.com/teeks99/boost-python-test-docker + - cxx: g++ + docker-img: teeks99/boost-python-test:gcc-15_1.89.0 + + container: + image: ${{ matrix.docker-img }} + + steps: + - uses: actions/checkout@v5 + + - name: setup prerequisites + run: | + # Warning: this is not necessarily the same Python version as the one configured above ! + python3 -m pip install -U faber --break-system-packages + - name: build + run: | + ${{ matrix.python }} --version + ${{ matrix.cxx }} --version + faber -v + sed -e "s/\$PYTHON/${{ matrix.python }}/g" .ci/faber > ~/.faber + faber \ + --with-boost-include=${BOOST_PY_DEPS} \ + --builddir=build \ + cxx.name=${{ matrix.cxx }} \ + cxxflags=-std=${{ matrix.std }} \ + cppflags=-std=${{ matrix.std }} \ + -j`nproc` + - name: test + run: | + faber \ + --with-boost-include=${BOOST_PY_DEPS} \ + --builddir=build \ + cxx.name=${{ matrix.cxx }} \ + cxxflags=-std=${{ matrix.std }} \ + cppflags=-std=${{ matrix.std }} \ + -j`nproc` \ + test.report diff --git a/.github/workflows/test-ubuntu.yml b/.github/workflows/test-ubuntu.yml index 6f94c2d186..abb0f2d59a 100644 --- a/.github/workflows/test-ubuntu.yml +++ b/.github/workflows/test-ubuntu.yml @@ -1,6 +1,10 @@ +# Test on Ubuntu with various compiler and language standard versions. name: Test Ubuntu -on: [push, pull_request] +on: + push: + pull_request: + workflow_dispatch: jobs: build: @@ -9,49 +13,37 @@ jobs: strategy: fail-fast: false matrix: - python: [python, python3] + python-version: ['3.14'] cxx: [g++, clang++] std: [c++11, c++14, c++17] include: - # Add the appropriate docker image for each compiler. - # The images from teeks99/boost-python-test already have boost::python - # pre-reqs installed, see: - # https://github.com/teeks99/boost-python-test-docker - - cxx: clang++ - docker-img: teeks99/boost-python-test:clang-21_1.89.0 - - cxx: g++ - docker-img: teeks99/boost-python-test:gcc-15_1.89.0 + # Also test with free-threaded build of Python + - python-version: '3.14t' + cxx: clang++ + std: c++17 container: - image: ${{ matrix.docker-img }} + # Add the appropriate docker image for the compiler. + # The images from teeks99/boost-python-test already have boost::python + # pre-reqs installed, see: + # https://github.com/teeks99/boost-python-test-docker + image: ${{ matrix.cxx == 'g++' && + 'teeks99/boost-python-test:gcc-15_1.89.0' || + 'teeks99/boost-python-test:clang-21_1.89.0' }} steps: - uses: actions/checkout@v5 - + - name: setup python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - name: setup prerequisites run: | - # Warning: this is not necessarily the same Python version as the one configured above ! - python3 -m pip install -U faber --break-system-packages + echo "CXX=${{ matrix.cxx }}" >> "$GITHUB_ENV" + echo "CXX_STD=${{ matrix.std }}" >> "$GITHUB_ENV" - name: build run: | - ${{ matrix.python }} --version - ${{ matrix.cxx }} --version - faber -v - sed -e "s/\$PYTHON/${{ matrix.python }}/g" .ci/faber > ~/.faber - faber \ - --with-boost-include=${BOOST_PY_DEPS} \ - --builddir=build \ - cxx.name=${{ matrix.cxx }} \ - cxxflags=-std=${{ matrix.std }} \ - cppflags=-std=${{ matrix.std }} \ - -j`nproc` + .github/run-faber.sh - name: test run: | - faber \ - --with-boost-include=${BOOST_PY_DEPS} \ - --builddir=build \ - cxx.name=${{ matrix.cxx }} \ - cxxflags=-std=${{ matrix.std }} \ - cppflags=-std=${{ matrix.std }} \ - -j`nproc` \ - test.report + .github/run-faber.sh test.report From 32da86df269fb8f7ef777ddd8c3424d854e94455 Mon Sep 17 00:00:00 2001 From: Stefan Seefeld Date: Tue, 2 Dec 2025 08:38:34 -0500 Subject: [PATCH 7/7] Improve test coverage. --- .github/workflows/test-ubuntu-py2.yml | 58 --------------------------- .github/workflows/test-ubuntu.yml | 49 +++++++++++++++++++++- 2 files changed, 47 insertions(+), 60 deletions(-) delete mode 100644 .github/workflows/test-ubuntu-py2.yml diff --git a/.github/workflows/test-ubuntu-py2.yml b/.github/workflows/test-ubuntu-py2.yml deleted file mode 100644 index 801664c105..0000000000 --- a/.github/workflows/test-ubuntu-py2.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Test Ubuntu, Python 2.x - -on: - push: - pull_request: - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python: [python] - cxx: [g++] - std: [c++11] - include: - # Add the appropriate docker image for each compiler. - # The images from teeks99/boost-python-test already have boost::python - # pre-reqs installed, see: - # https://github.com/teeks99/boost-python-test-docker - - cxx: g++ - docker-img: teeks99/boost-python-test:gcc-15_1.89.0 - - container: - image: ${{ matrix.docker-img }} - - steps: - - uses: actions/checkout@v5 - - - name: setup prerequisites - run: | - # Warning: this is not necessarily the same Python version as the one configured above ! - python3 -m pip install -U faber --break-system-packages - - name: build - run: | - ${{ matrix.python }} --version - ${{ matrix.cxx }} --version - faber -v - sed -e "s/\$PYTHON/${{ matrix.python }}/g" .ci/faber > ~/.faber - faber \ - --with-boost-include=${BOOST_PY_DEPS} \ - --builddir=build \ - cxx.name=${{ matrix.cxx }} \ - cxxflags=-std=${{ matrix.std }} \ - cppflags=-std=${{ matrix.std }} \ - -j`nproc` - - name: test - run: | - faber \ - --with-boost-include=${BOOST_PY_DEPS} \ - --builddir=build \ - cxx.name=${{ matrix.cxx }} \ - cxxflags=-std=${{ matrix.std }} \ - cppflags=-std=${{ matrix.std }} \ - -j`nproc` \ - test.report diff --git a/.github/workflows/test-ubuntu.yml b/.github/workflows/test-ubuntu.yml index abb0f2d59a..31637de5ef 100644 --- a/.github/workflows/test-ubuntu.yml +++ b/.github/workflows/test-ubuntu.yml @@ -17,6 +17,21 @@ jobs: cxx: [g++, clang++] std: [c++11, c++14, c++17] include: + - python-version: '2.7' + cxx: g++ + std: c++11 + - python-version: '3.10' + cxx: g++ + std: c++17 + - python-version: '3.11' + cxx: g++ + std: c++17 + - python-version: '3.12' + cxx: g++ + std: c++17 + - python-version: '3.13' + cxx: g++ + std: c++17 # Also test with free-threaded build of Python - python-version: '3.14t' cxx: clang++ @@ -34,16 +49,46 @@ jobs: steps: - uses: actions/checkout@v5 - name: setup python + if: "${{ matrix.python-version != '2.7' }}" uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: setup prerequisites run: | + # Warning: this is not necessarily the same Python version as the one configured above ! + python3 -m pip install -U faber --break-system-packages echo "CXX=${{ matrix.cxx }}" >> "$GITHUB_ENV" echo "CXX_STD=${{ matrix.std }}" >> "$GITHUB_ENV" - - name: build + - name: build-py2 + if: "${{ matrix.python-version == '2.7' }}" + run: | + python --version + ${{ matrix.cxx }} --version + faber -v + sed -e "s/\$PYTHON/python/g" .ci/faber > ~/.faber + faber \ + --with-boost-include=${BOOST_PY_DEPS} \ + --builddir=build \ + cxx.name=${{ matrix.cxx }} \ + cxxflags=-std=${{ matrix.std }} \ + cppflags=-std=${{ matrix.std }} \ + -j`nproc` + - name: build-py3 + if: "${{ matrix.python-version != '2.7' }}" run: | .github/run-faber.sh - - name: test + - name: test-py2 + if: "${{ matrix.python-version == '2.7' }}" + run: | + faber \ + --with-boost-include=${BOOST_PY_DEPS} \ + --builddir=build \ + cxx.name=${{ matrix.cxx }} \ + cxxflags=-std=${{ matrix.std }} \ + cppflags=-std=${{ matrix.std }} \ + -j`nproc` \ + test.report + - name: test-py3 + if: "${{ matrix.python-version != '2.7' }}" run: | .github/run-faber.sh test.report