Improve C++ test infrastructure: progress reporter, timeouts, and skip hanging Move Subinterpreter test

.github/workflows/ci.yml

@@ -229,6 +229,7 @@ jobs:
       run: cmake --build . --target pytest
 
     - name: Compiled tests
+      timeout-minutes: 3
       run: cmake --build . --target cpptest
 
     - name: Interface test
@@ -334,6 +335,7 @@ jobs:
       run: cmake --build --preset default --target pytest
 
     - name: C++ tests
+      timeout-minutes: 3
       run: cmake --build --preset default --target cpptest
 
     - name: Visibility test
@@ -393,6 +395,7 @@ jobs:
       run: cmake --build build --target pytest
 
     - name: C++ tests
+      timeout-minutes: 3
       run: cmake --build build --target cpptest
 
     - name: Interface test
@@ -516,6 +519,7 @@ jobs:
       run: cmake --build build --target pytest
 
     - name: C++ tests
+      timeout-minutes: 3
       run: cmake --build build --target cpptest
 
     - name: Interface test
@@ -570,6 +574,7 @@ jobs:
       run: cmake --build build --target pytest
 
     - name: C++ tests
+      timeout-minutes: 3
       run: cmake --build build --target cpptest
 
     - name: Interface test
@@ -652,6 +657,7 @@ jobs:
         cmake --build build-11 --target check
 
     - name: C++ tests C++11
+      timeout-minutes: 3
       run: |
         set +e; source /opt/intel/oneapi/setvars.sh; set -e
         cmake --build build-11 --target cpptest
@@ -689,6 +695,7 @@ jobs:
         cmake --build build-17 --target check
 
     - name: C++ tests C++17
+      timeout-minutes: 3
       run: |
         set +e; source /opt/intel/oneapi/setvars.sh; set -e
         cmake --build build-17 --target cpptest
@@ -760,6 +767,7 @@ jobs:
       run: cmake --build build --target pytest
 
     - name: C++ tests
+      timeout-minutes: 3
       run: cmake --build build --target cpptest
 
     - name: Interface test
@@ -1001,6 +1009,7 @@ jobs:
       run: cmake --build build --target pytest
 
     - name: C++20 tests
+      timeout-minutes: 3
       run: cmake --build build --target cpptest -j 2
 
     - name: Interface test C++20
@@ -1077,6 +1086,7 @@ jobs:
       run: cmake --build build --target pytest -j 2
 
     - name: C++11 tests
+      timeout-minutes: 3
       run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build --target cpptest -j 2
 
     - name: Interface test C++11
@@ -1101,6 +1111,7 @@ jobs:
       run: cmake --build build2 --target pytest -j 2
 
     - name: C++14 tests
+      timeout-minutes: 3
       run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build2 --target cpptest -j 2
 
     - name: Interface test C++14
@@ -1125,6 +1136,7 @@ jobs:
       run: cmake --build build3 --target pytest -j 2
 
     - name: C++17 tests
+      timeout-minutes: 3
       run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build3 --target cpptest -j 2
 
     - name: Interface test C++17
@@ -1196,6 +1208,7 @@ jobs:
         run: cmake --build . --target pytest -j 2
 
       - name: C++ tests
+        timeout-minutes: 3
         run: cmake --build . --target cpptest -j 2
 
       - name: Interface test
@@ -1258,6 +1271,7 @@ jobs:
         run: cmake --build . --target pytest -j 2
 
       - name: C++ tests
+        timeout-minutes: 3
         run: cmake --build . --target cpptest -j 2
 
       - name: Interface test
@@ -1330,6 +1344,7 @@ jobs:
       run: cmake --build build --target pytest -j 2
 
     - name: C++ tests
+      timeout-minutes: 3
       run: PYTHONHOME=/clangarm64 PYTHONPATH=/clangarm64 cmake --build build --target cpptest -j 2
 
     - name: Interface test

.github/workflows/reusable-standard.yml

@@ -83,6 +83,7 @@ jobs:
         run: cmake --build build --target pytest
 
       - name: C++ tests
+        timeout-minutes: 3
         run: cmake --build build --target cpptest
 
       - name: Interface test

.github/workflows/upstream.yml

@@ -66,6 +66,7 @@ jobs:
       run: cmake --build build11 --target pytest -j 2
 
     - name: C++11 tests
+      timeout-minutes: 3
       run: cmake --build build11  --target cpptest -j 2
 
     - name: Interface test C++11
@@ -87,6 +88,7 @@ jobs:
       run: cmake --build build17 --target pytest
 
     - name: C++17 tests
+      timeout-minutes: 3
       run: cmake --build build17 --target cpptest
 
     # Third build - C++17 mode with unstable ABI

tests/pure_cpp/CMakeLists.txt

@@ -15,6 +15,8 @@ target_link_libraries(smart_holder_poc_test PRIVATE pybind11::headers Catch2::Ca
 add_custom_target(
   test_pure_cpp
   COMMAND "$<TARGET_FILE:smart_holder_poc_test>"
-  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+  USES_TERMINAL # Ensures output is shown immediately (not buffered and possibly lost on crash)
+)
 
 add_dependencies(check test_pure_cpp)

tests/test_cross_module_rtti/CMakeLists.txt

@@ -60,7 +60,9 @@ add_custom_target(
   test_cross_module_rtti
   COMMAND "$<TARGET_FILE:test_cross_module_rtti_main>"
   DEPENDS test_cross_module_rtti_main
-  WORKING_DIRECTORY "$<TARGET_FILE_DIR:test_cross_module_rtti_main>")
+  WORKING_DIRECTORY "$<TARGET_FILE_DIR:test_cross_module_rtti_main>"
+  USES_TERMINAL # Ensures output is shown immediately (not buffered and possibly lost on crash)
+)
 
 set_target_properties(test_cross_module_rtti_bindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY
                                                                  "${CMAKE_CURRENT_BINARY_DIR}")

tests/test_with_catch/CMakeLists.txt

@@ -47,7 +47,9 @@ add_custom_target(
   cpptest
   COMMAND "$<TARGET_FILE:test_with_catch>"
   DEPENDS test_with_catch
-  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+  USES_TERMINAL # Ensures output is shown immediately (not buffered and possibly lost on crash)
+)
 
 pybind11_add_module(external_module THIN_LTO external_module.cpp)
 set_target_properties(external_module PROPERTIES LIBRARY_OUTPUT_DIRECTORY

tests/test_with_catch/catch.cpp

@@ -3,6 +3,17 @@
 
 #include <pybind11/embed.h>
 
+#include <chrono>
+#include <csignal>
+#include <cstring>
+#include <ctime>
+#include <iomanip>
+#include <sstream>
+
+#ifndef _WIN32
+#    include <unistd.h>
+#endif
+
 // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to
 // catch 2.0.1; this should be fixed in the next catch release after 2.0.1).
 PYBIND11_WARNING_DISABLE_MSVC(4996)
@@ -13,11 +24,126 @@ PYBIND11_WARNING_DISABLE_MSVC(4996)
 #endif
 
 #define CATCH_CONFIG_RUNNER
+#define CATCH_CONFIG_DEFAULT_REPORTER "progress"
+#include "catch_skip.h"
+
 #include <catch.hpp>
 
 namespace py = pybind11;
 
+// Simple progress reporter that prints a line per test case.
+namespace {
+
+class ProgressReporter : public Catch::StreamingReporterBase<ProgressReporter> {
+public:
+    using StreamingReporterBase<ProgressReporter>::StreamingReporterBase;
+
+    static std::string getDescription() { return "Simple progress reporter (one line per test)"; }
+
+    void testCaseStarting(Catch::TestCaseInfo const &testInfo) override {
+        print_python_version_once();
+        auto &os = Catch::cout();
+        os << "[ RUN      ] " << testInfo.name << '\n';
+        os.flush();
+    }
+
+    void testCaseEnded(Catch::TestCaseStats const &stats) override {
+        bool failed = stats.totals.assertions.failed > 0;
+        auto &os = Catch::cout();
+        os << (failed ? "[  FAILED  ] " : "[       OK ] ") << stats.testInfo.name << '\n';
+        os.flush();
+    }
+
+    void noMatchingTestCases(std::string const &spec) override {
+        auto &os = Catch::cout();
+        os << "[  NO TEST ] no matching test cases for spec: " << spec << '\n';
+        os.flush();
+    }
+
+    void reportInvalidArguments(std::string const &arg) override {
+        auto &os = Catch::cout();
+        os << "[   ERROR  ] invalid Catch2 arguments: " << arg << '\n';
+        os.flush();
+    }
+
+    void assertionStarting(Catch::AssertionInfo const &) override {}
+
+    bool assertionEnded(Catch::AssertionStats const &) override { return false; }
+
+    void testRunEnded(Catch::TestRunStats const &stats) override {
+        auto &os = Catch::cout();
+        auto passed = stats.totals.testCases.passed;
+        auto failed = stats.totals.testCases.failed;
+        auto total = passed + failed;
+        auto assertions = stats.totals.assertions.passed + stats.totals.assertions.failed;
+        if (failed == 0) {
+            os << "[  PASSED  ] " << total << " test cases, " << assertions << " assertions.\n";
+        } else {
+            os << "[  FAILED  ] " << failed << " of " << total << " test cases, " << assertions
+               << " assertions.\n";
+        }
+        os.flush();
+    }
+
+private:
+    void print_python_version_once() {
+        if (printed_) {
+            return;
+        }
+        printed_ = true;
+        auto &os = Catch::cout();
+        os << "[ PYTHON   ] " << Py_GetVersion() << '\n';
+        os.flush();
+    }
+
+    bool printed_ = false;
+};
+
+} // namespace
+
+CATCH_REGISTER_REPORTER("progress", ProgressReporter)
+
+namespace {
+
+std::string get_utc_timestamp() {
+    auto now = std::chrono::system_clock::now();
+    auto time_t_now = std::chrono::system_clock::to_time_t(now);
+    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()) % 1000;
+
+    std::tm utc_tm{};
+#if defined(_WIN32)
+    gmtime_s(&utc_tm, &time_t_now);
+#else
+    gmtime_r(&time_t_now, &utc_tm);
+#endif
+
+    std::ostringstream oss;
+    oss << std::put_time(&utc_tm, "%Y-%m-%d %H:%M:%S") << '.' << std::setfill('0') << std::setw(3)
+        << ms.count() << 'Z';
+    return oss.str();
+}
+
+#ifndef _WIN32
+// Signal handler to print a message when the process is terminated.
+// Uses only async-signal-safe functions.
+void termination_signal_handler(int sig) {
+    const char *msg = "[  SIGNAL  ] Process received SIGTERM\n";
+    // write() is async-signal-safe, unlike std::cout
+    ssize_t written = write(STDOUT_FILENO, msg, strlen(msg));
+    (void) written; // suppress "unused variable" warnings
+    // Re-raise with default handler to get proper exit status
+    std::signal(sig, SIG_DFL);
+    std::raise(sig);
+}
+#endif
+
+} // namespace
+
 int main(int argc, char *argv[]) {
+#ifndef _WIN32
+    std::signal(SIGTERM, termination_signal_handler);
+#endif
+
     // Setup for TEST_CASE in test_interpreter.cpp, tagging on a large random number:
     std::string updated_pythonpath("pybind11_test_with_catch_PYTHONPATH_2099743835476552");
     const char *preexisting_pythonpath = getenv("PYTHONPATH");
@@ -35,9 +161,15 @@ int main(int argc, char *argv[]) {
     setenv("PYTHONPATH", updated_pythonpath.c_str(), /*replace=*/1);
 #endif
 
+    std::cout << "[ STARTING ] " << get_utc_timestamp() << '\n';
+    std::cout.flush();
+
     py::scoped_interpreter guard{};
 
     auto result = Catch::Session().run(argc, argv);
 
+    std::cout << "[ DONE     ] " << get_utc_timestamp() << " (result " << result << ")\n";
+    std::cout.flush();
+
     return result < 0xff ? result : 0xff;
 }

tests/test_with_catch/catch_skip.h

@@ -0,0 +1,16 @@
+// Macro to skip a test at runtime with a visible message.
+// Catch2 v2 doesn't have native skip support (v3 does with SKIP()).
+// The test will count as "passed" in totals, but the output clearly shows it was skipped.
+
+#pragma once
+
+#include <catch.hpp>
+
+#define PYBIND11_CATCH2_SKIP_IF(condition, reason)                                                \
+    do {                                                                                          \
+        if (condition) {                                                                          \
+            Catch::cout() << "[ SKIPPED  ] " << (reason) << '\n';                                 \
+            Catch::cout().flush();                                                                \
+            return;                                                                               \
+        }                                                                                         \
+    } while (0)

tests/test_with_catch/test_subinterpreter.cpp

@@ -6,6 +6,8 @@
 // catch 2.0.1; this should be fixed in the next catch release after 2.0.1).
 PYBIND11_WARNING_DISABLE_MSVC(4996)
 
+#    include "catch_skip.h"
+
 #    include <catch.hpp>
 #    include <cstdlib>
 #    include <fstream>
@@ -92,6 +94,13 @@ TEST_CASE("Single Subinterpreter") {
 
 #    if PY_VERSION_HEX >= 0x030D0000
 TEST_CASE("Move Subinterpreter") {
+    // Test is skipped on free-threaded Python 3.14+ due to a hang in Py_EndInterpreter()
+    // when the subinterpreter is destroyed from a different thread than it was created on.
+    // See: https://github.com/pybind/pybind11/pull/5940
+#        if PY_VERSION_HEX >= 0x030E0000 && defined(Py_GIL_DISABLED)
+    PYBIND11_CATCH2_SKIP_IF(true, "Skipped on free-threaded Python 3.14+ (see PR #5940)");
+#        endif
+
     std::unique_ptr<py::subinterpreter> sub(new py::subinterpreter(py::subinterpreter::create()));
 
     // on this thread, use the subinterpreter and import some non-trivial junk