diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index 8147e58d322667..759ec4277b8b7d 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -206,6 +206,9 @@ The ``ZoneInfo`` class has two alternate constructors: Objects created via this constructor cannot be pickled (see `pickling`_). + :exc:`ValueError` is raised if the data read from *file_obj* is not a valid + TZif file. + .. classmethod:: ZoneInfo.no_cache(key) An alternate constructor that bypasses the constructor's cache. It is diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 0d35eed38f303d..11f08031ec54f2 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -854,11 +854,13 @@ Optimizations * Builds using Visual Studio 2026 (MSVC 18) may now use the new :ref:`tail-calling interpreter `. - Results on an early experimental MSVC compiler reported roughly 15% speedup - on the geometric mean of pyperformance on Windows x86-64 over - the switch-case interpreter. We have - observed speedups ranging from 15% for large pure-Python libraries + Results on Visual Studio 18.1.1 report between + `15-20% `__ + speedup on the geometric mean of pyperformance on Windows x86-64 over + the switch-case interpreter on an AMD Ryzen 7 5800X. We have + observed speedups ranging from 14% for large pure-Python libraries to 40% for long-running small pure-Python scripts on Windows. + This was made possible by a new feature introduced in MSVC 18. (Contributed by Chris Eibl, Ken Jin, and Brandt Bucher in :gh:`143068`. Special thanks to the MSVC team including Hulon Jenkins.) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index c6c82038d7c85f..f6bdba3e9916c0 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -103,7 +103,6 @@ extern int _PyPerfTrampoline_SetCallbacks(_PyPerf_Callbacks *); extern void _PyPerfTrampoline_GetCallbacks(_PyPerf_Callbacks *); extern int _PyPerfTrampoline_Init(int activate); extern int _PyPerfTrampoline_Fini(void); -extern void _PyPerfTrampoline_FreeArenas(void); extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 818c4f159591fe..3fe1fdaa1589b6 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -87,7 +87,9 @@ struct _ceval_runtime_state { struct trampoline_api_st trampoline_api; FILE *map_file; Py_ssize_t persist_after_fork; - _PyFrameEvalFunction prev_eval_frame; + _PyFrameEvalFunction prev_eval_frame; + Py_ssize_t trampoline_refcount; + int code_watcher_id; #else int _not_used; #endif diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 17333d8c02255d..3cebd7883fcf29 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -2140,7 +2140,7 @@ def _communicate(self, input, endtime, orig_timeout): while selector.get_map(): timeout = self._remaining_time(endtime) - if timeout is not None and timeout < 0: + if timeout is not None and timeout <= 0: self._check_timeout(endtime, orig_timeout, stdout, stderr, skip_check_and_raise=True) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index e0baeece34c7b3..c42c0d4f5e9bc2 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -2104,6 +2104,23 @@ def make_case(): with self.assertRaises(BufferError): ba.rsplit(evil) + def test_extend_empty_buffer_overflow(self): + # gh-143003 + class EvilIter: + def __iter__(self): + return self + def __next__(self): + return next(source) + def __length_hint__(self): + return 0 + + # Use ASCII digits so float() takes the fast path that expects a NUL terminator. + source = iter(b'42') + ba = bytearray() + ba.extend(EvilIter()) + + self.assertRaises(ValueError, float, bytearray()) + def test_hex_use_after_free(self): # Prevent UAF in bytearray.hex(sep) with re-entrant sep.__len__. # Regression test for https://github.com/python/cpython/issues/143195. diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index b1a3a8e65a9802..fe1b5fbe00bbc4 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -1752,7 +1752,12 @@ def main_work(): unwinder_gil = RemoteUnwinder( p.pid, only_active_thread=True ) - gil_traces = _get_stack_trace_with_retry(unwinder_gil) + # Use condition to retry until we capture a thread holding the GIL + # (sampling may catch moments with no GIL holder on slow CI) + gil_traces = _get_stack_trace_with_retry( + unwinder_gil, + condition=lambda t: sum(len(i.threads) for i in t) >= 1, + ) # Count threads total_threads = sum( diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index bc3593ce4ba992..48bf246cadd2f8 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1173,7 +1173,13 @@ def test_flush_parameters(self): if hasattr(mmap, 'MS_INVALIDATE'): m.flush(PAGESIZE * 2, flags=mmap.MS_INVALIDATE) if hasattr(mmap, 'MS_ASYNC') and hasattr(mmap, 'MS_INVALIDATE'): - m.flush(0, PAGESIZE, flags=mmap.MS_ASYNC | mmap.MS_INVALIDATE) + if sys.platform == 'freebsd': + # FreeBSD doesn't support this combination + with self.assertRaises(OSError) as cm: + m.flush(0, PAGESIZE, flags=mmap.MS_ASYNC | mmap.MS_INVALIDATE) + self.assertEqual(cm.exception.errno, errno.EINVAL) + else: + m.flush(0, PAGESIZE, flags=mmap.MS_ASYNC | mmap.MS_INVALIDATE) @unittest.skipUnless(sys.platform == 'linux', 'Linux only') @support.requires_linux_version(5, 17, 0) diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 8f3ca59c9ef5ed..581072d0701d65 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -252,6 +252,8 @@ def test_bad_zones(self): bad_zones = [ b"", # Empty file b"AAAA3" + b" " * 15, # Bad magic + # Truncated V2 file (should not loop indefinitely) + b"TZif2" + (b"\x00" * 39) + b"TZif2" + (b"\x00" * 39) + b"\n" + b"Part", ] for bad_zone in bad_zones: diff --git a/Lib/zoneinfo/_common.py b/Lib/zoneinfo/_common.py index 03cc42149f9b74..59f3f0ce853f74 100644 --- a/Lib/zoneinfo/_common.py +++ b/Lib/zoneinfo/_common.py @@ -118,11 +118,10 @@ def get_abbr(idx): c = fobj.read(1) # Should be \n assert c == b"\n", c - tz_bytes = b"" - while (c := fobj.read(1)) != b"\n": - tz_bytes += c - - tz_str = tz_bytes + line = fobj.readline() + if not line.endswith(b"\n"): + raise ValueError("Invalid TZif file: unexpected end of file") + tz_str = line.rstrip(b"\n") else: tz_str = None diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-23-00-13-02.gh-issue-143003.92g5qW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-23-00-13-02.gh-issue-143003.92g5qW.rst new file mode 100644 index 00000000000000..30df3c53abd29f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-23-00-13-02.gh-issue-143003.92g5qW.rst @@ -0,0 +1,2 @@ +Fix an overflow of the shared empty buffer in :meth:`bytearray.extend` when +``__length_hint__()`` returns 0 for non-empty iterator. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-23-57-43.gh-issue-143228.m3EF9E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-23-57-43.gh-issue-143228.m3EF9E.rst new file mode 100644 index 00000000000000..893bc29543d91d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-23-57-43.gh-issue-143228.m3EF9E.rst @@ -0,0 +1,4 @@ +Fix use-after-free in perf trampoline when toggling profiling while +threads are running or during interpreter finalization with daemon threads +active. The fix uses reference counting to ensure trampolines are not freed +while any code object could still reference them. Pach by Pablo Galindo diff --git a/Misc/NEWS.d/next/Library/2025-12-27-00-14-56.gh-issue-142195.UgBEo5.rst b/Misc/NEWS.d/next/Library/2025-12-27-00-14-56.gh-issue-142195.UgBEo5.rst new file mode 100644 index 00000000000000..b2b1ffe7225bd7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-27-00-14-56.gh-issue-142195.UgBEo5.rst @@ -0,0 +1 @@ +Updated timeout evaluation logic in :mod:`subprocess` to be compatible with deterministic environments like Shadow where time moves exactly as requested. diff --git a/Misc/NEWS.d/next/Library/2025-12-28-13-49-06.gh-issue-143241.5H4b8d.rst b/Misc/NEWS.d/next/Library/2025-12-28-13-49-06.gh-issue-143241.5H4b8d.rst new file mode 100644 index 00000000000000..7170a06015ee7c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-28-13-49-06.gh-issue-143241.5H4b8d.rst @@ -0,0 +1,2 @@ +:mod:`zoneinfo`: fix infinite loop in :meth:`ZoneInfo.from_file +` when parsing a malformed TZif file. Patch by Fatih Celik. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 5262ac20c07300..7f09769e12f05f 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2223,7 +2223,6 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) Py_DECREF(bytearray_obj); return NULL; } - buf[len++] = value; Py_DECREF(item); if (len >= buf_size) { @@ -2233,7 +2232,7 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) Py_DECREF(bytearray_obj); return PyErr_NoMemory(); } - addition = len >> 1; + addition = len ? len >> 1 : 1; if (addition > PyByteArray_SIZE_MAX - len) buf_size = PyByteArray_SIZE_MAX; else @@ -2247,6 +2246,7 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) have invalidated it. */ buf = PyByteArray_AS_STRING(bytearray_obj); } + buf[len++] = value; } Py_DECREF(it); diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 335d8ac7dadd10..c0dc1f7a49bdca 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -204,6 +204,42 @@ enum perf_trampoline_type { #define persist_after_fork _PyRuntime.ceval.perf.persist_after_fork #define perf_trampoline_type _PyRuntime.ceval.perf.perf_trampoline_type #define prev_eval_frame _PyRuntime.ceval.perf.prev_eval_frame +#define trampoline_refcount _PyRuntime.ceval.perf.trampoline_refcount +#define code_watcher_id _PyRuntime.ceval.perf.code_watcher_id + +static void free_code_arenas(void); + +static void +perf_trampoline_reset_state(void) +{ + free_code_arenas(); + if (code_watcher_id >= 0) { + PyCode_ClearWatcher(code_watcher_id); + code_watcher_id = -1; + } + extra_code_index = -1; +} + +static int +perf_trampoline_code_watcher(PyCodeEvent event, PyCodeObject *co) +{ + if (event != PY_CODE_EVENT_DESTROY) { + return 0; + } + if (extra_code_index == -1) { + return 0; + } + py_trampoline f = NULL; + int ret = _PyCode_GetExtra((PyObject *)co, extra_code_index, (void **)&f); + if (ret != 0 || f == NULL) { + return 0; + } + trampoline_refcount--; + if (trampoline_refcount == 0) { + perf_trampoline_reset_state(); + } + return 0; +} static void perf_map_write_entry(void *state, const void *code_addr, @@ -407,6 +443,7 @@ py_trampoline_evaluator(PyThreadState *ts, _PyInterpreterFrame *frame, perf_code_arena->code_size, co); _PyCode_SetExtra((PyObject *)co, extra_code_index, (void *)new_trampoline); + trampoline_refcount++; f = new_trampoline; } assert(f != NULL); @@ -433,6 +470,7 @@ int PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *co) } trampoline_api.write_state(trampoline_api.state, new_trampoline, perf_code_arena->code_size, co); + trampoline_refcount++; return _PyCode_SetExtra((PyObject *)co, extra_code_index, (void *)new_trampoline); } @@ -487,6 +525,10 @@ _PyPerfTrampoline_Init(int activate) { #ifdef PY_HAVE_PERF_TRAMPOLINE PyThreadState *tstate = _PyThreadState_GET(); + if (code_watcher_id == 0) { + // Initialize to -1 since 0 is a valid watcher ID + code_watcher_id = -1; + } if (!activate) { _PyInterpreterState_SetEvalFrameFunc(tstate->interp, prev_eval_frame); perf_status = PERF_STATUS_NO_INIT; @@ -504,6 +546,13 @@ _PyPerfTrampoline_Init(int activate) if (new_code_arena() < 0) { return -1; } + code_watcher_id = PyCode_AddWatcher(perf_trampoline_code_watcher); + if (code_watcher_id < 0) { + PyErr_FormatUnraisable("Failed to register code watcher for perf trampoline"); + free_code_arenas(); + return -1; + } + trampoline_refcount = 1; // Base refcount held by the system perf_status = PERF_STATUS_OK; } #endif @@ -525,17 +574,19 @@ _PyPerfTrampoline_Fini(void) trampoline_api.free_state(trampoline_api.state); perf_trampoline_type = PERF_TRAMPOLINE_UNSET; } - extra_code_index = -1; + + // Prevent new trampolines from being created perf_status = PERF_STATUS_NO_INIT; -#endif - return 0; -} -void _PyPerfTrampoline_FreeArenas(void) { -#ifdef PY_HAVE_PERF_TRAMPOLINE - free_code_arenas(); + // Decrement base refcount. If refcount reaches 0, all code objects are already + // dead so clean up now. Otherwise, watcher remains active to clean up when last + // code object dies; extra_code_index stays valid so watcher can identify them. + trampoline_refcount--; + if (trampoline_refcount == 0) { + perf_trampoline_reset_state(); + } #endif - return; + return 0; } int diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 45b585faf9c980..bb663db195c089 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1944,7 +1944,6 @@ finalize_interp_clear(PyThreadState *tstate) _PyArg_Fini(); _Py_ClearFileSystemEncoding(); _PyPerfTrampoline_Fini(); - _PyPerfTrampoline_FreeArenas(); } finalize_interp_types(tstate->interp);