From 69f54ce4523174860a92271e3cec5c498f89627f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 27 Nov 2025 19:00:02 +0100 Subject: [PATCH 1/3] gh-140210: Make test_sysconfig.test_parse_makefile_renamed_vars ignore environment variables (#140213) The test did not expect it could be run with e.g. CFLAGS set to a custom value. --- Lib/test/test_sysconfig.py | 6 ++++-- .../Tests/2025-10-16-15-08-58.gh-issue-140210.P9vUP8.rst | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2025-10-16-15-08-58.gh-issue-140210.P9vUP8.rst diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 8101657b04a5c6..502103ce629358 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -20,7 +20,7 @@ ) from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, - change_cwd) + change_cwd, EnvironmentVarGuard) from test.support.venv import VirtualEnvironmentMixin import sysconfig @@ -807,7 +807,9 @@ def test_parse_makefile_renamed_vars(self): print("PY_LDFLAGS=-lm", file=makefile) print("var2=$(LDFLAGS)", file=makefile) print("var3=$(CPPFLAGS)", file=makefile) - vars = _parse_makefile(TESTFN) + with EnvironmentVarGuard() as env: + env.clear() + vars = _parse_makefile(TESTFN) self.assertEqual(vars, { 'var1': '-Wall', 'CFLAGS': '-Wall', diff --git a/Misc/NEWS.d/next/Tests/2025-10-16-15-08-58.gh-issue-140210.P9vUP8.rst b/Misc/NEWS.d/next/Tests/2025-10-16-15-08-58.gh-issue-140210.P9vUP8.rst new file mode 100644 index 00000000000000..f2064bfd377e69 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-10-16-15-08-58.gh-issue-140210.P9vUP8.rst @@ -0,0 +1,2 @@ +Make ``test_sysconfig.test_parse_makefile_renamed_vars`` less fragile by +clearing the environment variables before parsing the Makefile. From 656a64b37f817cc8fe36ee17f332100482185cce Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 27 Nov 2025 11:17:59 -0800 Subject: [PATCH 2/3] gh-141930: Use the regular IO stack to write .pyc files for a better error message on failure (GH-141931) * Use open() to write the bytecode * Convert to unittest style asserts * Tweak news, thanks @vstinner * Tidy * reword NEWS, avoid word "retried" --- Lib/importlib/_bootstrap_external.py | 8 +-- Lib/test/test_importlib/test_util.py | 71 ++++++++++++++----- ...-11-24-21-09-30.gh-issue-141930.hIIzSd.rst | 2 + 3 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 192c0261408ead..2f9307cba4f086 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -208,12 +208,8 @@ def _write_atomic(path, data, mode=0o666): try: # We first write data to a temporary file, and then use os.replace() to # perform an atomic rename. - with _io.FileIO(fd, 'wb') as file: - bytes_written = file.write(data) - if bytes_written != len(data): - # Raise an OSError so the 'except' below cleans up the partially - # written file. - raise OSError("os.write() didn't write the full pyc file") + with _io.open(fd, 'wb') as file: + file.write(data) _os.replace(path_tmp, path) except OSError: try: diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index a77ce234deec58..0adab8d14e0452 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -788,31 +788,70 @@ def test_complete_multi_phase_init_module(self): self.run_with_own_gil(script) -class MiscTests(unittest.TestCase): - def test_atomic_write_should_notice_incomplete_writes(self): +class PatchAtomicWrites: + def __init__(self, truncate_at_length, never_complete=False): + self.truncate_at_length = truncate_at_length + self.never_complete = never_complete + self.seen_write = False + self._children = [] + + def __enter__(self): import _pyio oldwrite = os.write - seen_write = False - - truncate_at_length = 100 # Emulate an os.write that only writes partial data. def write(fd, data): - nonlocal seen_write - seen_write = True - return oldwrite(fd, data[:truncate_at_length]) + if self.seen_write and self.never_complete: + return None + self.seen_write = True + return oldwrite(fd, data[:self.truncate_at_length]) # Need to patch _io to be _pyio, so that io.FileIO is affected by the # os.write patch. - with (support.swap_attr(_bootstrap_external, '_io', _pyio), - support.swap_attr(os, 'write', write)): - with self.assertRaises(OSError): - # Make sure we write something longer than the point where we - # truncate. - content = b'x' * (truncate_at_length * 2) - _bootstrap_external._write_atomic(os_helper.TESTFN, content) - assert seen_write + self.children = [ + support.swap_attr(_bootstrap_external, '_io', _pyio), + support.swap_attr(os, 'write', write) + ] + for child in self.children: + child.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for child in self.children: + child.__exit__(exc_type, exc_val, exc_tb) + + +class MiscTests(unittest.TestCase): + + def test_atomic_write_retries_incomplete_writes(self): + truncate_at_length = 100 + length = truncate_at_length * 2 + + with PatchAtomicWrites(truncate_at_length=truncate_at_length) as cm: + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * length + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) + + self.assertEqual(os.stat(support.os_helper.TESTFN).st_size, length) + os.unlink(support.os_helper.TESTFN) + + def test_atomic_write_errors_if_unable_to_complete(self): + truncate_at_length = 100 + + with ( + PatchAtomicWrites( + truncate_at_length=truncate_at_length, never_complete=True, + ) as cm, + self.assertRaises(OSError) + ): + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * (truncate_at_length * 2) + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) with self.assertRaises(OSError): os.stat(support.os_helper.TESTFN) # Check that the file did not get written. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst new file mode 100644 index 00000000000000..06a12f98224e88 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst @@ -0,0 +1,2 @@ +When importing a module, use Python's regular file object to ensure that +writes to ``.pyc`` files are complete or an appropriate error is raised. From 5ec03cf3b086fd01614cbd97bb99d45aac3668fe Mon Sep 17 00:00:00 2001 From: dgpb <3577712+dg-pb@users.noreply.github.com> Date: Fri, 28 Nov 2025 00:22:21 +0200 Subject: [PATCH 3/3] gh-133228: c-analyzer clang preprocessor (GH-133229) * impl * included 2 failures to tsvs next to similar entries * added fix/hack for curses.h fails * fix leftover from debug --- .../c_parser/preprocessor/__init__.py | 3 +- .../c-analyzer/c_parser/preprocessor/clang.py | 104 ++++++++++++++++++ Tools/c-analyzer/c_parser/preprocessor/gcc.py | 15 ++- Tools/c-analyzer/cpython/_parser.py | 4 + Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + Tools/c-analyzer/cpython/ignored.tsv | 1 + 6 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 Tools/c-analyzer/c_parser/preprocessor/clang.py diff --git a/Tools/c-analyzer/c_parser/preprocessor/__init__.py b/Tools/c-analyzer/c_parser/preprocessor/__init__.py index 30a86cbd7dc494..f8d2f805cb1b19 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/__init__.py +++ b/Tools/c-analyzer/c_parser/preprocessor/__init__.py @@ -16,6 +16,7 @@ from . import ( pure as _pure, gcc as _gcc, + clang as _clang, ) @@ -234,7 +235,7 @@ def handling_errors(ignore_exc=None, *, log_err=None): 'bcpp': None, # aliases/extras: 'gcc': _gcc.preprocess, - 'clang': None, + 'clang': _clang.preprocess, } diff --git a/Tools/c-analyzer/c_parser/preprocessor/clang.py b/Tools/c-analyzer/c_parser/preprocessor/clang.py new file mode 100644 index 00000000000000..574a23f8f6d6f9 --- /dev/null +++ b/Tools/c-analyzer/c_parser/preprocessor/clang.py @@ -0,0 +1,104 @@ +import os.path +import re, sys + +from . import common as _common +from . import gcc as _gcc + +_normpath = _gcc._normpath + +TOOL = 'clang' + +META_FILES = { + '', + '', +} + + +def preprocess(filename, + incldirs=None, + includes=None, + macros=None, + samefiles=None, + cwd=None, + ): + if not cwd or not os.path.isabs(cwd): + cwd = os.path.abspath(cwd or '.') + filename = _normpath(filename, cwd) + + postargs = _gcc.POST_ARGS + basename = os.path.basename(filename) + dirname = os.path.basename(os.path.dirname(filename)) + if (basename not in _gcc.FILES_WITHOUT_INTERNAL_CAPI + and dirname not in _gcc.DIRS_WITHOUT_INTERNAL_CAPI): + postargs += ('-DPy_BUILD_CORE=1',) + + text = _common.preprocess( + TOOL, + filename, + incldirs=incldirs, + includes=includes, + macros=macros, + #preargs=PRE_ARGS, + postargs=postargs, + executable=['clang'], + compiler='unix', + cwd=cwd, + ) + return _iter_lines(text, filename, samefiles, cwd) + + +EXIT_MARKERS = {'# 2 "" 2', '# 3 "" 2', '# 4 "" 2'} + + +def _iter_lines(text, reqfile, samefiles, cwd, raw=False): + lines = iter(text.splitlines()) + + # The first line is special. + # The subsequent lines are consistent. + firstlines = [ + f'# 1 "{reqfile}"', + '# 1 "" 1', + '# 1 "" 3', + '# 370 "" 3', + '# 1 "" 1', + '# 1 "" 2', + ] + for expected in firstlines: + line = next(lines) + if line != expected: + raise NotImplementedError((line, expected)) + + # Do all the CLI-provided includes. + filter_reqfile = (lambda f: _gcc._filter_reqfile(f, reqfile, samefiles)) + make_info = (lambda lno: _common.FileInfo(reqfile, lno)) + last = None + for line in lines: + assert last != reqfile, (last,) + # NOTE:condition is clang specific + if not line: + continue + lno, included, flags = _gcc._parse_marker_line(line, reqfile) + if not included: + raise NotImplementedError((line,)) + if included == reqfile: + # This will be the last one. + assert 2 in flags, (line, flags) + else: + # NOTE:first condition is specific to clang + if _normpath(included, cwd) == reqfile: + assert 1 in flags or 2 in flags, (line, flags, included, reqfile) + else: + assert 1 in flags, (line, flags, included, reqfile) + yield from _gcc._iter_top_include_lines( + lines, + _normpath(included, cwd), + cwd, + filter_reqfile, + make_info, + raw, + EXIT_MARKERS + ) + last = included + # The last one is always the requested file. + # NOTE:_normpath is clang specific + assert _normpath(included, cwd) == reqfile, (line,) diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index d20cd19f6e6d5e..4a55a1a24ee1be 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -65,6 +65,8 @@ '-E', ) +EXIT_MARKERS = {'# 0 "" 2', '# 1 "" 2'} + def preprocess(filename, incldirs=None, @@ -138,6 +140,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False): filter_reqfile, make_info, raw, + EXIT_MARKERS ) last = included # The last one is always the requested file. @@ -146,7 +149,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False): def _iter_top_include_lines(lines, topfile, cwd, filter_reqfile, make_info, - raw): + raw, exit_markers): partial = 0 # depth files = [topfile] # We start at 1 in case there are source lines (including blank ones) @@ -154,12 +157,20 @@ def _iter_top_include_lines(lines, topfile, cwd, # _parse_marker_line() that the preprocessor reported lno as 1. lno = 1 for line in lines: - if line == '# 0 "" 2' or line == '# 1 "" 2': + if line in exit_markers: # We're done with this top-level include. return _lno, included, flags = _parse_marker_line(line) if included: + # HACK: + # Mixes curses.h and ncurses.h marker lines + # gcc silently passes this, while clang fails + # See: /Include/py_curses.h #if-elif directives + # And compare with preprocessor output + if os.path.basename(included) == 'curses.h': + included = os.path.join(os.path.dirname(included), 'ncurses.h') + lno = _lno included = _normpath(included, cwd) # We hit a marker line. diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index fd198d7d06c96f..d348a99fff7a11 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -340,6 +340,10 @@ def format_tsv_lines(lines): # Catch-alls: _abs('Include/**/*.h'): (5_000, 500), + + # Specific to clang + _abs('Modules/selectmodule.c'): (40_000, 3000), + _abs('Modules/_testcapi/pyatomic.c'): (30_000, 1000), } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 3c3cb2f9c86f16..301784f773d31f 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -400,6 +400,7 @@ Modules/_tkinter.c - tcl_lock - Modules/_tkinter.c - excInCmd - Modules/_tkinter.c - valInCmd - Modules/_tkinter.c - trbInCmd - +Modules/socketmodule.c - netdb_lock - ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index bd4a8cf0d3e65c..adb183000deeff 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -16,6 +16,7 @@ filename funcname name reason ## indicators for resource availability/capability # (set during first init) Python/bootstrap_hash.c py_getrandom getrandom_works - +Python/bootstrap_hash.c py_getentropy getentropy_works - Python/fileutils.c - _Py_open_cloexec_works - Python/fileutils.c set_inheritable ioctl_works - # (set lazily, *after* first init)