[pull] main from python:main

Lib/importlib/_bootstrap_external.py

@@ -208,12 +208,8 @@ def _write_atomic(path, data, mode=0o666):
     try:
         # We first write data to a temporary file, and then use os.replace() to
         # perform an atomic rename.
-        with _io.FileIO(fd, 'wb') as file:
-            bytes_written = file.write(data)
-        if bytes_written != len(data):
-            # Raise an OSError so the 'except' below cleans up the partially
-            # written file.
-            raise OSError("os.write() didn't write the full pyc file")
+        with _io.open(fd, 'wb') as file:
+            file.write(data)
         _os.replace(path_tmp, path)
     except OSError:
         try:

Lib/test/test_importlib/test_util.py

@@ -788,31 +788,70 @@ def test_complete_multi_phase_init_module(self):
             self.run_with_own_gil(script)
 
 
-class MiscTests(unittest.TestCase):
-    def test_atomic_write_should_notice_incomplete_writes(self):
+class PatchAtomicWrites:
+    def __init__(self, truncate_at_length, never_complete=False):
+        self.truncate_at_length = truncate_at_length
+        self.never_complete = never_complete
+        self.seen_write = False
+        self._children = []
+
+    def __enter__(self):
         import _pyio
 
         oldwrite = os.write
-        seen_write = False
-
-        truncate_at_length = 100
 
         # Emulate an os.write that only writes partial data.
         def write(fd, data):
-            nonlocal seen_write
-            seen_write = True
-            return oldwrite(fd, data[:truncate_at_length])
+            if self.seen_write and self.never_complete:
+                return None
+            self.seen_write = True
+            return oldwrite(fd, data[:self.truncate_at_length])
 
         # Need to patch _io to be _pyio, so that io.FileIO is affected by the
         # os.write patch.
-        with (support.swap_attr(_bootstrap_external, '_io', _pyio),
-              support.swap_attr(os, 'write', write)):
-            with self.assertRaises(OSError):
-                # Make sure we write something longer than the point where we
-                # truncate.
-                content = b'x' * (truncate_at_length * 2)
-                _bootstrap_external._write_atomic(os_helper.TESTFN, content)
-        assert seen_write
+        self.children = [
+            support.swap_attr(_bootstrap_external, '_io', _pyio),
+            support.swap_attr(os, 'write', write)
+        ]
+        for child in self.children:
+            child.__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        for child in self.children:
+            child.__exit__(exc_type, exc_val, exc_tb)
+
+
+class MiscTests(unittest.TestCase):
+
+    def test_atomic_write_retries_incomplete_writes(self):
+        truncate_at_length = 100
+        length = truncate_at_length * 2
+
+        with PatchAtomicWrites(truncate_at_length=truncate_at_length) as cm:
+            # Make sure we write something longer than the point where we
+            # truncate.
+            content = b'x' * length
+            _bootstrap_external._write_atomic(os_helper.TESTFN, content)
+        self.assertTrue(cm.seen_write)
+
+        self.assertEqual(os.stat(support.os_helper.TESTFN).st_size, length)
+        os.unlink(support.os_helper.TESTFN)
+
+    def test_atomic_write_errors_if_unable_to_complete(self):
+        truncate_at_length = 100
+
+        with (
+            PatchAtomicWrites(
+                truncate_at_length=truncate_at_length, never_complete=True,
+            ) as cm,
+            self.assertRaises(OSError)
+        ):
+            # Make sure we write something longer than the point where we
+            # truncate.
+            content = b'x' * (truncate_at_length * 2)
+            _bootstrap_external._write_atomic(os_helper.TESTFN, content)
+        self.assertTrue(cm.seen_write)
 
         with self.assertRaises(OSError):
             os.stat(support.os_helper.TESTFN) # Check that the file did not get written.

Lib/test/test_sysconfig.py

@@ -20,7 +20,7 @@
 )
 from test.support.import_helper import import_module
 from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink,
-                                    change_cwd)
+                                    change_cwd, EnvironmentVarGuard)
 from test.support.venv import VirtualEnvironmentMixin
 
 import sysconfig
@@ -807,7 +807,9 @@ def test_parse_makefile_renamed_vars(self):
             print("PY_LDFLAGS=-lm", file=makefile)
             print("var2=$(LDFLAGS)", file=makefile)
             print("var3=$(CPPFLAGS)", file=makefile)
-        vars = _parse_makefile(TESTFN)
+        with EnvironmentVarGuard() as env:
+            env.clear()
+            vars = _parse_makefile(TESTFN)
         self.assertEqual(vars, {
             'var1': '-Wall',
             'CFLAGS': '-Wall',

Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst

@@ -0,0 +1,2 @@
+When importing a module, use Python's regular file object to ensure that
+writes to ``.pyc`` files are complete or an appropriate error is raised.

Misc/NEWS.d/next/Tests/2025-10-16-15-08-58.gh-issue-140210.P9vUP8.rst

@@ -0,0 +1,2 @@
+Make ``test_sysconfig.test_parse_makefile_renamed_vars`` less fragile by
+clearing the environment variables before parsing the Makefile.

Tools/c-analyzer/c_parser/preprocessor/__init__.py

@@ -16,6 +16,7 @@
 from . import (
     pure as _pure,
     gcc as _gcc,
+    clang as _clang,
 )
 
 
@@ -234,7 +235,7 @@ def handling_errors(ignore_exc=None, *, log_err=None):
     'bcpp': None,
     # aliases/extras:
     'gcc': _gcc.preprocess,
-    'clang': None,
+    'clang': _clang.preprocess,
 }
 
 

Tools/c-analyzer/c_parser/preprocessor/clang.py

@@ -0,0 +1,104 @@
+import os.path
+import re, sys
+
+from . import common as _common
+from . import gcc as _gcc
+
+_normpath = _gcc._normpath
+
+TOOL = 'clang'
+
+META_FILES = {
+    '<built-in>',
+    '<command line>',
+}
+
+
+def preprocess(filename,
+               incldirs=None,
+               includes=None,
+               macros=None,
+               samefiles=None,
+               cwd=None,
+               ):
+    if not cwd or not os.path.isabs(cwd):
+        cwd = os.path.abspath(cwd or '.')
+    filename = _normpath(filename, cwd)
+
+    postargs = _gcc.POST_ARGS
+    basename = os.path.basename(filename)
+    dirname = os.path.basename(os.path.dirname(filename))
+    if (basename not in _gcc.FILES_WITHOUT_INTERNAL_CAPI
+       and dirname not in _gcc.DIRS_WITHOUT_INTERNAL_CAPI):
+        postargs += ('-DPy_BUILD_CORE=1',)
+
+    text = _common.preprocess(
+        TOOL,
+        filename,
+        incldirs=incldirs,
+        includes=includes,
+        macros=macros,
+        #preargs=PRE_ARGS,
+        postargs=postargs,
+        executable=['clang'],
+        compiler='unix',
+        cwd=cwd,
+    )
+    return _iter_lines(text, filename, samefiles, cwd)
+
+
+EXIT_MARKERS = {'# 2 "<built-in>" 2', '# 3 "<built-in>" 2', '# 4 "<built-in>" 2'}
+
+
+def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
+    lines = iter(text.splitlines())
+
+    # The first line is special.
+    # The subsequent lines are consistent.
+    firstlines = [
+        f'# 1 "{reqfile}"',
+        '# 1 "<built-in>" 1',
+        '# 1 "<built-in>" 3',
+        '# 370 "<built-in>" 3',
+        '# 1 "<command line>" 1',
+        '# 1 "<built-in>" 2',
+    ]
+    for expected in firstlines:
+        line = next(lines)
+        if line != expected:
+            raise NotImplementedError((line, expected))
+
+    # Do all the CLI-provided includes.
+    filter_reqfile = (lambda f: _gcc._filter_reqfile(f, reqfile, samefiles))
+    make_info = (lambda lno: _common.FileInfo(reqfile, lno))
+    last = None
+    for line in lines:
+        assert last != reqfile, (last,)
+        # NOTE:condition is clang specific
+        if not line:
+            continue
+        lno, included, flags = _gcc._parse_marker_line(line, reqfile)
+        if not included:
+            raise NotImplementedError((line,))
+        if included == reqfile:
+            # This will be the last one.
+            assert 2 in flags, (line, flags)
+        else:
+            # NOTE:first condition is specific to clang
+            if _normpath(included, cwd) == reqfile:
+                assert 1 in flags or 2 in flags, (line, flags, included, reqfile)
+            else:
+                assert 1 in flags, (line, flags, included, reqfile)
+        yield from _gcc._iter_top_include_lines(
+            lines,
+            _normpath(included, cwd),
+            cwd,
+            filter_reqfile,
+            make_info,
+            raw,
+            EXIT_MARKERS
+        )
+        last = included
+    # The last one is always the requested file.
+    # NOTE:_normpath is clang specific
+    assert _normpath(included, cwd) == reqfile, (line,)

Tools/c-analyzer/c_parser/preprocessor/gcc.py

@@ -65,6 +65,8 @@
     '-E',
 )
 
+EXIT_MARKERS = {'# 0 "<command-line>" 2', '# 1 "<command-line>" 2'}
+
 
 def preprocess(filename,
                incldirs=None,
@@ -138,6 +140,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
             filter_reqfile,
             make_info,
             raw,
+            EXIT_MARKERS
         )
         last = included
     # The last one is always the requested file.
@@ -146,20 +149,28 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
 
 def _iter_top_include_lines(lines, topfile, cwd,
                             filter_reqfile, make_info,
-                            raw):
+                            raw, exit_markers):
     partial = 0  # depth
     files = [topfile]
     # We start at 1 in case there are source lines (including blank ones)
     # before the first marker line.  Also, we already verified in
     # _parse_marker_line() that the preprocessor reported lno as 1.
     lno = 1
     for line in lines:
-        if line == '# 0 "<command-line>" 2' or line == '# 1 "<command-line>" 2':
+        if line in exit_markers:
             # We're done with this top-level include.
             return
 
         _lno, included, flags = _parse_marker_line(line)
         if included:
+            # HACK:
+            # Mixes curses.h and ncurses.h marker lines
+            # gcc silently passes this, while clang fails
+            # See: /Include/py_curses.h #if-elif directives
+            # And compare with preprocessor output
+            if os.path.basename(included) == 'curses.h':
+                included = os.path.join(os.path.dirname(included), 'ncurses.h')
+
             lno = _lno
             included = _normpath(included, cwd)
             # We hit a marker line.

Tools/c-analyzer/cpython/_parser.py

@@ -340,6 +340,10 @@ def format_tsv_lines(lines):
 
     # Catch-alls:
     _abs('Include/**/*.h'): (5_000, 500),
+
+    # Specific to clang
+    _abs('Modules/selectmodule.c'): (40_000, 3000),
+    _abs('Modules/_testcapi/pyatomic.c'): (30_000, 1000),
 }
 
 

Tools/c-analyzer/cpython/globals-to-fix.tsv

@@ -400,6 +400,7 @@ Modules/_tkinter.c	-	tcl_lock	-
 Modules/_tkinter.c	-	excInCmd	-
 Modules/_tkinter.c	-	valInCmd	-
 Modules/_tkinter.c	-	trbInCmd	-
+Modules/socketmodule.c 	- 	netdb_lock 	-
 
 
 ##################################

Tools/c-analyzer/cpython/ignored.tsv

@@ -16,6 +16,7 @@ filename	funcname	name	reason
 ## indicators for resource availability/capability
 # (set during first init)
 Python/bootstrap_hash.c	py_getrandom	getrandom_works	-
+Python/bootstrap_hash.c	py_getentropy	getentropy_works	-
 Python/fileutils.c	-	_Py_open_cloexec_works	-
 Python/fileutils.c	set_inheritable	ioctl_works	-
 # (set lazily, *after* first init)