[pull] main from python:main

Doc/library/xmlrpc.client.rst

@@ -472,7 +472,7 @@ remote server into a single request [#]_.
 
    Create an object used to boxcar method calls. *server* is the eventual target of
    the call. Calls can be made to the result object, but they will immediately
-   return ``None``, and only store the call name and parameters in the
+   return ``None``, and only store the call name and arguments in the
    :class:`MultiCall` object. Calling the object itself causes all stored calls to
    be transmitted as a single ``system.multicall`` request. The result of this call
    is a :term:`generator`; iterating over this generator yields the individual

Include/cpython/object.h

@@ -436,8 +436,6 @@ PyAPI_FUNC(void) _Py_NO_RETURN _PyObject_AssertFailed(
 PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op);
 PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate);
 
-PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count);
-
 /* For backwards compatibility with the old trashcan mechanism */
 #define Py_TRASHCAN_BEGIN(op, dealloc)
 #define Py_TRASHCAN_END

Include/internal/pycore_ceval.h

@@ -263,6 +263,11 @@ static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate)  {
 #endif
 }
 
+// Export for test_peg_generator
+PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(
+    PyThreadState *tstate,
+    int margin_count);
+
 static inline void _Py_LeaveRecursiveCall(void)  {
 }
 

Include/internal/pycore_pymem.h

@@ -70,6 +70,27 @@ static inline int _PyMem_IsPtrFreed(const void *ptr)
 #endif
 }
 
+// Similar to _PyMem_IsPtrFreed() but expects an 'unsigned long' instead of a
+// pointer.
+static inline int _PyMem_IsULongFreed(unsigned long value)
+{
+#if SIZEOF_LONG == 8
+    return (value == 0
+            || value == (unsigned long)0xCDCDCDCDCDCDCDCD
+            || value == (unsigned long)0xDDDDDDDDDDDDDDDD
+            || value == (unsigned long)0xFDFDFDFDFDFDFDFD
+            || value == (unsigned long)0xFFFFFFFFFFFFFFFF);
+#elif SIZEOF_LONG == 4
+    return (value == 0
+            || value == (unsigned long)0xCDCDCDCD
+            || value == (unsigned long)0xDDDDDDDD
+            || value == (unsigned long)0xFDFDFDFD
+            || value == (unsigned long)0xFFFFFFFF);
+#else
+#  error "unknown long size"
+#endif
+}
+
 extern int _PyMem_GetAllocatorName(
     const char *name,
     PyMemAllocatorName *allocator);

Include/internal/pycore_tracemalloc.h

@@ -30,8 +30,8 @@ struct _PyTraceMalloc_Config {
 };
 
 
-/* Pack the frame_t structure to reduce the memory footprint on 64-bit
-   architectures: 12 bytes instead of 16. */
+/* Pack the tracemalloc_frame and tracemalloc_traceback structures to reduce
+   the memory footprint on 64-bit architectures: 12 bytes instead of 16. */
 #if defined(_MSC_VER)
 #pragma pack(push, 4)
 #endif
@@ -46,18 +46,22 @@ tracemalloc_frame {
     PyObject *filename;
     unsigned int lineno;
 };
-#ifdef _MSC_VER
-#pragma pack(pop)
-#endif
 
-struct tracemalloc_traceback {
+struct
+#ifdef __GNUC__
+__attribute__((packed))
+#endif
+tracemalloc_traceback {
     Py_uhash_t hash;
     /* Number of frames stored */
     uint16_t nframe;
     /* Total number of frames the traceback had */
     uint16_t total_nframe;
-    struct tracemalloc_frame frames[1];
+    struct tracemalloc_frame frames[];
 };
+#ifdef _MSC_VER
+#pragma pack(pop)
+#endif
 
 
 struct _tracemalloc_runtime_state {
@@ -95,7 +99,7 @@ struct _tracemalloc_runtime_state {
        Protected by TABLES_LOCK(). */
     _Py_hashtable_t *domains;
 
-    struct tracemalloc_traceback empty_traceback;
+    struct tracemalloc_traceback *empty_traceback;
 
     Py_tss_t reentrant_key;
 };

Python/traceback.c

@@ -1094,6 +1094,9 @@ tstate_is_freed(PyThreadState *tstate)
     if (_PyMem_IsPtrFreed(tstate->interp)) {
         return 1;
     }
+    if (_PyMem_IsULongFreed(tstate->thread_id)) {
+        return 1;
+    }
     return 0;
 }
 
@@ -1113,7 +1116,7 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header)
     }
 
     if (tstate_is_freed(tstate)) {
-        PUTS(fd, "  <tstate is freed>\n");
+        PUTS(fd, "  <freed thread state>\n");
         return;
     }
 
@@ -1138,12 +1141,16 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header)
             PUTS(fd, "  <freed frame>\n");
             break;
         }
+        // Read frame->previous early since memory can be freed during
+        // dump_frame()
+        _PyInterpreterFrame *previous = frame->previous;
+
         if (dump_frame(fd, frame) < 0) {
             PUTS(fd, "  <invalid frame>\n");
             break;
         }
 
-        frame = frame->previous;
+        frame = previous;
         if (frame == NULL) {
             break;
         }
@@ -1240,7 +1247,9 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current)
                         tstate->thread_id,
                         sizeof(unsigned long) * 2);
 
-    write_thread_name(fd, tstate);
+    if (!_PyMem_IsULongFreed(tstate->thread_id)) {
+        write_thread_name(fd, tstate);
+    }
 
     PUTS(fd, " (most recent call first):\n");
 }
@@ -1298,7 +1307,6 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp,
         return "unable to get the thread head state";
 
     /* Dump the traceback of each thread */
-    tstate = PyInterpreterState_ThreadHead(interp);
     unsigned int nthreads = 0;
     _Py_BEGIN_SUPPRESS_IPH
     do
@@ -1309,11 +1317,18 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp,
             PUTS(fd, "...\n");
             break;
         }
+
+        if (tstate_is_freed(tstate)) {
+            PUTS(fd, "<freed thread state>\n");
+            break;
+        }
+
         write_thread_id(fd, tstate, tstate == current_tstate);
         if (tstate == current_tstate && tstate->interp->gc.collecting) {
             PUTS(fd, "  Garbage-collecting\n");
         }
         dump_traceback(fd, tstate, 0);
+
         tstate = PyThreadState_Next(tstate);
         nthreads++;
     } while (tstate != NULL);

Python/tracemalloc.c

@@ -46,7 +46,7 @@ typedef struct tracemalloc_frame frame_t;
 typedef struct tracemalloc_traceback traceback_t;
 
 #define TRACEBACK_SIZE(NFRAME) \
-        (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME - 1))
+        (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME))
 
 static const int MAX_NFRAME = UINT16_MAX;
 
@@ -329,8 +329,9 @@ traceback_new(void)
     traceback->nframe = 0;
     traceback->total_nframe = 0;
     traceback_get_frames(traceback);
-    if (traceback->nframe == 0)
-        return &tracemalloc_empty_traceback;
+    if (traceback->nframe == 0) {
+        return tracemalloc_empty_traceback;
+    }
     traceback->hash = traceback_hash(traceback);
 
     /* intern the traceback */
@@ -754,12 +755,18 @@ _PyTraceMalloc_Init(void)
         return _PyStatus_NO_MEMORY();
     }
 
-    tracemalloc_empty_traceback.nframe = 1;
-    tracemalloc_empty_traceback.total_nframe = 1;
+    assert(tracemalloc_empty_traceback == NULL);
+    tracemalloc_empty_traceback = raw_malloc(TRACEBACK_SIZE(1));
+    if (tracemalloc_empty_traceback  == NULL) {
+        return _PyStatus_NO_MEMORY();
+    }
+
+    tracemalloc_empty_traceback->nframe = 1;
+    tracemalloc_empty_traceback->total_nframe = 1;
     /* borrowed reference */
-    tracemalloc_empty_traceback.frames[0].filename = &_Py_STR(anon_unknown);
-    tracemalloc_empty_traceback.frames[0].lineno = 0;
-    tracemalloc_empty_traceback.hash = traceback_hash(&tracemalloc_empty_traceback);
+    tracemalloc_empty_traceback->frames[0].filename = &_Py_STR(anon_unknown);
+    tracemalloc_empty_traceback->frames[0].lineno = 0;
+    tracemalloc_empty_traceback->hash = traceback_hash(tracemalloc_empty_traceback);
 
     tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED;
     return _PyStatus_OK();
@@ -782,6 +789,9 @@ tracemalloc_deinit(void)
     _Py_hashtable_destroy(tracemalloc_filenames);
 
     PyThread_tss_delete(&tracemalloc_reentrant_key);
+
+    raw_free(tracemalloc_empty_traceback);
+    tracemalloc_empty_traceback = NULL;
 }
 
 

Tools/peg_generator/pegen/c_generator.py

@@ -31,6 +31,7 @@
 
 EXTENSION_PREFIX = """\
 #include "pegen.h"
+#include "pycore_ceval.h"
 
 #if defined(Py_DEBUG) && defined(Py_BUILD_CORE)
 #  define D(x) if (p->debug) { x; }