Merge remote-tracking branch 'upstream/main' into binary-op

Author: aisk

Doc/faq/programming.rst

@@ -1226,13 +1226,13 @@ This converts the list into a set, thereby removing duplicates, and then back
 into a list.
 
 
-How do you remove multiple items from a list
---------------------------------------------
+How do you remove multiple items from a list?
+---------------------------------------------
 
 As with removing duplicates, explicitly iterating in reverse with a
 delete condition is one possibility.  However, it is easier and faster
 to use slice replacement with an implicit or explicit forward iteration.
-Here are three variations.::
+Here are three variations::
 
    mylist[:] = filter(keep_function, mylist)
    mylist[:] = (x for x in mylist if keep_condition)

Include/internal/pycore_opcode_metadata.h

@@ -22,6 +22,7 @@
 NLCRE = re.compile(r'\r\n|\r|\n')
 fcre = re.compile(r'^From ', re.MULTILINE)
 NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
+NEWLINE_WITHOUT_FWSP_BYTES = re.compile(br'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
 
 
 class Generator:
@@ -429,7 +430,16 @@ def _write_headers(self, msg):
         # This is almost the same as the string version, except for handling
         # strings with 8bit bytes.
         for h, v in msg.raw_items():
-            self._fp.write(self.policy.fold_binary(h, v))
+            folded = self.policy.fold_binary(h, v)
+            if self.policy.verify_generated_headers:
+                linesep = self.policy.linesep.encode()
+                if not folded.endswith(linesep):
+                    raise HeaderWriteError(
+                        f'folded header does not end with {linesep!r}: {folded!r}')
+                if NEWLINE_WITHOUT_FWSP_BYTES.search(folded.removesuffix(linesep)):
+                    raise HeaderWriteError(
+                        f'folded header contains newline: {folded!r}')
+            self._fp.write(folded)
         # A blank line always separates headers from body
         self.write(self._NL)
 

Include/internal/pycore_uop_ids.h

@@ -2897,6 +2897,7 @@ def testfunc(n):
         self.assertIn("_POP_TOP_NOP", uops)
         self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2)
 
+<<<<<<< HEAD
     def test_binary_op_refcount_elimination(self):
         class CustomAdder:
             def __init__(self, val):
@@ -2917,6 +2918,22 @@ def testfunc(n):
         self.assertIsNotNone(ex)
         uops = get_opnames(ex)
         self.assertIn("_BINARY_OP", uops)
+=======
+    def test_binary_op_extend_float_long_add_refcount_elimination(self):
+        def testfunc(n):
+            a = 1.5
+            b = 2
+            res = 0.0
+            for _ in range(n):
+                res = a + b
+            return res
+
+        res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD)
+        self.assertEqual(res, 3.5)
+        self.assertIsNotNone(ex)
+        uops = get_opnames(ex)
+        self.assertIn("_BINARY_OP_EXTEND", uops)
+>>>>>>> upstream/main
         self.assertIn("_POP_TOP_NOP", uops)
         self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2)
 

Include/internal/pycore_uop_metadata.h

@@ -313,7 +313,7 @@ def test_flatten_unicode_linesep(self):
         self.assertEqual(s.getvalue(), self.typ(expected))
 
     def test_verify_generated_headers(self):
-        """gh-121650: by default the generator prevents header injection"""
+        # gh-121650: by default the generator prevents header injection
         class LiteralHeader(str):
             name = 'Header'
             def fold(self, **kwargs):
@@ -334,6 +334,8 @@ def fold(self, **kwargs):
 
                 with self.assertRaises(email.errors.HeaderWriteError):
                     message.as_string()
+                with self.assertRaises(email.errors.HeaderWriteError):
+                    message.as_bytes()
 
 
 class TestBytesGenerator(TestGeneratorBase, TestEmailBase):

Lib/email/generator.py

@@ -296,7 +296,7 @@ def test_short_maxlen_error(self):
                     policy.fold("Subject", subject)
 
     def test_verify_generated_headers(self):
-        """Turning protection off allows header injection"""
+        # Turning protection off allows header injection
         policy = email.policy.default.clone(verify_generated_headers=False)
         for text in (
             'Header: Value\r\nBad: Injection\r\n',
@@ -319,6 +319,10 @@ def fold(self, **kwargs):
                     message.as_string(),
                     f"{text}\nBody",
                 )
+                self.assertEqual(
+                    message.as_bytes(),
+                    f"{text}\nBody".encode(),
+                )
 
     # XXX: Need subclassing tests.
     # For adding subclassed objects, make sure the usual rules apply (subclass

Lib/test/test_capi/test_opt.py

@@ -0,0 +1 @@
+Eliminate redundant refcounting from ``BINARY_OP_EXTEND``.

Lib/test/test_email/test_generator.py

@@ -0,0 +1,4 @@
+:mod:`~email.generator.BytesGenerator` will now refuse to serialize (write) headers
+that are unsafely folded or delimited; see
+:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas
+Bloemsaat and Petr Viktorin in :gh:`121650`).