Merge branch 'master' into 3.6

Author: slozier

README.md

@@ -86,4 +86,4 @@ Binaries of IronPython 3 can be downloaded from the [release page](https://githu
 See the [building document](Documentation/building.md). Since the main development is on Windows, bugs on other platforms may inadvertently be introduced - please report them!
 
 ## Supported Platforms
-IronPython 3 targets .NET Framework 4.6, .NET Standard 2.0, .NET Core 3.1 and .NET 6.0. The support for .NET and .NET Core follow the lifecycle defined on [.NET and .NET Core Support Policy](https://dotnet.microsoft.com/platform/support/policy/dotnet-core).
+IronPython 3 targets .NET Framework 4.6.2, .NET Standard 2.0, .NET Core 3.1 and .NET 6.0. The support for .NET and .NET Core follow the lifecycle defined on [.NET and .NET Core Support Policy](https://dotnet.microsoft.com/platform/support/policy/dotnet-core).

Src/DLR

@@ -485,12 +485,13 @@ TLSv1.1    no    no    yes    no     yes      no
                        TLSv1.2    no    no    yes    no      no     yes
              */
 
+#pragma warning disable CA5397 // Do not use deprecated SslProtocols values
+#pragma warning disable CS0618 // Type or member is obsolete
+#pragma warning disable SYSLIB0039 // Type or member is obsolete
+
             private static SslProtocols GetProtocolType(int protocol, int options) {
                 SslProtocols result = SslProtocols.None;
-
                 switch (protocol) {
-#pragma warning disable CA5397 // Do not use deprecated SslProtocols values
-#pragma warning disable CS0618 // Type or member is obsolete
                     case PythonSsl.PROTOCOL_SSLv2:
                         result = SslProtocols.Ssl2;
                         break;
@@ -500,33 +501,31 @@ private static SslProtocols GetProtocolType(int protocol, int options) {
                     case PythonSsl.PROTOCOL_SSLv23:
                         result = SslProtocols.Ssl2 | SslProtocols.Ssl3 | SslProtocols.Tls | SslProtocols.Tls11 | SslProtocols.Tls12;
                         break;
-#pragma warning restore CS0618 // Type or member is obsolete
                     case PythonSsl.PROTOCOL_TLSv1:
                         result = SslProtocols.Tls;
                         break;
                     case PythonSsl.PROTOCOL_TLSv1_1:
                         result = SslProtocols.Tls11;
                         break;
-#pragma warning restore CA5397 // Do not use deprecated SslProtocols values
                     case PythonSsl.PROTOCOL_TLSv1_2:
                         result = SslProtocols.Tls12;
                         break;
                     default:
                         throw new InvalidOperationException("bad ssl protocol type: " + protocol);
                 }
                 // Filter out requested protocol exclusions:
-#pragma warning disable CA5397 // Do not use deprecated SslProtocols values
-#pragma warning disable CS0618 // Type or member is obsolete
                 result &= (options & PythonSsl.OP_NO_SSLv3) != 0 ? ~SslProtocols.Ssl3 : ~SslProtocols.None;
                 result &= (options & PythonSsl.OP_NO_SSLv2) != 0 ? ~SslProtocols.Ssl2 : ~SslProtocols.None;
-#pragma warning restore CS0618 // Type or member is obsolete
                 result &= (options & PythonSsl.OP_NO_TLSv1) != 0 ? ~SslProtocols.Tls : ~SslProtocols.None;
                 result &= (options & PythonSsl.OP_NO_TLSv1_1) != 0 ? ~SslProtocols.Tls11 : ~SslProtocols.None;
                 result &= (options & PythonSsl.OP_NO_TLSv1_2) != 0 ? ~SslProtocols.Tls12 : ~SslProtocols.None;
-#pragma warning restore CA5397 // Do not use deprecated SslProtocols values
                 return result;
             }
 
+#pragma warning restore SYSLIB0039 // Type or member is obsolete
+#pragma warning restore CS0618 // Type or member is obsolete
+#pragma warning restore CA5397 // Do not use deprecated SslProtocols values
+
             public PythonTuple cipher() {
                 if (_sslStream != null && _sslStream.IsAuthenticated) {
                     return PythonTuple.MakeTuple(
@@ -540,19 +539,23 @@ public PythonTuple cipher() {
 
             public object compression() => null; // TODO
 
-            private string ProtocolToPython() {
-                switch (_sslStream.SslProtocol) {
 #pragma warning disable CA5397 // Do not use deprecated SslProtocols values
 #pragma warning disable CS0618 // Type or member is obsolete
+#pragma warning disable SYSLIB0039 // Type or member is obsolete
+
+            private string ProtocolToPython() {
+                switch (_sslStream.SslProtocol) {
                     case SslProtocols.Ssl2: return "SSLv2";
                     case SslProtocols.Ssl3: return "TLSv1/SSLv3";
-#pragma warning restore CS0618 // Type or member is obsolete
                     case SslProtocols.Tls: return "TLSv1";
-#pragma warning restore CA5397 // Do not use deprecated SslProtocols values
                     default: return _sslStream.SslProtocol.ToString();
                 }
             }
 
+#pragma warning restore SYSLIB0039 // Type or member is obsolete
+#pragma warning restore CS0618 // Type or member is obsolete
+#pragma warning restore CA5397 // Do not use deprecated SslProtocols values
+
             public object peer_certificate(bool binary_form) {
                 var peerCert = _sslStream?.RemoteCertificate;
 

Src/IronPython.Modules/_ssl.cs

@@ -747,23 +747,25 @@ static string ValidateString(object? str) {
                 void AppendGroup(StringBuilder sb, int index) => sb.Append(_m.Groups[index].Value);
             }
 
-            private static bool IsGroupNumber(string name) {
-                foreach (char c in name) {
-                    if (!char.IsNumber(c)) return false;
-                }
-                return true;
-            }
-
             [return: DictionaryTypeInfo(typeof(string), typeof(object))]
             public PythonDictionary groupdict(object? @default = null) {
                 string[] groupNames = this.re._re.GetGroupNames();
                 Debug.Assert(groupNames.Length == this._m.Groups.Count);
                 PythonDictionary d = new PythonDictionary();
                 for (int i = 0; i < groupNames.Length; i++) {
-                    if (IsGroupNumber(groupNames[i])) continue; // python doesn't report group numbers
-                    d[groupNames[i]] = re.GetGroupValue(_m.Groups[i], @default);
+                    var groupName = groupNames[i];
+                    if (IsGroupNumber(groupName)) continue; // python doesn't report group numbers
+                    if (groupName.StartsWith(_mangledNamedGroup, StringComparison.Ordinal)) continue; // don't include unnamed groups
+                    d[groupName] = re.GetGroupValue(_m.Groups[i], @default);
                 }
                 return d;
+
+                static bool IsGroupNumber(string name) {
+                    foreach (char c in name) {
+                        if (!char.IsNumber(c)) return false;
+                    }
+                    return true;
+                }
             }
 
             [return: SequenceTypeInfo(typeof(int))]

Src/IronPython.Modules/re.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Runtime.CompilerServices;
@@ -97,15 +98,16 @@ internal override bool HasLateBoundVariableSets {
             }
         }
 
-        internal override bool ExposesLocalVariable(PythonVariable variable) => true;
+        internal override bool ExposesLocalVariable(PythonVariable variable) {
+            Debug.Assert(variable.Name == "__class__");
+            return true;
+        }
 
-        private bool needClassCell { get; set; }
 
         internal override bool TryBindOuter(ScopeStatement from, PythonReference reference, out PythonVariable variable) {
             if (reference.Name == "__class__") {
-                needClassCell = true;
+                ClassVariable = variable = EnsureClassVariable();
                 ClassCellVariable = EnsureVariable("__classcell__");
-                ClassVariable = variable = EnsureVariable(reference.Name);
                 variable.AccessedInNestedScope = true;
                 from.AddFreeVariable(variable, true);
                 for (ScopeStatement scope = from.Parent; scope != this; scope = scope.Parent) {
@@ -124,18 +126,19 @@ internal override bool TryBindOuter(ScopeStatement from, PythonReference referen
             // Python semantics: The variables bound local in the class
             // scope are accessed by name - the dictionary behavior of classes
             if (TryGetVariable(reference.Name, out variable)) {
-                // TODO: This results in doing a dictionary lookup to get/set the local,
-                // when it should probably be an uninitialized check / global lookup for gets
-                // and a direct set
-                if (variable.Kind == VariableKind.Global) {
+                if (variable.Kind is VariableKind.Global) {
                     AddReferencedGlobal(reference.Name);
-                } else if (variable.Kind == VariableKind.Local) {
+                } else if (variable.Kind is VariableKind.Local) {
+                    // TODO: This results in doing a dictionary lookup to get/set the local,
+                    // when it should probably be an uninitialized check / global lookup for gets
+                    // and a direct set
                     return null;
-                }
-
-                if (variable.Kind != VariableKind.Nonlocal) {
+                } else if (variable.Kind is VariableKind.Attribute) {
+                    return null; // fall back on LookupName/SetName in local context dict, which is faster than LookupGlobalVariable
+                } else if (variable.Kind is VariableKind.Parameter) {
                     return variable;
                 }
+                // else NonLocal: continue binding
             }
 
             // Try to bind in outer scopes, if we have an unqualified exec we need to leave the
@@ -150,6 +153,20 @@ internal override bool TryBindOuter(ScopeStatement from, PythonReference referen
             return null;
         }
 
+        internal override PythonVariable EnsureVariable(string name) {
+            if (TryGetVariable(name, out PythonVariable variable)) {
+                return variable;
+            }
+            return CreateVariable(name, VariableKind.Attribute);
+        }
+
+        internal PythonVariable EnsureClassVariable() {
+            if (TryGetVariable("$__class__", out PythonVariable variable)) {
+                return variable;
+            }
+            return CreateVariable("__class__", VariableKind.Local, "$__class__");
+        }
+
         internal override Ast LookupVariableExpression(PythonVariable variable) {
             // Emulates opcode LOAD_CLASSDEREF
             return Ast.Call(
@@ -242,6 +259,9 @@ static MSAst.Expression UnpackKeywordsHelper(MSAst.Expression context, ReadOnlyS
             }
         }
 
+        private MSAst.Expression SetLocalName(string name, MSAst.Expression expression)
+            => Ast.Call(AstMethods.SetName, LocalContext, Ast.Constant(name), expression);
+
         private Microsoft.Scripting.Ast.LightExpression<Func<CodeContext, CodeContext>> MakeClassBody() {
             // we always need to create a nested context for class defs            
 
@@ -260,44 +280,41 @@ private Microsoft.Scripting.Ast.LightExpression<Func<CodeContext, CodeContext>>
             var createLocal = CreateLocalContext(_parentContextParam);
 
             init.Add(Ast.Assign(LocalCodeContextVariable, createLocal));
-            // __classcell__ == ClosureCell(__class__)
-            if (needClassCell) {
-                var exp = (ClosureExpression) GetVariableExpression(ClassVariable!);
-                MSAst.Expression assignClassCell = AssignValue(GetVariableExpression(ClassCellVariable!), exp.ClosureCell);
-                init.Add(assignClassCell);
-            }
-
-            List<MSAst.Expression> statements = new List<MSAst.Expression>();
-            // Create the body
-            MSAst.Expression bodyStmt = Body;
-
 
             // __module__ = __name__
-            MSAst.Expression modStmt = AssignValue(GetVariableExpression(ModVariable!), GetVariableExpression(ModuleNameVariable!));
+            MSAst.Expression modStmt = SetLocalName("__module__", AstUtils.Convert(GetVariableExpression(ModuleNameVariable!), typeof(object)));
+
+            // TODO: set __qualname__
 
+            // __doc__ = """..."""
+            MSAst.Expression? docStmt = null;
             string doc = GetDocumentation(Body);
-            if (doc != null) {
-                statements.Add(
-                    AssignValue(
-                        GetVariableExpression(DocVariable!),
-                        AstUtils.Constant(doc)
-                    )
-                );
+            if (doc is not null) {
+                docStmt = SetLocalName("__doc__", Ast.Constant(doc, typeof(object)));
             }
 
+            // Create the body
+            MSAst.Expression bodyStmt = Body;
             if (Body.CanThrow && GlobalParent.PyContext.PythonOptions.Frames) {
                 bodyStmt = AddFrame(LocalContext, FuncCodeExpr, bodyStmt);
                 locals.Add(FunctionStackVariable);
             }
 
+            // __classcell__ == ClosureCell(__class__)
+            MSAst.Expression? assignClassCellStmt = null;
+            if (ClassVariable is not null) {
+                var exp = (ClosureExpression)GetVariableExpression(ClassVariable);
+                assignClassCellStmt = AssignValue(GetVariableExpression(ClassCellVariable!), exp.ClosureCell);
+            }
+
             bodyStmt = WrapScopeStatements(
                 Ast.Block(
                     Ast.Block(init),
-                    statements.Count == 0 ?
-                        EmptyBlock :
-                        Ast.Block(new ReadOnlyCollection<MSAst.Expression>(statements)),
                     modStmt,
+                    // __qualname__
+                    docStmt is not null ? docStmt : AstUtils.Empty(),
                     bodyStmt,
+                    assignClassCellStmt is not null ? assignClassCellStmt : AstUtils.Empty(),
                     LocalContext
                 ),
                 Body.CanThrow

Src/IronPython/Compiler/Ast/ClassDefinition.cs

@@ -118,20 +118,20 @@ private void Bind(PythonAst unboundAst) {
             // Find all scopes and variables
             unboundAst.Walk(this);
 
-            // Bind
+            // Bind scopes
             foreach (ScopeStatement scope in _scopes) {
                 scope.Bind(this);
             }
 
-            // Finish the globals
+            // Bind globals
             unboundAst.Bind(this);
 
-            // Finish Binding w/ outer most scopes first.
+            // Finish binding w/ outer most scopes first.
             for (int i = _scopes.Count - 1; i >= 0; i--) {
                 _scopes[i].FinishBind(this);
             }
 
-            // Finish the globals
+            // Finish globals
             unboundAst.FinishBind(this);
 
             // Run flow checker

Src/IronPython/Compiler/Ast/PythonNameBinder.cs

@@ -311,7 +311,7 @@ internal virtual MSAst.Expression GetParentClosureTuple() {
             throw new NotSupportedException();
         }
 
-        private bool TryGetAnyVariable(string name, out PythonVariable variable) {
+        internal bool TryGetVariable(string name, out PythonVariable variable) {
             if (Variables != null) {
                 return Variables.TryGetValue(name, out variable);
             } else {
@@ -320,15 +320,6 @@ private bool TryGetAnyVariable(string name, out PythonVariable variable) {
             }
         }
 
-        internal bool TryGetVariable(string name, out PythonVariable variable) {
-            if (TryGetAnyVariable(name, out variable)) {
-                return true;
-            } else {
-                variable = null;
-                return false;
-            }
-        }
-
         internal virtual bool TryBindOuter(ScopeStatement from, PythonReference reference, out PythonVariable variable) {
             // Hide scope contents by default (only functions expose their locals)
             variable = null;
@@ -381,7 +372,7 @@ internal virtual void FinishBind(PythonNameBinder binder) {
             if (Variables != null) {
                 foreach (PythonVariable variable in Variables.Values) {
                     if (!HasClosureVariable(closureVariables, variable) &&
-                        variable.Kind is not VariableKind.Global and not VariableKind.Nonlocal &&
+                        variable.Kind is VariableKind.Local or VariableKind.Parameter &&
                         (variable.AccessedInNestedScope || ExposesLocalVariable(variable))) {
 
                         if (closureVariables == null) {
@@ -398,6 +389,8 @@ variable.Kind is not VariableKind.Global and not VariableKind.Nonlocal &&
                         } else {
                             _variableMapping[variable] = Ast.Parameter(typeof(object), variable.Name);
                         }
+                    } else if (variable.Kind == VariableKind.Attribute) {
+                        _variableMapping[variable] = new LookupGlobalVariable(LocalContext, variable.Name, isLocal: true); // TODO: If no user-supplied dictionary is in place, optimize to use more efficient access, see CollectableCompilationMode
                     }
                 }
             }
@@ -461,15 +454,15 @@ private bool TryGetNonlocalStatement(string name, out NonlocalStatement node) {
             return _nonlocalVars?.TryGetValue(name, out node) ?? false;
         }
 
-        internal PythonVariable/*!*/ CreateVariable(string name, VariableKind kind) {
+        internal PythonVariable/*!*/ CreateVariable(string name, VariableKind kind, string key = null) {
             EnsureVariables();
-            Debug.Assert(!Variables.ContainsKey(name));
+            Debug.Assert(!Variables.ContainsKey(key ?? name));
             PythonVariable variable;
-            Variables[name] = variable = new PythonVariable(name, kind, this);
+            Variables[key ?? name] = variable = new PythonVariable(name, kind, this);
             return variable;
         }
 
-        internal PythonVariable/*!*/ EnsureVariable(string name) {
+        internal virtual PythonVariable/*!*/ EnsureVariable(string name) {
             PythonVariable variable;
             if (!TryGetVariable(name, out variable)) {
                 return CreateVariable(name, VariableKind.Local);

Src/IronPython/Compiler/Ast/ScopeStatement.cs

@@ -35,6 +35,15 @@ public enum VariableKind {
         ///
         /// Provides a by-reference access to a local variable in an outer scope.
         /// </summary>
-        Nonlocal
+        Nonlocal,
+
+        /// <summary>
+        /// Attrribute variable.
+        ///
+        /// Like a local variable, but is stored directly in the context dictionary,
+        /// rather than a closure cell.
+        /// Should only appear in a class lambda.
+        /// </summary>
+        Attribute
     }
 }

Src/IronPython/Compiler/Ast/VariableKind.cs

@@ -1991,12 +1991,7 @@ public void __init__(
                         throw PythonOps.ValueError(string.Format("illegal newline value: " + newline));
                 }
 
-                if (encoding == null) {
-                    encoding = PythonLocale.PreferredEncoding;
-                    if (encoding == "") {
-                        encoding = "ascii";
-                    }
-                }
+                encoding ??= PythonLocale.PreferredEncoding;
 
                 if (errors == null) {
                     errors = "strict";