Merge branch 'master' into 3.6

Author: slozier

Src/IronPython/Compiler/Ast/AstMethods.cs

@@ -72,7 +72,7 @@ internal static class AstMethods {
         public static readonly MethodInfo MakeSet = GetMethod((Func<object[], SetCollection>)PythonOps.MakeSet);
         public static readonly MethodInfo MakeEmptySet = GetMethod((Func<SetCollection>)PythonOps.MakeEmptySet);
         public static readonly MethodInfo MakeHomogeneousDictFromItems = GetMethod((Func<object[], PythonDictionary>)PythonOps.MakeHomogeneousDictFromItems);
-        public static readonly MethodInfo CreateLocalContext = GetMethod((Func<CodeContext, MutableTuple, string[], CodeContext>)PythonOps.CreateLocalContext);
+        public static readonly MethodInfo CreateLocalContext = GetMethod((Func<CodeContext, MutableTuple, string[], int, int, bool, CodeContext>)PythonOps.CreateLocalContext);
         public static readonly MethodInfo UpdateStackTrace = GetMethod((Action<Exception, CodeContext, FunctionCode, int>)PythonOps.UpdateStackTrace);
         public static readonly MethodInfo ForLoopDispose = GetMethod((Action<KeyValuePair<IEnumerator, IDisposable>>)PythonOps.ForLoopDispose);
         public static readonly MethodInfo GetClosureTupleFromContext = GetMethod((Func<CodeContext, MutableTuple>)PythonOps.GetClosureTupleFromContext);

Src/IronPython/Compiler/Ast/CallExpression.cs

@@ -24,7 +24,6 @@ namespace IronPython.Compiler.Ast {
     public class CallExpression : Expression, IInstructionProvider {
         private readonly Expression[] _args;
         private readonly Keyword[] _kwargs;
-        private Expression[]? _implicitArgs;
 
         public CallExpression(Expression target, IReadOnlyList<Expression>? args, IReadOnlyList<Keyword>? kwargs) {
             Target = target;
@@ -38,10 +37,6 @@ public CallExpression(Expression target, IReadOnlyList<Expression>? args, IReadO
 
         public IReadOnlyList<Keyword> Kwargs => _kwargs;
 
-        internal void SetImplicitArgs(params Expression[] args) {
-            _implicitArgs = args;
-        }
-
         public bool NeedsLocalsDictionary() {
             if (!(Target is NameExpression nameExpr)) return false;
 
@@ -61,10 +56,6 @@ public override MSAst.Expression Reduce() {
             ReadOnlySpan<Expression> args = _args;
             ReadOnlySpan<Keyword> kwargs = _kwargs;
 
-            if (args.Length == 0 && _implicitArgs != null) {
-                args = _implicitArgs;
-            }
-
             // count splatted list args and find the lowest index of a list argument, if any
             ScanArgs(args, out var numList, out int firstListPos);
             Debug.Assert(numList == 0 || firstListPos < args.Length);
@@ -188,9 +179,6 @@ static MSAst.Expression UnpackDictHelper(MSAst.Expression context, ReadOnlySpan<
 
         void IInstructionProvider.AddInstructions(LightCompiler compiler) {
             ReadOnlySpan<Expression> args = _args;
-            if (args.Length == 0 && _implicitArgs != null) {
-                args = _implicitArgs;
-            }
 
             if (Kwargs.Count > 0) {
                 compiler.Compile(Reduce());
@@ -430,11 +418,6 @@ public override int Run(InterpretedFrame frame) {
         public override void Walk(PythonWalker walker) {
             if (walker.Walk(this)) {
                 Target?.Walk(walker);
-                if (_implicitArgs is not null) {
-                    foreach (var arg in _implicitArgs) {
-                        arg.Walk(walker);
-                    }
-                }
                 foreach (var arg in _args) {
                     arg.Walk(walker);
                 }

Src/IronPython/Compiler/Ast/ClassDefinition.cs

@@ -35,8 +35,8 @@ public class ClassDefinition : ScopeStatement {
 
         private static int _classId;
 
-        private static readonly MSAst.ParameterExpression _parentContextParam = Ast.Parameter(typeof(CodeContext), "$parentContext");
-        private static readonly MSAst.Expression _tupleExpression = MSAst.Expression.Call(AstMethods.GetClosureTupleFromContext, _parentContextParam);
+        private static readonly MSAst.ParameterExpression _outerContextParam = Ast.Parameter(typeof(CodeContext), "$outerContext");
+        private static readonly MSAst.Expression _tupleExpression = MSAst.Expression.Call(AstMethods.GetClosureTupleFromContext, _outerContextParam);
 
         public ClassDefinition(string name, IReadOnlyList<Expression>? bases, IReadOnlyList<Keyword>? keywords, Statement? body = null) {
             _name = name;
@@ -128,6 +128,7 @@ internal override bool TryBindOuter(ScopeStatement from, PythonReference referen
             if (TryGetVariable(reference.Name, out variable)) {
                 if (variable.Kind is VariableKind.Global) {
                     AddReferencedGlobal(reference.Name);
+                    return variable;
                 } else if (variable.Kind is VariableKind.Local) {
                     // TODO: This results in doing a dictionary lookup to get/set the local,
                     // when it should probably be an uninitialized check / global lookup for gets
@@ -271,13 +272,13 @@ private Microsoft.Scripting.Ast.LightExpression<Func<CodeContext, CodeContext>>
             locals.Add(LocalCodeContextVariable);
             locals.Add(PythonAst._globalContext);
 
-            init.Add(Ast.Assign(PythonAst._globalContext, new GetGlobalContextExpression(_parentContextParam)));
+            init.Add(Ast.Assign(PythonAst._globalContext, new GetGlobalContextExpression(_outerContextParam)));
 
             GlobalParent.PrepareScope(locals, init);
 
             CreateVariables(locals, init);
 
-            var createLocal = CreateLocalContext(_parentContextParam);
+            var createLocal = CreateLocalContext(_outerContextParam, newNamespace: false);
 
             init.Add(Ast.Assign(LocalCodeContextVariable, createLocal));
 
@@ -327,7 +328,7 @@ private Microsoft.Scripting.Ast.LightExpression<Func<CodeContext, CodeContext>>
                     bodyStmt
                 ),
                 Name + "$" + Interlocked.Increment(ref _classId),
-                new[] { _parentContextParam }
+                new[] { _outerContextParam }
                 );
 
             return lambda;

Src/IronPython/Compiler/Ast/FunctionDefinition.cs

@@ -145,7 +145,10 @@ internal override int KwOnlyArgCount {
         internal PythonVariable PythonVariable { get; set; }
 
         internal override bool ExposesLocalVariable(PythonVariable variable) {
-            return NeedsLocalsDictionary; 
+            return NeedsLocalsDictionary
+                || (ContainsSuperCall && variable.Kind is VariableKind.Parameter
+                        && _parameters is not null && _parameters.Length > 0
+                        && _parameters[0].PythonVariable == variable);
         }
 
         internal override FunctionAttributes Flags {
@@ -258,7 +261,7 @@ internal override void Bind(PythonNameBinder binder) {
 
         internal override void FinishBind(PythonNameBinder binder) {
             foreach (var param in _parameters) {
-                _variableMapping[param.PythonVariable] = param.FinishBind(NeedsLocalsDictionary);
+                _variableMapping[param.PythonVariable] = param.FinishBind(forceClosureCell: ExposesLocalVariable(param.PythonVariable));
             }
             base.FinishBind(binder);
         }

Src/IronPython/Compiler/Ast/Parameter.cs

@@ -60,8 +60,8 @@ public Expression DefaultValue {
 
         internal PythonVariable PythonVariable { get; set; }
 
-        internal MSAst.Expression FinishBind(bool needsLocalsDictionary) {
-            if (PythonVariable.AccessedInNestedScope || needsLocalsDictionary) {
+        internal MSAst.Expression FinishBind(bool forceClosureCell) {
+            if (PythonVariable.AccessedInNestedScope || forceClosureCell) {
                 ParameterExpression = Expression.Parameter(typeof(object), Name);
                 var cell = Ast.Parameter(typeof(ClosureCell), Name);
                 return new ClosureExpression(PythonVariable, cell, ParameterExpression);

Src/IronPython/Compiler/Ast/PythonNameBinder.cs

@@ -851,13 +851,13 @@ public override bool Walk(ComprehensionFor node) {
         public override bool Walk(CallExpression node) {
             node.Parent = _currentScope;
 
-            if (node.Target is NameExpression nameExpr && nameExpr.Name == "super" && _currentScope is FunctionDefinition func) {
-                _currentScope.Reference("__class__");
-                if (node.Args.Count == 0 && node.Kwargs.Count == 0 && func.ParameterNames.Length > 0) {
-                    node.SetImplicitArgs(new NameExpression("__class__"), new NameExpression(func.ParameterNames[0]));
-                }
+            if (node.Target is NameExpression nameExpr && nameExpr.Name == "super"
+                && _currentScope is not ClassDefinition
+                && node.Args.Count == 0 && node.Kwargs.Count == 0) {
+
+                _currentScope.ContainsSuperCall = true;
             }
-            return base.Walk(node);
+            return true;
         }
 
         #endregion

Src/IronPython/Compiler/Ast/PythonReference.cs

@@ -2,6 +2,8 @@
 // The .NET Foundation licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
+#nullable enable
+
 namespace IronPython.Compiler.Ast {
     /// <summary>
     /// Represents a reference to a name.  A PythonReference is created for each name
@@ -14,6 +16,6 @@ public PythonReference(string name) {
 
         public string Name { get; }
 
-        internal PythonVariable PythonVariable { get; set; }
+        internal PythonVariable? PythonVariable { get; set; }
     }
 }

Src/IronPython/Compiler/Ast/ScopeStatement.cs

@@ -93,6 +93,13 @@ internal bool IsClosure {
         /// </summary>
         internal bool NeedsLocalsDictionary { get; set; }
 
+        /// <summary>
+        /// True if this scope contains a parameterless call to super().
+        ///
+        /// It is used to ensure that the first argument is accessible through a closure cell.
+        /// </summary>
+        internal bool ContainsSuperCall{ get; set; }
+
         public virtual string Name {
             get {
                 return "<unknown>";
@@ -124,7 +131,7 @@ internal virtual bool IsGlobal {
 
         internal bool NeedsLocalContext {
             get {
-                return NeedsLocalsDictionary || ContainsNestedFreeVariables;
+                return NeedsLocalsDictionary || ContainsNestedFreeVariables || ContainsSuperCall;
             }
         }
 
@@ -350,6 +357,7 @@ internal virtual void FinishBind(PythonNameBinder binder) {
                 Debug.Assert(parentClosure != null);
 
                 foreach (var variable in FreeVariables) {
+                    Debug.Assert(!HasClosureVariable(closureVariables, variable));
                     for (int i = 0; i < parentClosure.Length; i++) {
                         if (parentClosure[i].Variable == variable) {
                             Ast prop = LocalParentTuple;
@@ -362,22 +370,19 @@ internal virtual void FinishBind(PythonNameBinder binder) {
                     }
                     Debug.Assert(_variableMapping.ContainsKey(variable));
 
-                    if (closureVariables == null) {
-                        closureVariables = new List<ClosureInfo>();
-                    }
-                    closureVariables.Add(new ClosureInfo(variable, !(this is ClassDefinition)));
+                    closureVariables ??= new List<ClosureInfo>();
+                    closureVariables.Add(new ClosureInfo(variable, this is not ClassDefinition));
                 }
             }
 
             if (Variables != null) {
                 foreach (PythonVariable variable in Variables.Values) {
-                    if (!HasClosureVariable(closureVariables, variable) &&
-                        variable.Kind is VariableKind.Local or VariableKind.Parameter &&
+                    if (variable.Kind is VariableKind.Local or VariableKind.Parameter &&
                         (variable.AccessedInNestedScope || ExposesLocalVariable(variable))) {
 
-                        if (closureVariables == null) {
-                            closureVariables = new List<ClosureInfo>();
-                        }
+                        Debug.Assert(!HasClosureVariable(closureVariables, variable));
+
+                        closureVariables ??= new List<ClosureInfo>();
                         closureVariables.Add(new ClosureInfo(variable, true));
                     }
 
@@ -390,7 +395,9 @@ variable.Kind is VariableKind.Local or VariableKind.Parameter &&
                             _variableMapping[variable] = Ast.Parameter(typeof(object), variable.Name);
                         }
                     } else if (variable.Kind == VariableKind.Attribute) {
-                        _variableMapping[variable] = new LookupGlobalVariable(LocalContext, variable.Name, isLocal: true); // TODO: If no user-supplied dictionary is in place, optimize to use more efficient access, see CollectableCompilationMode
+                        // If no user-supplied dictionary is in place, a more efficient access is possible, see CollectableCompilationMode
+                        // However, this would probably have a negligible effect in this case.
+                        _variableMapping[variable] = new LookupGlobalVariable(LocalContext, variable.Name, isLocal: true);
                     }
                 }
             }
@@ -430,12 +437,12 @@ internal void AddGlobalVariable(PythonVariable variable) {
         }
 
         internal PythonReference Reference(string name) {
-            if (_references == null) {
-                _references = new Dictionary<string, PythonReference>(StringComparer.Ordinal);
-            }
-            PythonReference reference;
-            if (!_references.TryGetValue(name, out reference)) {
+            _references ??= new Dictionary<string, PythonReference>(StringComparer.Ordinal);
+            if (!_references.TryGetValue(name, out PythonReference reference)) {
                 _references[name] = reference = new PythonReference(name);
+                if (name == "super" && this is not ClassDefinition) {
+                    Reference("__class__");
+                }
             }
             return reference;
         }
@@ -670,16 +677,28 @@ internal MSAst.Expression FuncCodeExpr {
             }
         }
 
-        internal MSAst.MethodCallExpression CreateLocalContext(MSAst.Expression parentContext) {
-            var closureVariables = _closureVariables;
-            if (_closureVariables == null) {
-                closureVariables = new ClosureInfo[0];
+        internal MSAst.MethodCallExpression CreateLocalContext(MSAst.Expression parentContext, bool newNamespace = true) {
+            var closureVariables = _closureVariables ?? Array.Empty<ClosureInfo>();
+            
+            int numFreeVars = FreeVariables?.Count ?? 0;
+            int firstArgIdx = -1;
+            if ((NeedsLocalsDictionary || ContainsSuperCall) && ArgCount > 0) {
+                for (int idx = numFreeVars; idx < closureVariables.Length; idx++) {
+                    if (closureVariables[idx].Variable.Kind == VariableKind.Parameter) {
+                        firstArgIdx = idx;
+                        break;
+                    }
+                }
             }
+
             return Ast.Call(
                 AstMethods.CreateLocalContext,
                 parentContext,
                 MutableTuple.Create(ArrayUtils.ConvertAll(closureVariables, x => GetClosureCell(x))),
-                Ast.Constant(ArrayUtils.ConvertAll(closureVariables, x => x.AccessedInScope ? x.Variable.Name : null))
+                Ast.Constant(ArrayUtils.ConvertAll(closureVariables, x => x.AccessedInScope ? x.Variable.Name : null)),
+                AstUtils.Constant(numFreeVars),
+                AstUtils.Constant(firstArgIdx),
+                AstUtils.Constant(newNamespace)
             );
         }
 

Src/IronPython/Runtime/CommonDictionaryStorage.cs

@@ -67,7 +67,7 @@ internal class CommonDictionaryStorage : DictionaryStorage, ISerializable, IDese
         public CommonDictionaryStorage() { }
 
         /// <summary>
-        /// Creates a new dictionary storage with no buckets
+        /// Creates a new dictionary storage with buckets
         /// </summary>
         public CommonDictionaryStorage(int count) {
             _buckets = new Bucket[(int)(count / Load + 2)];

Src/IronPython/Runtime/CustomDictionaryStorage.cs

@@ -11,7 +11,17 @@
 
 namespace IronPython.Runtime {
     internal abstract class CustomDictionaryStorage : DictionaryStorage {
-        private readonly CommonDictionaryStorage/*!*/ _storage = new CommonDictionaryStorage();
+        private readonly CommonDictionaryStorage/*!*/ _storage;
+
+        public CustomDictionaryStorage() {
+            _storage = new();
+        }
+
+        protected CustomDictionaryStorage(CommonDictionaryStorage storage) {
+            _storage = storage;
+        }
+
+        public CommonDictionaryStorage Storage => _storage;
 
         public override void Add(ref DictionaryStorage storage, object key, object value) {
             Add(key, value);