Execute class body lambda in outer context (#1542)

* Create class context as argument for class body lambda

* Execute class body lambda in outer context

Author: BCSharp

Src/IronPython/Compiler/Ast/AstMethods.cs

@@ -72,7 +72,7 @@ internal static class AstMethods {
         public static readonly MethodInfo MakeSet = GetMethod((Func<object[], SetCollection>)PythonOps.MakeSet);
         public static readonly MethodInfo MakeEmptySet = GetMethod((Func<SetCollection>)PythonOps.MakeEmptySet);
         public static readonly MethodInfo MakeHomogeneousDictFromItems = GetMethod((Func<object[], PythonDictionary>)PythonOps.MakeHomogeneousDictFromItems);
-        public static readonly MethodInfo CreateLocalContext = GetMethod((Func<CodeContext, MutableTuple, string[], int, int, CodeContext>)PythonOps.CreateLocalContext);
+        public static readonly MethodInfo CreateLocalContext = GetMethod((Func<CodeContext, MutableTuple, string[], int, int, bool, CodeContext>)PythonOps.CreateLocalContext);
         public static readonly MethodInfo UpdateStackTrace = GetMethod((Action<Exception, CodeContext, FunctionCode, int>)PythonOps.UpdateStackTrace);
         public static readonly MethodInfo ForLoopDispose = GetMethod((Action<KeyValuePair<IEnumerator, IDisposable>>)PythonOps.ForLoopDispose);
         public static readonly MethodInfo GetClosureTupleFromContext = GetMethod((Func<CodeContext, MutableTuple>)PythonOps.GetClosureTupleFromContext);

Src/IronPython/Compiler/Ast/ClassDefinition.cs

@@ -35,8 +35,8 @@ public class ClassDefinition : ScopeStatement {
 
         private static int _classId;
 
-        private static readonly MSAst.ParameterExpression _parentContextParam = Ast.Parameter(typeof(CodeContext), "$parentContext");
-        private static readonly MSAst.Expression _tupleExpression = MSAst.Expression.Call(AstMethods.GetClosureTupleFromContext, _parentContextParam);
+        private static readonly MSAst.ParameterExpression _outerContextParam = Ast.Parameter(typeof(CodeContext), "$outerContext");
+        private static readonly MSAst.Expression _tupleExpression = MSAst.Expression.Call(AstMethods.GetClosureTupleFromContext, _outerContextParam);
 
         public ClassDefinition(string name, IReadOnlyList<Expression>? bases, IReadOnlyList<Keyword>? keywords, Statement? body = null) {
             _name = name;
@@ -272,13 +272,13 @@ private Microsoft.Scripting.Ast.LightExpression<Func<CodeContext, CodeContext>>
             locals.Add(LocalCodeContextVariable);
             locals.Add(PythonAst._globalContext);
 
-            init.Add(Ast.Assign(PythonAst._globalContext, new GetGlobalContextExpression(_parentContextParam)));
+            init.Add(Ast.Assign(PythonAst._globalContext, new GetGlobalContextExpression(_outerContextParam)));
 
             GlobalParent.PrepareScope(locals, init);
 
             CreateVariables(locals, init);
 
-            var createLocal = CreateLocalContext(_parentContextParam);
+            var createLocal = CreateLocalContext(_outerContextParam, newNamespace: false);
 
             init.Add(Ast.Assign(LocalCodeContextVariable, createLocal));
 
@@ -328,7 +328,7 @@ private Microsoft.Scripting.Ast.LightExpression<Func<CodeContext, CodeContext>>
                     bodyStmt
                 ),
                 Name + "$" + Interlocked.Increment(ref _classId),
-                new[] { _parentContextParam }
+                new[] { _outerContextParam }
                 );
 
             return lambda;

Src/IronPython/Compiler/Ast/ScopeStatement.cs

@@ -677,7 +677,7 @@ internal MSAst.Expression FuncCodeExpr {
             }
         }
 
-        internal MSAst.MethodCallExpression CreateLocalContext(MSAst.Expression parentContext) {
+        internal MSAst.MethodCallExpression CreateLocalContext(MSAst.Expression parentContext, bool newNamespace = true) {
             var closureVariables = _closureVariables ?? Array.Empty<ClosureInfo>();
 
             int numFreeVars = FreeVariables?.Count ?? 0;
@@ -697,7 +697,8 @@ internal MSAst.MethodCallExpression CreateLocalContext(MSAst.Expression parentCo
                 MutableTuple.Create(ArrayUtils.ConvertAll(closureVariables, x => GetClosureCell(x))),
                 Ast.Constant(ArrayUtils.ConvertAll(closureVariables, x => x.AccessedInScope ? x.Variable.Name : null)),
                 AstUtils.Constant(numFreeVars),
-                AstUtils.Constant(firstArgIdx)
+                AstUtils.Constant(firstArgIdx),
+                AstUtils.Constant(newNamespace)
             );
         }
 

Src/IronPython/Runtime/CommonDictionaryStorage.cs

@@ -67,7 +67,7 @@ internal class CommonDictionaryStorage : DictionaryStorage, ISerializable, IDese
         public CommonDictionaryStorage() { }
 
         /// <summary>
-        /// Creates a new dictionary storage with no buckets
+        /// Creates a new dictionary storage with buckets
         /// </summary>
         public CommonDictionaryStorage(int count) {
             _buckets = new Bucket[(int)(count / Load + 2)];

Src/IronPython/Runtime/CustomDictionaryStorage.cs

@@ -11,7 +11,17 @@
 
 namespace IronPython.Runtime {
     internal abstract class CustomDictionaryStorage : DictionaryStorage {
-        private readonly CommonDictionaryStorage/*!*/ _storage = new CommonDictionaryStorage();
+        private readonly CommonDictionaryStorage/*!*/ _storage;
+
+        public CustomDictionaryStorage() {
+            _storage = new();
+        }
+
+        protected CustomDictionaryStorage(CommonDictionaryStorage storage) {
+            _storage = storage;
+        }
+
+        public CommonDictionaryStorage Storage => _storage;
 
         public override void Add(ref DictionaryStorage storage, object key, object value) {
             Add(key, value);

Src/IronPython/Runtime/FunctionCode.cs

@@ -585,7 +585,7 @@ internal object Call(CodeContext/*!*/ context) {
                 throw PythonOps.TypeError("cannot exec code object that contains free variables: {0}", co_freevars.__repr__(context));
             }
 
-            if (Target == null || (Target.GetMethodInfo() != null && Target.GetMethodInfo().DeclaringType == typeof(PythonCallTargets))) {
+            if (Target == null || (Target.GetMethodInfo()?.DeclaringType == typeof(PythonCallTargets))) {
                 UpdateDelegate(context.LanguageContext, true);
             }
 

Src/IronPython/Runtime/Operations/PythonOps.cs

@@ -1458,9 +1458,23 @@ public static void InitializeForFinalization(CodeContext/*!*/ context, object ne
                 }
             } // else metaclass is expected to be a callable and overrides any inherited metaclass through any bases
 
+            // Prepare class context
+            // The class context is like the parent context but with its own attribute dict.
+            // Or another way: the class context is like the local context of the class lambda but without the __class__ variable.
+            var attrStorage = new CommonDictionaryStorage();
+            PythonDictionary attrDict = parentContext.Dict._storage switch {
+                // If the parent context dict is backed by RuntimeVariablesDictionaryStorage,
+                // the class context dict also has to be backed by RuntimeVariablesDictionaryStorage so that the closure is preserved.
+                RuntimeVariablesDictionaryStorage parentStorage =>
+                    new PythonDictionary(new RuntimeVariablesDictionaryStorage(parentStorage, attrStorage)),
+                // Otherwise a standard dict suffices.
+                _ => new PythonDictionary(attrStorage),
+            };
+            CodeContext classContext = new CodeContext(attrDict, parentContext.ModuleContext);
+
             // Call class body lambda
-            CodeContext classContext = func(parentContext);
-            PythonDictionary vars = classContext.Dict;
+            CodeContext localContext = func(classContext);
+            PythonDictionary vars = localContext.Dict;
 
             // Prepare classdict
             // TODO: prepared classdict should be used by `func` (PEP 3115)
@@ -3622,10 +3636,19 @@ public static object SetName(CodeContext/*!*/ context, string name, object value
 
         #region Global Access
 
-        public static CodeContext/*!*/ CreateLocalContext(CodeContext/*!*/ outerContext, MutableTuple boxes, string[] args, int numFreeVars, int arg0Idx) {
+        public static CodeContext/*!*/ CreateLocalContext(CodeContext/*!*/ outerContext, MutableTuple boxes, string[] args, int numFreeVars, int arg0Idx, bool newAttribStorage) {
+            CommonDictionaryStorage? attribs = null;
+            if (!newAttribStorage) {
+                attribs = outerContext.Dict._storage switch {
+                    CustomDictionaryStorage vars => vars.Storage,
+                    CommonDictionaryStorage commonStorage => commonStorage,
+                    _ => new()
+                };
+            }
+
             return new CodeContext(
                 new PythonDictionary(
-                    new RuntimeVariablesDictionaryStorage(boxes, args, numFreeVars, arg0Idx)
+                    new RuntimeVariablesDictionaryStorage(boxes, args, numFreeVars, arg0Idx, attribs ?? new())
                 ),
                 outerContext.ModuleContext
             );

Src/IronPython/Runtime/RuntimeVariablesDictionaryStorage.cs

@@ -16,7 +16,13 @@ internal class RuntimeVariablesDictionaryStorage : CustomDictionaryStorage {
         private readonly int _numFreeVars;
         private readonly int _arg0Idx;
 
-        public RuntimeVariablesDictionaryStorage(MutableTuple boxes, string[] args, int numFreeVars, int arg0Idx) {
+        public RuntimeVariablesDictionaryStorage(RuntimeVariablesDictionaryStorage vars, CommonDictionaryStorage storage)
+            : this(vars._boxes, vars._args, vars._numFreeVars, vars._arg0Idx, storage) {
+        }
+
+        public RuntimeVariablesDictionaryStorage(MutableTuple boxes, string[] args, int numFreeVars, int arg0Idx, CommonDictionaryStorage storage)
+            : base(storage) {
+
             Debug.Assert(0 <= numFreeVars && numFreeVars <= args.Length);
             Debug.Assert(arg0Idx == -1 || numFreeVars <= arg0Idx && arg0Idx < args.Length);
 

Tests/modules/system_related/test_sys_getframe.py

@@ -126,10 +126,7 @@ class x(object):
             abc = get_odd_code()
             defined = 42
 
-        if is_cli:
-            verify(x.abc, False)
-        else:
-            verify(x.abc, True)
+        verify(x.abc, True)
 
         class x(object):
             abc = get_odd_code()