Simplify _stdlib tests (#1631)

* Simplify _stdlib tests

* Fix failing tests

Author: slozier

Src/IronPython/Lib/iptest/__init__.py

@@ -2,7 +2,7 @@
 # The .NET Foundation licenses this file to you under the Apache 2.0 License.
 # See the LICENSE file in the project root for more information.
 
-from .ipunittest import IronPythonTestCase, stdout_trapper, stderr_trapper, path_modifier, retryOnFailure, run_test, skipUnlessIronPython, source_root, expectedFailureIf
+from .ipunittest import IronPythonTestCase, stdout_trapper, stderr_trapper, path_modifier, retryOnFailure, run_test, skipUnlessIronPython, source_root, expectedFailureIf, generate_suite
 from .test_env import *
 from .type_util import *
 from .misc_util import ip_supported_encodings

Src/IronPython/Lib/iptest/ipunittest.py

@@ -112,9 +112,9 @@ def load_ironpython_test(*args):
     clr.AddReference("Microsoft.Dynamic")
     clr.AddReference("IronPython")
 
-    if args: 
+    if args:
         return clr.LoadAssemblyFromFileWithPath(_iron_python_test_dll)
-    else: 
+    else:
         clr.AddReferenceToFileAndPath(_iron_python_test_dll)
 
 class IronPythonTestCase(unittest.TestCase, FileUtil, ProcessUtil):
@@ -195,6 +195,36 @@ def run_test(name):
         from test import support
         support.run_unittest(name)
 
+def _flatten_suite(suite):
+    tests = []
+    for t in suite:
+        if isinstance(t, unittest.BaseTestSuite):
+            tests.extend(_flatten_suite(t))
+        else:
+            tests.append(t)
+    return tests
+
+def generate_suite(tests, failing_tests, skip_tests=[]):
+    all_tests = _flatten_suite(tests)
+    unknown_tests = []
+
+    for t in skip_tests:
+        try:
+            all_tests.remove(t)
+        except ValueError:
+            unknown_tests.append(t)
+
+    for t in failing_tests:
+        try:
+            all_tests[all_tests.index(t)] = unittest.expectedFailure(t)
+        except:
+            unknown_tests.append(t)
+
+    if unknown_tests:
+        raise ValueError("Unknown tests:\n - {}".format('\n - '.join(str(t) for t in unknown_tests)))
+
+    return unittest.TestSuite(all_tests)
+
 # class testpath:
 #     # find the ironpython root directory
 #     rowan_root          = get_environ_variable("dlr_root")
@@ -220,13 +250,13 @@ def run_test(name):
 #     lib_testdir         = path_combine(external_dir, '27/Lib')
 #     private_testdir     = path_combine(external_dir, '27/Lib/test')
 
-#     if is_cli: 
+#     if is_cli:
 #         ipython_executable  = sys.executable
 #         if is_posix:
 #             cpython_executable  = '/usr/bin/python2.7'
 #         else:
 #             cpython_executable  = path_combine(external_dir, '27/python.exe')
-#     else: 
+#     else:
 #         ipython_executable  = path_combine(sys.prefix, 'ipy.exe')
 #         cpython_executable  = sys.executable
 

Src/StdLib/Lib/test/test_socket.py

@@ -5123,7 +5123,8 @@ def testTypes(self):
                     source.close()
 
 
-def test_main():
+# ironpython: refactored to use load_tests protocol
+def load_tests(*args):
     tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest,
              TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, UDPTimeoutTest ]
 
@@ -5177,8 +5178,11 @@ def test_main():
         TestSocketSharing,
     ])
 
+    return unittest.TestSuite([unittest.makeSuite(test) for test in tests])
+
+def test_main():
     thread_info = support.threading_setup()
-    support.run_unittest(*tests)
+    support.run_unittest(__name__)
     support.threading_cleanup(*thread_info)
 
 if __name__ == "__main__":

Tests/gen_stdlib_test.py

@@ -22,21 +22,24 @@
 ## Run selected tests from {name} from StdLib
 ##
 
-import unittest
-import sys
-
-from iptest import run_test
+from iptest import is_ironpython, generate_suite, run_test
 
 import {package}test.{name}
 
 def load_tests(loader, standard_tests, pattern):
-    if sys.implementation.name == 'ironpython':
-        suite = unittest.TestSuite()
-{tests}
-        return suite
+    tests = loader.loadTestsFromModule({package}test.{name})
+
+    if is_ironpython:
+        {tests}
+
+        failing_tests = []
+
+        skip_tests = []
+
+        return generate_suite(tests, failing_tests, skip_tests)
 
     else:
-        return loader.loadTestsFromModule({package}test.{name}, pattern)
+        return tests
 
 run_test(__name__)
 """
@@ -51,24 +54,34 @@ def load_tests(loader, standard_tests, pattern):
     sys.path.insert(0, os.path.abspath(os.path.join(__file__, "../../Src/StdLib/Lib")))
     module = importlib.import_module("{package}test.{name}".format(name=name, package=package + "." if package else ""))
 
-    existing_tests = {}
-    try:
-        re_failure = re.compile(r'^\s*suite\.addTest\(unittest\.expectedFailure\((.*)\)\)( #.*)?$')
-        re_ok = re.compile(r'^\s*suite\.addTest\((.*)\)( #.*)?$')
-        with open(filepath, "r", encoding="utf-8") as f:
-            for line in f:
-                match = re_failure.match(line) or re_ok.match(line)
-                if match:
-                    existing_tests[match.group(1)] = match.group(0)
-    except FileNotFoundError:
-        pass
-
     tests = []
     for suite in unittest.defaultTestLoader.loadTestsFromModule(module):
         for test in suite:
             tests.append("{}('{}')".format(unittest.util.strclass(test.__class__), test._testMethodName))
 
-    tests = [existing_tests.get(t, "        suite.addTest({})".format(t)) for t in tests]
+    if os.path.exists(filepath):
+        with open(filepath, "r", encoding="utf-8") as f:
+            lines = list(f)
+
+        existing_tests = set()
+
+        first = 0
+        for i, line in enumerate(lines):
+            if "{" in line: raise NotImplementedError
+
+            if not first:
+                if line.startswith("    if is_ironpython:"):
+                    first = i + 1
+            else:
+                if line.startswith("        return generate_suite("):
+                    break
+                existing_tests.add(line.split("#")[0].strip().rstrip(","))
+
+        tests = [t for t in tests if t not in existing_tests]
+
+        if tests:
+            lines.insert(first, "        {tests}\n")
+        template = "".join(lines)
 
     with open(filepath, "w", encoding="utf-8") as f:
-        f.write(template.format(name=name, package=package + "." if package else "", tests="\n".join(tests)))
+        f.write(template.format(name=name, package=package + "." if package else "", tests="\n        ".join(tests)))

Tests/modules/system_related/test_resource_stdlib.py

@@ -6,29 +6,23 @@
 ## Run selected tests from test_resource from StdLib
 ##
 
-import unittest
-import sys
-
-from iptest import run_test
+from iptest import is_ironpython, generate_suite, run_test
 
 import test.test_resource
 
 def load_tests(loader, standard_tests, pattern):
-    if sys.implementation.name == 'ironpython':
-        suite = unittest.TestSuite()
-        suite.addTest(test.test_resource.ResourceTest('test_args'))
-        suite.addTest(test.test_resource.ResourceTest('test_freebsd_contants'))
-        #suite.addTest(test.test_resource.ResourceTest('test_fsize_enforced')) # TODO: handle SIGXFSZ
-        suite.addTest(test.test_resource.ResourceTest('test_fsize_ismax'))
-        suite.addTest(test.test_resource.ResourceTest('test_fsize_toobig'))
-        suite.addTest(test.test_resource.ResourceTest('test_getrusage'))
-        suite.addTest(test.test_resource.ResourceTest('test_linux_constants'))
-        suite.addTest(test.test_resource.ResourceTest('test_pagesize'))
-        suite.addTest(test.test_resource.ResourceTest('test_prlimit'))
-        suite.addTest(test.test_resource.ResourceTest('test_setrusage_refcount'))
-        return suite
+    tests = loader.loadTestsFromModule(test.test_resource)
+
+    if is_ironpython:
+        failing_tests = []
+
+        skip_tests = [
+            test.test_resource.ResourceTest('test_fsize_enforced') # TODO: handle SIGXFSZ
+        ]
+
+        return generate_suite(tests, failing_tests, skip_tests)
 
     else:
-        return loader.loadTestsFromModule(test.test_resource, pattern)
+        return tests
 
 run_test(__name__)

Tests/modules/type_related/test_bitfields_ctypes_stdlib.py

@@ -6,43 +6,25 @@
 ## Run selected tests from test_bitfields from StdLib
 ##
 
-import unittest
-import sys
-
-from iptest import run_test
+from iptest import is_ironpython, generate_suite, run_test, is_windows
 
 import ctypes.test.test_bitfields
 
 def load_tests(loader, standard_tests, pattern):
-    if sys.implementation.name == 'ironpython':
-        suite = unittest.TestSuite()
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_anon_bitfields'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_c_wchar'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_longlong'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_mixed_2'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_mixed_3'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_multi_bitfields_size'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_nonint_types'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_signed'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_single_bitfield_size'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_uint32'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_uint32_swap_big_endian'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_uint32_swap_little_endian'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_uint64'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_ulonglong'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_unsigned'))
-        suite.addTest(ctypes.test.test_bitfields.C_Test('test_ints'))
-        if sys.platform.startswith('win'):
-            suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_mixed_1'))
-            suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_mixed_4'))
-            suite.addTest(ctypes.test.test_bitfields.C_Test('test_shorts'))
-        else: # https://github.com/IronLanguages/ironpython3/issues/1442
-            suite.addTest(unittest.expectedFailure(ctypes.test.test_bitfields.BitFieldTest('test_mixed_1')))
-            suite.addTest(unittest.expectedFailure(ctypes.test.test_bitfields.BitFieldTest('test_mixed_4')))
-            suite.addTest(unittest.expectedFailure(ctypes.test.test_bitfields.C_Test('test_shorts')))
-        return suite
+    tests = loader.loadTestsFromModule(ctypes.test.test_bitfields)
+
+    if is_ironpython:
+        failing_tests = []
+        if not is_windows: # https://github.com/IronLanguages/ironpython3/issues/1442
+            failing_tests += [
+                ctypes.test.test_bitfields.BitFieldTest('test_mixed_1'),
+                ctypes.test.test_bitfields.BitFieldTest('test_mixed_4'),
+                ctypes.test.test_bitfields.C_Test('test_shorts'),
+            ]
+
+        return generate_suite(tests, failing_tests)
 
     else:
-        return loader.loadTestsFromModule(ctypes.test.test_bitfields, pattern)
+        return tests
 
 run_test(__name__)

Tests/test_builtin_stdlib.py

@@ -6,85 +6,36 @@
 ## Run selected tests from test_builtin from StdLib
 ##
 
-import unittest
-import sys
+from iptest import is_ironpython, generate_suite, run_test, is_netcoreapp
 
-from iptest import run_test, is_netcoreapp
-
-import builtins
-import doctest
 import test.test_builtin
 
 def load_tests(loader, standard_tests, pattern):
-    if sys.implementation.name == 'ironpython':
-        suite = unittest.TestSuite()
-        suite.addTest(test.test_builtin.BuiltinTest('test_abs'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_all'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_any'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_ascii'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_bin'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_bytearray_translate'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_callable'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_chr'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_cmp'))
-        suite.addTest(unittest.expectedFailure(test.test_builtin.BuiltinTest('test_compile')))
-        suite.addTest(test.test_builtin.BuiltinTest('test_construct_singletons'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_delattr'))
-        suite.addTest(unittest.expectedFailure(test.test_builtin.BuiltinTest('test_dir')))
-        suite.addTest(test.test_builtin.BuiltinTest('test_divmod'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_eval'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_exec'))
-        suite.addTest(unittest.expectedFailure(test.test_builtin.BuiltinTest('test_exec_globals')))
-        suite.addTest(test.test_builtin.BuiltinTest('test_exec_redirected'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_filter'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_filter_pickle'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_format'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_general_eval'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_getattr'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_hasattr'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_hash'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_hex'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_id'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_import'))
-        suite.addTest(unittest.expectedFailure(test.test_builtin.BuiltinTest('test_input')))
-        suite.addTest(test.test_builtin.BuiltinTest('test_isinstance'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_issubclass'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_iter'))
-        suite.addTest(unittest.expectedFailure(test.test_builtin.BuiltinTest('test_len')))
-        suite.addTest(test.test_builtin.BuiltinTest('test_map'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_map_pickle'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_max'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_min'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_neg'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_next'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_oct'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_open'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_open_default_encoding'))
-        suite.addTest(unittest.expectedFailure(test.test_builtin.BuiltinTest('test_open_non_inheritable'))) # https://github.com/IronLanguages/ironpython3/issues/1225
-        suite.addTest(test.test_builtin.BuiltinTest('test_ord'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_pow'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_repr'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_round'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_round_large'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_setattr'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_sum'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_type'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_vars'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_zip'))
-        suite.addTest(test.test_builtin.BuiltinTest('test_zip_pickle'))
-        suite.addTest(unittest.expectedFailure(test.test_builtin.PtyTests('test_input_no_stdout_fileno')))
-        suite.addTest(unittest.expectedFailure(test.test_builtin.PtyTests('test_input_tty')))
-        suite.addTest(unittest.expectedFailure(test.test_builtin.PtyTests('test_input_tty_non_ascii')))
-        suite.addTest(unittest.expectedFailure(test.test_builtin.PtyTests('test_input_tty_non_ascii_unicode_errors')))
-        if not is_netcoreapp:
-            suite.addTest(test.test_builtin.ShutdownTest('test_cleanup'))
-        suite.addTest(test.test_builtin.TestSorted('test_baddecorator'))
-        suite.addTest(test.test_builtin.TestSorted('test_basic'))
-        suite.addTest(test.test_builtin.TestSorted('test_inputtypes'))
-        suite.addTest(doctest.DocTestSuite(builtins))
-        return suite
+    tests = loader.loadTestsFromModule(test.test_builtin)
+
+    if is_ironpython:
+        failing_tests = [
+            test.test_builtin.BuiltinTest('test_compile'),
+            test.test_builtin.BuiltinTest('test_dir'),
+            test.test_builtin.BuiltinTest('test_exec_globals'),
+            test.test_builtin.BuiltinTest('test_input'),
+            test.test_builtin.BuiltinTest('test_len'),
+            test.test_builtin.BuiltinTest('test_open_non_inheritable'), # https://github.com/IronLanguages/ironpython3/issues/1225
+            test.test_builtin.PtyTests('test_input_no_stdout_fileno'),
+            test.test_builtin.PtyTests('test_input_tty'),
+            test.test_builtin.PtyTests('test_input_tty_non_ascii'),
+            test.test_builtin.PtyTests('test_input_tty_non_ascii_unicode_errors'),
+        ]
+
+        skip_tests = []
+        if is_netcoreapp:
+            skip_tests += [
+                test.test_builtin.ShutdownTest('test_cleanup'),
+            ]
+
+        return generate_suite(tests, failing_tests, skip_tests)
 
     else:
-        return loader.loadTestsFromModule(test.test_builtin, pattern)
+        return tests
 
 run_test(__name__)