Simplify _stdlib tests (#1634)

* Simplify _stdlib tests

* Skip datetime tests

* Re-enable test

Author: slozier

Src/IronPython/Lib/iptest/__init__.py

@@ -2,7 +2,7 @@
 # The .NET Foundation licenses this file to you under the Apache 2.0 License.
 # See the LICENSE file in the project root for more information.
 
-from .ipunittest import IronPythonTestCase, stdout_trapper, stderr_trapper, path_modifier, retryOnFailure, run_test, skipUnlessIronPython, source_root, expectedFailureIf
+from .ipunittest import IronPythonTestCase, stdout_trapper, stderr_trapper, path_modifier, retryOnFailure, run_test, skipUnlessIronPython, source_root, expectedFailureIf, generate_suite
 from .test_env import *
 from .type_util import *
 from .misc_util import ip_supported_encodings

Src/IronPython/Lib/iptest/ipunittest.py

@@ -112,9 +112,9 @@ def load_ironpython_test(*args):
     clr.AddReference("Microsoft.Dynamic")
     clr.AddReference("IronPython")
 
-    if args: 
+    if args:
         return clr.LoadAssemblyFromFileWithPath(_iron_python_test_dll)
-    else: 
+    else:
         clr.AddReferenceToFileAndPath(_iron_python_test_dll)
 
 class IronPythonTestCase(unittest.TestCase, FileUtil, ProcessUtil):
@@ -195,6 +195,37 @@ def run_test(name):
         from test import support
         support.run_unittest(name)
 
+def _flatten_suite(suite):
+    tests = []
+    for t in suite:
+        if isinstance(t, unittest.BaseTestSuite):
+            tests.extend(_flatten_suite(t))
+        else:
+            tests.append(t)
+    return tests
+
+def generate_suite(tests, failing_tests, skip_tests=[]):
+    all_tests = _flatten_suite(tests)
+    test_indices = {t: i for i, t in enumerate(all_tests)}
+    unknown_tests = []
+
+    for t in skip_tests:
+        try:
+            all_tests[test_indices.pop(t)] = None
+        except ValueError:
+            unknown_tests.append(t)
+
+    for t in failing_tests:
+        try:
+            all_tests[test_indices.pop(t)] = unittest.expectedFailure(t)
+        except ValueError:
+            unknown_tests.append(t)
+
+    if unknown_tests:
+        raise ValueError("Unknown tests:\n - {}".format('\n - '.join(str(t) for t in unknown_tests)))
+
+    return unittest.TestSuite(t for t in all_tests if t is not None)
+
 # class testpath:
 #     # find the ironpython root directory
 #     rowan_root          = get_environ_variable("dlr_root")
@@ -220,13 +251,13 @@ def run_test(name):
 #     lib_testdir         = path_combine(external_dir, '27/Lib')
 #     private_testdir     = path_combine(external_dir, '27/Lib/test')
 
-#     if is_cli: 
+#     if is_cli:
 #         ipython_executable  = sys.executable
 #         if is_posix:
 #             cpython_executable  = '/usr/bin/python2.7'
 #         else:
 #             cpython_executable  = path_combine(external_dir, '27/python.exe')
-#     else: 
+#     else:
 #         ipython_executable  = path_combine(sys.prefix, 'ipy.exe')
 #         cpython_executable  = sys.executable
 

Src/IronPythonTest/Cases/IronPythonCasesManifest.ini

@@ -108,6 +108,7 @@ Timeout=300000 # 5 minute timeout - sometimes slow
 [IronPython.test_sqlite3_stdlib]
 IsolationLevel=PROCESS
 RunCondition=NOT $(IS_OSX) # https://github.com/IronLanguages/ironpython3/issues/1483
+NotParallelSafe=true
 
 [IronPython.test_stdconsole]
 Timeout=600000 # 10 minute timeout

Src/StdLib/Lib/test/test_socket.py

@@ -5632,7 +5632,9 @@ def test_new_tcp_flags(self):
         self.assertEqual([], unknown,
             "New TCP flags were discovered. See bpo-32394 for more information")
 
-def test_main():
+
+# ironpython: refactored to use load_tests protocol
+def load_tests(*args):
     tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest,
              TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, UDPTimeoutTest ]
 
@@ -5689,8 +5691,11 @@ def test_main():
     ])
     tests.append(TestMSWindowsTCPFlags)
 
+    return unittest.TestSuite([unittest.makeSuite(test) for test in tests])
+
+def test_main():
     thread_info = support.threading_setup()
-    support.run_unittest(*tests)
+    support.run_unittest(__name__)
     support.threading_cleanup(*thread_info)
 
 if __name__ == "__main__":

Tests/gen_stdlib_test.py

@@ -22,21 +22,24 @@
 ## Run selected tests from {name} from StdLib
 ##
 
-import unittest
-import sys
-
-from iptest import run_test
+from iptest import is_ironpython, generate_suite, run_test
 
 import {package}test.{name}
 
 def load_tests(loader, standard_tests, pattern):
-    if sys.implementation.name == 'ironpython':
-        suite = unittest.TestSuite()
-{tests}
-        return suite
+    tests = loader.loadTestsFromModule({package}test.{name}, pattern=pattern)
+
+    if is_ironpython:
+        {tests}
+
+        failing_tests = []
+
+        skip_tests = []
+
+        return generate_suite(tests, failing_tests, skip_tests)
 
     else:
-        return loader.loadTestsFromModule({package}test.{name}, pattern)
+        return tests
 
 run_test(__name__)
 """
@@ -51,24 +54,34 @@ def load_tests(loader, standard_tests, pattern):
     sys.path.insert(0, os.path.abspath(os.path.join(__file__, "../../Src/StdLib/Lib")))
     module = importlib.import_module("{package}test.{name}".format(name=name, package=package + "." if package else ""))
 
-    existing_tests = {}
-    try:
-        re_failure = re.compile(r'^\s*suite\.addTest\(unittest\.expectedFailure\((.*)\)\)( #.*)?$')
-        re_ok = re.compile(r'^\s*suite\.addTest\((.*)\)( #.*)?$')
-        with open(filepath, "r", encoding="utf-8") as f:
-            for line in f:
-                match = re_failure.match(line) or re_ok.match(line)
-                if match:
-                    existing_tests[match.group(1)] = match.group(0)
-    except FileNotFoundError:
-        pass
-
     tests = []
     for suite in unittest.defaultTestLoader.loadTestsFromModule(module):
         for test in suite:
             tests.append("{}('{}')".format(unittest.util.strclass(test.__class__), test._testMethodName))
 
-    tests = [existing_tests.get(t, "        suite.addTest({})".format(t)) for t in tests]
+    if os.path.exists(filepath):
+        with open(filepath, "r", encoding="utf-8") as f:
+            lines = list(f)
+
+        existing_tests = set()
+
+        first = 0
+        for i, line in enumerate(lines):
+            if "{" in line: raise NotImplementedError
+
+            if not first:
+                if line.startswith("    if is_ironpython:"):
+                    first = i + 1
+            else:
+                if line.startswith("        return generate_suite("):
+                    break
+                existing_tests.add(line.split("#")[0].strip().rstrip(","))
+
+        tests = [t for t in tests if t not in existing_tests]
+
+        if tests:
+            lines.insert(first, "        {tests}\n")
+        template = "".join(lines)
 
     with open(filepath, "w", encoding="utf-8") as f:
-        f.write(template.format(name=name, package=package + "." if package else "", tests="\n".join(tests)))
+        f.write(template.format(name=name, package=package + "." if package else "", tests="\n        ".join(tests)))

Tests/modules/system_related/test_resource_stdlib.py

@@ -6,30 +6,23 @@
 ## Run selected tests from test_resource from StdLib
 ##
 
-import unittest
-import sys
-
-from iptest import run_test
+from iptest import is_ironpython, generate_suite, run_test
 
 import test.test_resource
 
 def load_tests(loader, standard_tests, pattern):
-    if sys.implementation.name == 'ironpython':
-        suite = unittest.TestSuite()
-        suite.addTest(test.test_resource.ResourceTest('test_args'))
-        suite.addTest(test.test_resource.ResourceTest('test_freebsd_contants'))
-        #suite.addTest(test.test_resource.ResourceTest('test_fsize_enforced')) # TODO: handle SIGXFSZ
-        suite.addTest(test.test_resource.ResourceTest('test_fsize_ismax'))
-        suite.addTest(test.test_resource.ResourceTest('test_fsize_toobig'))
-        suite.addTest(test.test_resource.ResourceTest('test_getrusage'))
-        suite.addTest(test.test_resource.ResourceTest('test_linux_constants'))
-        suite.addTest(test.test_resource.ResourceTest('test_pagesize'))
-        suite.addTest(test.test_resource.ResourceTest('test_prlimit'))
-        suite.addTest(test.test_resource.ResourceTest('test_prlimit_refcount'))
-        suite.addTest(test.test_resource.ResourceTest('test_setrusage_refcount'))
-        return suite
+    tests = loader.loadTestsFromModule(test.test_resource, pattern=pattern)
+
+    if is_ironpython:
+        failing_tests = []
+
+        skip_tests = [
+            test.test_resource.ResourceTest('test_fsize_enforced') # TODO: handle SIGXFSZ
+        ]
+
+        return generate_suite(tests, failing_tests, skip_tests)
 
     else:
-        return loader.loadTestsFromModule(test.test_resource, pattern)
+        return tests
 
 run_test(__name__)

Tests/modules/type_related/test_bitfields_ctypes_stdlib.py

@@ -6,43 +6,25 @@
 ## Run selected tests from test_bitfields from StdLib
 ##
 
-import unittest
-import sys
-
-from iptest import run_test
+from iptest import is_ironpython, generate_suite, run_test, is_windows
 
 import ctypes.test.test_bitfields
 
 def load_tests(loader, standard_tests, pattern):
-    if sys.implementation.name == 'ironpython':
-        suite = unittest.TestSuite()
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_anon_bitfields'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_c_wchar'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_longlong'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_mixed_2'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_mixed_3'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_multi_bitfields_size'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_nonint_types'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_signed'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_single_bitfield_size'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_uint32'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_uint32_swap_big_endian'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_uint32_swap_little_endian'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_uint64'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_ulonglong'))
-        suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_unsigned'))
-        suite.addTest(ctypes.test.test_bitfields.C_Test('test_ints'))
-        if sys.platform.startswith('win'):
-            suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_mixed_1'))
-            suite.addTest(ctypes.test.test_bitfields.BitFieldTest('test_mixed_4'))
-            suite.addTest(ctypes.test.test_bitfields.C_Test('test_shorts'))
-        else: # https://github.com/IronLanguages/ironpython3/issues/1442
-            suite.addTest(unittest.expectedFailure(ctypes.test.test_bitfields.BitFieldTest('test_mixed_1')))
-            suite.addTest(unittest.expectedFailure(ctypes.test.test_bitfields.BitFieldTest('test_mixed_4')))
-            suite.addTest(unittest.expectedFailure(ctypes.test.test_bitfields.C_Test('test_shorts')))
-        return suite
+    tests = loader.loadTestsFromModule(ctypes.test.test_bitfields, pattern=pattern)
+
+    if is_ironpython:
+        failing_tests = []
+        if not is_windows: # https://github.com/IronLanguages/ironpython3/issues/1442
+            failing_tests += [
+                ctypes.test.test_bitfields.BitFieldTest('test_mixed_1'),
+                ctypes.test.test_bitfields.BitFieldTest('test_mixed_4'),
+                ctypes.test.test_bitfields.C_Test('test_shorts'),
+            ]
+
+        return generate_suite(tests, failing_tests)
 
     else:
-        return loader.loadTestsFromModule(ctypes.test.test_bitfields, pattern)
+        return tests
 
 run_test(__name__)

Tests/test_abc_stdlib.py

@@ -6,40 +6,21 @@
 ## Run selected tests from test_abc from StdLib
 ##
 
-import unittest
-import sys
-
-from iptest import run_test
+from iptest import is_ironpython, generate_suite, run_test
 
 import test.test_abc
 
 def load_tests(loader, standard_tests, pattern):
-    if sys.implementation.name == 'ironpython':
-        suite = unittest.TestSuite()
-        suite.addTest(test.test_abc.TestABC('test_ABC_helper'))
-        suite.addTest(test.test_abc.TestABC('test_abstractclassmethod_basics'))
-        suite.addTest(test.test_abc.TestABC('test_abstractmethod_basics'))
-        suite.addTest(test.test_abc.TestABC('test_abstractmethod_integration'))
-        suite.addTest(test.test_abc.TestABC('test_abstractproperty_basics'))
-        suite.addTest(test.test_abc.TestABC('test_abstractstaticmethod_basics'))
-        suite.addTest(test.test_abc.TestABC('test_all_new_methods_are_called'))
-        suite.addTest(test.test_abc.TestABC('test_customdescriptors_with_abstractmethod'))
-        suite.addTest(test.test_abc.TestABC('test_descriptors_with_abstractmethod'))
-        suite.addTest(test.test_abc.TestABC('test_isinstance_invalidation'))
-        suite.addTest(test.test_abc.TestABC('test_metaclass_abc'))
-        suite.addTest(test.test_abc.TestABC('test_register_as_class_deco'))
-        suite.addTest(test.test_abc.TestABC('test_register_non_class'))
-        suite.addTest(test.test_abc.TestABC('test_registration_basics'))
-        suite.addTest(test.test_abc.TestABC('test_registration_builtins'))
-        suite.addTest(test.test_abc.TestABC('test_registration_edge_cases'))
-        suite.addTest(test.test_abc.TestABC('test_registration_transitiveness'))
-        suite.addTest(unittest.expectedFailure(test.test_abc.TestABCWithInitSubclass('test_works_with_init_subclass'))) # https://github.com/IronLanguages/ironpython3/issues/1448
-        suite.addTest(test.test_abc.TestLegacyAPI('test_abstractclassmethod_basics'))
-        suite.addTest(test.test_abc.TestLegacyAPI('test_abstractproperty_basics'))
-        suite.addTest(test.test_abc.TestLegacyAPI('test_abstractstaticmethod_basics'))
-        return suite
+    tests = loader.loadTestsFromModule(test.test_abc, pattern=pattern)
+
+    if is_ironpython:
+        failing_tests = [
+            test.test_abc.TestABCWithInitSubclass('test_works_with_init_subclass'), # https://github.com/IronLanguages/ironpython3/issues/1448
+        ]
+
+        return generate_suite(tests, failing_tests)
 
     else:
-        return loader.loadTestsFromModule(test.test_abc, pattern)
+        return tests
 
 run_test(__name__)