Merge pull request #892 from Rubtsowa/remove_old_pipeline_config

remove skip_sdc_jit

Author: AlexanderKalistratov

sdc/__init__.py

@@ -51,27 +51,25 @@
 
 from ._version import get_versions
 
-if not sdc.config.config_pipeline_hpat_default:
-    """
-    Overload Numba function to allow call SDC pass in Numba compiler pipeline
-    Functions are:
-    - Numba DefaultPassBuilder define_nopython_pipeline()
-
-    TODO: Needs to detect 'import Pandas' and align initialization according to it
-    """
-
-    # sdc.config.numba_compiler_define_nopython_pipeline_orig = \
-    # numba.core.compiler.DefaultPassBuilder.define_nopython_pipeline
-    # numba.core.compiler.DefaultPassBuilder.define_nopython_pipeline = \
-    # sdc.datatypes.hpat_pandas_dataframe_pass.sdc_nopython_pipeline_lite_register
-
-    import sdc.rewrites.dataframe_constructor
-    import sdc.rewrites.read_csv_consts
-    import sdc.rewrites.dataframe_getitem_attribute
-    import sdc.datatypes.hpat_pandas_functions
-    import sdc.datatypes.hpat_pandas_dataframe_functions
-else:
-    import sdc.compiler
+"""
+Overload Numba function to allow call SDC pass in Numba compiler pipeline
+Functions are:
+- Numba DefaultPassBuilder define_nopython_pipeline()
+
+TODO: Needs to detect 'import Pandas' and align initialization according to it
+"""
+
+# sdc.config.numba_compiler_define_nopython_pipeline_orig = \
+# numba.core.compiler.DefaultPassBuilder.define_nopython_pipeline
+# numba.core.compiler.DefaultPassBuilder.define_nopython_pipeline = \
+# sdc.datatypes.hpat_pandas_dataframe_pass.sdc_nopython_pipeline_lite_register
+
+import sdc.rewrites.dataframe_constructor
+import sdc.rewrites.read_csv_consts
+import sdc.rewrites.dataframe_getitem_attribute
+import sdc.datatypes.hpat_pandas_functions
+import sdc.datatypes.hpat_pandas_dataframe_functions
+
 
 multithread_mode = False
 

sdc/config.py

@@ -56,11 +56,6 @@ def strtobool(val):
 because decorator called later then modules have been initialized
 '''
 
-config_pipeline_hpat_default = strtobool(os.getenv('SDC_CONFIG_PIPELINE_SDC', 'False'))
-'''
-Default value used to select compiler pipeline in a function decorator
-'''
-
 config_use_parallel_overloads = strtobool(os.getenv('SDC_AUTO_PARALLEL', 'True'))
 '''
 Default value used to select whether auto parallel would be applied to sdc functions
@@ -71,16 +66,6 @@ def strtobool(val):
 Default value used to select whether sdc functions would inline
 '''
 
-if not config_pipeline_hpat_default:
-    # avoid using MPI transport if no SDC compiler pipeline used
-    config_transport_mpi_default = False
-    config_transport_mpi = config_transport_mpi_default
-
-numba_compiler_define_nopython_pipeline_orig = None
-'''
-Default value for a pointer intended to use as Numba.DefaultPassBuilder.define_nopython_pipeline() in overloaded function
-'''
-
 test_expected_failure = strtobool(os.getenv('SDC_TEST_EXPECTED_FAILURE', 'False'))
 '''
 If True then replaces skip decorators to expectedFailure decorator.

sdc/datatypes/hpat_pandas_dataframe_types.py

@@ -41,8 +41,6 @@
 from numba.core.datamodel import register_default, StructModel
 from numba.core.typing.templates import signature, infer_global, AbstractTemplate
 
-from sdc.config import config_pipeline_hpat_default
-
 
 class DataFrameTypeIterator(types.SimpleIteratorType):
     """
@@ -111,23 +109,23 @@ def iterator_type(self):
         return DataFrameTypeIterator(self)
 
 
-if not config_pipeline_hpat_default:
-    @register_model(DataFrameType)
-    class DataFrameTypeModel(StructModel):
-        """
-        Model for DataFrameType type
-        All members must be the same as main type for this model
+@register_model(DataFrameType)
+class DataFrameTypeModel(StructModel):
+    """
+    Model for DataFrameType type
+    All members must be the same as main type for this model
 
-        Test: python -m sdc.runtests sdc.tests.test_dataframe.TestDataFrame.test_create_numeric_column
-        """
+    Test: python -m sdc.runtests sdc.tests.test_dataframe.TestDataFrame.test_create_numeric_column
+    """
+
+    def __init__(self, dmm, fe_type):
+        members = [
+            ('data', fe_type.data)
+        ]
+        models.StructModel.__init__(self, dmm, fe_type, members)
 
-        def __init__(self, dmm, fe_type):
-            members = [
-                ('data', fe_type.data)
-            ]
-            models.StructModel.__init__(self, dmm, fe_type, members)
 
-    make_attribute_wrapper(DataFrameType, 'data', '_data')
+make_attribute_wrapper(DataFrameType, 'data', '_data')
 
 
 @intrinsic
@@ -164,65 +162,65 @@ def _hpat_pandas_dataframe_init_codegen(context, builder, signature, args):
     return sig, _hpat_pandas_dataframe_init_codegen
 
 
-if not config_pipeline_hpat_default:
-    @overload(pandas.DataFrame)
-    def hpat_pandas_dataframe(data=None, index=None, columns=None, dtype=None, copy=False):
-        """
-        Special Numba procedure to overload Python type pandas.DataFrame::ctor() with Numba registered model
-        """
+@overload(pandas.DataFrame)
+def hpat_pandas_dataframe(data=None, index=None, columns=None, dtype=None, copy=False):
+    """
+    Special Numba procedure to overload Python type pandas.DataFrame::ctor() with Numba registered model
+    """
 
-        if isinstance(data, types.DictType):
-            def hpat_pandas_dataframe_impl(data=None, index=None, columns=None, dtype=None, copy=False):
-                series_dict = {}
-                series_list = []
+    if isinstance(data, types.DictType):
+        def hpat_pandas_dataframe_impl(data=None, index=None, columns=None, dtype=None, copy=False):
+            series_dict = {}
+            series_list = []
 
-                for key, value in data.items():
-                    """
-                    Convert input dictionary with:
-                        key - unicode string
-                        value - array
-                    into dictinary of pandas.Series with same names and values
-                    """
+            for key, value in data.items():
+                """
+                Convert input dictionary with:
+                    key - unicode string
+                    value - array
+                into dictinary of pandas.Series with same names and values
+                """
 
-                    series_item = pandas.Series(data=value, name=key)
-                    series_dict[key] = series_item
-                    series_list.append(series_item)
+                series_item = pandas.Series(data=value, name=key)
+                series_dict[key] = series_item
+                series_list.append(series_item)
 
-                # return _hpat_pandas_dataframe_init(series_dict)
-                return _hpat_pandas_dataframe_init(series_list)
+            # return _hpat_pandas_dataframe_init(series_dict)
+            return _hpat_pandas_dataframe_init(series_list)
 
-            return hpat_pandas_dataframe_impl
+        return hpat_pandas_dataframe_impl
 
-    @box(DataFrameType)
-    def hpat_pandas_dataframe_box(typ, val, c):
-        """
-        This method is to copy data from JITted region data structure
-        to new Python object data structure.
-        Python object data structure has creating in this procedure.
-        """
 
-        dataframe = cgutils.create_struct_proxy(typ)(c.context, c.builder, value=val)
+@box(DataFrameType)
+def hpat_pandas_dataframe_box(typ, val, c):
+    """
+    This method is to copy data from JITted region data structure
+    to new Python object data structure.
+    Python object data structure has creating in this procedure.
+    """
 
-        ir_ptr_data = c.box(typ.data, dataframe.data)
+    dataframe = cgutils.create_struct_proxy(typ)(c.context, c.builder, value=val)
 
-        dataframe_ctor_args = c.pyapi.tuple_pack([ir_ptr_data, ])
-        # dataframe_ctor_kwargs = c.pyapi.dict_pack([("data", ir_ptr_data), ])
-        """
-        It is better to use kwargs but it fails into SIGSEGV
-        """
+    ir_ptr_data = c.box(typ.data, dataframe.data)
 
-        dataframe_ctor_fn = c.pyapi.unserialize(c.pyapi.serialize_object(pandas.DataFrame))
-        """
-        Create a pandas.DataFrame ctor() function pointer
-        """
+    dataframe_ctor_args = c.pyapi.tuple_pack([ir_ptr_data, ])
+    # dataframe_ctor_kwargs = c.pyapi.dict_pack([("data", ir_ptr_data), ])
+    """
+    It is better to use kwargs but it fails into SIGSEGV
+    """
 
-        df_obj = c.pyapi.call(dataframe_ctor_fn, dataframe_ctor_args)  # kws=dataframe_ctor_kwargs)
-        """
-        Call pandas.DataFrame function pointer with parameters
-        """
+    dataframe_ctor_fn = c.pyapi.unserialize(c.pyapi.serialize_object(pandas.DataFrame))
+    """
+    Create a pandas.DataFrame ctor() function pointer
+    """
+
+    df_obj = c.pyapi.call(dataframe_ctor_fn, dataframe_ctor_args)  # kws=dataframe_ctor_kwargs)
+    """
+    Call pandas.DataFrame function pointer with parameters
+    """
 
-        c.pyapi.decref(ir_ptr_data)
-        c.pyapi.decref(dataframe_ctor_args)
-        c.pyapi.decref(dataframe_ctor_fn)
+    c.pyapi.decref(ir_ptr_data)
+    c.pyapi.decref(dataframe_ctor_args)
+    c.pyapi.decref(dataframe_ctor_fn)
 
-        return df_obj
+    return df_obj

sdc/decorators.py

@@ -40,44 +40,7 @@ def jit(signature_or_function=None, **options):
         '''
         options['nopython'] = True
 
-    if not sdc.config.config_pipeline_hpat_default:
-        '''
-        Use Numba compiler pipeline
-        '''
-        return numba.jit(signature_or_function, **options)
-
-    _locals = options.pop('locals', {})
-    assert isinstance(_locals, dict)
-
-    # put pivots in locals TODO: generalize numba.jit options
-    pivots = options.pop('pivots', {})
-    assert isinstance(pivots, dict)
-    for var, vals in pivots.items():
-        _locals[var + ":pivot"] = vals
-
-    distributed = set(options.pop('distributed', set()))
-    assert isinstance(distributed, (set, list))
-    _locals["##distributed"] = distributed
-
-    threaded = set(options.pop('threaded', set()))
-    assert isinstance(threaded, (set, list))
-    _locals["##threaded"] = threaded
-
-    options['locals'] = _locals
-
-    #options['parallel'] = True
-    options['parallel'] = {'comprehension': True,
-                           'setitem': False,  # FIXME: support parallel setitem
-                           'reduction': True,
-                           'numpy': True,
-                           'stencil': True,
-                           'fusion': True,
-                           }
-
-    # Option MPI is boolean and true by default
-    # it means MPI transport will be used
-    mpi_transport_requested = options.pop('MPI', sdc.config.config_transport_mpi_default)
-    if not isinstance(mpi_transport_requested, (int, bool)):
-        raise ValueError("Option MPI or SDC_CONFIG_MPI environment variable should be boolean")
-
-    return numba.jit(signature_or_function, pipeline_class=sdc.compiler.SDCPipeline, **options)
+    '''
+    Use Numba compiler pipeline
+    '''
+    return numba.jit(signature_or_function, **options)

sdc/distributed_lower.py

@@ -620,9 +620,3 @@ def lower_hpat_finalize(context, builder, sig, args):
 @numba.njit
 def call_finalize():
     hpat_finalize()
-
-
-if sdc.config.config_pipeline_hpat_default:
-    atexit.register(call_finalize)
-    # flush output before finalize
-    atexit.register(sys.stdout.flush)