Merge branch 'master' into benchamrk_ci_integration

Author: samir-nasibli

sdc/datatypes/hpat_pandas_dataframe_pass.py

@@ -65,7 +65,8 @@ def run_pass(self, state):
                 out_nodes = [inst]
 
                 if isinstance(inst, ir.Assign):
-                    self.state.func_ir._definitions[inst.target.name].remove(inst.value)
+                    if inst.value in self.state.func_ir._definitions[inst.target.name]:
+                        self.state.func_ir._definitions[inst.target.name].remove(inst.value)
                     out_nodes = self._run_assign(inst)
 
                 if isinstance(out_nodes, list):

sdc/datatypes/hpat_pandas_series_functions.py

@@ -1178,7 +1178,7 @@ def hpat_pandas_series_corr(self, other, method='pearson', min_periods=None):
     if not isinstance(other.data.dtype, types.Number):
         ty_checker.raise_exc(other.data, 'number', 'other.data')
 
-    if not isinstance(min_periods, (types.Integer, types.Omitted, types.NoneType)):
+    if not isinstance(min_periods, (int, types.Integer, types.Omitted, types.NoneType)) and min_periods is not None:
         ty_checker.raise_exc(min_periods, 'int64', 'min_periods')
 
     def hpat_pandas_series_corr_impl(self, other, method='pearson', min_periods=None):
@@ -1200,7 +1200,20 @@ def hpat_pandas_series_corr_impl(self, other, method='pearson', min_periods=None
         if len(self_arr) < min_periods:
             return numpy.nan
 
-        return numpy.corrcoef(self_arr, other_arr)[0, 1]
+        new_self = pandas.Series(self_arr)
+        new_other = pandas.Series(other_arr)
+
+        n = new_self.count()
+        ma = new_self.sum()
+        mb = new_other.sum()
+        a = n * (self_arr * other_arr).sum() - ma * mb
+        b1 = n * (self_arr * self_arr).sum() - ma * ma
+        b2 = n * (other_arr * other_arr).sum() - mb * mb
+
+        if b1 == 0 or b2 == 0:
+            return numpy.nan
+
+        return a / numpy.sqrt(b1 * b2)
 
     return hpat_pandas_series_corr_impl
 
@@ -3106,10 +3119,10 @@ def hpat_pandas_series_argsort_idx_impl(self, axis=0, kind='quicksort', order=No
                 sort_nona = numpy.argsort(self._data[~na_data_arr])
             q = 0
             for id, i in enumerate(sort):
-                if id not in list(sort[len(self._data) - na:]):
-                    result[id] = sort_nona[id-q]
-                else:
+                if id in set(sort[len(self._data) - na:]):
                     q += 1
+                else:
+                    result[id] = sort_nona[id - q]
             for i in sort[len(self._data) - na:]:
                 result[i] = -1
 
@@ -3133,10 +3146,10 @@ def hpat_pandas_series_argsort_noidx_impl(self, axis=0, kind='quicksort', order=
             sort_nona = numpy.argsort(self._data[~na_data_arr])
         q = 0
         for id, i in enumerate(sort):
-            if id not in list(sort[len(self._data) - na:]):
-                result[id] = sort_nona[id - q]
-            else:
+            if id in set(sort[len(self._data) - na:]):
                 q += 1
+            else:
+                result[id] = sort_nona[id - q]
         for i in sort[len(self._data) - na:]:
             result[i] = -1
 
@@ -3543,7 +3556,15 @@ def hpat_pandas_series_cov_impl(self, other, min_periods=None):
         if len(self_arr) < min_periods:
             return numpy.nan
 
-        return numpy.cov(self_arr, other_arr)[0, 1]
+        new_self = pandas.Series(self_arr)
+
+        ma = new_self.mean()
+        mb = other.mean()
+
+        if numpy.isinf(mb):
+            return numpy.nan
+
+        return ((self_arr - ma) * (other_arr - mb)).sum() / (new_self.count() - 1.0)
 
     return hpat_pandas_series_cov_impl
 

sdc/datatypes/hpat_pandas_stringmethods_functions.py

@@ -83,6 +83,7 @@ def hpat_pandas_stringmethods_upper_impl(self):
 
 import numba
 from numba.extending import overload_method
+from numba.errors import TypingError
 
 from sdc.datatypes.hpat_pandas_stringmethods_types import StringMethodsType
 
@@ -186,6 +187,42 @@ def hpat_pandas_stringmethods_{methodname}_impl(self{methodparams}):
 """
 
 
+@overload_method(StringMethodsType, 'len')
+def hpat_pandas_stringmethods_len(self):
+    """
+    Pandas Series method :meth:`pandas.core.strings.StringMethods.len()` implementation.
+
+    Note: Unicode type of list elements are supported only. Numpy.NaN is not supported as elements.
+
+    .. only:: developer
+
+    Test: python -m sdc.runtests sdc.tests.test_series.TestSeries.test_series_str_len1
+
+    Parameters
+    ----------
+    self: :class:`pandas.core.strings.StringMethods`
+        input arg
+
+    Returns
+    -------
+    :obj:`pandas.Series`
+         returns :obj:`pandas.Series` object
+    """
+
+    if not isinstance(self, StringMethodsType):
+        raise TypingError('Method len(). The object must be a pandas.core.strings. Given: {}'.format(self))
+
+    def hpat_pandas_stringmethods_len_impl(self):
+        item_count = len(self._data)
+        result = numpy.empty(item_count, numba.types.int64)
+        for idx, item in enumerate(self._data._data):
+            result[idx] = len(item)
+
+        return pandas.Series(result, name=self._data._name)
+
+    return hpat_pandas_stringmethods_len_impl
+
+
 def _hpat_pandas_stringmethods_autogen(method_name):
     """"
     The function generates a function for 'method_name' from source text that is created on the fly.
@@ -231,7 +268,7 @@ def _hpat_pandas_stringmethods_autogen(method_name):
     This is the list of function which are autogenerated to be used from Numba directly.
 """
 
-_hpat_pandas_stringmethods_autogen_exceptions = ['split', 'len', 'get', 'replace']
+_hpat_pandas_stringmethods_autogen_exceptions = ['split', 'get', 'replace']
 
 for method_name in _hpat_pandas_stringmethods_autogen_methods:
     if not (method_name.startswith('__') or method_name in _hpat_pandas_stringmethods_autogen_exceptions):

sdc/hiframes/dataframe_pass.py

@@ -76,6 +76,18 @@ def __init__(self, state):
         self.state = state
 
     def run_pass(self):
+        """
+        The function could return exxeption. It means that the IR transformation can not be completed.
+        This is acceptable behaviour.
+        """
+
+        try:
+            self.run_pass_throw()
+            return True
+        except ValueError:
+            return False
+
+    def run_pass_throw(self):
         blocks = self.state.func_ir.blocks
         # topo_order necessary so DataFrame data replacement optimization can
         # be performed in one pass
@@ -109,8 +121,7 @@ def run_pass(self):
                         # TODO: add this to dead_branch_prune pass
                         for inst in self.state.func_ir.blocks[dead_label].body:
                             if is_assign(inst):
-                                self.state.func_ir._definitions[inst.target.name].remove(
-                                    inst.value)
+                                self.state.func_ir._definitions[inst.target.name].remove(inst.value)
 
                         del self.state.func_ir.blocks[dead_label]
                     else:
@@ -124,9 +135,7 @@ def run_pass(self):
                         used_vars = set()
                         new_body = []
                         for inst in reversed(block.body):
-                            if (is_assign(inst)
-                                    and inst.target.name not in used_vars
-                                    and inst.target.name in jmp_defs):
+                            if (is_assign(inst) and inst.target.name not in used_vars and inst.target.name in jmp_defs):
                                 self.state.func_ir._definitions[inst.target.name].remove(inst.value)
                                 continue
                             used_vars.update(v.name for v in inst.list_vars())
@@ -140,7 +149,8 @@ def run_pass(self):
                 out_nodes = [inst]
 
                 if isinstance(inst, ir.Assign):
-                    self.state.func_ir._definitions[inst.target.name].remove(inst.value)
+                    if inst.value in self.state.func_ir._definitions[inst.target.name]:
+                        self.state.func_ir._definitions[inst.target.name].remove(inst.value)
                     out_nodes = self._run_assign(inst)
                 elif isinstance(inst, (ir.SetItem, ir.StaticSetItem)):
                     out_nodes = self._run_setitem(inst)

sdc/hiframes/pd_series_ext.py

@@ -756,7 +756,7 @@ def resolve_rename(self, ary, args, kws):
     Functions which are still overloaded by HPAT compiler pipeline
 """
 
-str2str_methods_excluded = ['upper', 'lower', 'lstrip', 'rstrip', 'strip']
+str2str_methods_excluded = ['upper', 'len', 'lower', 'lstrip', 'rstrip', 'strip']
 """
     Functions which are used from Numba directly by calling from StringMethodsType
 

sdc/tests/test_base.py

@@ -0,0 +1,26 @@
+import unittest
+import warnings
+
+
+class TestCase(unittest.TestCase):
+    """Base class for all tests"""
+
+    def numba_jit(self, *args, **kwargs):
+        import numba
+        if 'nopython' in kwargs:
+            warnings.warn('nopython is set to True and is ignored', RuntimeWarning)
+        if 'parallel' in kwargs:
+            warnings.warn('parallel is set to True and is ignored', RuntimeWarning)
+        kwargs.update({'nopython': True, 'parallel': True})
+        return numba.jit(*args, **kwargs)
+
+    def sdc_jit(self, *args, **kwargs):
+        import sdc
+        return sdc.jit(*args, **kwargs)
+
+    def jit(self, *args, **kwargs):
+        from sdc import config
+        if config.config_pipeline_hpat_default:
+            return self.sdc_jit(*args, **kwargs)
+        else:
+            return self.numba_jit(*args, **kwargs)