Update pandas 1.0.5 (#905)

* Update Pandas 1.0.5 (#899)

* Update Pandas 1.0.5

* Skip failed tests with Pandas 1.0

* fix rolling count (#900)

* fix rolling count

* add if in gen_impl

* fix rolling.count

* fix rolling apply mean (#902)

* fix test_series_getitem_idx_bool_array2 (#903)

* Fix RangeIndex tests (#904)

* fix tests for series (#907)

* Remove skip_pandas1 decorator (#909)

Co-authored-by: Rubtsowa <36762665+Rubtsowa@users.noreply.github.com>
Co-authored-by: Angelina Kharchevnikova <angelina.kharchevnikova@intel.com>

Author: 3 people

README.rst

@@ -34,13 +34,13 @@ Distribution includes Intel® SDC for Python 3.6 and Python 3.7 for Windows and
 
 Intel® SDC conda package can be installed using the steps below::
 
-    > conda create -n sdc-env python=<3.7 or 3.6> pyarrow=0.17.0 pandas=0.25.3 -c anaconda -c conda-forge
+    > conda create -n sdc-env python=<3.7 or 3.6> pyarrow=0.17.0 pandas=1.0.5 -c anaconda -c conda-forge
     > conda activate sdc-env
     > conda install sdc -c intel/label/beta -c intel -c defaults -c conda-forge --override-channels
 
 Intel® SDC wheel package can be installed using the steps below::
 
-    > conda create -n sdc-env python=<3.7 or 3.6> pip pyarrow=0.17.0 pandas=0.25.3 -c anaconda -c conda-forge
+    > conda create -n sdc-env python=<3.7 or 3.6> pip pyarrow=0.17.0 pandas=1.0.5 -c anaconda -c conda-forge
     > conda activate sdc-env
     > pip install --index-url https://pypi.anaconda.org/intel/label/beta/simple --extra-index-url https://pypi.anaconda.org/intel/simple --extra-index-url https://pypi.org/simple sdc
 
@@ -82,7 +82,7 @@ Building on Linux with setuptools
 
     export PYVER=<3.6 or 3.7>
     export NUMPYVER=<1.16 or 1.17>
-    conda create -n sdc-env -q -y -c intel/label/beta -c defaults -c intel -c conda-forge python=$PYVER numpy=$NUMPYVER tbb-devel tbb4py numba=0.49 pandas=0.25.3 pyarrow=0.17.0 gcc_linux-64 gxx_linux-64
+    conda create -n sdc-env -q -y -c intel/label/beta -c defaults -c intel -c conda-forge python=$PYVER numpy=$NUMPYVER tbb-devel tbb4py numba=0.49 pandas=1.0.5 pyarrow=0.17.0 gcc_linux-64 gxx_linux-64
     source activate sdc-env
     git clone https://github.com/IntelPython/sdc.git
     cd sdc
@@ -120,7 +120,7 @@ Building on Windows with setuptools
 
     set PYVER=<3.6 or 3.7>
     set NUMPYVER=<1.16 or 1.17>
-    conda create -n sdc-env -c intel/label/beta -c defaults -c intel -c conda-forge python=%PYVER% numpy=%NUMPYVER% tbb-devel tbb4py numba=0.49 pandas=0.25.3 pyarrow=0.17.0
+    conda create -n sdc-env -c intel/label/beta -c defaults -c intel -c conda-forge python=%PYVER% numpy=%NUMPYVER% tbb-devel tbb4py numba=0.49 pandas=1.0.5 pyarrow=0.17.0
     conda activate sdc-env
     set INCLUDE=%INCLUDE%;%CONDA_PREFIX%\Library\include
     set LIB=%LIB%;%CONDA_PREFIX%\Library\lib

conda-recipe/meta.yaml

@@ -1,5 +1,5 @@
 {% set NUMBA_VERSION = "==0.50.1" %}
-{% set PANDAS_VERSION = "==0.25.3" %}
+{% set PANDAS_VERSION = "==1.0.5" %}
 {% set PYARROW_VERSION = "==0.17.0" %}
 
 package:

sdc/datatypes/hpat_pandas_series_functions.py

@@ -407,7 +407,7 @@ def hpat_pandas_series_getitem_idx_slice_impl(self, idx):
         def hpat_pandas_series_getitem_idx_list_impl(self, idx):
 
             if len(self) != len(idx):
-                raise IndexError("Item wrong length")
+                raise IndexError('Boolean index has wrong length')
 
             return pandas.Series(
                 data=numpy_like.getitem_by_mask(self._data, idx),

sdc/datatypes/hpat_pandas_series_rolling_functions.py

@@ -235,21 +235,24 @@ def put_corr(x, y, nfinite, result):
 
 
 @sdc_register_jitable
-def pop_count(value, nfinite, result):
+def pop_count(value, counter, result):
     """Calculate the window count without old value."""
     if numpy.isnan(value):
-        return nfinite, result
+        return counter, result
 
-    return nfinite, result - 1
+    return counter, result - 1
 
 
 @sdc_register_jitable
-def put_count(value, nfinite, result):
+def put_count(value, counter, result):
     """Calculate the window count with new value."""
+    # rolling.count() fills front of the resulting array by nans according to min_periods using counter
+    # Ex: s = pd.Series([1, 2, np.nan]) ->  s.rolling(window=3, min_periods=2).count() -> pd.Series([np.nan, 2.0, 2.0])
+    counter += 1
     if numpy.isnan(value):
-        return nfinite, result
+        return counter, result
 
-    return nfinite, result + 1
+    return counter, result + 1
 
 
 @sdc_register_jitable
@@ -731,7 +734,7 @@ def impl(self, ddof=1):
 
 
 sdc_pandas_series_rolling_count_impl = gen_sdc_pandas_series_rolling_impl(
-    pop_count, put_count, get_result=result, init_result=0.)
+    pop_count, put_count, init_result=0.)
 sdc_pandas_series_rolling_kurt_impl = gen_sdc_pandas_series_rolling_impl(
     pop_kurt, put_kurt, get_result=kurt_result_or_nan,
     init_result=(0., 0., 0., 0.))
@@ -771,8 +774,7 @@ def hpat_pandas_rolling_series_apply_impl(self, func, raw=None):
         output_arr = numpy.empty(length, dtype=float64)
 
         def culc_apply(arr, func, minp):
-            finite_arr = arr.copy()
-            finite_arr[numpy.isinf(arr)] = numpy.nan
+            finite_arr = arr[numpy.isfinite(arr)]
             if len(finite_arr) < minp:
                 return numpy.nan
             else:

sdc/extensions/indexes/range_index_ext.py

@@ -97,6 +97,7 @@ def pd_range_index_overload(start=None, stop=None, step=None, dtype=None, copy=F
         raise SDCLimitation(f"{_func_name} Unsupported parameter. Given 'fastpath': {fastpath}")
 
     dtype_is_np_int64 = dtype is types.NumberClass(types.int64)
+    dtype_is_np_int32 = dtype is types.NumberClass(types.int32)
     dtype_is_unicode_str = isinstance(dtype, (types.UnicodeType, types.StringLiteral))
     if not _check_dtype_param_type(dtype):
         ty_checker.raise_exc(dtype, 'int64 dtype', 'dtype')
@@ -125,9 +126,12 @@ def pd_range_index_ctor_impl(start=None, stop=None, step=None, dtype=None, copy=
 
         if not (dtype is None
                 or dtype_is_unicode_str and dtype == 'int64'
-                or dtype_is_np_int64):
-            raise TypeError("Invalid to pass a non-int64 dtype to RangeIndex")
+                or dtype_is_unicode_str and dtype == 'int32'
+                or dtype_is_np_int64
+                or dtype_is_np_int32):
+            raise ValueError("Incorrect `dtype` passed: expected signed integer")
 
+        # TODO: add support of int32 type
         _start = types.int64(start) if start is not None else types.int64(0)
 
         if stop is None:

sdc/tests/test_indexes.py

@@ -195,7 +195,7 @@ def test_impl(stop, dtype):
         sdc_func = self.jit(test_impl)
 
         n = 11
-        supported_dtypes = [None, np.int64, 'int64']
+        supported_dtypes = [None, np.int64, 'int64', np.int32, 'int32']
         for dtype in supported_dtypes:
             with self.subTest(dtype=dtype):
                 result = sdc_func(n, dtype)
@@ -208,7 +208,7 @@ def test_impl(stop, dtype):
         sdc_func = self.jit(test_impl)
 
         n = 11
-        invalid_dtypes = ['float', np.int32, 'int32']
+        invalid_dtypes = ['float']
         for dtype in invalid_dtypes:
             with self.subTest(dtype=dtype):
                 with self.assertRaises(Exception) as context:

sdc/tests/test_series.py

@@ -148,7 +148,7 @@ def series_values_from_argsort_result(series, argsorted):
 def restore_series_sort_values(series, my_result_index, ascending):
     value_dict = {}
     nan_list = []
-    data = np.copy(series.data)
+    data = np.copy(series.values)
     index = np.copy(series.index)
     for value in range(len(data)):
         # if np.isnan(data[value]):
@@ -996,7 +996,7 @@ def test_impl(A, i):
         S = pd.Series(np.arange(n), name='A')
         # SDC and pandas results differ due to type limitation requirements:
         # SDC returns Series of one element, whereas pandas returns scalar, hence we align result
-        result = hpat_func(S, i).data[0]
+        result = hpat_func(S, i).values[0]
         result_ref = test_impl(S, i)
         self.assertEqual(result, result_ref)
 
@@ -1096,7 +1096,7 @@ def test_impl(S, key):
         indices = [[2, 3, 5], [2, 3, 5], [2, 3, 5]]
         for key, index in zip(keys, indices):
             S = pd.Series([11, 22, 33], index, name='A')
-            np.testing.assert_array_equal(jit_impl(S, key).data, np.array(test_impl(S, key)))
+            np.testing.assert_array_equal(jit_impl(S, key).values, np.array(test_impl(S, key)))
 
     def test_series_getitem_duplicate_index(self):
         def test_impl(A):
@@ -1162,7 +1162,7 @@ def test_impl(S, key):
         indices = [[2, 3, 5], [2, 2, 2], [2, 4, 15]]
         for key, data, index in zip(keys, all_data, indices):
             S = pd.Series(data, index, name='A')
-            np.testing.assert_array_equal(jit_impl(S, key).data, np.array(test_impl(S, key)))
+            np.testing.assert_array_equal(jit_impl(S, key).values, np.array(test_impl(S, key)))
 
     def test_series_loc_str(self):
         def test_impl(A):
@@ -4048,7 +4048,7 @@ def test_impl(series, ascending, kind):
                     if kind == 'mergesort':
                         pd.testing.assert_series_equal(ref_result, jit_result)
                     else:
-                        np.testing.assert_array_equal(ref_result.data, jit_result.data)
+                        np.testing.assert_array_equal(ref_result.values, jit_result.values)
                         self.assertEqual(ref, jit)
 
     @skip_parallel

setup.py

@@ -373,7 +373,7 @@ def run(self):
       package_data={'sdc.tests': ['*.bz2'], },
       install_requires=[
           'numpy>=1.16',
-          'pandas==0.25.3',
+          'pandas>=1.0',
           'pyarrow==0.17.0',
           'numba>=0.50.1,<0.51',
           'tbb'