Move to pandas=1.2.0 (#959)

* Move to pandas=1.2.0

Motivation: use latest versions of dependencies.

* More failed tests are fixed

* Fixing doc build

Author: kozlov-alexey

README.rst

@@ -34,13 +34,13 @@ Distribution includes Intel® SDC for Python 3.6 and Python 3.7 for Windows and
 
 Intel® SDC conda package can be installed using the steps below::
 
-    > conda create -n sdc-env python=<3.7 or 3.6> pyarrow=0.17.0 pandas=1.0.5 -c anaconda -c conda-forge
+    > conda create -n sdc-env python=<3.7 or 3.6> pyarrow=0.17.0 pandas=1.2.0 -c anaconda -c conda-forge
     > conda activate sdc-env
     > conda install sdc -c intel/label/beta -c intel -c defaults -c conda-forge --override-channels
 
 Intel® SDC wheel package can be installed using the steps below::
 
-    > conda create -n sdc-env python=<3.7 or 3.6> pip pyarrow=0.17.0 pandas=1.0.5 -c anaconda -c conda-forge
+    > conda create -n sdc-env python=<3.7 or 3.6> pip pyarrow=0.17.0 pandas=1.2.0 -c anaconda -c conda-forge
     > conda activate sdc-env
     > pip install --index-url https://pypi.anaconda.org/intel/label/beta/simple --extra-index-url https://pypi.anaconda.org/intel/simple --extra-index-url https://pypi.org/simple sdc
 
@@ -82,7 +82,7 @@ Building on Linux with setuptools
 
     export PYVER=<3.6 or 3.7>
     export NUMPYVER=<1.16 or 1.17>
-    conda create -n sdc-env -q -y -c intel/label/beta -c defaults -c intel -c conda-forge python=$PYVER numpy=$NUMPYVER tbb-devel tbb4py numba=0.52 pandas=1.0.5 pyarrow=0.17.0 gcc_linux-64 gxx_linux-64
+    conda create -n sdc-env -q -y -c intel/label/beta -c defaults -c intel -c conda-forge python=$PYVER numpy=$NUMPYVER tbb-devel tbb4py numba=0.52 pandas=1.2.0 pyarrow=0.17.0 gcc_linux-64 gxx_linux-64
     source activate sdc-env
     git clone https://github.com/IntelPython/sdc.git
     cd sdc
@@ -120,7 +120,7 @@ Building on Windows with setuptools
 
     set PYVER=<3.6 or 3.7>
     set NUMPYVER=<1.16 or 1.17>
-    conda create -n sdc-env -c intel/label/beta -c defaults -c intel -c conda-forge python=%PYVER% numpy=%NUMPYVER% tbb-devel tbb4py numba=0.52 pandas=1.0.5 pyarrow=0.17.0
+    conda create -n sdc-env -c intel/label/beta -c defaults -c intel -c conda-forge python=%PYVER% numpy=%NUMPYVER% tbb-devel tbb4py numba=0.52 pandas=1.2.0 pyarrow=0.17.0
     conda activate sdc-env
     set INCLUDE=%INCLUDE%;%CONDA_PREFIX%\Library\include
     set LIB=%LIB%;%CONDA_PREFIX%\Library\lib

conda-recipe/meta.yaml

@@ -1,5 +1,5 @@
 {% set NUMBA_VERSION = "==0.52.0" %}
-{% set PANDAS_VERSION = "==1.0.5" %}
+{% set PANDAS_VERSION = "==1.2.0" %}
 {% set PYARROW_VERSION = "==0.17.0" %}
 
 package:

docs/source/_templates/_api_ref.pandas.window_templ.rst

@@ -51,8 +51,8 @@ Exponentially-weighted moving window functions
 ----------------------------------------------
 
 .. sdc_toctree
-   EWM.mean
-   EWM.std
-   EWM.var
-   EWM.corr
-   EWM.cov
+   ewm.ExponentialMovingWindow.mean
+   ewm.ExponentialMovingWindow.std
+   ewm.ExponentialMovingWindow.var
+   ewm.ExponentialMovingWindow.corr
+   ewm.ExponentialMovingWindow.cov

docs/source/getting_started.rst

@@ -41,14 +41,14 @@ Distribution includes Intel SDC for Python 3.6 and 3.7 for Windows and Linux pla
 Intel SDC conda package can be installed using the steps below:
 ::
 
-    > conda create -n sdc_env python=<3.7 or 3.6> pyarrow=0.17.0 pandas=0.25.3 -c anaconda -c conda-forge
+    > conda create -n sdc_env python=<3.7 or 3.6> pyarrow=0.17.0 pandas=1.2.0 -c anaconda -c conda-forge
     > conda activate sdc_env
     > conda install sdc -c intel/label/beta -c intel -c defaults -c conda-forge --override-channels
 
 Intel SDC wheel package can be installed using the steps below:
 ::
 
-    > conda create -n sdc_env python=<3.7 or 3.6> pip pyarrow=0.17.0 pandas=0.25.3 -c anaconda -c conda-forge
+    > conda create -n sdc_env python=<3.7 or 3.6> pip pyarrow=0.17.0 pandas=1.2.0 -c anaconda -c conda-forge
     > conda activate sdc_env
     > pip install --index-url https://pypi.anaconda.org/intel/label/beta/simple --extra-index-url https://pypi.anaconda.org/intel/simple --extra-index-url https://pypi.org/simple sdc
 

requirements.txt

@@ -1,5 +1,5 @@
 numpy>=1.16
-pandas==1.0.5
+pandas==1.2.0
 pyarrow==0.17.0
 numba==0.52.0
 tbb

sdc/datatypes/hpat_pandas_series_functions.py

@@ -776,7 +776,7 @@ def sdc_pandas_series_setitem_idx_int_series_align_impl(self, idx, value):
                     raise ValueError("Reindexing only valid with uniquely valued Index objects")
 
                 if len(valid_indices_masked) != idx_size:
-                    raise ValueError("Reindexing not possible: idx has index not found in Series")
+                    raise KeyError("Reindexing not possible: idx has index not found in Series")
 
                 if value_is_scalar == True:  # noqa
                     self._data[valid_indices_positions] = _value
@@ -810,7 +810,7 @@ def sdc_pandas_series_setitem_idx_str_series_align_impl(self, idx, value):
                         set_positions[i] = map_index_to_position[index_value]
 
                 if number_of_found != idx_data_size:
-                    raise ValueError("Reindexing not possible: idx has index not found in Series")
+                    raise KeyError("Reindexing not possible: idx has index not found in Series")
 
                 if value_is_series == True:  # noqa
                     self._data[set_positions] = value._data

sdc/tests/test_rolling.py

@@ -1149,7 +1149,6 @@ def test_impl(df, other, pairwise):
             hpat_func(df, other, True)
         self.assertIn(msg_tmpl.format('False, None'), str(raises.exception))
 
-    @unittest.expectedFailure
     def test_df_rolling_cov_issue_floating_point_rounding(self):
         """
             Cover issue of different float rounding in Python and SDC/Numba:

sdc/tests/test_series.py

@@ -2138,16 +2138,26 @@ def test_series_value_counts_numeric_dropna_false(self):
         def test_impl(S):
             return S.value_counts(dropna=False)
 
-        data_to_test = [[1, 2, 3, 1, 1, 3],
-                        [1, 2, 3, np.nan, 1, 3, np.nan, np.inf],
-                        [0.1, 3., np.nan, 3., 0.1, 3., np.nan, np.inf, 0.1, 0.1]]
+        data_to_test = [
+            [1, 2, 3, 1, 1, 3],
+            [1, 2, 3, np.nan, 1, 3, np.nan, np.inf],
+            [0.1, 3., np.nan, 3., 0.1, 3., np.nan, np.inf, 0.1, 0.1]
+        ]
 
         hpat_func = self.jit(test_impl)
 
         for data in data_to_test:
             with self.subTest(series_data=data):
                 S = pd.Series(data)
-                pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
+                result = hpat_func(S)
+                result_ref = test_impl(S)
+
+                # order within groups of same counts may be different since
+                # pandas impl uses sort_values() with default kind='quicksort'
+                pd.testing.assert_series_equal(
+                    result.sort_index(),
+                    result_ref.sort_index()
+                )
 
     def test_series_value_counts_str_dropna_false(self):
         def test_impl(S):
@@ -3933,7 +3943,10 @@ def test_impl_non_literal_kind(A, param_value):
             np.unique(np.random.randint(0, 100, n)),
             ['ac', 'c', 'cb', 'ca', None, 'da', 'cc', 'ddd', 'd']
         ]
-        kind_values = ['quicksort', 'mergesort']
+        kind_values = [
+            'quicksort',
+            'mergesort'
+            ]
         for data in data_to_test:
             S = pd.Series(data=data)
             for kind in kind_values:
@@ -4013,6 +4026,7 @@ def test_impl(series, ascending, kind):
                         np.testing.assert_array_equal(ref_result.data, jit_result.data)
                         self.assertEqual(ref, jit)
 
+    @skip_numba_jit("BUG: sort with kind=mergesort, ascending=False not stable for string data (SAT-3828)")
     def test_series_sort_values_full_unicode4(self):
         def test_impl(series, ascending, kind):
             return series.sort_values(axis=0, ascending=ascending, kind=literally(kind), na_position='last')
@@ -4023,17 +4037,17 @@ def test_impl(series, ascending, kind):
 
         for data in all_data:
             series = pd.Series(data * 3)
-            for ascending in [True, False]:
-                for kind in ['quicksort', 'mergesort']:
-                    ref_result = test_impl(series, ascending, kind=kind)
-                    jit_result = hpat_func(series, ascending, kind=kind)
-                    ref = restore_series_sort_values(series, ref_result.index, ascending)
-                    jit = restore_series_sort_values(series, jit_result.index, ascending)
+            for ascending, kind in product([True, False], ['quicksort', 'mergesort']):
+                with self.subTest(data=data, ascending=ascending, kind=kind):
+                    result = hpat_func(series, ascending, kind=kind)
+                    result_ref = test_impl(series, ascending, kind=kind)
                     if kind == 'mergesort':
-                        pd.testing.assert_series_equal(ref_result, jit_result)
+                        pd.testing.assert_series_equal(result, result_ref)
                     else:
-                        np.testing.assert_array_equal(ref_result.values, jit_result.values)
-                        self.assertEqual(ref, jit)
+                        np.testing.assert_array_equal(result.values, result_ref.values)
+                        jit = restore_series_sort_values(series, result.index, ascending)
+                        ref = restore_series_sort_values(series, result_ref.index, ascending)
+                        self.assertEqual(jit, ref)
 
     @skip_parallel
     def test_series_sort_values_full_idx(self):
@@ -5316,6 +5330,7 @@ def test_impl(A, i, value):
         test_impl(S2, idx, value)
         pd.testing.assert_series_equal(S1, S2)
 
+    @unittest.expectedFailure   # FIXME_Pandas#37427 (since pandas=1.1 setitem does diff things for diff dtypes)
     def test_series_setitem_idx_str_series(self):
         """ Verifies Series.setitem for idx operand of type pandas.Series and string dtype called on
             integer Series with index of matching dtype and scalar and non scalar assigned values """
@@ -5331,6 +5346,7 @@ def test_series_setitem_idx_str_series(self):
                           pd.Series(assigned_values)]
         self._test_series_setitem([series_data], [series_index], [idx], values_to_test, np.intp)
 
+    @unittest.expectedFailure   # FIXME_Pandas#37427 (since pandas=1.1 setitem does diff things for diff dtypes)
     def test_series_setitem_idx_float_series(self):
         """ Verifies Series.setitem for idx operand of type pandas.Series and float dtype called on
             integer Series with index of matching dtype and scalar and non scalar assigned values """

setup.py

@@ -375,7 +375,7 @@ def run(self):
       package_data={'sdc.tests': ['*.bz2'], },
       install_requires=[
           'numpy>=1.16',
-          'pandas>=1.0',
+          'pandas>=1.2.0',
           'pyarrow==0.17.0',
           'numba>=0.52.0,<0.53',
           'tbb'