Merge pull request #33 from IntelPython/master

Merge changes from SDC master

Author: Vyacheslav-Smirnov

.coveragerc

@@ -1,28 +1 @@
-# Compiled python modules.
-*.pyc
-
-# Setuptools distribution folder.
-/dist
-
-# cache
-hpat/__pycache__
-hpat/hiframes/__pycache__
-
-# Python egg metadata, regenerated from source files by setuptools.
-/*.egg-info
-/*.egg
-
-*.o
-*.so
-
 build/
-docs/_build/
-docs/gh-pages/
-*.DS_Store
-
-# data files for testing
-*.pq
-*.csv
-*.parquet
-*.hdf5
-tmp_tests/

.gitignore

@@ -4,9 +4,15 @@ Intel® Scalable Dataframe Compiler
 
 .. image:: https://travis-ci.com/IntelPython/sdc.svg?branch=master
     :target: https://travis-ci.com/IntelPython/sdc
+    :alt: Travis CI
+
+.. image:: https://dev.azure.com/IntelPython/HPAT/_apis/build/status/IntelPython.sdc?branchName=master
+    :target: https://dev.azure.com/IntelPython/HPAT/_build/latest?definitionId=2&branchName=master
+    :alt: Azure Pipelines
 
 .. image:: https://coveralls.io/repos/github/IntelPython/sdc/badge.svg?branch=master
     :target: https://coveralls.io/github/IntelPython/sdc?branch=master
+    :alt: Coveralls
 
 A compiler-based framework for big data in Python
 #################################################
@@ -162,6 +168,5 @@ Running unit tests
 ------------------
 ::
 
-    conda install h5py
     python sdc/tests/gen_test_data.py
     python -m unittest

README.rst

@@ -1,5 +1,4 @@
 echo on
 
-set HDF5_DIR="%LIBRARY_PREFIX%"
 "%PYTHON%" setup.py build install --single-version-externally-managed --record=record.txt
 if errorlevel 1 exit 1

buildscripts/sdc-conda-recipe/bld.bat

@@ -1,4 +1,4 @@
 set -ex
 
-HDF5_DIR="${PREFIX}" MACOSX_DEPLOYMENT_TARGET=10.9 \
+MACOSX_DEPLOYMENT_TARGET=10.9 \
 $PYTHON setup.py build install --single-version-externally-managed --record=record.txt

buildscripts/sdc-conda-recipe/build.sh

@@ -33,8 +33,6 @@ requirements:
     - pyarrow {{ PYARROW_VERSION }}
     - arrow-cpp {{ ARROW_CPP_VERSION }}
     - boost
-    - hdf5
-    - h5py
     - mpich                           # [not win]
     - impi_rt                         # [win]
     - impi-devel                      # [win]
@@ -55,7 +53,6 @@ requirements:
 
 test:
   requires:
-    - h5py
     - scipy
   imports:
     - sdc
@@ -78,8 +75,6 @@ outputs:
         - pyarrow {{ PYARROW_VERSION }}
         - arrow-cpp {{ PYARROW_VERSION }}
         - boost
-        - hdf5
-        - h5py
         - mpich                           # [not win]
         - impi_rt                         # [win]
         - impi-devel                      # [win]

buildscripts/sdc-conda-recipe/meta.yaml

@@ -108,17 +108,21 @@
 
 
     if run_coverage == 'True':
+        if platform.system() == 'Windows':
+            format_print('Coverage can be run only on Linux of mac for now')
+            sys.exit(0)
+
+        coverage_omit = './sdc/tests/*'
+        coverage_cmd = ' && '.join(['coverage erase',
+                                   f'coverage run --source=./sdc --omit {coverage_omit} ./sdc/runtests.py',
+                                   'coveralls -v'])
+
         format_print('Run coverage')
         format_print(f'Assume that SDC is installed in develop build-mode to {develop_env} environment', new_block=False)
         format_print('Install scipy and coveralls')
         run_command(f'{develop_env_activate} && conda install -q -y scipy coveralls')
-        os.environ['PYTHONPATH'] = '.'
-        os.environ['HDF5_DIR'] = conda_prefix
-        try:
-            run_command(f'{develop_env_activate} && python -m sdc.tests.gen_test_data && coverage erase && coverage run -m sdc.runtests && coveralls -v')
-        except:
-            format_print('Coverage fails')
-            print(traceback.format_exc())
+        run_command(f'{develop_env_activate} && python -m sdc.tests.gen_test_data')
+        run_command(f'{develop_env_activate} && {coverage_cmd}')
         sys.exit(0)
 
     if test_mode == 'develop':

buildscripts/test.py

@@ -2,72 +2,76 @@
 import matplotlib.pyplot as plt
 
 data_performance = {
-'read_parquet': ((0.804, 2.939, 6.235, 20.91), 
-	(0.51, 1.662, 3.151, 8.555)),
-'read_csv': ((19.732, 10.466, 6.277, 6.947)),
-'describe': ((0.823, 1.713, 3.324, 7.103), 
-	(0.704, 1.396, 2.89, 6.485)),
+'read_csv': ((10.324, 20.793, 41.413, 84.63), 
+    (0.914,  1.925, 3.988, 8.149)),
+'describe': ((0.374, 0.681, 1.426, 2.875), 
+    (0.235, 0.597, 1.192, 2.264)),
 'v_counts': ((0.867, 1.777, 3.447, 6.799), 
-	(0.699, 1.401, 2.914, 6.276)),
+    (0.699, 1.401, 2.914, 6.276)),
+'statistics': ((2.1, 4.6, 8.3, 20.2), 
+    (0.3, 0.7, 1.6, 3.2)),
+'sum': ((0.9, 1.2, 2.7, 5.9), 
+    (0.1, 0.4, 0.8, 1.9)),
 }
 
 plot_params = {
-	'read_parquet':(('1m', '2m', '4m', '8m'), 'Data size', 'Performance: Pandas vs SDC', True, 'upper left'),
-	'read_csv':(('1 node', '2 nodes', '4 nodes', '8 nodes'), 'Number of processes', 'SDC Scalability', False, 'upper right'),
-	'describe':(('1m', '2m', '4m', '8m'), 'Data size', 'Performance: Pandas vs SDC', True, 'upper left'),
-	'v_counts':(('1m', '2m', '4m', '8m'), 'Data size', 'Data size', 'Performance: Pandas vs SDC', True, 'upper left'),
+    'read_csv':(('1m', '2m', '4m', '8m'), 'Data size', 'Performance: Pandas vs SDC', True, 'upper left'),
+    'describe':(('1m', '2m', '4m', '8m'), 'Data size', 'Performance: Pandas vs SDC', True, 'upper left'),
+    'v_counts':(('1m', '2m', '4m', '8m'), 'Data size', 'Data size', 'Performance: Pandas vs SDC', True, 'upper left'),
+    'statistics':(('10m', '20m', '40m', '80m'), 'Data size', 'Performance: Pandas vs SDC', True, 'upper left'),
+    'sum':(('10m', '20m', '40m', '80m'), 'Data size', 'Performance: Pandas vs SDC', True, 'upper left'),
 }
 
 
 class Plotter:
-	def __init__(self, func_id='read_parquet'):
-		self.func_id = func_id
-		self.x_labels, self.x_title, self.title, self.is_compared, self.label_position = plot_params[self.func_id]
-		self.ngroups = len(self.x_labels)
+    def __init__(self, func_id='read_parquet'):
+        self.func_id = func_id
+        self.x_labels, self.x_title, self.title, self.is_compared, self.label_position = plot_params[self.func_id]
+        self.ngroups = len(self.x_labels)
 
-	def autolabel(self, rects, ax):
-		for rect in rects:
-			height = rect.get_height()
-			ax.annotate('{}'.format(height),
-				xy=(rect.get_x() + rect.get_width() / 2, height),
-				xytext=(0, 3),  # 3 points vertical offset
-				textcoords="offset points",
-				ha='center', va='bottom', fontsize=12)
+    def autolabel(self, rects, ax):
+        for rect in rects:
+            height = rect.get_height()
+            ax.annotate('{}'.format(height),
+                xy=(rect.get_x() + rect.get_width() / 2, height),
+                xytext=(0, 3), # 3 points vertical offset
+                textcoords="offset points",
+                ha='center', va='bottom', fontsize=12)
 
-	def plot_performance(self):
+    def plot_performance(self):
 
-		plt.figure(figsize = (16, 8))
-		# create plot
-		index = np.arange(self.ngroups)
-		bar_width = 0.35
-		opacity = 0.8
+        plt.figure(figsize = (16, 8))
+        # create plot
+        index = np.arange(self.ngroups)
+        bar_width = 0.35
+        opacity = 0.8
 
-		plt.xlabel(self.x_title, fontsize=16)
-		plt.ylabel('Time, s', fontsize=16)
-		plt.title(self.title, fontsize=18)
-		
-		plt.tick_params(labelsize=12)
+        plt.xlabel(self.x_title, fontsize=16)
+        plt.ylabel('Time, s', fontsize=16)
+        plt.title(self.title, fontsize=18)
+        
+        plt.tick_params(labelsize=12)
 
-		if self.is_compared:
-			data_pandas, data_sdc = data_performance[self.func_id]
-			rects_pandas = plt.bar(index + bar_width, data_pandas, bar_width,
-			alpha=opacity,
-			label='Pandas')
+        if self.is_compared:
+            data_pandas, data_sdc = data_performance[self.func_id]
+            rects_pandas = plt.bar(index + bar_width, data_pandas, bar_width,
+            alpha=opacity,
+            label='Pandas')
 
-			plt.xticks(index + bar_width, self.x_labels)
-		else:
-			data_sdc = data_performance[self.func_id]
-			plt.xticks(index, self.x_labels)
+            plt.xticks(index + bar_width, self.x_labels)
+        else:
+            data_sdc = data_performance[self.func_id]
+            plt.xticks(index, self.x_labels)
 
-		rects_sdc = plt.bar(index, data_sdc, bar_width,
-		alpha=opacity,
-		label='SDC')
+        rects_sdc = plt.bar(index, data_sdc, bar_width,
+        alpha=opacity,
+        label='SDC')
 
-		if self.is_compared:
-			self.autolabel(rects_pandas, plt)
+        if self.is_compared:
+            self.autolabel(rects_pandas, plt)
 
-		plt.legend(fontsize=16, loc=self.label_position)
-		self.autolabel(rects_sdc, plt)
+        plt.legend(fontsize=16, loc=self.label_position)
+        self.autolabel(rects_sdc, plt)
 
-		plt.tight_layout()
-		plt.show()
+        plt.tight_layout()
+        plt.show()