Skip to content
This repository was archived by the owner on Feb 2, 2024. It is now read-only.

Commit a0d6c0f

Browse files
author
Ehsan Totoni
committed
support Global and FreeVar in parquet file
1 parent 552628a commit a0d6c0f

2 files changed

Lines changed: 28 additions & 1 deletion

File tree

hpat/io/parquet_pio.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,8 @@ def gen_parquet_read(self, file_name, lhs):
9090

9191
if table_types is None:
9292
fname_def = guard(get_definition, self.func_ir, file_name)
93-
if not isinstance(fname_def, ir.Const) or not isinstance(fname_def.value, str):
93+
if (not isinstance(fname_def, (ir.Const, ir.Global, ir.FreeVar))
94+
or not isinstance(fname_def.value, str)):
9495
raise ValueError("Parquet schema not available")
9596
file_name_str = fname_def.value
9697
col_names, col_types = parquet_file_schema(file_name_str)

hpat/tests/test_io.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010
get_start_end)
1111

1212

13+
kde_file = 'kde.parquet'
14+
15+
1316
class TestIO(unittest.TestCase):
1417

1518
def setUp(self):
@@ -190,6 +193,29 @@ def test_impl():
190193
self.assertEqual(count_array_REPs(), 0)
191194
self.assertEqual(count_parfor_REPs(), 0)
192195

196+
def test_pq_read_global_str1(self):
197+
def test_impl():
198+
df = pd.read_parquet(kde_file)
199+
X = df['points']
200+
return X.sum()
201+
202+
hpat_func = hpat.jit(test_impl)
203+
np.testing.assert_almost_equal(hpat_func(), test_impl())
204+
self.assertEqual(count_array_REPs(), 0)
205+
self.assertEqual(count_parfor_REPs(), 0)
206+
207+
def test_pq_read_freevar_str1(self):
208+
kde_file2 = 'kde.parquet'
209+
def test_impl():
210+
df = pd.read_parquet(kde_file2)
211+
X = df['points']
212+
return X.sum()
213+
214+
hpat_func = hpat.jit(test_impl)
215+
np.testing.assert_almost_equal(hpat_func(), test_impl())
216+
self.assertEqual(count_array_REPs(), 0)
217+
self.assertEqual(count_parfor_REPs(), 0)
218+
193219
def test_pd_read_parquet(self):
194220
def test_impl():
195221
df = pd.read_parquet('kde.parquet')

0 commit comments

Comments
 (0)