Fix segfault in NRT_dealloc for str_arr getitem (#955)

* Fix segfault in NRT_dealloc for str_arr getitem

Motivation: legacy SDC implementation copied Numba internals (MemInfo
struct and API functions), which if changed on the Numba side (as it's done
in IntelPython/Numba where external_allocator is added) may cause segfaults.
This PR removes duplication of Numba internals and moves hstr extension to
NRT API functions

* Updating conda-recipe with build requirement on Numba

Author: kozlov-alexey

conda-recipe/meta.yaml

@@ -20,6 +20,7 @@ requirements:
     - {{ compiler('cxx') }}    # [not osx]
     - wheel
     - python
+    - numba   {{ NUMBA_VERSION }}
 
   host:
     - python

sdc/_meminfo.h

@@ -26,8 +26,9 @@
 
 #include <Python.h>
 #include <iostream>
+#include <stdlib.h>
 
-#include "_meminfo.h"
+#include "numba/core/runtime/nrt_external.h"
 
 #ifndef Py_UNREACHABLE
 #define Py_UNREACHABLE() abort()
@@ -37,6 +38,7 @@
 
 typedef struct
 {
+    NRT_api_functions* nrt;
     NRT_MemInfo* buffer;
     void* data;
     enum PyUnicode_Kind kind;
@@ -120,20 +122,28 @@ void _C_UnicodeWriter_Init(_C_UnicodeWriter* writer)
 #include "stringlib/undef.h"
 
 static inline int _C_UnicodeWriter_WriteCharInline(_C_UnicodeWriter* writer, Py_UCS4 ch);
-static int _copy_characters(NRT_MemInfo* to,
+static int _copy_characters(NRT_api_functions* nrt,
+                            NRT_MemInfo* to,
                             Py_ssize_t to_start,
                             NRT_MemInfo* from,
                             Py_ssize_t from_start,
                             Py_ssize_t how_many,
                             unsigned int from_kind,
                             unsigned int to_kind);
 
+
+static void str_data_dtor(void* data_ptr)
+{
+    free(data_ptr);
+}
+
 // similar to PyUnicode_New()
 NRT_MemInfo* alloc_writer(_C_UnicodeWriter* writer, Py_ssize_t newlen, Py_UCS4 maxchar)
 {
     enum PyUnicode_Kind kind;
     int is_ascii = 0;
     Py_ssize_t char_size;
+    auto nrt = writer->nrt;
 
     if (maxchar < 128)
     {
@@ -161,20 +171,22 @@ NRT_MemInfo* alloc_writer(_C_UnicodeWriter* writer, Py_ssize_t newlen, Py_UCS4 m
         kind = PyUnicode_4BYTE_KIND;
         char_size = 4;
     }
-    NRT_MemInfo* newbuffer = NRT_MemInfo_alloc_safe((newlen + 1) * char_size);
-    if (newbuffer == NULL)
+
+    char* str_data = (char*)malloc((newlen + 1) * char_size);
+    if (str_data == NULL)
     {
         return NULL;
     }
 
+    auto newbuffer = nrt->manage_memory(str_data, str_data_dtor);
     if (writer->buffer != NULL)
     {
-        _copy_characters(newbuffer, 0, writer->buffer, 0, writer->pos, writer->kind, kind);
-        NRT_MemInfo_call_dtor(writer->buffer);
+        _copy_characters(nrt, newbuffer, 0, writer->buffer, 0, writer->pos, writer->kind, kind);
+        nrt->release(writer->buffer);
     }
     writer->buffer = newbuffer;
     writer->maxchar = KIND_MAX_CHAR_VALUE(kind);
-    writer->data = writer->buffer->data;
+    writer->data = nrt->get_data(writer->buffer);
 
     if (!writer->readonly)
     {
@@ -356,19 +368,22 @@ static Py_ssize_t ascii_decode(const char* start, const char* end, Py_UCS1* dest
     return p - start;
 }
 
+
 // ported from CPython PyUnicode_DecodeUTF8Stateful: https://github.com/python/cpython/blob/31e8d69bfe7cf5d4ffe0967cb225d2a8a229cc97/Objects/unicodeobject.c#L4813
-void decode_utf8(const char* s, Py_ssize_t size, int* kind, int* is_ascii, int* length, NRT_MemInfo** meminfo)
+void decode_utf8(const char* s, Py_ssize_t size, int* kind, int* is_ascii, int* length, NRT_MemInfo** meminfo, void* nrt_table)
 {
     _C_UnicodeWriter writer;
     const char* end = s + size;
+    auto nrt = (NRT_api_functions*)nrt_table;
 
     const char* errmsg = "";
     *is_ascii = 0;
 
     if (size == 0)
     {
-        (*meminfo) = NRT_MemInfo_alloc_safe(1);
-        ((char*)((*meminfo)->data))[0] = 0;
+        char* str_data = (char*)malloc(1);
+        (*meminfo) = nrt->manage_memory(str_data, str_data_dtor);
+        ((char*)(nrt->get_data(*meminfo)))[0] = 0;
         *kind = PyUnicode_1BYTE_KIND;
         *is_ascii = 1;
         *length = 0;
@@ -379,9 +394,10 @@ void decode_utf8(const char* s, Py_ssize_t size, int* kind, int* is_ascii, int*
     if (size == 1 && (unsigned char)s[0] < 128)
     {
         // TODO interning
-        (*meminfo) = NRT_MemInfo_alloc_safe(2);
-        ((char*)((*meminfo)->data))[0] = s[0];
-        ((char*)((*meminfo)->data))[1] = 0;
+        char* str_data = (char*)malloc(2);
+        (*meminfo) = nrt->manage_memory(str_data, str_data_dtor);
+        ((char*)(nrt->get_data(*meminfo)))[0] = s[0];
+        ((char*)(nrt->get_data(*meminfo)))[1] = 0;
         *kind = PyUnicode_1BYTE_KIND;
         *is_ascii = 1;
         *length = 1;
@@ -390,6 +406,7 @@ void decode_utf8(const char* s, Py_ssize_t size, int* kind, int* is_ascii, int*
 
     _C_UnicodeWriter_Init(&writer);
     writer.min_length = size;
+    writer.nrt = nrt;
     if (_C_UnicodeWriter_Prepare(&writer, writer.min_length, 127) == -1)
         goto onError;
 
@@ -469,7 +486,7 @@ void decode_utf8(const char* s, Py_ssize_t size, int* kind, int* is_ascii, int*
 
 onError:
     std::cerr << "utf8 decode error:" << errmsg << std::endl;
-    NRT_MemInfo_call_dtor(writer.buffer);
+    nrt->release(*meminfo);
     return;
 }
 
@@ -499,7 +516,8 @@ void decode_utf8(const char* s, Py_ssize_t size, int* kind, int* is_ascii, int*
             *_to++ = (to_type)*_iter++;                                                                                \
     } while (0)
 
-static int _copy_characters(NRT_MemInfo* to,
+static int _copy_characters(NRT_api_functions* nrt,
+                            NRT_MemInfo* to,
                             Py_ssize_t to_start,
                             NRT_MemInfo* from,
                             Py_ssize_t from_start,
@@ -516,8 +534,8 @@ static int _copy_characters(NRT_MemInfo* to,
     if (how_many == 0)
         return 0;
 
-    from_data = from->data;
-    to_data = to->data;
+    from_data = nrt->get_data(from);
+    to_data = nrt->get_data(to);
 
     if (from_kind == to_kind)
     {

sdc/_str_decode.cpp

@@ -1026,6 +1026,7 @@ def str_arr_getitem_by_array_impl(A, arg):
 def decode_utf8(typingctx, ptr_t, len_t=None):
     def codegen(context, builder, sig, args):
         ptr, length = args
+        nrt_table = context.nrt.get_nrt_api(builder)
 
         # create str and call decode with internal pointers
         uni_str = cgutils.create_struct_proxy(string_type)(context, builder)
@@ -1034,14 +1035,16 @@ def codegen(context, builder, sig, args):
                                                  lir.IntType(32).as_pointer(),
                                                  lir.IntType(32).as_pointer(),
                                                  lir.IntType(64).as_pointer(),
-                                                 uni_str.meminfo.type.as_pointer()])
+                                                 uni_str.meminfo.type.as_pointer(),
+                                                 lir.IntType(8).as_pointer()])
         fn_decode = builder.module.get_or_insert_function(
             fnty, name="decode_utf8")
         builder.call(fn_decode, [ptr, length,
                                  uni_str._get_ptr_by_name('kind'),
                                  uni_str._get_ptr_by_name('is_ascii'),
                                  uni_str._get_ptr_by_name('length'),
-                                 uni_str._get_ptr_by_name('meminfo')])
+                                 uni_str._get_ptr_by_name('meminfo'),
+                                 nrt_table])
         uni_str.hash = context.get_constant(_Py_hash_t, -1)
         uni_str.data = context.nrt.meminfo_data(builder, uni_str.meminfo)
         # Set parent to NULL

sdc/str_arr_ext.py

@@ -29,6 +29,7 @@
 import platform
 import os
 import sys
+import numba
 from docs.source.buildscripts.sdc_build_doc import SDCBuildDoc
 
 
@@ -185,14 +186,15 @@ def check_file_at_path(path2file):
                      )
 
 str_libs = np_compile_args['libraries']
+numba_include_path = numba.extending.include_path()
 
 ext_str = Extension(name="sdc.hstr_ext",
                     sources=["sdc/_str_ext.cpp"],
                     libraries=str_libs,
                     define_macros=np_compile_args['define_macros'],
                     extra_compile_args=eca,
                     extra_link_args=ela,
-                    include_dirs=np_compile_args['include_dirs'] + ind,
+                    include_dirs=np_compile_args['include_dirs'] + ind + [numba_include_path],
                     library_dirs=np_compile_args['library_dirs'] + lid,
                     )