Segfault using make_dataset (issue with importlib)

[arachid1]

Hello, I'm dealing with a segfault when I run:

python /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py --list_file audio/training_list.txt --featurizer compiled_featurizer/mfcc --window_size 40 --shift 40
Transforming training_list.txt with window size=40 and shift=40
Segmentation fault: 11

I used pdb and traced the segmentation fault in the following sequence of files:

make_dataset.py (line 162)

transform = featurizer.AudioTransform(featurizer_path, 0)

then, in featurizer.py (line 35)

if os.path.splitext(model_path)[1] == ".ell":
            import compute_ell_model as ell
            self.model = ell.ComputeModel(model_path)
            self.using_map = True
        else:
            import compiled_ell_model as ell
            self.model = ell.CompiledModel(model_path) # <- this line 

then in compilled_ell_model.py (line 29)

self.module = importlib.import_module(model_name)

This is the output i get with pdb:

alirachidi:BPS alirachidi$ python -m pdb /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py --list_file audio/training_list.txt --featurizer compiled_featurizer/mfcc --window_size 40 --shift40
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py(13)<module>()
-> """
(Pdb) break /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py:163
Breakpoint 1 at /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py:163
(Pdb) continue
Transforming training_list.txt with window size=40 and shift=40
Segmentation fault: 11
alirachidi:BPS alirachidi$ python -m pdb /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py --list_file audio/training_list.txt --featurizer compiled_featurizer/mfcc --window_size 40 --shift40
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py(13)<module>()
-> """
(Pdb) break /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py:162
Breakpoint 1 at /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py:162
(Pdb) continue
Transforming training_list.txt with window size=40 and shift=40
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py(162)_get_dataset()
-> transform = featurizer.AudioTransform(featurizer_path, 0)
(Pdb) step
--Call--
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/featurizer.py(19)__init__()
-> def __init__(self, model_path, output_window_size):
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/featurizer.py(28)__init__()
-> self.using_map = False
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/featurizer.py(29)__init__()
-> if os.path.splitext(model_path)[1] == ".ell":
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/featurizer.py(34)__init__()
-> import compiled_ell_model as ell
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/featurizer.py(35)__init__()
-> self.model = ell.CompiledModel(model_path)
(Pdb) step
--Call--
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/compiled_ell_model.py(17)__init__()
-> def __init__(self, model_path):
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/compiled_ell_model.py(18)__init__()
-> self.module = None
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/compiled_ell_model.py(19)__init__()
-> parent_dir = os.path.dirname(model_path)
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/compiled_ell_model.py(20)__init__()
-> if not os.path.isdir(parent_dir):
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/compiled_ell_model.py(23)__init__()
-> sys.path += [parent_dir]
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/compiled_ell_model.py(24)__init__()
-> sys.path += [os.path.join(parent_dir, "build")]
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/compiled_ell_model.py(25)__init__()
-> sys.path += [os.path.join(parent_dir, "build", "release")]
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/compiled_ell_model.py(26)__init__()
-> sys.path += [os.path.join(parent_dir, "build", "debug")]
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/compiled_ell_model.py(28)__init__()
-> model_name = os.path.basename(model_path)
(Pdb) step
--Call--
> /opt/anaconda3/envs/KWS/lib/python3.6/posixpath.py(144)basename()
-> def basename(p):
(Pdb) next
> /opt/anaconda3/envs/KWS/lib/python3.6/posixpath.py(146)basename()
-> p = os.fspath(p)
(Pdb) next
> /opt/anaconda3/envs/KWS/lib/python3.6/posixpath.py(147)basename()
-> sep = _get_sep(p)
(Pdb) next
> /opt/anaconda3/envs/KWS/lib/python3.6/posixpath.py(148)basename()
-> i = p.rfind(sep) + 1
(Pdb) next
> /opt/anaconda3/envs/KWS/lib/python3.6/posixpath.py(149)basename()
-> return p[i:]
(Pdb) next
--Return--
> /opt/anaconda3/envs/KWS/lib/python3.6/posixpath.py(149)basename()->'mfcc'
-> return p[i:]
(Pdb) next
> /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/compiled_ell_model.py(29)__init__()
-> self.module = importlib.import_module(model_name)
(Pdb) step
--Call--
> /opt/anaconda3/envs/KWS/lib/python3.6/importlib/__init__.py(108)import_module()
-> def import_module(name, package=None):
(Pdb) step
> /opt/anaconda3/envs/KWS/lib/python3.6/importlib/__init__.py(116)import_module()
-> level = 0
(Pdb) step
> /opt/anaconda3/envs/KWS/lib/python3.6/importlib/__init__.py(117)import_module()
-> if name.startswith('.'):
(Pdb) next
> /opt/anaconda3/envs/KWS/lib/python3.6/importlib/__init__.py(126)import_module()
-> return _bootstrap._gcd_import(name[level:], package, level)
(Pdb) next
Segmentation fault: 11

I attempted another run with pdb earlier and it seems to fail at the load_unlocked function.

For now, I'm passing the .ell file (to enter the first if statement as you can see in the third piece of code) and it doesn't work. The code holds and doesn't generate a npz file:

alirachidi:BPS alirachidi$ python /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py --list_file audio/training_list.txt --featurizer ./featurizer.ell --window_size 40 --shift 40
Transforming training_list.txt with window size=40 and shift=40
Featurizer requires auto-scaling of audio input to float range [-1, 1]
Transforming 1340 files from bed

I tried updating my importlib but that doesn't solve it. Not exactly sure what the issue is.

Thanks in advance!

[arachid1]

Report with -Xfaulthandler:

alirachidi:BPS alirachidi$ python -Xfaulthandler /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py --list_file audio/training_list.txt --featurizer compiled_featurizer/mfcc --window_size 40--shift 40
Transforming training_list.txt with window size=40 and shift=40
Fatal Python error: Segmentation fault

Current thread 0x0000000113246dc0 (most recent call first):
  File "<frozen importlib._bootstrap>", line 219 in _call_with_frames_removed
  File "<frozen importlib._bootstrap_external>", line 922 in create_module
  File "<frozen importlib._bootstrap>", line 571 in module_from_spec
  File "<frozen importlib._bootstrap>", line 658 in _load_unlocked
  File "<frozen importlib._bootstrap>", line 955 in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 971 in _find_and_load
  File "compiled_featurizer/mfcc.py", line 11 in <module>
  File "<frozen importlib._bootstrap>", line 219 in _call_with_frames_removed
  File "<frozen importlib._bootstrap_external>", line 678 in exec_module
  File "<frozen importlib._bootstrap>", line 665 in _load_unlocked
  File "<frozen importlib._bootstrap>", line 955 in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 971 in _find_and_load
  File "<frozen importlib._bootstrap>", line 994 in _gcd_import
  File "/opt/anaconda3/envs/KWS/lib/python3.6/importlib/__init__.py", line 126 in import_module
  File "/Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/../compiled_ell_model.py", line 29 in __init__
  File "/Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/../featurizer.py", line 35 in __init__
  File "/Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py", line 162 in _get_dataset
  File "/Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py", line 249 in make_dataset
  File "/Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tools/utilities/pythonlibs/audio/training/make_dataset.py", line 283 in <module>
Segmentation fault: 11