[Bert/PyT] Replace conda install of jemalloc with apt package manager

Author: jojennin

PyTorch/LanguageModeling/BERT/Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get install -y iputils-ping
 COPY . .
 
 # Install lddl
-RUN conda install -y jemalloc
+RUN apt-get install -y libjemalloc-dev
 RUN pip install git+https://github.com/NVIDIA/lddl.git
 RUN python -m nltk.downloader punkt
 

PyTorch/LanguageModeling/BERT/run.sub

@@ -241,7 +241,7 @@ if [ ! -d "${host_pretrain_parquet}" ] || [ -z "$(ls -A "${host_pretrain_parquet
   fi
   # Should we use jemalloc for the LDDL preprocessor?
   if [ "${USE_JEMALLOC}" == "true" ]; then
-    readonly use_jemalloc_flag="--export=ALL,LD_PRELOAD=/opt/conda/lib/libjemalloc.so"
+    readonly use_jemalloc_flag="--export=ALL,LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so"
   elif [ "${USE_JEMALLOC}" == "false" ]; then
     readonly use_jemalloc_flag=""
   else

PyTorch/LanguageModeling/BERT/scripts/run_pretraining.sh

@@ -90,7 +90,7 @@ if [ ! -d "${DATA_DIR_PHASE1}" ] || [ -z "$(ls -A ${DATA_DIR_PHASE1})" ]; then
          --oversubscribe \
          --allow-run-as-root \
          -np ${num_dask_workers} \
-         -x LD_PRELOAD=/opt/conda/lib/libjemalloc.so \
+         -x LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so \
             preprocess_bert_pretrain \
                --schedule mpi \
                --vocab-file ${VOCAB_FILE} \
@@ -255,7 +255,7 @@ if [ ! -d "${DATA_DIR_PHASE2}" ] || [ -z "$(ls -A ${DATA_DIR_PHASE2})" ]; then
          --oversubscribe \
          --allow-run-as-root \
          -np ${num_dask_workers} \
-         -x LD_PRELOAD=/opt/conda/lib/libjemalloc.so \
+         -x LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so \
             preprocess_bert_pretrain \
                --schedule mpi \
                --vocab-file ${VOCAB_FILE} \