Update script version for Watson Discovery 4.7.0 on CP4D

Author: KosukeOkamoto

discovery-data/latest/all-backup-restore.sh

@@ -51,13 +51,13 @@ rm -rf ${TMP_WORK_DIR}
 mkdir -p ${TMP_WORK_DIR}
 mkdir -p ${BACKUP_RESTORE_LOG_DIR}
 
-wd_version=${WD_VERSION:-`get_version`}
+wd_version=${WD_VERSION:-$(get_version)}
 
 setup_etcd_env
 
 # backup etcd
 if [ ${COMMAND} = 'backup' ] ; then
-  BACKUP_FILE=${BACKUP_FILE:-"etcd_snapshot_`date "+%Y%m%d_%H%M%S"`.db"}
+  BACKUP_FILE=${BACKUP_FILE:-"etcd_snapshot_$(date "+%Y%m%d_%H%M%S").db"}
   brlog "INFO" "Start backup etcd..."
   run_cmd_in_pod ${ETCD_POD} "rm -rf ${ETCD_BACKUP_DIR} ${ETCD_BACKUP} && \
   mkdir -p ${ETCD_BACKUP_DIR} && \
@@ -70,19 +70,19 @@ if [ ${COMMAND} = 'backup' ] ; then
 
   if "${ARCHIVE_ON_LOCAL}" ; then 
     brlog "INFO" "Transferring backup files"
-    mkdir -p "`dirname ${TMP_WORK_DIR}${ETCD_BACKUP_DIR}`"
+    mkdir -p "$(dirname ${TMP_WORK_DIR}${ETCD_BACKUP_DIR})"
     kube_cp_to_local -r ${ETCD_POD} "${TMP_WORK_DIR}${ETCD_BACKUP_DIR}" "${ETCD_BACKUP_DIR}" ${OC_ARGS}
     oc ${OC_ARGS} exec ${ETCD_POD} -- bash -c "rm -rf ${ETCD_BACKUP_DIR}"
     brlog "INFO" "Archiving data"
-    tar ${ETCD_TAR_OPTIONS[@]} -cf "${BACKUP_FILE}" -C "${TMP_WORK_DIR}${ETCD_BACKUP_DIR}" .
+    tar "${ETCD_TAR_OPTIONS[@]}" -cf "${BACKUP_FILE}" -C "${TMP_WORK_DIR}${ETCD_BACKUP_DIR}" .
   else
     brlog "INFO" "Archiving data..."
     run_cmd_in_pod ${ETCD_POD} "tar ${ETCD_ARCHIVE_OPTION} -cf ${ETCD_BACKUP} -C ${ETCD_BACKUP_DIR} ." ${OC_ARGS}
     brlog "INFO" "Trasnfering archive..."
     kube_cp_to_local ${ETCD_POD} "${BACKUP_FILE}" "${ETCD_BACKUP}" ${OC_ARGS}
     oc ${OC_ARGS} exec ${ETCD_POD} --  bash -c "rm -rf ${ETCD_BACKUP_DIR} ${ETCD_BACKUP}"
   fi
-  if "${VERIFY_DATASTORE_ARCHIVE}" && brlog "INFO" "Verifying backup archive" && ! tar ${ETCD_TAR_OPTIONS[@]} -tf ${BACKUP_FILE} &> /dev/null ; then
+  if "${VERIFY_DATASTORE_ARCHIVE}" && brlog "INFO" "Verifying backup archive" && ! tar "${ETCD_TAR_OPTIONS[@]}" -tf ${BACKUP_FILE} &> /dev/null ; then
     brlog "ERROR" "Backup file is broken, or does not exist."
     exit 1
   fi
@@ -105,7 +105,7 @@ if [ ${COMMAND} = 'restore' ] ; then
   if "${ARCHIVE_ON_LOCAL}" ; then
     brlog "INFO" "Extracting archive"
     mkdir -p "${TMP_WORK_DIR}${ETCD_BACKUP_DIR}"
-    tar ${ETCD_TAR_OPTIONS[@]} -xf ${BACKUP_FILE} -C "${TMP_WORK_DIR}${ETCD_BACKUP_DIR}"
+    tar "${ETCD_TAR_OPTIONS[@]}" -xf ${BACKUP_FILE} -C "${TMP_WORK_DIR}${ETCD_BACKUP_DIR}"
     brlog "INFO" "Transferring backup files"
     kube_cp_from_local -r ${ETCD_POD} "${TMP_WORK_DIR}${ETCD_BACKUP_DIR}" "${ETCD_BACKUP_DIR}" ${OC_ARGS}
   else

discovery-data/latest/elastic-backup-restore.sh

@@ -11,13 +11,12 @@ OC_ARGS="${OC_ARGS:-}"
 MINIO_BACKUP="minio_backup.tar.gz"
 MINIO_BACKUP_DIR="${MINIO_BACKUP_DIR:-minio_backup}"
 TMP_WORK_DIR="tmp/minio_workspace"
-MINIO_JOB_FILE="${SCRIPT_DIR}/src/minio-backup-restore-job.yml"
 BACKUP_RESTORE_IN_POD=${BACKUP_RESTORE_IN_POD-false}
 CURRENT_COMPONENT="minio"
 MINIO_ELASTIC_BACKUP=${MINIO_ELASTIC_BACKUP:-false}
 DISABLE_MC_MULTIPART=${DISABLE_MC_MULTIPART:-true}
 ELASTIC_BACKUP_BUCKET="elastic-backup"
-SED_REG_OPT="`get_sed_reg_opt`"
+SED_REG_OPT="$(get_sed_reg_opt)"
 SCRIPT_DIR=${SCRIPT_DIR}
 
 printUsage() {
@@ -44,7 +43,7 @@ done
 brlog "INFO" "MinIO:"
 brlog "INFO" "Tenant name: $TENANT_NAME"
 
-BACKUP_FILE=${BACKUP_FILE:-"minio_`date "+%Y%m%d_%H%M%S"`.tar.gz"}
+BACKUP_FILE=${BACKUP_FILE:-"minio_$(date "+%Y%m%d_%H%M%S").tar.gz"}
 
 MINIO_ARCHIVE_OPTION="${MINIO_ARCHIVE_OPTION-$DATASTORE_ARCHIVE_OPTION}"
 if [ -n "${MINIO_ARCHIVE_OPTION}" ] ; then
@@ -55,7 +54,7 @@ fi
 VERIFY_ARCHIVE=${VERIFY_ARCHIVE:-true}
 VERIFY_DATASTORE_ARCHIVE=${VERIFY_DATASTORE_ARCHIVE:-$VERIFY_ARCHIVE}
 
-setup_minio_env
+setup_s3_env
 
 rm -rf ${TMP_WORK_DIR}
 
@@ -64,7 +63,7 @@ mkdir -p "${BACKUP_RESTORE_LOG_DIR}"
 
 if "${BACKUP_RESTORE_IN_POD}" ; then
   BACKUP_RESTORE_DIR_IN_POD="/tmp/backup-restore-workspace"
-  launch_minio_pod
+  launch_s3_pod
   _oc_cp "${SCRIPT_DIR}/src" ${POD}:${BACKUP_RESTORE_DIR_IN_POD}/ ${OC_ARGS}
   _oc_cp "${SCRIPT_DIR}/lib" ${POD}:${BACKUP_RESTORE_DIR_IN_POD}/ ${OC_ARGS}
   _oc_cp "${SCRIPT_DIR}/src/minio-backup-restore-in-pod.sh" "${POD}:${BACKUP_RESTORE_DIR_IN_POD}/run.sh" ${OC_ARGS}
@@ -89,14 +88,14 @@ if "${BACKUP_RESTORE_IN_POD}" ; then
   if [ "${COMMAND}" = "backup" ] ; then
     brlog "INFO" "Transferring backup data"
     kube_cp_to_local ${POD} "${BACKUP_FILE}" "${BACKUP_RESTORE_DIR_IN_POD}/${MINIO_BACKUP}" ${OC_ARGS}
-    if "${VERIFY_DATASTORE_ARCHIVE}" && brlog "INFO" "Verifying backup archive" && ! tar ${MINIO_TAR_OPTIONS[@]} -tf ${BACKUP_FILE} &> /dev/null ; then
+    if "${VERIFY_DATASTORE_ARCHIVE}" && brlog "INFO" "Verifying backup archive" && ! tar "${MINIO_TAR_OPTIONS[@]}" -tf ${BACKUP_FILE} &> /dev/null ; then
       brlog "ERROR" "Backup file is broken, or does not exist."
       oc ${OC_ARGS} exec ${POD} -- bash -c "cd ${BACKUP_RESTORE_DIR_IN_POD}; ls | xargs rm -rf"
       exit 1
     fi
   fi
   oc ${OC_ARGS} exec ${POD} -- bash -c "cd ${BACKUP_RESTORE_DIR_IN_POD}; ls | xargs rm -rf"
-  oc ${OC_ARGS} delete -f "${MINIO_JOB_FILE}"
+  oc ${OC_ARGS} delete -f "${S3_JOB_FILE}"
   rm -rf ${TMP_WORK_DIR}
   if [ -z "$(ls tmp)" ] ; then
     rm -rf tmp
@@ -114,32 +113,35 @@ fi
 export MINIO_CONFIG_DIR="${PWD}/${TMP_WORK_DIR}/.mc"
 MC_OPTS=(--config-dir ${MINIO_CONFIG_DIR} --insecure)
 
+BUCKET_SUFFIX="$(get_bucket_suffix)"
+
 # backup
 if [ "${COMMAND}" = "backup" ] ; then
   brlog "INFO" "Start backup minio"
   brlog "INFO" "Backup data..."
   start_minio_port_forward
-  ${MC} ${MC_OPTS[@]} --quiet config host add wdminio ${MINIO_ENDPOINT_URL} ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} > /dev/null
-  EXCLUDE_OBJECTS=`cat "${SCRIPT_DIR}/src/minio_exclude_paths"`
-  for bucket in `${MC} ${MC_OPTS[@]} ls wdminio | sed ${SED_REG_OPT} "s|.*[0-9]+B\ (.*)/.*|\1|g" | grep -v ${ELASTIC_BACKUP_BUCKET}`
+  ${MC} "${MC_OPTS[@]}" --quiet config host add wdminio ${S3_ENDPOINT_URL} ${S3_ACCESS_KEY} ${S3_SECRET_KEY} > /dev/null
+  EXCLUDE_OBJECTS=$(cat "${SCRIPT_DIR}/src/minio_exclude_paths")
+  for bucket in $(${MC} "${MC_OPTS[@]}" ls wdminio | sed ${SED_REG_OPT} "s|.*[0-9]+B\ (.*)/.*|\1|g" | grep -v ${ELASTIC_BACKUP_BUCKET})
   do
     EXTRA_MC_MIRROR_COMMAND=""
     ORG_IFS=${IFS}
     IFS=$'\n'
     for line in ${EXCLUDE_OBJECTS}
     do
-      if [[ ${line} == ${bucket}* ]] ; then
-        if [ "${line#$bucket }" = "*" ] ; then
+      base_bucket_name=${bucket%"${BUCKET_SUFFIX}"}
+      if [[ ${line} == ${base_bucket_name}* ]] ; then
+        if [ "${line#"$base_bucket_name" }" = "*" ] ; then
           brlog "DEBUG" "SKIP ${bucket}"
           continue 2
         fi
-        EXTRA_MC_MIRROR_COMMAND="--exclude ${line#$bucket } ${EXTRA_MC_MIRROR_COMMAND}"
+        EXTRA_MC_MIRROR_COMMAND="--exclude ${line#"${base_bucket_name}" } ${EXTRA_MC_MIRROR_COMMAND}"
       fi
     done
     IFS=${ORG_IFS}
     cd ${TMP_WORK_DIR}
     set +e
-    ${MC} ${MC_OPTS[@]} --quiet mirror ${EXTRA_MC_MIRROR_COMMAND} wdminio/${bucket} ${MINIO_BACKUP_DIR}/${bucket} &>> "${SCRIPT_DIR}/${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
+    ${MC} "${MC_OPTS[@]}" --quiet mirror ${EXTRA_MC_MIRROR_COMMAND} wdminio/${bucket} ${MINIO_BACKUP_DIR}/${bucket} &>> "${SCRIPT_DIR}/${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
     RC=$?
     echo "RC=${RC}" >> "${SCRIPT_DIR}/${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
     if [ $RC -ne 0 ] ; then
@@ -151,8 +153,8 @@ if [ "${COMMAND}" = "backup" ] ; then
   done
   stop_minio_port_forward
   brlog "INFO" "Archiving data..."
-  tar ${MINIO_TAR_OPTIONS[@]} -cf ${BACKUP_FILE} -C ${TMP_WORK_DIR}/${MINIO_BACKUP_DIR} .
-  if "${VERIFY_DATASTORE_ARCHIVE}" && brlog "INFO" "Verifying backup archive" && ! tar ${MINIO_TAR_OPTIONS[@]} -tf ${BACKUP_FILE} &> /dev/null ; then
+  tar "${MINIO_TAR_OPTIONS[@]}" -cf ${BACKUP_FILE} -C ${TMP_WORK_DIR}/${MINIO_BACKUP_DIR} .
+  if "${VERIFY_DATASTORE_ARCHIVE}" && brlog "INFO" "Verifying backup archive" && ! tar "${MINIO_TAR_OPTIONS[@]}" -tf ${BACKUP_FILE} &> /dev/null ; then
     brlog "ERROR" "Backup file is broken, or does not exist."
     exit 1
   fi
@@ -172,21 +174,26 @@ if [ "${COMMAND}" = "restore" ] ; then
   fi
   brlog "INFO" "Start restore minio: ${BACKUP_FILE}"
   brlog "INFO" "Extracting archive..."
-  tar ${MINIO_TAR_OPTIONS[@]} -xf ${BACKUP_FILE} -C ${TMP_WORK_DIR}/${MINIO_BACKUP_DIR}
+  tar "${MINIO_TAR_OPTIONS[@]}" -xf ${BACKUP_FILE} -C ${TMP_WORK_DIR}/${MINIO_BACKUP_DIR}
   brlog "INFO" "Restoring data..."
   start_minio_port_forward
-  ${MC} ${MC_OPTS[@]} --quiet config host add wdminio ${MINIO_ENDPOINT_URL} ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} > /dev/null
-  for bucket in `ls ${TMP_WORK_DIR}/${MINIO_BACKUP_DIR}`
+  ${MC} "${MC_OPTS[@]}" --quiet config host add wdminio ${S3_ENDPOINT_URL} ${S3_ACCESS_KEY} ${S3_SECRET_KEY} > /dev/null
+  for bucket_path in "${TMP_WORK_DIR}/${MINIO_BACKUP_DIR}"/*
   do
-    if ${MC} ${MC_OPTS[@]} ls wdminio | grep ${bucket} > /dev/null ; then
-      if [ -n "`${MC} ${MC_OPTS[@]} ls wdminio/${bucket}/`" ] ; then
-        ${MC} ${MC_OPTS[@]} --quiet rm --recursive --force --dangerous "wdminio/${bucket}/" > /dev/null
+    bucket="$(basename "${bucket_path}")"
+    if [ -n "${BUCKET_SUFFIX}" ] && [[ "${bucket}" != *"${BUCKET_SUFFIX}"  ]] ; then
+      mv "${TMP_WORK_DIR}/${MINIO_BACKUP_DIR}/${bucket}" "${TMP_WORK_DIR}/${MINIO_BACKUP_DIR}/${bucket}${BUCKET_SUFFIX}"
+      bucket="${bucket}${BUCKET_SUFFIX}"
+    fi
+    if ${MC} "${MC_OPTS[@]}" ls wdminio | grep ${bucket} > /dev/null ; then
+      if [ -n "$(${MC} "${MC_OPTS[@]}" ls wdminio/${bucket}/)" ] ; then
+        ${MC} "${MC_OPTS[@]}" --quiet rm --recursive --force --dangerous "wdminio/${bucket}/" > /dev/null
       fi
       if [ "${bucket}" = "discovery-dfs" ] ; then
         continue
       fi
       set +e
-      ${MC} ${MC_OPTS[@]} --quiet mirror ${TMP_WORK_DIR}/${MINIO_BACKUP_DIR}/${bucket} wdminio/${bucket} &>> "${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
+      ${MC} "${MC_OPTS[@]}" --quiet mirror ${TMP_WORK_DIR}/${MINIO_BACKUP_DIR}/${bucket} wdminio/${bucket} &>> "${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
       RC=$?
       echo "RC=${RC}" >> "${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
       if [ $RC -ne 0 ] ; then
@@ -198,6 +205,8 @@ if [ "${COMMAND}" = "restore" ] ; then
   done
   stop_minio_port_forward
   brlog "INFO" "Done"
+  brlog "INFO" "Restart setup jobs"
+  restart_job "enrichment-model-copy orchestrator-setup"
   brlog "INFO" "Applying updates"
   . ./lib/restore-updates.bash
   minio_updates

discovery-data/latest/etcd-backup-restore.sh

@@ -58,7 +58,6 @@ CURRENT_COMPONENT="migration"
 TENANT_NAME="${instance_name:-wd}"
 MINIO_MT_MIGRATION_SCRIPTS="${SCRIPT_DIR}/src/minio-mt-migration.sh"
 BACKUP_RESTORE_DIR_IN_POD="/tmp/"
-MINIO_JOB_FILE="${SCRIPT_DIR}/src/minio-backup-restore-job.yml"
 OC_ARGS="${nsopt:-}"
 
 
@@ -85,8 +84,8 @@ run_pg_job
 PG_POD=${POD}
 
 
-setup_minio_env
-launch_minio_pod
+setup_s3_env
+launch_s3_pod
 MC_POD=${POD}
 
 ELASTIC_POD=$(oc get pods ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l tenant=${TENANT_NAME},app=elastic,ibm-es-data=True)
@@ -117,7 +116,9 @@ do
   ##########################
   brlog "INFO" "    Migrating dadmin tables"
 
-  standard_mt_tables=(api_usage datasets projects rcs rds trials wd_collection_document_status wd_collection_job_status wd_collections wd_collections_enrichment_job_status wd_collections_project wd_collections_status wd_converter_configs wd_converter_options wd_converters wd_crawler_configs wd_crawlers wd_dataset_document_status wd_datasets wd_datasets_collection wd_datasets_labeler wd_datasets_project wd_datasets_status wd_enrichments wd_enrichments_lang wd_enrichments_project wd_enrichments_stats wd_export_status wd_fileresources wd_key_value_store_global wd_labelers wd_labelers_project wd_rcs wd_rds wd_search_user_identity wd_total_collection_document_count wd_ui_resources)
+  CMD="psql -d dadmin -t -A -c \"SELECT table_name FROM information_schema.columns WHERE table_schema = 'public' AND column_name = 'tenant_id' ORDER BY table_name\""
+  standard_mt_tables=($(oc exec ${OC_ARGS} "${PG_POD}" -- bash -c "${CMD}"))
+  echo "Updating tables with tenant_id: ${standard_mt_tables[*]}"
   foreign_key_tables=(wd_collections wd_collections_status wd_collections_project wd_datasets_collection wd_collection_document_status wd_collections_enrichment_job_status wd_datasets wd_datasets_status wd_datasets_project wd_enrichments wd_enrichments_lang wd_enrichments_project)
 
   SQL="BEGIN;"
@@ -224,20 +225,30 @@ do
 
   brlog "INFO" "    Migrating MinIO contents"
 
-  run_script_in_pod ${MC_POD} "${SCRIPT_DIR}/src/minio-mt-migration.sh" "-s ${src} -t ${dst}"
+  bucket_suffix="$(get_bucket_suffix)"
+  if [ -n "${bucket_suffix}" ] ; then
+    run_script_in_pod ${MC_POD} "${SCRIPT_DIR}/src/minio-mt-migration.sh" "-s ${src} -t ${dst} --suffix ${bucket_suffix}"
+  else
+    run_script_in_pod ${MC_POD} "${SCRIPT_DIR}/src/minio-mt-migration.sh" "-s ${src} -t ${dst}"
+  fi
 
   ##############
   # ElasticSearch
   ##############
 
   brlog "INFO" "    Migrating ElasticSearch index"
+  if [ "$(oc ${OC_ARGS} get elasticsearchcluster ${TENANT_NAME} -o jsonpath='{.spec.nodes[?(@.name=="data")].replicas}')" = "1" ] ; then
+    replica=0
+  else
+    replica=1
+  fi
   _oc_cp "${SCRIPT_DIR}/src/tenant_index_template.json" "${ELASTIC_POD}:/tmp/tenant_index_template.json" ${OC_ARGS} -c elasticsearch
-  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/src/elastic-mt-migration.sh" "-s ${src} -t ${dst} --template /tmp/tenant_index_template.json" -c elasticsearch
+  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/src/elastic-mt-migration.sh" "-s ${src} -t ${dst} --template /tmp/tenant_index_template.json --replica ${replica}" -c elasticsearch
 
 done
 
 oc ${OC_ARGS} delete -f "${PG_JOB_FILE}"
-oc ${OC_ARGS} delete -f "${MINIO_JOB_FILE}"
+oc ${OC_ARGS} delete -f "${S3_JOB_FILE}"
 
 rm -rf ${TMP_WORK_DIR}
 if [ -z "$(ls tmp)" ] ; then

discovery-data/latest/lib/function.bash

@@ -59,10 +59,10 @@ do
   fi
 done
 
-WD_VERSION=${WD_VERSION:-`get_version`}
+WD_VERSION=${WD_VERSION:-$(get_version)}
 PG_POD=""
 
-if [ `compare_version "${WD_VERSION}" "4.0.0"` -ge 0 ] ; then
+if [ $(compare_version "${WD_VERSION}" "4.0.0") -ge 0 ] ; then
   PG_JOB_TEMPLATE="${SCRIPT_DIR}/src/backup-restore-job-template.yml"
   PG_JOB_FILE="${SCRIPT_DIR}/src/pg-backup-restore-job.yml"
   PG_BACKUP_RESTORE_JOB="wd-discovery-postgres-backup-restore"
@@ -78,8 +78,8 @@ if require_tenant_backup ; then
 
   brlog "INFO" "Restore tenants information"
 
-  for tenants_file in `ls -t tmp_wd_tenants_*.txt` ; do
-    if [ -n "`cat ${tenants_file}`" ] ; then
+  for tenants_file in $(ls -t tmp_wd_tenants_*.txt) ; do
+    if [ -n "$(cat ${tenants_file})" ] ; then
       kube_cp_from_local ${PG_POD} "${tenants_file}" "/tmp/tenants" ${OC_ARGS}
       fetch_cmd_result ${PG_POD} 'export PGUSER=${PGUSER:-$STKEEPER_PG_SU_USERNAME} && \
         export PGPASSWORD=${PGPASSWORD:-$STKEEPER_PG_SU_PASSWORD} && \
@@ -98,7 +98,7 @@ fi
 ## End restore tenants
 
 ## Set default as tenant ID
-if [ `compare_version "${WD_VERSION}" "2.2.1"` -ge 0 ] && [ `compare_version "${WD_VERSION}" "4.0.5"` -le 0 ] && [ `compare_version "${BACKUP_FILE_VERSION}" "2.2.0"` -le 0 ] ; then
+if [ $(compare_version "${WD_VERSION}" "2.2.1") -ge 0 ] && [ $(compare_version "${WD_VERSION}" "4.0.5") -le 0 ] && [ $(compare_version "${BACKUP_FILE_VERSION}" "2.2.0") -le 0 ] ; then
   brlog "INFO" "Update tenant id to default"
   fetch_cmd_result ${PG_POD} 'export PGUSER=${PGUSER:-$STKEEPER_PG_SU_USERNAME} && \
       export PGPASSWORD=${PGPASSWORD:-$STKEEPER_PG_SU_PASSWORD} && \
@@ -121,7 +121,7 @@ fetch_cmd_result ${PG_POD} 'export PGUSER=${PGUSER:-$STKEEPER_PG_SU_USERNAME} &&
 
 ## End update ranker
 
-if [ `compare_version "${WD_VERSION}" "4.0.0"` -ge 0 ] ; then
+if [ $(compare_version "${WD_VERSION}" "4.0.0") -ge 0 ] ; then
   oc ${OC_ARGS} delete -f $PG_JOB_FILE
 fi