Merge pull request #277 from yuki462-b/discovery-scripts-5.2.0

kvineetnair · web-flow · commit dece7e288ce7 · 2025-06-19T19:04:21.000+05:30
Merging as discussed on Slack.
diff --git a/discovery-data/latest/all-backup-restore.sh b/discovery-data/latest/all-backup-restore.sh
@@ -381,8 +381,10 @@ if ! "${BACKUP_RESTORE_IN_POD:-false}" ; then
 fi
 
 if [ ${COMMAND} = 'backup' ] ; then
-  ELASTIC_VERSION=$(get_elastic_version)
-  validate_elastic_version "$ELASTIC_VERSION"
+  if [ $(compare_version "${WD_VERSION}" "5.2.0") -lt 0 ] ; then
+    ELASTIC_VERSION=$(get_elastic_version)
+    validate_elastic_version "${ELASTIC_VERSION}"
+  fi 
   if [ $(compare_version "${WD_VERSION}" "4.0.6") -ge 0 ] ; then
     create_backup_instance_mappings
   fi
@@ -391,15 +393,17 @@ if [ ${COMMAND} = 'backup' ] ; then
       quiesce
     fi
   fi
-  if [[ ${ELASTIC_VERSION} = "ES6" ]] && [[ ${REINDEX_OLD_INDEX} = "true" ]]; then
-    reindex_elastic_es6
-  fi 
+  if [ $(compare_version "${WD_VERSION}" "5.2.0") -lt 0 ] ; then
+    if [[ ${ELASTIC_VERSION} = "ES6" ]] && [[ ${REINDEX_OLD_INDEX} = "true" ]]; then
+      reindex_elastic_es6
+    fi 
+  fi
+
   if [ $(compare_version "${WD_VERSION}" "2.1.3") -ge 0 ] ; then
     ALL_COMPONENT=("wddata" "etcd" "postgresql" "elastic" "minio")
   else
     ALL_COMPONENT=("wddata" "etcd" "hdp" "postgresql" "elastic")
   fi
-
   export ALL_COMPONENT
   export VERIFY_COMPONENT=( "${ALL_COMPONENT[@]}")
   BACKUP_FILE=${BACKUP_FILE:-"watson-discovery_$(date "+%Y%m%d_%H%M%S").backup"}
diff --git a/discovery-data/latest/elastic-backup-restore.sh b/discovery-data/latest/elastic-backup-restore.sh
diff --git a/discovery-data/latest/lib/function.bash b/discovery-data/latest/lib/function.bash
@@ -238,6 +238,19 @@ else
   TRANSFER_TAR_OPTIONS=("")
 fi
 
+# Extract gzip compressed file into plain tar file.
+gzip_to_plain_tar() {
+  brlog "DEBUG" "Recompressing file"
+  local original_backup="$1"
+  local recompressed_backup="$2"
+  gunzip -c "${original_backup}" > "${recompressed_backup}"
+  # Debug.
+  original_backup_info=$(ls -lh "${original_backup}")
+  recompressed_backup_info=$(ls -lh "${recompressed_backup}")
+  brlog "DEBUG" "Original backup: ${original_backup_info}"
+  brlog "DEBUG" "Recompressed backup: ${recompressed_backup_info}"
+}
+
 kube_cp_from_local(){
   IS_RECURSIVE=false
   if [ "$1" = "-r" ] ; then
@@ -413,6 +426,18 @@ get_mc(){
   fi
 }
 
+setup_mc() {
+  mkdir -p ${TMP_WORK_DIR}/.mc
+  if [ -n "${MC_COMMAND+UNDEF}" ] ; then
+    MC=${MC_COMMAND}
+  else
+    get_mc ${TMP_WORK_DIR}
+    MC=${TMP_WORK_DIR}/mc
+  fi
+  export MINIO_CONFIG_DIR="${PWD}/${TMP_WORK_DIR}/.mc"
+  MC_OPTS=(--config-dir ${MINIO_CONFIG_DIR} --quiet --insecure)
+}
+
 # Compare two timestamps (i.e. 2024-06-12T14:34:03Z).
 # Return 1 if timestamp1 > timestamp2 else 0.
 compare_timestamps() {
@@ -1254,7 +1279,8 @@ check_datastore_available(){
 # Assume there are only version 6 or 7 indices in the cluster.
 get_elastic_version() {
   ELASTIC_POD=$(get_elastic_pod)
-  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/reindex_es6_indices.sh" "--dry-run" -c elasticsearch ${OC_ARGS}
+  ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
+  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/reindex_es6_indices.sh" "--dry-run" -c "${ELASTIC_POD_CONTAINER}" ${OC_ARGS}
   result=$(get_last_cmd_result_in_pod)
 
   if echo "$result" | grep -q "Skip ElasticSearch 6 index"; then
@@ -1287,7 +1313,8 @@ validate_elastic_version() {
 # Reindex elastic search indices from version 6 to 7.
 reindex_elastic_es6() {
   brlog "INFO" "Reindexing ..."
-  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/reindex_es6_indices.sh" "" -c elasticsearch ${OC_ARGS}
+  ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
+  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/reindex_es6_indices.sh" "" -c "${ELASTIC_POD_CONTAINER}" ${OC_ARGS}
   brlog "INFO" "Reindex completed. Waiting for ElasticSearch status to be green ..."
   # Wait a few moments for elastic search to be ready after the reindex.
   # TODO: implement waiting logic inside of reindex_es6_indices.sh, and remove the sleep below from this function.
@@ -1341,12 +1368,49 @@ check_postgres_available(){
 }
 
 get_elastic_pod(){
-  echo "$(oc get pods ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l tenant=${TENANT_NAME},app=elastic,ibm-es-data=True)"
+  local wd_version=${WD_VERSION:-$(get_version)}
+  if [ $(compare_version ${wd_version} "5.2.0") -lt 0 ] ; then
+    echo "$(oc get pods ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l tenant=${TENANT_NAME},app=elastic,ibm-es-data=True)"
+  else 
+    echo "$(oc get pods ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l tenant=${TENANT_NAME},app=opensearch)"
+  fi
+}
+
+get_elastic_pod_container() {
+  local wd_version=${WD_VERSION:-$(get_version)}
+  if [ $(compare_version ${wd_version} "5.2.0") -lt 0 ] ; then
+    echo "elasticsearch"
+  else
+    echo "opensearch"
+  fi
+}
+
+get_elastic_repo() {
+  local wd_version=${WD_VERSION:-$(get_version)}
+  if [ $(compare_version ${wd_version} "5.2.0") -lt 0 ] ; then
+    echo "my_backup"
+  else
+    echo "cloudpak"
+  fi
+}
+
+get_elastic_repo_location() {
+  local wd_version=${WD_VERSION:-$(get_version)}
+  if [ $(compare_version ${wd_version} "5.2.0") -lt 0 ] ; then
+    echo "/workdir/shared_storage"
+  else
+    echo "/workdir/snapshot_storage"
+  fi
+}
+
+get_opensearch_cluster() {
+  echo "$(oc get cluster.opensearch ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l tenant=${TENANT_NAME},app=opensearch)"
 }
 
 check_elastic_available(){
   ELASTIC_POD=$(get_elastic_pod)
-  oc exec ${OC_ARGS} "${ELASTIC_POD}" -c elasticsearch -- bash -c 'export ELASTIC_ENDPOINT=https://localhost:9200 && \
+  ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
+  oc exec ${OC_ARGS} "${ELASTIC_POD}" -c "${ELASTIC_POD_CONTAINER}" -- bash -c 'export ELASTIC_ENDPOINT=https://localhost:9200 && \
   curl -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_cluster/health" | grep "\"status\":\"yellow\"\\|\"status\":\"green\"" > /dev/null' || return 1
   return 0
 }
@@ -1355,8 +1419,9 @@ check_s3_available(){
   setup_s3_env
   local wd_version="${WD_VERSION:-$(get_version)}"
   if [ $(compare_version ${wd_version} "4.7.0") -lt 0 ] ; then
-    ELASTIC_POD=$(oc get pods ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l tenant=${TENANT_NAME},app=elastic,ibm-es-data=True)
-    oc exec ${OC_ARGS} "${ELASTIC_POD}" -c elasticsearch -- bash -c "curl -ks 'https://${S3_SVC}:${S3_PORT}/minio/health/ready' -w '%{http_code}' -o /dev/null | grep 200 > /dev/null" || return 1
+    ELASTIC_POD=$(get_elastic_pod)
+    ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
+    oc exec ${OC_ARGS} "${ELASTIC_POD}" -c "${ELASTIC_POD_CONTAINER}" -- bash -c "curl -ks 'https://${S3_SVC}:${S3_PORT}/minio/health/ready' -w '%{http_code}' -o /dev/null | grep 200 > /dev/null" || return 1
     return 0
   else
     launch_s3_pod
@@ -1487,10 +1552,12 @@ check_instance_mappings(){
 
 get_instance_tuples(){
   ELASTIC_POD=$(get_elastic_pod)
+  ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
+
   local file_name="$(basename "${MAPPING_FILE}")"
-  _oc_cp "${MAPPING_FILE}" "${ELASTIC_POD}:/tmp/mapping.json" -c elasticsearch
-  local mappings=( $(fetch_cmd_result ${ELASTIC_POD} "jq -r '.instance_mappings[] | \"\(.source_instance_id),\(.dest_instance_id)\"' /tmp/mapping.json" -c elasticsearch) )
-  oc exec -c elasticsearch ${ELASTIC_POD} -- bash -c "rm -f /tmp/mapping.json"
+  _oc_cp "${MAPPING_FILE}" "${ELASTIC_POD}:/tmp/mapping.json" -c "${ELASTIC_POD_CONTAINER}"
+  local mappings=( $(fetch_cmd_result ${ELASTIC_POD} "jq -r '.instance_mappings[] | \"\(.source_instance_id),\(.dest_instance_id)\"' /tmp/mapping.json" -c "${ELASTIC_POD_CONTAINER}") )
+  oc exec -c "${ELASTIC_POD_CONTAINER}" ${ELASTIC_POD} -- bash -c "rm -f /tmp/mapping.json"
   for map in "${mappings[@]}"
   do
     ORG_IFS=${IFS}
@@ -1642,6 +1709,16 @@ get_bucket_suffix(){
   echo "${suffix}"
 }
 
+get_snapshot_repo_size() {
+  local wd_version=${WD_VERSION:-$(get_version)}
+  if [ $(compare_version "${wd_version}" "5.2.0") -lt 0 ] ; then
+    echo "$(oc ${OC_ARGS} get elasticsearchcluster ${TENANT_NAME} -o jsonpath='{.spec.snapshotRepo.size}')"
+  else
+    opensearch_cluster=$(get_opensearch_cluster)
+    echo "$(oc ${OC_ARGS} get cluster.opensearch ${opensearch_cluster} -o jsonpath='{.spec.backup.sizeLimit}')"
+  fi 
+}
+
 create_elastic_shared_pvc(){
   local wd_version=${WD_VERSION:-$(get_version)}
   if [ $(compare_version "${wd_version}" "4.7.0") -ge 0 ] ; then
@@ -1667,7 +1744,7 @@ create_elastic_shared_pvc(){
         brlog "ERROR" "numfmt command is not available. Please install numfmt."
         exit 1
       fi
-      local snapshot_repo_size="$(oc ${OC_ARGS} get elasticsearchcluster ${TENANT_NAME} -o jsonpath='{.spec.snapshotRepo.size}')"
+      local snapshot_repo_size=$(get_snapshot_repo_size)
       local size_array=( $(echo "${snapshot_repo_size}" | awk 'match($0, /([[:digit:]]+)([[:alpha:]]+)/, array) {print array[1], array[2]}') )
       ELASTIC_SHARED_PVC_SIZE="$((size_array[0]*2))${size_array[1]}"
       ELASTIC_SHARED_PVC_DEFAULT_NAME="${TENANT_NAME}-discovery-backup-restore-pvc"
@@ -1685,6 +1762,15 @@ spec:
   storageClassName: ${FILE_STORAGE_CLASS}
 EOF
       ELASTIC_SHARED_PVC="${ELASTIC_SHARED_PVC_DEFAULT_NAME}"
+
+      if [ $(compare_version "${wd_version}" "5.2.0") -ge 0 ] ; then
+        # Need those label when using custom shared PVC for opensearch.
+        # https://pages.github.ibm.com/CloudPakOpenContent/ibm-opensearch-operator/api/crds/#clusterspecbackup
+        oc label pvc $ELASTIC_SHARED_PVC icpdsupport/empty-on-nd-backup=false
+        oc label pvc $ELASTIC_SHARED_PVC icpdsupport/ignore-on-nd-backup=false
+      fi 
+
+      brlog "DEBUG" "Created '$ELASTIC_SHARED_PVC' PVC for elasticsearch backup/restore."
     fi
   fi
 }
diff --git a/discovery-data/latest/lib/restore-updates.bash b/discovery-data/latest/lib/restore-updates.bash
@@ -3,21 +3,24 @@
 set -euo pipefail
 
 UPDATE_DIR="$SCRIPT_DIR/src"
+source "${SCRIPT_DIR}/lib/function.bash"
+
+ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
 
 elastic_updates(){
   if ls "$UPDATE_DIR"/*.elupdate &> /dev/null ; then
     for ELASTIC_COMMAND in "$UPDATE_DIR"/*.elupdate; do
       value=$(<${ELASTIC_COMMAND})
-      oc ${OC_ARGS} exec ${ELASTIC_POD} -c elasticsearch -- bash -c "${value}" >> "${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
+      oc ${OC_ARGS} exec ${ELASTIC_POD} -c "${ELASTIC_POD_CONTAINER}" -- bash -c "${value}" >> "${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
     done
   fi
   if [ $(compare_version "${BACKUP_FILE_VERSION:-$(get_backup_version)}" "2.1.2") -le 0 ] ; then 
     if ls "$UPDATE_DIR"/*.elupdate_script &> /dev/null ; then
       for ELASTIC_COMMAND in "$UPDATE_DIR"/*.elupdate_script; do
         UPDATE_SCRIPT_NAME=$(basename ${ELASTIC_COMMAND})
         _oc_cp ${ELASTIC_COMMAND} ${ELASTIC_POD}:/tmp/${UPDATE_SCRIPT_NAME} ${OC_ARGS}
-        oc ${OC_ARGS} exec ${ELASTIC_POD} -c elasticsearch -- bash /tmp/${UPDATE_SCRIPT_NAME}
-        oc ${OC_ARGS} exec ${ELASTIC_POD} -c elsaticsearch -- rm /tmp/${UPDATE_SCRIPT_NAME}
+        oc ${OC_ARGS} exec ${ELASTIC_POD} -c "${ELASTIC_POD_CONTAINER}" -- bash /tmp/${UPDATE_SCRIPT_NAME}
+        oc ${OC_ARGS} exec ${ELASTIC_POD} -c "${ELASTIC_POD_CONTAINER}" -- rm /tmp/${UPDATE_SCRIPT_NAME}
       done
     fi
   fi
diff --git a/discovery-data/latest/mt-mt-migration.sh b/discovery-data/latest/mt-mt-migration.sh
@@ -88,7 +88,8 @@ setup_s3_env
 launch_s3_pod
 MC_POD=${POD}
 
-ELASTIC_POD=$(oc get pods ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l tenant=${TENANT_NAME},app=elastic,ibm-es-data=True)
+ELASTIC_POD=$(get_elastic_pod)
+ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
 
 brlog "INFO" "Start MT migration"
 
@@ -242,8 +243,8 @@ do
   else
     replica=1
   fi
-  _oc_cp "${SCRIPT_DIR}/src/tenant_index_template.json" "${ELASTIC_POD}:/tmp/tenant_index_template.json" ${OC_ARGS} -c elasticsearch
-  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/src/elastic-mt-migration.sh" "-s ${src} -t ${dst} --template /tmp/tenant_index_template.json --replica ${replica}" -c elasticsearch
+  _oc_cp "${SCRIPT_DIR}/src/tenant_index_template.json" "${ELASTIC_POD}:/tmp/tenant_index_template.json" ${OC_ARGS} -c "${ELASTIC_POD_CONTAINER}"
+  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/src/elastic-mt-migration.sh" "-s ${src} -t ${dst} --template /tmp/tenant_index_template.json --replica ${replica}" -c "${ELASTIC_POD_CONTAINER}"
 
 done
 
diff --git a/discovery-data/latest/src/elastic-backup-restore-in-pod.sh b/discovery-data/latest/src/elastic-backup-restore-in-pod.sh
@@ -22,6 +22,7 @@ ELASTIC_REQUEST_TIMEOUT="30m"
 TMP_WORK_DIR="/tmp/backup-restore-workspace"
 CURRENT_COMPONENT="elastic"
 ELASTIC_LOG="${TMP_WORK_DIR}/elastic.log"
+ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
 
 # Need to export HOME on OCP 4.18　which is used to configure the default mc config directory.
 export HOME="${TMP_WORK_DIR}"
@@ -106,7 +107,7 @@ EOF
       break;
     else
       # comment out the progress because it shows always 0 until it complete.
-      # brlog "INFO" "Progress: $(fetch_cmd_result ${ELASTIC_POD} 'curl -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_snapshot/'${ELASTIC_REPO}'/'${ELASTIC_SNAPSHOT}'" | jq -c ".snapshots[0].shards"' ${OC_ARGS} -c elasticsearch)"
+      # brlog "INFO" "Progress: $(fetch_cmd_result ${ELASTIC_POD} 'curl -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_snapshot/'${ELASTIC_REPO}'/'${ELASTIC_SNAPSHOT}'" | jq -c ".snapshots[0].shards"' ${OC_ARGS} -c "${ELASTIC_POD_CONTAINER}")"
       sleep ${ELASTIC_STATUS_CHECK_INTERVAL}
     fi
   done
@@ -197,7 +198,7 @@ EOF
   if [ "${ELASTIC_WAIT_GREEN_STATE}" = "true" ] ; then
     brlog "INFO" "Wait for the ElasticSearch to be Green State"
     while true;
-    cluster_status=$(fetch_cmd_result ${ELASTIC_POD} 'curl -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_cluster/health" | jq -r ".status"' ${OC_ARGS} -c elasticsearch)
+    cluster_status=$(fetch_cmd_result ${ELASTIC_POD} 'curl -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_cluster/health" | jq -r ".status"' ${OC_ARGS} -c "${ELASTIC_POD_CONTAINER}")
     do
       if [ "${cluster_status}" = "green" ] ; then
         break;
diff --git a/discovery-data/latest/st-mt-migration.sh b/discovery-data/latest/st-mt-migration.sh
@@ -54,10 +54,11 @@ fetch_cmd_result ${PG_POD} "PGUSER=${PGUSER} PGPASSWORD=${PGPASSWORD} psql -d da
 brlog "INFO" "Correct ElasticSearch Index"
 
 ELASTIC_POD=${ELASTIC_POD:-$(get_elastic_pod)}
-ARCHIVED_SETTINGS=$(fetch_cmd_result ${ELASTIC_POD} 'curl -XGET -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_settings" | jq -r "if ( . | length == 0 ) then \"null\" else .[].settings.archived end" | uniq' -c elasticsearch ${OC_ARGS})
+ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
+ARCHIVED_SETTINGS=$(fetch_cmd_result ${ELASTIC_POD} 'curl -XGET -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_settings" | jq -r "if ( . | length == 0 ) then \"null\" else .[].settings.archived end" | uniq' -c "${ELASTIC_POD_CONTAINER}" ${OC_ARGS})
 if [ "${ARCHIVED_SETTINGS}" != "null" ] ; then
   brlog "INFO" "Found archived Index."
-  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/src/reindex_archived_indices.sh" "" -c elasticsearch ${OC_ARGS}
+  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/src/reindex_archived_indices.sh" "" -c "${ELASTIC_POD_CONTAINER}" ${OC_ARGS}
 else
   brlog "INFO" "No archived Index. Skip."
 fi
@@ -95,4 +96,4 @@ oc annotate ${OC_ARGS} wd ${TENANT_NAME} --overwrite watsonDiscoveryMigrationSou
 
 if ! "${was_quiesced}" ; then
   unquiesce
-fi
+fi
diff --git a/discovery-data/latest/version.txt b/discovery-data/latest/version.txt
@@ -1,2 +1,2 @@
 The Backup and Restore Scripts for the Watson Discovery on CP4D.
-Scripts Version: 5.1.2
+Scripts Version: 5.2.0

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`The Backup and Restore Scripts for the Watson Discovery on CP4D.`
`2`		`-Scripts Version: 5.1.2`
	`2`	`+Scripts Version: 5.2.0`