Merge pull request #283 from yuki462-b/5.3.0-backup-scripts

feat: updating watson discovery backup restore scripts for 5.3.0.

Author: kvineetnair

discovery-data/latest/all-backup-restore.sh

@@ -42,8 +42,8 @@ Basically, you don't need these advanced options.
     --datastore-archive-option="<tar_option>"  Tar options for compression used on archiving the backup files of ElasticSearch, MinIO and internal configuration. Default "-z".
     --postgresql-archive-option="<tar_option>" Tar options for compression used on archiving the backup files of postgres. Note that the backup files of postgresql are archived on its pod by default. Default "-z".
     --etcd-archive-option="<tar_option>"       Tar options used on archiving the backup files of etcd. Note that the backup files of etcd are archived on its pod by default. Default "-z".
-    --reindex-old-index                        Reindex elastic search indices from version 6 to 7 before making the backup. If existing elastic search indices version is already 7, this flag does nothing. They have to be reindexed if you want to restore to 4.8.x. Reindex actually won't change your data, but it takes time. If you want to make the backup without reindex, use '--ignore-old-index' instead.
-    --ignore-old-index                         Whether to skip reindexing old (version 6) elastic search indices before making the backup. You can proceed backup without reindex, but you can't restore the backup data to 4.8.x.
+    --reindex-old-index                        Reindex old Elasticsearch indices before making a backup. If you want to restore into 4.8.6 and later, you'll need to reindex to elasticsearch7. If you want to restore into 5.3.0 and later, you'll need to reindex to opensearch2. Reindex actually won't change your data, but it takes time. If you want to make the backup without reindex, use '--ignore-old-index' instead.
+    --ignore-old-index                         Whether to skip reindexing old elastic search indices before making the backup.
     --skip-verify-archive                      Skip the all verifying process of the archive.
     --skip-verify-backup                       Skip verifying the backup file.
     --skip-verify-datastore-archive            Skip verifying the archive of datastores.
@@ -381,9 +381,9 @@ if ! "${BACKUP_RESTORE_IN_POD:-false}" ; then
 fi
 
 if [ ${COMMAND} = 'backup' ] ; then
-  if [ $(compare_version "${WD_VERSION}" "5.2.0") -lt 0 ] ; then
-    ELASTIC_VERSION=$(get_elastic_version)
-    validate_elastic_version "${ELASTIC_VERSION}"
+  if [ $(compare_version "${WD_VERSION}" "5.3.0") -lt 0 ] ; then
+    ELASTIC_INDEX_VERSION=$(get_ELASTIC_INDEX_VERSION)
+    validate_ELASTIC_INDEX_VERSION "${ELASTIC_INDEX_VERSION}"
   fi 
   if [ $(compare_version "${WD_VERSION}" "4.0.6") -ge 0 ] ; then
     create_backup_instance_mappings
@@ -393,10 +393,24 @@ if [ ${COMMAND} = 'backup' ] ; then
       quiesce
     fi
   fi
-  if [ $(compare_version "${WD_VERSION}" "5.2.0") -lt 0 ] ; then
-    if [[ ${ELASTIC_VERSION} = "ES6" ]] && [[ ${REINDEX_OLD_INDEX} = "true" ]]; then
-      reindex_elastic_es6
-    fi 
+  # Reindex old indices if requested (only for versions before OpenSearch 2 / 5.3.0)
+  if [[ ${REINDEX_OLD_INDEX} = "true" ]]; then
+    # Reindex ES6 -> ES7 (WD versions: 2.2.0 <= version < 5.2.0)
+    if [ $(compare_version "${WD_VERSION}" "2.2.0") -ge 0 ] && \
+       [ $(compare_version "${WD_VERSION}" "5.2.0") -lt 0 ] && \
+       [[ ${ELASTIC_INDEX_VERSION} = "ES6" ]]; then
+      reindex_elastic "${ELASTIC_INDEX_VERSION}"
+      ELASTIC_INDEX_VERSION=$(get_ELASTIC_INDEX_VERSION)
+      validate_ELASTIC_INDEX_VERSION "${ELASTIC_INDEX_VERSION}"
+    fi
+    # Reindex ES7 -> OS2 (WD versions: 4.8.6 <= version < 5.3.0)
+    if [ $(compare_version "${WD_VERSION}" "4.8.6") -ge 0 ] && \
+       [ $(compare_version "${WD_VERSION}" "5.3.0") -lt 0 ] && \
+       [[ ${ELASTIC_INDEX_VERSION} = "ES7" ]]; then
+      reindex_elastic "${ELASTIC_INDEX_VERSION}"
+      ELASTIC_INDEX_VERSION=$(get_ELASTIC_INDEX_VERSION)
+      validate_ELASTIC_INDEX_VERSION "${ELASTIC_INDEX_VERSION}"
+    fi
   fi
 
   if [ $(compare_version "${WD_VERSION}" "2.1.3") -ge 0 ] ; then

discovery-data/latest/elastic-backup-restore.sh

@@ -312,7 +312,7 @@ if [ $(compare_version ${WD_VERSION} "5.2.0") -ge 0 ]; then
   brlog "INFO" "Scale down opensearch operator"
   OPENSEARCH_CLUSTER=$(get_opensearch_cluster)
   OPENSEARCH_VERSION=$(oc ${OC_ARGS} get cluster.opensearch ${OPENSEARCH_CLUSTER} -o jsonpath='{.status.release}{"\n"}')
-  OPENSEARCH_OPERATOR_DEPLOY=( $(oc get deploy -A -l "olm.owner=ibm-opensearch-operator.v${OPENSEARCH_VERSION}" | tail -n1 | awk '{print $1,$2}') )
+  OPENSEARCH_OPERATOR_DEPLOY=( $(get_opensearch_operator_deployment) )
   oc ${OC_ARGS} scale deploy -n "${OPENSEARCH_OPERATOR_DEPLOY[0]}" "${OPENSEARCH_OPERATOR_DEPLOY[1]}" --replicas=0
   trap_add "oc ${OC_ARGS} scale deploy -n ${OPENSEARCH_OPERATOR_DEPLOY[0]} ${OPENSEARCH_OPERATOR_DEPLOY[1]} --replicas=1"
 fi
@@ -599,7 +599,8 @@ EOF
     run_cmd_in_pod ${ELASTIC_POD} "${restore_whitelist_indices_cmd}" ${OC_ARGS} -c "${ELASTIC_POD_CONTAINER}"
     result=$(get_last_cmd_result_in_pod)
     brlog "DEBUG" "restore whitelist indices result: ${result}"
-    if ! echo "${result}" | grep -Eq '"total":1,"failed":0,"successful":1'; then
+    # TODO handle multiple whitelist indicies case.
+    if ! echo "${result}" | grep -Eq '"total":1,"failed":0,"successful":1|"index_not_found_exception"'; then
       brlog "ERROR" "Could not restore the ${whitelist_indices} index due to the following error: ${result}. Please check the cluster."
       clean_up
       exit 1

discovery-data/latest/etcd-backup-restore.sh

@@ -54,18 +54,15 @@ mkdir -p ${BACKUP_RESTORE_LOG_DIR}
 wd_version=${WD_VERSION:-$(get_version)}
 
 setup_etcd_env
+ETCD_VARIABLES=$(export_etcd_variable_command)
 
 # backup etcd
 if [ ${COMMAND} = 'backup' ] ; then
   BACKUP_FILE=${BACKUP_FILE:-"etcd_snapshot_$(date "+%Y%m%d_%H%M%S").db"}
   brlog "INFO" "Start backup etcd..."
   run_cmd_in_pod ${ETCD_POD} "rm -rf ${ETCD_BACKUP_DIR} ${ETCD_BACKUP} && \
   mkdir -p ${ETCD_BACKUP_DIR} && \
-  export ETCDCTL_USER='${ETCD_USER}:${ETCD_PASSWORD}' && \
-  export ETCDCTL_CERT='/etc/etcdtls/operator/etcd-tls/etcd-client.crt' && \
-  export ETCDCTL_CACERT='/etc/etcdtls/operator/etcd-tls/etcd-client-ca.crt' && \
-  export ETCDCTL_KEY='/etc/etcdtls/operator/etcd-tls/etcd-client.key' && \
-  export ETCDCTL_ENDPOINTS='https://${ETCD_SERVICE}:2379' && \
+  ${ETCD_VARIABLES} && \
   etcdctl get --prefix '/' -w fields > ${ETCD_BACKUP_FILE}" ${OC_ARGS}
 
   if "${ARCHIVE_ON_LOCAL}" ; then 
@@ -116,11 +113,7 @@ if [ ${COMMAND} = 'restore' ] ; then
   fi
   brlog "INFO" "Restoring data..."
   cmd='export ETCDCTL_API=3 && \
-  export ETCDCTL_USER='${ETCD_USER}':'${ETCD_PASSWORD}' && \
-  export ETCDCTL_CERT=/etc/etcdtls/operator/etcd-tls/etcd-client.crt && \
-  export ETCDCTL_CACERT=/etc/etcdtls/operator/etcd-tls/etcd-client-ca.crt && \
-  export ETCDCTL_KEY=/etc/etcdtls/operator/etcd-tls/etcd-client.key && \
-  export ETCDCTL_ENDPOINTS=https://'${ETCD_SERVICE}':2379 && \
+  '${ETCD_VARIABLES}' && \
   export ETCD_BACKUP='${ETCD_BACKUP_FILE}' && \
   etcdctl del --prefix "/" && \
   cat ${ETCD_BACKUP} | \

discovery-data/latest/lib/function.bash

@@ -150,6 +150,20 @@ get_version(){
   fi
 }
 
+# Return 0 if wd is installed with helm (non-OLM method). Otherwise 1.
+is_managed_by_helm() {
+  if [ $(compare_version "${WD_VERSION}" "5.3.0") -lt 0 ] ; then
+    return 1
+  else
+    label=$(oc get wd ${OC_ARGS} ${TENANT_NAME} -o jsonpath="{.metadata.labels.app\.kubernetes\.io/managed-by}")
+    if [[ "$label" == "Helm" ]]; then
+      return 0  # Managed by Helm.
+    else
+      return 1  # Managed by Ansible.
+    fi
+  fi
+}
+
 # Return the major version, for instance if version is 4.x.y, return 4.
 major_version() {
   local version="$1"
@@ -715,14 +729,19 @@ get_migrator_tag(){
 
 get_migrator_image(){
   local wd_version=${WD_VERSION:-$(get_version)}
+  DISCOVERY_REGISTRY_NAMESPACE="cp/watson-discovery"
   if [ $(compare_version "${wd_version}" "4.6.0") -le 0 ] ; then
     echo "$(get_migrator_repo):${MIGRATOR_TAG:-$(get_migrator_tag)}"
   else
     utils_repo="$(oc get watsondiscoveryapi ${TENANT_NAME} -o jsonpath='{.spec.shared.dockerRegistryPrefix}')"
     utils_image="$(oc get watsondiscoveryapi ${TENANT_NAME} -o jsonpath='{.spec.shared.initContainer.utils.image.name}')"
     utils_tag="$(oc get watsondiscoveryapi ${TENANT_NAME} -o jsonpath='{.spec.shared.initContainer.utils.image.tag}')"
     utils_digest="$(oc get watsondiscoveryapi ${TENANT_NAME} -o jsonpath='{.spec.shared.initContainer.utils.image.digest}')"
-    echo "${utils_repo}/${utils_image}:${utils_tag}@${utils_digest}"
+    if [ $(compare_version "${wd_version}" "5.3.0") -lt 0 ] ; then
+      echo "${utils_repo}/${utils_image}:${utils_tag}@${utils_digest}"
+    else 
+      echo "${utils_repo}/${DISCOVERY_REGISTRY_NAMESPACE}/${utils_image}:${utils_tag}@${utils_digest}"
+    fi
   fi
 }
 
@@ -1320,62 +1339,124 @@ check_datastore_available(){
 }
 
 # Get elastic indices and return its version as a string. Return the oldest index version. 
-# Assume there are only version 6 or 7 indices in the cluster.
-get_elastic_version() {
+get_ELASTIC_INDEX_VERSION() {
   ELASTIC_POD=$(get_elastic_pod)
   ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
-  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/reindex_es6_indices.sh" "--dry-run" -c "${ELASTIC_POD_CONTAINER}" ${OC_ARGS}
+  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/reindex_es7_indices.sh" "--check-only" -c "${ELASTIC_POD_CONTAINER}" ${OC_ARGS}
   result=$(get_last_cmd_result_in_pod)
 
   if echo "$result" | grep -q "Skip ElasticSearch 6 index"; then
     echo "ES6"
-  elif echo "$result" | grep -q "Skip ElasticSearch 7 index"; then
+  elif echo "$result" | grep -q "Detected ElasticSearch 7 index"; then
     echo "ES7"
-  elif echo "$result" | grep -q "ElasticSearch has no index"; then
+  elif echo "$result" | grep -q "Skip OpenSearch 2 index"; then
+    echo "OS2"
+  elif echo "$result" | grep -q "OpenSearch has no index"; then
     echo "NoIndex"
   else
     echo "Unknown"
   fi  
 }
 
-validate_elastic_version() {
-  ELASTIC_VERSION="${1:-$(get_elastic_version)}"
+validate_ELASTIC_INDEX_VERSION() {
+  local wd_version="${WD_VERSION:-$(get_version)}"
+  ELASTIC_INDEX_VERSION="${1:-$(get_ELASTIC_INDEX_VERSION)}"
 
-  if [[ ${ELASTIC_VERSION} = "ES6" ]]; then
+  if [[ ${ELASTIC_INDEX_VERSION} = "ES6" ]]; then
     brlog "WARN" "Elasticsearch 6 detected."
     if [[ ${IGNORE_OLD_INDEX} = "true" ]]; then
-      brlog "WARN" "Ignoring old index."
+      brlog "WARN" "Ignoring elasticsearch 6 index."
     elif [[ ${REINDEX_OLD_INDEX} = "true" ]]; then
-      brlog "INFO" "Elasticsearch 6 will be reindexed after quiescing."
+      if [ $(compare_version "${WD_VERSION}" "2.2.0") -ge 0 ] && [ $(compare_version "${WD_VERSION}" "5.2.0") -lt 0 ] ; then
+        brlog "INFO" "Elasticsearch 6 will be reindexed after quiescing."
+      else
+        brlog "ERROR" "Detected old index created in ElasticSearch 6 that is not supported in Watson Discovery 4.8.x. They have to be reindexed into elasticsearch7 if you want to restore to 4.8.x or higher. Please reindex into elasticsearch7 using Watson Discovery verison less than 5.2.2. If you would like to ignore the error, run backup script with '--ignore-old-index' option."
+        exit 1
+      fi
     else
       brlog "ERROR" "Detected old index created in ElasticSearch 6 that is not supported in Watson Discovery 4.8.x. They have to be reindexed if you want to restore to 4.8.x or higher. Reindex actually won't change your data, but it takes time. Run the backup script with '--reindex-old-index' option if you want to reindex them. You can proceed the backup without reindex, but you can't restore the backup data to 4.8.x. If you would like to ignore the error, run backup script with '--ignore-old-index' option."
       exit 1
     fi
   fi 
+
+  if [[ ${ELASTIC_INDEX_VERSION} = "ES7" ]]; then
+    brlog "WARN" "Elasticsearch 7 detected."
+    if [[ ${IGNORE_OLD_INDEX} = "true" ]]; then
+      brlog "WARN" "Ignoring elasticsearch 7 index."
+    elif [[ ${REINDEX_OLD_INDEX} = "true" ]]; then
+      if [ $(compare_version "${WD_VERSION}" "4.8.6") -ge 0 ] && [ $(compare_version "${WD_VERSION}" "5.3.0") -lt 0 ] ; then
+        brlog "INFO" "Elasticsearch 7 will be reindexed after quiescing."
+      else 
+        brlog "ERROR" "Detected old index created in ElasticSearch 7 that is not supported in Watson Discovery 5.3.x. They have to be reindexed into opensearch2 if you want to restore to 5.3.x or higher. Please reindex into opensearch2 using Watson Discovery versions between 4.8.6 and below 5.3.0. If you would like to ignore the error, run backup script with '--ignore-old-index' option."
+        exit 1
+      fi 
+    else
+      brlog "ERROR" "Detected old index created in ElasticSearch 7 that is not supported in Watson Discovery 5.3.x. They have to be reindexed if you want to restore to 5.3.x or higher. Reindex actually won't change your data, but it takes time. Run the backup script with '--reindex-old-index' option if you want to reindex them. You can proceed the backup without reindex, but you can't restore the backup data to 5.3.x. If you would like to ignore the error, run backup script with '--ignore-old-index' option."
+      exit 1
+    fi
+  fi 
 }
 
-# Reindex elastic search indices from version 6 to 7.
-reindex_elastic_es6() {
+reindex_elastic() {
+  local reindex_source_version=$1
   brlog "INFO" "Reindexing ..."
   ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
-  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/reindex_es6_indices.sh" "" -c "${ELASTIC_POD_CONTAINER}" ${OC_ARGS}
+  reindex_script=""
+  if [[ ${reindex_source_version} = "ES6" ]]; then
+    reindex_script="reindex_es6_indices.sh"
+  elif [[ ${reindex_source_version} = "ES7" ]]; then
+    reindex_script="reindex_es7_indices.sh"
+  else
+    brlog "ERROR" "Unsupported reindex source version: ${reindex_source_version}"
+    exit 1
+  fi
+
+  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/${reindex_script}" "" -c "${ELASTIC_POD_CONTAINER}" ${OC_ARGS}
   brlog "INFO" "Reindex completed. Waiting for ElasticSearch status to be green ..."
   # Wait a few moments for elastic search to be ready after the reindex.
   # TODO: implement waiting logic inside of reindex_es6_indices.sh, and remove the sleep below from this function.
   sleep 60
   check_elastic_available || { brlog "ERROR" "ElasticSearch is unavailable"; exit 1; }
 }
 
+get_etcd_pod(){
+  local wd_version=${WD_VERSION:-$(get_version)}
+  if [ $(compare_version ${wd_version} "5.3.0") -lt 0 ] ; then
+    echo "$(oc get pods ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l etcd_cluster=${TENANT_NAME}-discovery-etcd)"
+  else 
+    echo "$(oc get pods ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l icpdsupport/podSelector=discovery-etcd)"
+  fi
+}
+
+export_etcd_variable_command(){
+  local wd_version=${WD_VERSION:-$(get_version)}
+  setup_etcd_env
+  if [ $(compare_version ${wd_version} "5.3.0") -lt 0 ] ; then
+    echo "export ETCDCTL_USER='${ETCD_USER}:${ETCD_PASSWORD}' && \
+      export ETCDCTL_CERT='/etc/etcdtls/operator/etcd-tls/etcd-client.crt' && \
+      export ETCDCTL_CACERT='/etc/etcdtls/operator/etcd-tls/etcd-client-ca.crt' && \
+      export ETCDCTL_KEY='/etc/etcdtls/operator/etcd-tls/etcd-client.key' && \
+      export ETCDCTL_ENDPOINTS='https://${ETCD_SERVICE}:2379'"
+  else
+    # In WD 5.3.0+, certificate environment variables are already set in the etcd StatefulSet.
+    echo "export ETCDCTL_USER='${ETCD_USER}:${ETCD_PASSWORD}' && \
+      export ETCDCTL_ENDPOINTS='https://${ETCD_SERVICE}:2379'"
+  fi
+}
 
 check_etcd_available(){
+  local wd_version=${WD_VERSION:-$(get_version)}
   setup_etcd_env
-  ETCD_POD=$(oc get pods ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l etcd_cluster=${TENANT_NAME}-discovery-etcd)
-  oc exec ${OC_ARGS} "${ETCD_POD}" -- bash -c "export ETCDCTL_USER='${ETCD_USER}:${ETCD_PASSWORD}' && \
-  export ETCDCTL_CERT='/etc/etcdtls/operator/etcd-tls/etcd-client.crt' && \
-  export ETCDCTL_CACERT='/etc/etcdtls/operator/etcd-tls/etcd-client-ca.crt' && \
-  export ETCDCTL_KEY='/etc/etcdtls/operator/etcd-tls/etcd-client.key' && \
-  export ETCDCTL_ENDPOINTS='https://${ETCD_SERVICE}:2379' && \
-  etcdctl endpoint health > /dev/null" || return 1
+  ETCD_POD=$(get_etcd_pod)
+  ETCD_VARIABLES=$(export_etcd_variable_command)
+  if [ $(compare_version ${wd_version} "5.3.0") -lt 0 ] ; then
+    oc exec ${OC_ARGS} "${ETCD_POD}" -- bash -c "${ETCD_VARIABLES} && \
+    etcdctl endpoint health > /dev/null" || return 1
+  else 
+    oc exec ${OC_ARGS} "${ETCD_POD}" -- bash -c "${ETCD_VARIABLES} && \
+    etcdctl endpoint health" || return 1
+  fi 
+  
   return 0
 }
 
@@ -1385,7 +1466,7 @@ setup_etcd_env(){
   ETCD_SECRET=$(oc get secret ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l "tenant=${TENANT_NAME},app in (etcd,etcd-root)")
   ETCD_USER=$(oc get secret ${OC_ARGS} ${ETCD_SECRET} --template '{{.data.username}}' | base64 --decode)
   ETCD_PASSWORD=$(oc get secret ${OC_ARGS} ${ETCD_SECRET} --template '{{.data.password}}' | base64 --decode)
-  ETCD_POD=$(oc get pods ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l etcd_cluster=${TENANT_NAME}-discovery-etcd)
+  ETCD_POD=$(get_etcd_pod)
 }
 
 setup_pg_env(){
@@ -1451,6 +1532,15 @@ get_opensearch_cluster() {
   echo "$(oc get cluster.opensearch ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l tenant=${TENANT_NAME},app=opensearch)"
 }
 
+get_opensearch_operator_deployment() {
+  local wd_version=${WD_VERSION:-$(get_version)}
+  if [ $(compare_version ${wd_version} "5.3.0") -ge 0 ] && is_managed_by_helm; then
+    echo "$(oc get deploy -A -l icpdsupport/addOnId=opencontent-opensearch | tail -n1 | awk '{print $1,$2}')"
+  else
+    echo "$(oc get deploy -A -l "olm.owner=ibm-opensearch-operator.v${OPENSEARCH_VERSION}" | tail -n1 | awk '{print $1,$2}')"
+  fi
+}
+
 check_elastic_available(){
   ELASTIC_POD=$(get_elastic_pod)
   ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)

discovery-data/latest/lib/restore-updates.bash

@@ -6,6 +6,7 @@ UPDATE_DIR="$SCRIPT_DIR/src"
 source "${SCRIPT_DIR}/lib/function.bash"
 
 ELASTIC_POD_CONTAINER=$(get_elastic_pod_container)
+ETCD_VARIABLES=$(export_etcd_variable_command)
 
 elastic_updates(){
   if ls "$UPDATE_DIR"/*.elupdate &> /dev/null ; then
@@ -31,11 +32,7 @@ etcd_updates(){
     for ETCD_COMMAND in "$UPDATE_DIR"/*.etcdupdate; do
       value=$(<${ETCD_COMMAND})
       oc ${OC_ARGS} exec ${ETCD_POD} -- sh -c "export ETCDCTL_API=3 && \
-        export ETCDCTL_USER=${ETCD_USER}:${ETCD_PASSWORD} && \
-        export ETCDCTL_CERT=/etc/etcdtls/operator/etcd-tls/etcd-client.crt && \
-        export ETCDCTL_CACERT=/etc/etcdtls/operator/etcd-tls/etcd-client-ca.crt && \
-        export ETCDCTL_KEY=/etc/etcdtls/operator/etcd-tls/etcd-client.key && \
-        export ETCDCTL_ENDPOINTS=https://${ETCD_SERVICE}:2379 && \
+        ${ETCD_VARIABLES} && \
         ${value}" >> "${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
     done
   fi