Skip to content

Commit 1fb3301

Browse files
KosukeOkamotoKosukeOkamoto
committed
Update script version for Watson Discovery 4.7.3 on CP4D
1 parent 03168f5 commit 1fb3301

8 files changed

Lines changed: 29 additions & 16 deletions

‎discovery-data/latest/elastic-backup-restore.sh‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,13 +158,12 @@ if "${BACKUP_RESTORE_IN_POD}" && [ $(compare_version ${WD_VERSION} "4.7.0") -lt
158158
add_secret_env_to_job_yaml "S3_SECRET_KEY" "${S3_SECRET}" "secretkey" "${ELASTIC_JOB_FILE}"
159159
add_env_to_job_yaml "ELASTIC_ARCHIVE_OPTION" "${ELASTIC_ARCHIVE_OPTION}" "${ELASTIC_JOB_FILE}"
160160
add_env_to_job_yaml "ELASTIC_STATUS_CHECK_INTERVAL" "${ELASTIC_STATUS_CHECK_INTERVAL}" "${ELASTIC_JOB_FILE}"
161-
add_env_to_job_yaml "ELASTIC_ARCHIVE_OPTION" "${ELASTIC_ARCHIVE_OPTION}" "${ELASTIC_JOB_FILE}"
162161
add_env_to_job_yaml "ELASTIC_WAIT_GREEN_STATE" "${ELASTIC_WAIT_GREEN_STATE}" "${ELASTIC_JOB_FILE}"
163162
add_env_to_job_yaml "DISABLE_MC_MULTIPART" "${DISABLE_MC_MULTIPART}" "${ELASTIC_JOB_FILE}"
164163
add_env_to_job_yaml "TZ" "${TZ_OFFSET}" "${ELASTIC_JOB_FILE}"
165164
add_env_to_job_yaml "KEEP_SNAPSHOT" "${KEEP_SNAPSHOT}" "${ELASTIC_JOB_FILE}"
166165
add_env_to_job_yaml "ELASTIC_MAX_WAIT_RECOVERY_SECONDS" "${ELASTIC_MAX_WAIT_RECOVERY_SECONDS}" "${ELASTIC_JOB_FILE}"
167-
add_volume_to_job_yaml "backup-restore-workspace" "${JOB_VOLUME_NAME}" "${TMP_PVC_NAME:-emptyDir}" "${ELASTIC_JOB_FILE}"
166+
add_volume_to_job_yaml "backup-restore-workspace" "${TMP_PVC_NAME:-emptyDir}" "${ELASTIC_JOB_FILE}"
168167

169168
oc ${OC_ARGS} delete -f "${ELASTIC_JOB_FILE}" &> /dev/null || true
170169
oc ${OC_ARGS} apply -f "${ELASTIC_JOB_FILE}"

‎discovery-data/latest/lib/function.bash‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1080,6 +1080,7 @@ launch_s3_pod(){
10801080
if [ -n "${MINIO_ARCHIVE_OPTION:+UNDEF}" ] ; then add_env_to_job_yaml "MINIO_ARCHIVE_OPTION" "${MINIO_ARCHIVE_OPTION}" "${S3_JOB_FILE}"; fi
10811081
if [ -n "${DISABLE_MC_MULTIPART:+UNDEF}" ] ; then add_env_to_job_yaml "DISABLE_MC_MULTIPART" "${DISABLE_MC_MULTIPART}" "${S3_JOB_FILE}"; fi
10821082
add_env_to_job_yaml "TZ" "${TZ_OFFSET}" "${S3_JOB_FILE}"
1083+
add_env_to_job_yaml "WD_VERSION" "${wd_version}" "${S3_JOB_FILE}"
10831084
add_volume_to_job_yaml "backup-restore-workspace" "${TMP_PVC_NAME:-emptyDir}" "${S3_JOB_FILE}"
10841085
if [ $(compare_version ${wd_version} "4.7.0") -ge 0 ] ; then
10851086
BUCKET_SUFFIX="$(get_bucket_suffix)"

‎discovery-data/latest/minio-backup-restore.sh‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,9 +122,13 @@ if [ "${COMMAND}" = "backup" ] ; then
122122
start_minio_port_forward
123123
${MC} "${MC_OPTS[@]}" --quiet config host add wdminio ${S3_ENDPOINT_URL} ${S3_ACCESS_KEY} ${S3_SECRET_KEY} > /dev/null
124124
EXCLUDE_OBJECTS=$(cat "${SCRIPT_DIR}/src/minio_exclude_paths")
125+
if [ $(compare_version "$(get_version)" "4.7.0") -ge 0 ] ; then
126+
EXCLUDE_OBJECTS+=$'\n'
127+
EXCLUDE_OBJECTS+="$(cat "${SCRIPT_DIR}/src/mcg_exclude_paths")"
128+
fi
125129
for bucket in $(${MC} "${MC_OPTS[@]}" ls wdminio | sed ${SED_REG_OPT} "s|.*[0-9]+B\ (.*)/.*|\1|g" | grep -v ${ELASTIC_BACKUP_BUCKET})
126130
do
127-
EXTRA_MC_MIRROR_COMMAND=""
131+
EXTRA_MC_MIRROR_COMMAND=()
128132
ORG_IFS=${IFS}
129133
IFS=$'\n'
130134
for line in ${EXCLUDE_OBJECTS}
@@ -135,13 +139,13 @@ if [ "${COMMAND}" = "backup" ] ; then
135139
brlog "DEBUG" "SKIP ${bucket}"
136140
continue 2
137141
fi
138-
EXTRA_MC_MIRROR_COMMAND="--exclude ${line#"${base_bucket_name}" } ${EXTRA_MC_MIRROR_COMMAND}"
142+
EXTRA_MC_MIRROR_COMMAND+=( "--exclude" "${line#"$base_bucket_name" }" )
139143
fi
140144
done
141145
IFS=${ORG_IFS}
142146
cd ${TMP_WORK_DIR}
143147
set +e
144-
${MC} "${MC_OPTS[@]}" --quiet mirror ${EXTRA_MC_MIRROR_COMMAND} wdminio/${bucket} ${MINIO_BACKUP_DIR}/${bucket} &>> "${SCRIPT_DIR}/${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
148+
${MC} "${MC_OPTS[@]}" --quiet mirror "${EXTRA_MC_MIRROR_COMMAND[@]}" wdminio/${bucket} ${MINIO_BACKUP_DIR}/${bucket} &>> "${SCRIPT_DIR}/${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
145149
RC=$?
146150
echo "RC=${RC}" >> "${SCRIPT_DIR}/${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
147151
if [ $RC -ne 0 ] ; then

‎discovery-data/latest/mt-mt-migration.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,10 @@ do
119119
CMD="psql -d dadmin -t -A -c \"SELECT table_name FROM information_schema.columns WHERE table_schema = 'public' AND column_name = 'tenant_id' ORDER BY table_name\""
120120
standard_mt_tables=($(oc exec ${OC_ARGS} "${PG_POD}" -- bash -c "${CMD}"))
121121
echo "Updating tables with tenant_id: ${standard_mt_tables[*]}"
122-
foreign_key_tables=(wd_collections wd_collections_status wd_collections_project wd_datasets_collection wd_collection_document_status wd_collections_enrichment_job_status wd_datasets wd_datasets_status wd_datasets_project wd_enrichments wd_enrichments_lang wd_enrichments_project wd_collection_stats)
123-
122+
foreign_key_tables=(wd_collections wd_collections_status wd_collections_project wd_datasets_collection wd_collection_document_status wd_collections_enrichment_job_status wd_datasets wd_datasets_status wd_datasets_project wd_enrichments wd_enrichments_lang wd_enrichments_project)
123+
if [ $(compare_version "${WD_VERSION}" "4.7.0") -ge 0 ] ; then
124+
foreign_key_tables+=( wd_collection_stats )
125+
fi
124126
SQL="BEGIN;"
125127

126128
for table in "${foreign_key_tables[@]}"; do

‎discovery-data/latest/src/elastic-backup-restore-in-pod.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ ELASTIC_LOG="${TMP_WORK_DIR}/elastic.log"
2525

2626
export MINIO_CONFIG_DIR="${TMP_WORK_DIR}/.mc"
2727
MC_OPTS=(--config-dir ${MINIO_CONFIG_DIR} --insecure)
28-
MC_MIRROR_OPTS=( "" )
28+
MC_MIRROR_OPTS=()
2929
if "${DISABLE_MC_MULTIPART:-true}" ; then
30-
MC_MIRROR_OPTS=( "${MC_MIRROR_OPTS[@]}" --disable-multipart )
30+
MC_MIRROR_OPTS+=( "--disable-multipart" )
3131
fi
3232
MC=mc
3333

‎discovery-data/latest/src/mcg_exclude_paths‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# bucket paths
2+
common mt/__built-in-tenant__/fileResource*
3+
common watson_nlp_models*

‎discovery-data/latest/src/minio-backup-restore-in-pod.sh‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ mkdir -p ${TMP_WORK_DIR}/.mc
3737
MC=mc
3838
export MINIO_CONFIG_DIR="${TMP_WORK_DIR}/.mc"
3939
MC_OPTS=(--config-dir "${MINIO_CONFIG_DIR}" --insecure)
40-
MC_MIRROR_OPTS=( "" )
40+
MC_MIRROR_OPTS=()
4141
if "${DISABLE_MC_MULTIPART:-true}" ; then
42-
MC_MIRROR_OPTS=( "${MC_MIRROR_OPTS[@]}" --disable-multipart )
42+
MC_MIRROR_OPTS+=( "--disable-multipart" )
4343
fi
4444

4545
BUCKET_SUFFIX="${BUCKET_SUFFIX:-}"
@@ -50,9 +50,13 @@ if [ "${COMMAND}" = "backup" ] ; then
5050
brlog "INFO" "Backup data..."
5151
${MC} "${MC_OPTS[@]}" --quiet config host add wdminio ${S3_ENDPOINT_URL} ${S3_ACCESS_KEY} ${S3_SECRET_KEY} > /dev/null
5252
EXCLUDE_OBJECTS=$(cat "${SCRIPT_DIR}/src/minio_exclude_paths")
53+
if [ $(compare_version "$(get_version)" "4.7.0") -ge 0 ] ; then
54+
EXCLUDE_OBJECTS+=$'\n'
55+
EXCLUDE_OBJECTS+="$(cat "${SCRIPT_DIR}/src/mcg_exclude_paths")"
56+
fi
5357
for bucket in $(${MC} "${MC_OPTS[@]}" ls wdminio | sed ${SED_REG_OPT} "s|.*[0-9]+B\ (.*)/.*|\1|g" | grep -v ${ELASTIC_BACKUP_BUCKET})
5458
do
55-
EXTRA_MC_MIRROR_COMMAND=""
59+
EXTRA_MC_MIRROR_COMMAND=()
5660
ORG_IFS=${IFS}
5761
IFS=$'\n'
5862
base_bucket_name=${bucket%"${BUCKET_SUFFIX}"}
@@ -63,15 +67,15 @@ if [ "${COMMAND}" = "backup" ] ; then
6367
brlog "DEBUG" "SKIP ${bucket}"
6468
continue 2
6569
fi
66-
EXTRA_MC_MIRROR_COMMAND="--exclude ${line#"$base_bucket_name" } ${EXTRA_MC_MIRROR_COMMAND}"
70+
EXTRA_MC_MIRROR_COMMAND+=( "--exclude" "${line#"$base_bucket_name" }" )
6771
fi
6872
done
6973
IFS=${ORG_IFS}
7074
cd ${TMP_WORK_DIR}
7175
set +e
7276
while true;
7377
do
74-
${MC} "${MC_OPTS[@]}" --quiet mirror ${MC_MIRROR_OPTS[@]} ${EXTRA_MC_MIRROR_COMMAND} wdminio/${bucket} ${MINIO_BACKUP_DIR}/${bucket} 2>&1
78+
${MC} "${MC_OPTS[@]}" --quiet mirror "${MC_MIRROR_OPTS[@]}" "${EXTRA_MC_MIRROR_COMMAND[@]}" wdminio/${bucket} ${MINIO_BACKUP_DIR}/${bucket} 2>&1
7579
RC=$?
7680
echo "RC=${RC}"
7781
if [ $RC -eq 0 ] ; then
@@ -126,7 +130,7 @@ if [ "${COMMAND}" = "restore" ] ; then
126130
fi
127131
while true;
128132
do
129-
${MC} "${MC_OPTS[@]}" ${MC_MIRROR_COMMAND} --quiet ${MC_MIRROR_OPTS[@]} ${TMP_WORK_DIR}/${MINIO_BACKUP_DIR}/${bucket}/ wdminio/${bucket}/ 2>&1
133+
${MC} "${MC_OPTS[@]}" ${MC_MIRROR_COMMAND} --quiet "${MC_MIRROR_OPTS[@]}" ${TMP_WORK_DIR}/${MINIO_BACKUP_DIR}/${bucket}/ wdminio/${bucket}/ 2>&1
130134
RC=$?
131135
echo "RC=${RC}"
132136
if [ $RC -eq 0 ] ; then

‎discovery-data/latest/version.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
The Backup and Restore Scripts for the Watson Discovery on CP4D.
2-
Scripts Version: 4.7.1
2+
Scripts Version: 4.7.3

0 commit comments

Comments
 (0)