Fix Readme
Abstract
Apply this hotfix after completing the watsonx Orchestrate installation for Version 5.3.1, or after upgrading from a previous release to Version 5.3.1.
Content
export PROJECT_CPD_INST_OPERATORS=<enter your IBM Software Hub operator project>
export PROJECT_CPD_INST_OPERANDS=<enter your IBM Software Hub operand project>Install skopeo: https://github.com/containers/skopeo/blob/main/install.md
export LOCAL_REGISTRY="your_local_registry"
export LOCAL_USER="your_local_registry_username"
export LOCAL_PASS="your_local_registry_password"
export IBM_ENTITLEMENT_KEY="your_ibm_entitlement_key"
export AUTH_JSON_PATH="${HOME}/.airgap/auth.json"
mkdir -p "$(dirname "${AUTH_JSON_PATH}")"
skopeo login cp.icr.io --username cp --password "${IBM_ENTITLEMENT_KEY}" --authfile "${AUTH_JSON_PATH}"
skopeo login "${LOCAL_REGISTRY}" --username "${LOCAL_USER}" --password "${LOCAL_PASS}" --authfile "${AUTH_JSON_PATH}"Copy operator images
# ibm-watsonx-orchestrate-operator (changed)
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://icr.io/cpopen/ibm-watsonx-orchestrate-operator@sha256:a2368d54ee8e05624ccda69fdd485cf5f3e586dc3ed4b6d32ab4c9dce6117276 \
docker://$LOCAL_REGISTRY/cpopen/ibm-watsonx-orchestrate-operator@sha256:a2368d54ee8e05624ccda69fdd485cf5f3e586dc3ed4b6d32ab4c9dce6117276
# ibm-wxo-component-operator (changed)
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://icr.io/cpopen/ibm-wxo-component-operator@sha256:7c82560e02eb4eabac988acb78e979ac3c035f6d1b1b93e9b1808b186b73e6a6 \
docker://$LOCAL_REGISTRY/cpopen/ibm-wxo-component-operator@sha256:7c82560e02eb4eabac988acb78e979ac3c035f6d1b1b93e9b1808b186b73e6a6
Copy operand images
# agent-analytics
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/agent-analytics@sha256:21a5bf18b80c5533f8328335a2b36d8bfbcb08a5a71a12b3bb491385510e10bd \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/agent-analytics@sha256:21a5bf18b80c5533f8328335a2b36d8bfbcb08a5a71a12b3bb491385510e10bd
# ai-gateway
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/ai-gateway@sha256:93bda853a0c7d9ea764c739c3fdfc3b73ef55df21d2c4ed1ab1588172f5d7064 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/ai-gateway@sha256:93bda853a0c7d9ea764c739c3fdfc3b73ef55df21d2c4ed1ab1588172f5d7064
# channel-integrations
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/channel-integrations@sha256:5db27d33535eb84857888ce5e74c9ec8b7a594e203d5b2265ffb45a25081df9d \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/channel-integrations@sha256:5db27d33535eb84857888ce5e74c9ec8b7a594e203d5b2265ffb45a25081df9d
# data-exhaust-producer
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/data-exhaust-producer@sha256:924b0e206d6c3937c700710c457f19e5cc8b220e8b44541e19b5d453ae945316 \
docker://${LOCAL_REGISTRY}/cp/data-exhaust-producer@sha256:924b0e206d6c3937c700710c457f19e5cc8b220e8b44541e19b5d453ae945316
# platformuiproxy
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/platformuiproxy@sha256:a784ce03cf7094a8424d73505bfc4db1effb1d48e1a1348f61aa32c79a98ddfc \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/platformuiproxy@sha256:a784ce03cf7094a8424d73505bfc4db1effb1d48e1a1348f61aa32c79a98ddfc
# skill-server
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/skill-server@sha256:1c09616441bd87e0bd983900098e078efb188f7c2122df615dbc4d0e00c26418 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/skill-server@sha256:1c09616441bd87e0bd983900098e078efb188f7c2122df615dbc4d0e00c26418
# socket-handler
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/socket-handler@sha256:a6097d3383d3737563b782e25bb7486d4019d88f7a3d82020f49bbe3c407f7b6 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/socket-handler@sha256:a6097d3383d3737563b782e25bb7486d4019d88f7a3d82020f49bbe3c407f7b6
# tools-runtime
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/tools-runtime@sha256:c980de7897b82744fc24d6155ef26b8f59080cd3a715ccd3d014050d790255c5 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/tools-runtime@sha256:c980de7897b82744fc24d6155ef26b8f59080cd3a715ccd3d014050d790255c5
# tools-runtime-manager
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/tools-runtime-manager@sha256:a1005b72cb3d982c89ab9f2a2c00ebc9c5afe909ac696374f40ce419699f2eb9 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/tools-runtime-manager@sha256:a1005b72cb3d982c89ab9f2a2c00ebc9c5afe909ac696374f40ce419699f2eb9
# tools-runtime-scheduler
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/tools-runtime-scheduler@sha256:81b0f04540d8b3f8fe229f0a9527a9cafbb605a499e5e275ffabcb51ed67a3a0 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/tools-runtime-scheduler@sha256:81b0f04540d8b3f8fe229f0a9527a9cafbb605a499e5e275ffabcb51ed67a3a0
# wo-ootb-primitive-skills
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/wo-ootb-primitive-skills@sha256:faabf8d9626ced20e9d7df15924e0fa5668708ffff828fe104db056a449ab435 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/wo-ootb-primitive-skills@sha256:faabf8d9626ced20e9d7df15924e0fa5668708ffff828fe104db056a449ab435
# wxo-builder
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/wxo-builder@sha256:708eead121b369a7fe58c1609e82644f04235b990a5e7117426ee138822676a0 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/wxo-builder@sha256:708eead121b369a7fe58c1609e82644f04235b990a5e7117426ee138822676a0
# wxo-chat
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/wxo-chat@sha256:7a31b346c68fbe93c6e5d0c871799ac32922aa54f90ba64a4c78de7ad4c7913a \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/wxo-chat@sha256:7a31b346c68fbe93c6e5d0c871799ac32922aa54f90ba64a4c78de7ad4c7913a
# wxo-connections
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/wxo-connections@sha256:f2bc260c3df0047a934de7a5824a7084c56d9606614d7a0e7884a6520edaea32 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/wxo-connections@sha256:f2bc260c3df0047a934de7a5824a7084c56d9606614d7a0e7884a6520edaea32
# wxo-flow-runtime
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/wxo-flow-runtime@sha256:3f00fe511a696d430f1b839adf1be520adae43b36bf844923b13eafc858e27a1 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/wxo-flow-runtime@sha256:3f00fe511a696d430f1b839adf1be520adae43b36bf844923b13eafc858e27a1
# wxo-server-conversation_controller
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/wxo-server-conversation_controller@sha256:6fc95de9e91f34a81bce84f3ef17d72856ffd9c4332d087c36dc84614bf2e44b \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/wxo-server-conversation_controller@sha256:6fc95de9e91f34a81bce84f3ef17d72856ffd9c4332d087c36dc84614bf2e44b
# wxo-server-server
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/wxo-server-server@sha256:b8c5c9219c1e39c06d9a73ccd35a031ea692365458d5934e5a6ceb63653adac2 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/wxo-server-server@sha256:b8c5c9219c1e39c06d9a73ccd35a031ea692365458d5934e5a6ceb63653adac2
# wxo-server-voice
skopeo copy --all --authfile "${AUTH_JSON_PATH}" --dest-tls-verify=false --src-tls-verify=false \
docker://cp.icr.io/cp/watsonx-orchestrate/wxo-server-voice@sha256:4f11c3ccbc0adbd1d2e76940250fe401252083772aadec32e90f835ba718e0b4 \
docker://${LOCAL_REGISTRY}/cp/watsonx-orchestrate/wxo-server-voice@sha256:4f11c3ccbc0adbd1d2e76940250fe401252083772aadec32e90f835ba718e0b4After copying, verify the images exist in $LOCAL_REGISTRY.
Create hotfix5310.sh with the contents below.
#!/bin/bash
-----------------------------------------------------------------------------
# watsonx Orchestrate 5.3.1 Hotfix
# - Verifies watsonx Orchestrate version from .status.versionStatus.status.
# - Images of Operators are replaced with the image from hotfix script.
# * HOTFIX_LABEL_VALUE for hotfix 0 is 5.3.1.0
# * If an existing label value matches x.x.x.x and is higher than HOTFIX_LABEL_VALUE,
# the script exits early after informing you
# - Deletes a fixed set of Jobs in the operands namespace and waits for all to reappear with
# new UIDs and succeed
# -----------------------------------------------------------------------------
# -----------------------------
# Patch operator deployment images
# -----------------------------
ts() { date +"%Y-%m-%d %H:%M:%S"; }
OPERATOR_IMAGES="icr.io/cpopen/ibm-watsonx-orchestrate-operator@sha256:a2368d54ee8e05624ccda69fdd485cf5f3e586dc3ed4b6d32ab4c9dce6117276
icr.io/cpopen/ibm-wxo-component-operator@sha256:7c82560e02eb4eabac988acb78e979ac3c035f6d1b1b93e9b1808b186b73e6a6"
if [ -z "${PROJECT_CPD_INST_OPERATORS:-}" ]; then
echo "[ERROR] PROJECT_CPD_INST_OPERATORS is not set. Exiting."
exit 1
fi
# Hotfix label configuration
HOTFIX_LABEL_KEY="${HOTFIX_LABEL_KEY:-hotfix}"
HOTFIX_LABEL_VALUE="${HOTFIX_LABEL_VALUE:-5.3.1.0}"
WO_CR_NAME=wo
is_semver4() {
v="$1"
printf '%s' "$v" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
}
# Backup dir for deployments
BACKUP_DIR="/tmp/wxo_deployment_backups"
mkdir -p "$BACKUP_DIR"
if [ -n "${OPERATOR_IMAGES:-}" ]; then
echo "[$(ts)] Updating operator deployment images in ${PROJECT_CPD_INST_OPERATORS}"
patched_deps=""
# Use a here-document to avoid subshell so patched_deps is preserved
while IFS= read -r img; do
[ -z "$img" ] && continue
# Extract base image name (e.g., ibm-wxo-component-operator)
base="$(basename "$img" | cut -d'@' -f1)"
echo "[$(ts)] Processing image: $img (base=$base)"
dep=""
case "$base" in
ibm-watsonx-orchestrate-operator)
dep="wo-operator"
;;
ibm-wxo-component-operator)
dep="ibm-wxo-componentcontroller-manager"
;;
*)
dep="$(oc -n "$PROJECT_CPD_INST_OPERATORS" get deploy --no-headers 2>/dev/null \
| grep "$base" | awk 'NR==1{print $1}')"
;;
esac
# Skip bundle/catalog images if they ever slip in
if echo "$base" | grep -Eq 'bundle|catalog'; then
echo "[$(ts)] Skipping $base (bundle/catalog image – not tied to deployment)"
continue
fi
if [ -z "$dep" ]; then
echo "[$(ts)] WARNING: no matching deployment found for $base — skipping"
continue
fi
# -----------------------------
# Backup deployment YAML
# -----------------------------
backup_file="${BACKUP_DIR}/${dep}-$(date +%Y%m%d%H%M%S).yaml"
if oc -n "$PROJECT_CPD_INST_OPERATORS" get deploy "$dep" -o yaml > "$backup_file" 2>/dev/null; then
echo "[$(ts)] Backed up deployment/$dep → $backup_file"
else
echo "[$(ts)] WARNING: failed to back up deployment/$dep"
fi
# Determine container name (assume first container is operator)
cname="$(oc -n "$PROJECT_CPD_INST_OPERATORS" get deploy "$dep" \
-o jsonpath='{.spec.template.spec.containers[0].name}' 2>/dev/null)"
if [ -z "$cname" ]; then
echo "[$(ts)] WARNING: cannot determine container name for $dep — skipping"
continue
fi
echo "[$(ts)] Patching deployment/$dep container '$cname' → $img"
if oc -n "$PROJECT_CPD_INST_OPERATORS" set image "deployment/$dep" "$cname=$img" >/dev/null 2>&1; then
echo "[$(ts)] ✓ Image updated for $dep"
patched_deps="$patched_deps $dep"
else
echo "[$(ts)] ✗ ERROR: failed to patch $dep"
fi
done <<EOF
$OPERATOR_IMAGES
EOF
# -----------------------------
# Label WO CR with configurable label and value
# -----------------------------
if [ -n "$WO_CR_NAME" ]; then
current_label="$(oc -n "$PROJECT_CPD_INST_OPERANDS" get wo "$WO_CR_NAME" -o jsonpath="{.metadata.labels.${HOTFIX_LABEL_KEY}}" 2>/dev/null || true)"
if [ "$current_label" = "$HOTFIX_LABEL_VALUE" ]; then
echo "[$(ts)] WO CR ${WO_CR_NAME} already labeled ${HOTFIX_LABEL_KEY}=${HOTFIX_LABEL_VALUE}"
else
echo "[$(ts)] Setting label ${HOTFIX_LABEL_KEY}=${HOTFIX_LABEL_VALUE} on WO CR ${WO_CR_NAME} in ns ${PROJECT_CPD_INST_OPERANDS}"
oc -n "$PROJECT_CPD_INST_OPERANDS" label wo "$WO_CR_NAME" "${HOTFIX_LABEL_KEY}=${HOTFIX_LABEL_VALUE}" --overwrite >/dev/null 2>&1 || true
new_label="$(oc -n "$PROJECT_CPD_INST_OPERANDS" get wo "$WO_CR_NAME" -o jsonpath="{.metadata.labels.${HOTFIX_LABEL_KEY}}" 2>/dev/null || true)"
if [ "$new_label" = "$HOTFIX_LABEL_VALUE" ]; then
echo "[$(ts)] Label set: ${HOTFIX_LABEL_KEY}=${HOTFIX_LABEL_VALUE}"
else
echo "[$(ts)] WARNING: could not confirm ${HOTFIX_LABEL_KEY}=${HOTFIX_LABEL_VALUE} label was set"
fi
fi
else
echo "[$(ts)] No WO CR found in ns ${PROJECT_CPD_INST_OPERANDS}, skipping label."
fi
# -----------------------------
# Verify patched deployments are healthy (1/1 or 2/2)
# -----------------------------
if [ -n "${patched_deps// /}" ]; then
echo "[$(ts)] Verifying rollout status for patched deployments..."
for dep in $patched_deps; do
echo "[$(ts)] Checking deployment/$dep..."
# Wait for rollout to complete
if oc -n "$PROJECT_CPD_INST_OPERATORS" rollout status deploy/"$dep" --timeout=300s; then
# Check Ready/Desired replica ratio
ratio="$(oc -n "$PROJECT_CPD_INST_OPERATORS" get deploy "$dep" \
-o jsonpath='{.status.readyReplicas}/{.status.replicas}' 2>/dev/null || echo '0/0')"
echo "[$(ts)] Ready/Desired: $ratio"
if [ "$ratio" = "1/1" ] || [ "$ratio" = "2/2" ]; then
echo "[$(ts)] ✓ Deployment $dep is healthy (pods up and running)."
else
echo "[$(ts)] ⚠ WARNING: Deployment $dep is not at 1/1 or 2/2; current $ratio"
fi
else
echo "[$(ts)] ✗ ERROR: rollout status for deployment/$dep did not complete successfully."
fi
done
else
echo "[$(ts)] No deployments were patched; skipping health verification."
fi
else
echo "[$(ts)] No OPERATOR_IMAGES specified — skipping operator image patch."
fi
# -----------------------------
# Final message
# -----------------------------
echo "------------------------------------------------------------------"
echo "[$(ts)] 5.3.1 Hotfix steps completed."
echo "Backups saved under ${BACKUP_DIR}"
echo "Monitor the watsonx Orchestrate CR status by running:"
echo " oc get wo -n ${PROJECT_CPD_INST_OPERANDS} -o yaml | grep -E 'watsonxOrchestrateStatus|${HOTFIX_LABEL_KEY}'"
echo "Ensure the watsonx Orchestrate CR status is 'Completed' and label ${HOTFIX_LABEL_KEY}=${HOTFIX_LABEL_VALUE} is present."
echo "------------------------------------------------------------------"
Make the script executable:
chmod 775 hotfix5310.shRun the script:
nohup sh hotfix5310.sh &Watch progress:
tail -f nohup.outVerify CR status and label:
oc get wo -n "${PROJECT_CPD_INST_OPERANDS}" -o yaml | grep hotfix
hotfix: 5.3.1.0
[{"Type":"MASTER","Line of Business":{"code":"LOB76","label":"Data Platform"},"Business Unit":{"code":"BU048","label":"IBM Software"},"Product":{"code":"SSVAUS","label":"IBM watsonx Orchestrate Cartridge for IBM Cloud Pak for Data"},"ARM Category":[{"code":"a8mKe0000008OVAIA2","label":"Operate-\u003EInstall"}],"Platform":[{"code":"PF025","label":"Platform Independent"}],"Version":""}]
Was this topic helpful?
Document Information
Modified date:
13 March 2026
UID
ibm17263409