Applying the watsonx Orchestrate 5.3.1 Patch 5 (5.3.3) Hotfix 0

#!/usr/bin/env bash
set -euo pipefail

# Function to print log messages with timestamp
log() {
  echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
}

PROJECT_CPD_INST_OPERATORS="${PROJECT_CPD_INST_OPERATORS:-}"
PROJECT_CPD_INST_OPERANDS="${PROJECT_CPD_INST_OPERANDS:-}"

if [[ -z "$PROJECT_CPD_INST_OPERATORS" ]]; then
  log "ERROR: PROJECT_CPD_INST_OPERATORS is not set."
  exit 1
fi

if [[ -z "$PROJECT_CPD_INST_OPERANDS" ]]; then
  log "ERROR: PROJECT_CPD_INST_OPERANDS is not set."
  exit 1
fi

# Hotfix label configuration
OPERATOR_PATCH_LABEL_KEY="${OPERATOR_PATCH_LABEL_KEY:-hotfix}"
OPERATOR_PATCH_LABEL_VALUE="${OPERATOR_PATCH_LABEL_VALUE:-5.3.3-hotfix0}"
WO_CR_NAME="wo"

# Make sure oc login is done
if ! oc whoami &>/dev/null; then
  log "ERROR: Not logged in to OpenShift. Please run 'oc login' first."
  exit 1
fi

log "✅ OpenShift login verified: $(oc whoami)"

# Backup dir for deployments
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CLUSTER_NAME="$(oc whoami --show-console | sed 's/.*console-openshift-console\.apps\.\([^.]*\)\..*/\1/')"
BACKUP_DIR="${SCRIPT_DIR}/wxo_deployment_backups/$CLUSTER_NAME"
mkdir -p "$BACKUP_DIR"

log "📁 Backup directory: $BACKUP_DIR"

# Check WXO version
log "🔍 Checking WXO version in namespace: $PROJECT_CPD_INST_OPERANDS"
WXO_VERSION=$(oc get wo -n "$PROJECT_CPD_INST_OPERANDS" -o jsonpath='{.items[0].spec.version}' 2>/dev/null || echo "")

if [[ -z "$WXO_VERSION" ]]; then
  log "ERROR: Unable to retrieve WXO version. Please ensure WatsonxOrchestrate resource exists."
  exit 1
fi

log "   Current WXO version: $WXO_VERSION"

if [[ "$WXO_VERSION" != "7.1.2" ]]; then
  log "ERROR: This hotfix can be applied only on 5.3.1 patch 5."
  log "       Expected: 7.1.2"
  log "       Found: $WXO_VERSION"
  exit 1
fi

log "✅ Version check passed (7.1.2)"
log ""

# Hardcode images here when you do not want to pass them as script arguments.
BOOTSTRAP_OPERATOR_IMAGE="icr.io/cpopen/ibm-wxo-component-operator@sha256:cdd2c4617d0f5e7b369866f815573d44f5b09cc6bfe2bbf8275e73388d85be5d"
COMPONENT_OPERATOR_IMAGE="icr.io/cpopen/ibm-watsonx-orchestrate-operator@sha256:0ec5bea7cedc70f2a68c6d7be3d1227533adaec874c95f17119344dfe5ea4fab"

if [[ $# -gt 1 ]]; then
  log "Usage: $0 [image1,image2,...]"
  exit 1
fi

if [[ $# -eq 1 ]]; then
  IFS=',' read -ra IMAGES <<< "$1"
else
  IMAGES=()
  [[ -n "$BOOTSTRAP_OPERATOR_IMAGE" ]] && IMAGES+=("$BOOTSTRAP_OPERATOR_IMAGE")
  [[ -n "$COMPONENT_OPERATOR_IMAGE" ]] && IMAGES+=("$COMPONENT_OPERATOR_IMAGE")

  if [[ ${#IMAGES[@]} -eq 0 ]]; then
    log "Usage: $0 [image1,image2,...]"
    log "Either pass images as an argument or hardcode BOOTSTRAP_OPERATOR_IMAGE / COMPONENT_OPERATOR_IMAGE in the script."
    exit 1
  fi
fi

# Track patched deployments for health verification
PATCHED_DEPLOYMENTS=()

for IMAGE in "${IMAGES[@]}"; do
  IMAGE="$(echo "$IMAGE" | xargs)"

  # Extract image name - handle both tag (:) and digest (@) formats
  IMAGE_NAME="$(basename "$IMAGE" | cut -d'@' -f1 | cut -d':' -f1)"

  # Map image → deployment
  case "$IMAGE_NAME" in
    ibm-wxo-component-operator)
      DEPLOYMENT="ibm-wxo-componentcontroller-manager"
      ;;
    ibm-watsonx-orchestrate-operator)
      DEPLOYMENT="wo-operator"
      ;;
    *)
      log "⚠️  No deployment mapping found for image: $IMAGE_NAME"
      continue
      ;;
  esac

  log "🔍 Checking deployment '$DEPLOYMENT' for image '$IMAGE_NAME'..."

  # Backup deployment YAML before patching
  BACKUP_FILE="${BACKUP_DIR}/${DEPLOYMENT}-$(date +%Y%m%d%H%M%S).yaml"
  if oc -n "$PROJECT_CPD_INST_OPERATORS" get deploy "$DEPLOYMENT" -o yaml > "$BACKUP_FILE" 2>/dev/null; then
    log "   Backed up deployment/$DEPLOYMENT → $BACKUP_FILE"
  else
    log "   WARNING: Failed to back up deployment/$DEPLOYMENT"
  fi

  CURRENT_IMAGE="$(oc get deploy "$DEPLOYMENT" -n "$PROJECT_CPD_INST_OPERATORS" \
    -o jsonpath='{.spec.template.spec.containers[0].image}')"

  if [[ "$CURRENT_IMAGE" != *"$IMAGE_NAME"* ]]; then
    log "⚠️  Image '$IMAGE_NAME' not found in deployment '$DEPLOYMENT'. Skipping."
    continue
  fi

  log "✅ Match found. Patching deployment '$DEPLOYMENT'"
  log "   Old: $CURRENT_IMAGE"
  log "   New: $IMAGE"

  if oc patch deploy "$DEPLOYMENT" -n "$PROJECT_CPD_INST_OPERATORS" \
    --type='json' \
    -p="[{
      \"op\": \"replace\",
      \"path\": \"/spec/template/spec/containers/0/image\",
      \"value\": \"$IMAGE\"
    }]"; then
    log "🚀 Successfully patched $DEPLOYMENT"
    PATCHED_DEPLOYMENTS+=("$DEPLOYMENT")
  else
    log "✗ ERROR: Failed to patch $DEPLOYMENT"
  fi
  log ""
done

# -----------------------------
# Label WO CR with operator patch label
# -----------------------------
if [[ -n "$WO_CR_NAME" ]]; then
  log "🏷️  Labeling WO CR with ${OPERATOR_PATCH_LABEL_KEY}=${OPERATOR_PATCH_LABEL_VALUE}..."
  
  CURRENT_LABEL="$(oc -n "$PROJECT_CPD_INST_OPERANDS" get wo "$WO_CR_NAME" \
    -o jsonpath="{.metadata.labels.${OPERATOR_PATCH_LABEL_KEY}}" 2>/dev/null || true)"
  
  if [[ "$CURRENT_LABEL" == "$OPERATOR_PATCH_LABEL_VALUE" ]]; then
    log "   WO CR ${WO_CR_NAME} already labeled ${OPERATOR_PATCH_LABEL_KEY}=${OPERATOR_PATCH_LABEL_VALUE}"
  else
    log "   Setting label ${OPERATOR_PATCH_LABEL_KEY}=${OPERATOR_PATCH_LABEL_VALUE} on WO CR ${WO_CR_NAME}"
    if oc -n "$PROJECT_CPD_INST_OPERANDS" label wo "$WO_CR_NAME" \
      "${OPERATOR_PATCH_LABEL_KEY}=${OPERATOR_PATCH_LABEL_VALUE}" --overwrite >/dev/null 2>&1; then
      
      NEW_LABEL="$(oc -n "$PROJECT_CPD_INST_OPERANDS" get wo "$WO_CR_NAME" \
        -o jsonpath="{.metadata.labels.${OPERATOR_PATCH_LABEL_KEY}}" 2>/dev/null || true)"
      
      if [[ "$NEW_LABEL" == "$OPERATOR_PATCH_LABEL_VALUE" ]]; then
        log "   ✅ Label set successfully: ${OPERATOR_PATCH_LABEL_KEY}=${OPERATOR_PATCH_LABEL_VALUE}"
      else
        log "   ⚠️  WARNING: Could not confirm label was set"
      fi
    else
      log "   ⚠️  WARNING: Failed to set label on WO CR"
    fi
  fi
else
  log "⚠️  No WO CR found, skipping label."
fi

log ""

# -----------------------------
# Verify patched deployments are healthy
# -----------------------------
if [[ ${#PATCHED_DEPLOYMENTS[@]} -gt 0 ]]; then
  log "🔍 Verifying rollout status for patched deployments..."
  
  for DEPLOYMENT in "${PATCHED_DEPLOYMENTS[@]}"; do
    log "   Checking deployment/$DEPLOYMENT..."
    
    # Wait for rollout to complete
    if oc -n "$PROJECT_CPD_INST_OPERATORS" rollout status deploy/"$DEPLOYMENT" --timeout=300s; then
      # Check Ready/Desired replica ratio
      RATIO="$(oc -n "$PROJECT_CPD_INST_OPERATORS" get deploy "$DEPLOYMENT" \
        -o jsonpath='{.status.readyReplicas}/{.status.replicas}' 2>/dev/null || echo '0/0')"
      log "   Ready/Desired: $RATIO"
      
      if [[ "$RATIO" == "1/1" ]] || [[ "$RATIO" == "2/2" ]]; then
        log "   ✅ Deployment $DEPLOYMENT is healthy (pods up and running)"
      else
        log "   ⚠️  WARNING: Deployment $DEPLOYMENT is not at 1/1 or 2/2; current $RATIO"
      fi
    else
      log "   ✗ ERROR: Rollout status for deployment/$DEPLOYMENT did not complete successfully"
    fi
  done
else
  log "ℹ️  No deployments were patched; skipping health verification."
fi

# -----------------------------
# Final message
# -----------------------------
log ""
log "------------------------------------------------------------------"
log "✅ Operator patch steps completed (${OPERATOR_PATCH_LABEL_VALUE})"
log ""
log "📁 Backups saved under: ${BACKUP_DIR}"
log ""
log "📊 Monitor the watsonx Orchestrate CR status by running:"
log "   oc get wo -n ${PROJECT_CPD_INST_OPERANDS} -o yaml | grep -E 'watsonxOrchestrateStatus|${OPERATOR_PATCH_LABEL_KEY}'"
log ""
log "✓ Ensure the watsonx Orchestrate CR status is 'Completed'"
log "✓ Ensure label ${OPERATOR_PATCH_LABEL_KEY}=${OPERATOR_PATCH_LABEL_VALUE} is present"
log ""
log "⏱️  It will take another 15–20 minutes for the updated components"
log "   to be applied and restarted."
log "------------------------------------------------------------------"