Restoring Planning Analytics

A Planning Analytics administrator can use the Planning Analytics administration console to restore the instance from an online backup.

Before you begin

  • Verify that the workstation where you run the installation is set up as a client workstation and includes the Red Hat® OpenShift® CLI.
  • Make sure that you are using the external version of the TM1 service. Export and import jobs for Planning Analytics support only the internal version of TM1.

    To determine if your TM1 service is external or internal, run the following command:

    oc get PAServiceInstance ${SERVICE_NAME} -n zen -o jsonpath='{.spec.tm1_internal_type}'

    If this command displays false as the output, your TM1 service is external.

Procedure

  1. Make sure that you are logged in to the Red Hat OpenShift cluster with the oc command line tool.
  2. Create the restore-pa.sh script to restore the instance.

    The restore-pa.sh script is a file with the following contents:

    #!/usr/bin/env bash
#
#       Licensed Materials - Property of IBM
#
#       IBM Cognos Products: pa
#
#       (C) Copyright IBM Corp. 2016
#
#       US Government Users Restricted Rights - Use, duplication or disclosure restricted by GSA ADP Schedule Contract with IBM Corp.
# -----------------------------------------------------------------------------
#

PA_KUBE_SCRIPT_DIR="$( cd "$(dirname "$0")" || exit ; pwd -P )"
PA_KUBE_TEMP_DIR=${PA_KUBE_SCRIPT_DIR}/.temp

set +e

KUBERNETES_CMD=kubectl
OPENSSL_CMD=openssl
UNZIP_CMD=unzip

PA_KUBE_NAMESPACE=${2:-zen}

EXTENSION=''
PA_KUBE_BACKUP_LOCATION=""
PA_KUBE_DECRYPT_BACKUP="false"

PA_KUBE_COUCHDB_POD=""
PA_KUBE_MONGO_POD=""
PA_KUBE_MYSQL_POD=""
PA_KUBE_REDIS_POD=""

PODS=( atlas async-service bss cdn gateway glass neo-idviz neo-provision pa-content pa-predict palm-service paw-ui-api plan-service prism-app prism-platform prism-proxy share-app share-platform share-proxy tm1proxy user-admin wa-proxy )

REPLICAS=()

#
# General messages
#

MSG_LOWERCASE_NO="n"
MSG_LOWERCASE_YES="y"
MSG_UPPERCASE_NO="N"
MSG_UPPERCASE_YES="Y"

#
# Restore messages
#
MSG_BACKUP_NAMESPACE_PROMPT="Enter the name of the namespace where Planning Analytics is deployed (default: '${1}'): "
MSG_PAW_LOCAL_BACKUP_INVALID_VERSION="No 'mongo.${1}' file found. The PAW Local backup must be created by a newer version."
MSG_RESTORE_CONFIGURATION="Restoring configuration"
MSG_RESTORE_COUCHDB="Restoring couchdb"
MSG_RESTORE="Restoring ${1}"
MSG_RESTORE_FROM_PAW_LOCAL_LINUX="Restoring from PAW Local Linux backup"
MSG_RESTORE_FROM_PAW_LOCAL_WINDOWS="Restoring from PAW Local Windows backup"
MSG_RESTORE_MONGO="Restoring mongo"
MSG_RESTORE_MYSQL="Restoring mysql"
MSG_RESTORE_PASSWORD="Decryption password: "
MSG_RESTORE_REDIS="Restoring redis"
MSG_RESTORE_USAGE="Usage: restore.sh <backup directory or zip file> <namespace>"
MSG_START_SERVICE="Starting service ${1}"
MSG_START_SERVICES="Starting services"
MSG_STOP_SERVICE="Stopping service ${1}"
MSG_STOP_SERVICES="Stopping services"
MSG_WAIT_FOR_SERVICE_TO_START="Waiting for service ${1} to start"
MSG_WAIT_FOR_SERVICE_TO_STOP="Waiting for service ${1} to stop"

function _exit {
  echo "$@"
  exit 1
}

function _getParameterValue {
  local msg=''
  local value=''

  msg=$( echo "${1}" "${2}" )
  read -er -p "${msg}" value
  value=${value//[[:space:]]}
  if [ "${value}" == "" ]; then
    value="${2}"
  fi

  PA_KUBE_CONFIGURATION_VALUE="${value}"
}

function _getPod {
  local idx=0
  local maxIdx=0	
  local pod=''
  local pods=()
  local result=''
  local container=''
  
  pod="${1}${PAW_INSTANCE_ID}"
  container="${2}"
  
  if ! read -r -a pods <<< "$( ${KUBERNETES_CMD} get pods -o=jsonpath='{range .items[*]}{.metadata.name}{.name}{" "}{end}' -l name="${pod}" -n "${PA_KUBE_NAMESPACE}" 2>&1 )"; then
    _exit "${pods[@]}"
  fi

  if [ "${#pods[@]}" != "1" ]; then
    _exit "${pods[@]}"
  fi
  
  maxIdx=${MAX_TIME:-30}
  while [ "${idx}" -lt "${maxIdx}" ]; do
	result=$( ${KUBERNETES_CMD} exec "${pods[0]}" -c "${container}" -n "${PA_KUBE_NAMESPACE}" -- /bin/bash -c 'ls >& /dev/null' 2>&1 )
    if [[ "${result}" != *"container not found"* ]]; then
	  break
    fi
    sleep 1
    (( idx=idx+1 ))
  done

  if [ "${idx}" -eq "${maxIdx}" ]; then
    _exit _exit "$( echo $MSG_CONTAINER_NOT_FOUND "${container}" )"
  fi
    
  echo "${pods[0]}"
}

function _getPods {
  PA_KUBE_COUCHDB_POD=$( _getPod 'couchdb-data1' 'couchdb1' )
  PA_KUBE_MONGO_POD=$( _getPod 'mongo-data1' 'mongo1' )
  PA_KUBE_MYSQL_POD=$( _getPod 'mysql-server' 'mysql-server' )
  PA_KUBE_REDIS_POD=$( _getPod 'redis-data1' 'redis' )
}

function _getReplicas {
  local pods=''

  local replicas=()
  for pod in "${PODS[@]}"; do
    pod="${pod}${PAW_INSTANCE_ID}"
    replica=$( ${KUBERNETES_CMD} get deployment "${pod}" --output='jsonpath={.spec.replicas}' -n "${PA_KUBE_NAMESPACE}" )
	if [ "${replica}" == "0" ]; then
		replica="1"
	fi
    REPLICAS+=("${replica}")
  done
}

function _log() {
  local msg=''
  local line=''
  local result=''

  if [ "${1:0:4}" == "$MSG_" ]; then
    msg=$( echo "$@" )
  else
    msg=$*
  fi

  if [ "$msg" != "" ]; then
    OLDIFS=${IFS}
    IFS=$'\n'

    case "${PA_LOG_MODE}" in
      console)
        for line in ${msg}; do
          echo "$( date +"%Y-%m-%d_:%H:%M:%S" ): ${line}"
        done
        ;;
      file)
        if [ ! -d "${PA_KUBE_LOG_DIR}" ]; then
          if ! result=$( mkdir -p "${PA_KUBE_LOG_DIR}" ); then
            _exit "${result}"
          fi
        fi
        for line in ${msg}; do
          echo "$( date +"${PA_KUBE_DATE_FORMAT}" ): ${line}" >> "${PA_KUBE_LOG_DIR}/${PA_KUBE_LOG_FILE}"
        done
        ;;
    esac

    IFS=${OLDIFS}
  fi
}

function _onExit {
  if [ -d "${PA_KUBE_TEMP_DIR}" ]; then
    rm -rf "${PA_KUBE_TEMP_DIR}"
  fi
}

#
# _prepareRestore
#
# ${1} - restore directory or zip file
#

function _prepareRestore {
  local result=''

  if ! result=$( mkdir -p "${PA_KUBE_TEMP_DIR}" ); then
    _exit "${result}"
  fi

  _log "${result}"

  if [[ "${1}" == *".zip" ]]; then
    if ! result=$( ${UNZIP_CMD} -d "${PA_KUBE_TEMP_DIR}" -j "${1}" ); then
      _exit "${result}"
    fi
  else
    if ! result=$( cp -r "${1}/." "${PA_KUBE_TEMP_DIR}" ); then
      _exit "${result}"
    fi
  fi
  
  if [ "${PA_KUBE_DECRYPT_BACKUP}" == "true" ]; then
    _getParameterValue $MSG_RESTORE_PASSWORD ""
  	for file in "${PA_KUBE_TEMP_DIR}"/*; do
  	  if ! result=$( ${OPENSSL_CMD} enc -d -base64 -in "${file}" -out "${file}.base64" -pass pass:"${PA_KUBE_CONFIGURATION_VALUE}"); then
  	    _exit "${result}"
  	  fi
  	  mv "${file}.base64" "${file}"
    done
  fi  
  
  EXTENSION="tgz"

  if [ -f "${PA_KUBE_TEMP_DIR}/bss.tgz" ]; then
    echo $MSG_RESTORE_FROM_PAW_LOCAL_LINUX
  fi

  if [ -f "${PA_KUBE_TEMP_DIR}/bss.zip" ]; then
    EXTENSION="zip"
    echo $MSG_RESTORE_FROM_PAW_LOCAL_WINDOWS
  fi

  if [ ! -f "${PA_KUBE_TEMP_DIR}/mysql.${EXTENSION}" ]; then
    _exit "$( echo $MSG_PAW_LOCAL_BACKUP_INVALID_VERSION "${EXTENSION}" )"
  fi
  
  _getPods
  _getReplicas
  _stopServices  
}

function _processArgs {
  if [ "${#@}" != "2" ]; then
    echo $MSG_RESTORE_USAGE
    exit 0
  fi

  PA_KUBE_BACKUP_LOCATION="${1}"
  PA_KUBE_NAMESPACE="${2}"
}

function _restoreCouchDb {
  local pods=()
  local result=''

  echo $MSG_RESTORE_COUCHDB

  if ! read -r -a pods <<< "$( ${KUBERNETES_CMD} get pods -o=jsonpath='{range .items[*]}{.metadata.name}{.name}{" "}{end}' -l name=couchdb-data1 -n "${PA_KUBE_NAMESPACE}" 2>&1 )"; then
    _exit "${pods[@]}"
  fi

  if [ "${#pods[@]}" != "1" ]; then
    _exit "${pods[@]}"
  fi


  if ! result=$( ${KUBERNETES_CMD} cp "${PA_KUBE_TEMP_DIR}/couchdb.${EXTENSION}" "${PA_KUBE_NAMESPACE}/${pods[0]}:/tmp/couchdb.${EXTENSION}" -c couchdb-data1 2>&1 ); then
    _exit "${result}"
  fi

  if ! result=$( ${KUBERNETES_CMD} exec "${pods[0]}" -c couchdb-data1 -n "${PA_KUBE_NAMESPACE}" -- /bin/bash -c '/tools/restore.sh' 2>&1 ); then
    _exit "${result}"
  fi
}

function _restoreMongo {
  local pods=()
  local result=''

  echo $MSG_RESTORE_MONGO

  if ! read -r -a pods <<< "$( ${KUBERNETES_CMD} get pods -o=jsonpath='{range .items[*]}{.metadata.name}{.name}{" "}{end}' -l name=mongo-data1 -n "${PA_KUBE_NAMESPACE}" 2>&1 )"; then
    _exit "${pods[@]}"
  fi

  if [ "${#pods[@]}" != "1" ]; then
    _exit "${pods[@]}"
  fi

  if ! result=$( ${KUBERNETES_CMD} cp "${PA_KUBE_TEMP_DIR}/mongo.${EXTENSION}"  "${PA_KUBE_NAMESPACE}/${pods[0]}:/tmp/mongo.${EXTENSION}" -c mongo-data1 2>&1 ); then
    _exit "${result}"
  fi

  if ! result=$( ${KUBERNETES_CMD} exec "${pods[0]}" -c mongo-data1 -n "${PA_KUBE_NAMESPACE}" -- /bin/bash -c '/tools/restore.sh' 2>&1 ); then
    _exit "${result}"
  fi

  _stopService mongo-data1
  _startService mongo-data1 1
}

function _restoreMySQL {
  local pods=()
  local result=''

  echo $MSG_RESTORE_MYSQL

  if ! read -r -a pods <<< "$( ${KUBERNETES_CMD} get pods -o=jsonpath='{range .items[*]}{.metadata.name}{.name}{" "}{end}' -l name=mysql-server -n "${PA_KUBE_NAMESPACE}" 2>&1 )"; then
    _exit "${pods[@]}"
  fi

  if ! result=$( ${KUBERNETES_CMD} cp "${PA_KUBE_TEMP_DIR}/mysql.${EXTENSION}" "${PA_KUBE_NAMESPACE}/${pods[0]}:/tmp/mysql.${EXTENSION}" -c mysql-server 2>&1 ); then
    _exit "${result}"
  fi

  if [ "${PA_KUBE_EXPLICIT_USERIDS}" == "true" ]; then
    if ! result=$( ${KUBERNETES_CMD} exec "${pods[0]}" -c mysql-server -n "${PA_KUBE_NAMESPACE}" -- /bin/bash -c "chown mysql:mysql /tmp/mysql.${EXTENSION}" 2>&1 ); then
      _exit "${result}"
    fi
  fi

  if ! result=$( ${KUBERNETES_CMD} exec "${pods[0]}" -c mysql-server -n "${PA_KUBE_NAMESPACE}" -- /bin/bash -c 'mysqladmin -u root --password=root flush-hosts;/tools/restore.sh' 2>&1 ); then
    _exit "${result}"
  fi
}

function _restoreRedis {
  local pods=()
  local result=''

  echo $MSG_RESTORE_REDIS

  _stopService redis-data2
  _stopService redis-data3

  _waitForRedisMaster

  if ! read -r -a pods <<< "$( ${KUBERNETES_CMD} get pods -o=jsonpath='{range .items[*]}{.metadata.name}{.name}{" "}{end}' -l name=redis-data1 -n "${PA_KUBE_NAMESPACE}" 2>&1 )"; then
    _exit "${pods[@]}"
  fi

  if [ "${#pods[@]}" != "1" ]; then
    _exit "${pods[@]}"
  fi


  if ! result=$( ${KUBERNETES_CMD} cp "${PA_KUBE_TEMP_DIR}/redis.${EXTENSION}" "${PA_KUBE_NAMESPACE}/${pods[0]}:/tmp/redis.${EXTENSION}" -c redis-data1 2>&1 ); then
    _exit "${result}"
  fi

  if ! result=$( ${KUBERNETES_CMD} exec "${pods[0]}" -c redis-data1 -n "${PA_KUBE_NAMESPACE}" -- /bin/bash -c '/tools/restore.sh' 2>&1 ); then
    _exit "${result}"
  fi

  _stopService redis-data1

  _startService redis-data1 1
  _startService redis-data2 1
  _startService redis-data3 1
}

function _startServices {
  local idx=''
  local pod=''

  idx=0
  for pod in "${PODS[@]}"; do
    pod="${pod}${PAW_INSTANCE_ID}"
	
    echo $MSG_START_SERVICE "${pod}"
	
    if ! result=$( ${KUBERNETES_CMD} scale --replicas="${REPLICAS[${idx}]}" deployment/"${pod}" -n "${PA_KUBE_NAMESPACE}" 2>&1 ); then
      _exit "${result}"
    fi		
    (( idx++ ))
  done
  
  for pod in "${PODS[@]}"; do
    _waitForServiceToStart "${pod}"
  done  
}

#
# _startService
#
# ${1} - service name
# ${2} - number of instances
#

function _startService {
  local pod=''
  local result=''

  pod="${1}${PAW_INSTANCE_ID}"
  
  echo $MSG_START_SERVICE "${pod}"

  if ! result=$( ${KUBERNETES_CMD} scale --replicas="${2}" deployment/"${pod}" -n "${PA_KUBE_NAMESPACE}" 2>&1 ); then
    _exit "${result}"
  fi

  result=$( ${KUBERNETES_CMD} get pods --selector=name="${pod}" --output=jsonpath='{.items..status.phase}' -n "${PA_KUBE_NAMESPACE}" 2>&1 )
  while [[ "${result}" != *"Running"* ]]; do
    sleep 1
    result=$( ${KUBERNETES_CMD} get pods --selector=name="${pod}" --output=jsonpath='{.items..status.phase}' -n "${PA_KUBE_NAMESPACE}" 2>&1 )
  done
}

function _stopServices {
  local pod=''

  for pod in "${PODS[@]}"; do
    pod="${pod}${PAW_INSTANCE_ID}"
	
    echo $MSG_STOP_SERVICE "${pod}"
	
    if ! result=$( ${KUBERNETES_CMD} scale --replicas=0 deployment/"${pod}" -n "${PA_KUBE_NAMESPACE}" 2>&1 ); then
      _exit "${result}"
    fi	  
  done
  
  for pod in "${PODS[@]}"; do
    pod="${pod}${PAW_INSTANCE_ID}"
    _waitForServiceToStop "${pod}"
  done
}

#
# _stopService
#
# ${1} - service name
#

function _stopService {
  local pod=''
  local result=''

  pod="${1}${PAW_INSTANCE_ID}"
  
  echo $MSG_STOP_SERVICE "${pod}"

  if ! result=$( ${KUBERNETES_CMD} scale --replicas=0 deployment/"${pod}" -n "${PA_KUBE_NAMESPACE}" 2>&1 ); then
    _exit "${result}"
  fi

  result=$( ${KUBERNETES_CMD} get pods --selector=name="${pod}" --output=jsonpath='{.items..status.phase}' -n "${PA_KUBE_NAMESPACE}" 2>&1 )
  while [[ "${result}" != "" ]]; do
    sleep 1
    result=$( ${KUBERNETES_CMD} get pods --selector=name="${pod}" --output=jsonpath='{.items..status.phase}' -n "${PA_KUBE_NAMESPACE}" 2>&1 )
  done
}

function _waitForRedisMaster {
  local idx=0
  local maxIdx=0
  local pod=''
  local pods=()
  local result=''
  local container=''
  
  pod="redis-data1${PAW_INSTANCE_ID}"
  container="redis-data1"
  
  if ! read -r -a pods <<< "$( ${KUBERNETES_CMD} get pods -o=jsonpath='{range .items[*]}{.metadata.name}{.name}{" "}{end}' -l name=${pod} -n "${PA_KUBE_NAMESPACE}" 2>&1 )"; then
    _exit "${pods[@]}"
  fi

  if [ "${#pods[@]}" != "1" ]; then
    _exit "${pods[@]}"
  fi

  maxIdx=${MAX_TIME:-30}
  while [ "${idx}" -lt "${maxIdx}" ]; do
    if result=$( ${KUBERNETES_CMD} exec "${pods[0]}" -c ${container} -n "${PA_KUBE_NAMESPACE}" -- /bin/bash -c '/tools/is-master.sh' 2>&1 ); then
      break
    fi
    sleep 1
    (( idx=idx+1 ))
  done

  if [ "${idx}" -eq "${maxIdx}" ]; then
    _exit _exit "$( echo $MSG_REDIS_MASTER_ERROR "${maxIdx}" )"
  fi
}

#
# _waitForServiceToStart
#
# ${1} - service name
#

function _waitForServiceToStart {
  local pod=''
  local result=''

  pod="${1}${PAW_INSTANCE_ID}"
  
  echo $MSG_WAIT_FOR_SERVICE_TO_START "${pod}"

  result=$( ${KUBERNETES_CMD} get pods --selector=name="${pod}" --output=jsonpath='{.items..status.phase}' -n "${PA_KUBE_NAMESPACE}" 2>&1 )
  while [[ "${result}" != *"Running"* ]]; do
    sleep 1
    result=$( ${KUBERNETES_CMD} get pods --selector=name="${pod}" --output=jsonpath='{.items..status.phase}' -n "${PA_KUBE_NAMESPACE}" 2>&1 )
  done
}

#
# _waitForServiceToStop
#
# ${1} - service name
#

function _waitForServiceToStop {
  local pod=''
  local result=''

  pod="${1}${PAW_INSTANCE_ID}"
  
  echo $MSG_WAIT_FOR_SERVICE_TO_STOP "${1}"

  result=$( ${KUBERNETES_CMD} get pods --selector=name="${pod}" --output=jsonpath='{.items..status.phase}' -n "${PA_KUBE_NAMESPACE}" 2>&1 )
  while [[ "${result}" != "" ]]; do
    sleep 1
    result=$( ${KUBERNETES_CMD} get pods --selector=name="${pod}" --output=jsonpath='{.items..status.phase}' -n "${PA_KUBE_NAMESPACE}" 2>&1 )
  done
}

#
# main
#

trap _onExit EXIT

_processArgs "${@}"

if [ -z "${PA_KUBE_BACKUP_LOCATION}" ]; then
  echo $MSG_RESTORE_USAGE
	exit 1
fi

if [ ! -e "${PA_KUBE_BACKUP_LOCATION}" ]; then
  echo $MSG_RESTORE_USAGE
	exit 1
fi

echo $MSG_RESTORE "${PA_KUBE_BACKUP_LOCATION}"

echo "Restoring to namespace $PA_KUBE_NAMESPACE"

_prepareRestore "${PA_KUBE_BACKUP_LOCATION}"
_restoreRedis
_restoreCouchDb
_restoreMongo
_restoreMySQL
_startServices
  3. Run the restore-pa.sh script with the directory that contains the backup as the first argument and the namespace where Planning Analytics is running as the second argument. 
    ./restore-pa.sh <backup-directory> <pa_namespace>

    For example,

    ./restore-pa.sh ./backup-2022-01-01 zen
  4. Wait for the Planning Analytics pods to start again.