Certificate rotation

When the service CA certificate is rotated then certain pods or services need to be restarted. The service CA certificate is often rotated when a cluster is upgraded, or when the certificate is close to expiry. You might see errors in the Resource Management UI, such as the following example.

Failed to connect to the Inventory Service using HTTPS. The signer certificate may be 
missing or invalid.

As a workaround you can use this script to complete a rolling restart. This will keep services running while new pods are started, then terminate the old pods.

#!/bin/bash

# set the namespace where your install exists:
NAMESPACE=netcool-operands
  
OCP_SERVICE_ANNOTATIONS=( "service\.alpha\.openshift\.io/serving-cert-secret-name"
                          "service\.beta\.openshift\.io/serving-cert-secret-name")
for ann in ${OCP_SERVICE_ANNOTATIONS[@]}; do
    echo "$(date) 🔎 Finding secrets that contain Openshift service serving certificates with annotation '$ann'"
    NOI_SECRETS=$(oc get svc --namespace ${NAMESPACE} -o jsonpath="{.items[*].metadata.annotations.${ann}}")

    for scrt in ${NOI_SECRETS[@]}; do
      echo "$(date) 🔎 Finding deployments that use secret $scrt..."
      NOI_DEPLOYMENTS+=($(oc get deployment --namespace ${NAMESPACE} -o go-template="
                                {{- range \$i, \$element := .items -}}
                                  {{- range \$j, \$volume := \$element.spec.template.spec.volumes -}}
                                    {{- if eq \$volume.secret.secretName \"$scrt\" -}}
                                      {{- printf \"%s \" \$element.metadata.name -}}
                                    {{- end -}}
                                    {{- range \$k,\$source := \$volume.projected.sources -}}
                                      {{- if eq \$source.secret.name \"$scrt\" -}}
                                        {{- printf \"%s \" \$element.metadata.name -}}
                                      {{- end -}}
                                    {{- end -}}
                                  {{- end -}}
                                {{- end -}}"))

      echo "$(date) 🔎 Finding statefulesets that use secret $scrt..."
      NOI_STS+=($(oc get statefulset --namespace ${NAMESPACE} -o go-template="
                    {{- range \$i, \$element := .items -}}
                      {{- range \$j, \$volume := \$element.spec.template.spec.volumes -}}
                        {{- if eq \$volume.secret.secretName \"$scrt\" -}}
                          {{- printf \"%s \" \$element.metadata.name -}}
                        {{- end -}}
                        {{- range \$k,\$source := \$volume.projected.sources -}}
                          {{- if eq \$source.secret.name \"$scrt\" -}}
                            {{- printf \"%s \" \$element.metadata.name -}}
                          {{- end -}}
                        {{- end -}}
                      {{- end -}}
                    {{- end -}}"))
    done
done

echo "$(date) 🔎 Finding configmaps that contain Openshift CA bundle"
NOI_CMS=$(oc get cm --namespace ${NAMESPACE} -o go-template="
                    {{- range \$i, \$item := .items -}}
                      {{- range \$k, \$v := \$item.metadata.annotations -}}
                        {{- if eq \$k \"service.beta.openshift.io/inject-cabundle\" -}}
                          {{- printf \"%s \" \$item.metadata.name -}}
                        {{- end -}}
                      {{- end -}}
                    {{- end -}}")

for cm in ${NOI_CMS[@]}; do
    echo "$(date) 🔎 Finding deployments that use configmap $cm..."
    NOI_DEPLOYMENTS+=($(oc get deployment --namespace ${NAMESPACE} -o go-template="
                              {{- range \$i, \$element := .items -}}
                                {{- range \$j, \$volume := \$element.spec.template.spec.volumes -}}
                                  {{- if eq \$volume.configMap.name \"$cm\" -}}
                                    {{- printf \"%s \" \$element.metadata.name -}}
                                  {{- end -}}
                                  {{- range \$k,\$source := \$volume.projected.sources -}}
                                    {{- if eq \$source.configMap.name \"$cm\" -}}
                                      {{- printf \"%s \" \$element.metadata.name -}}
                                    {{- end -}}
                                  {{- end -}}
                                {{- end -}}
                              {{- end -}}"))

    echo "$(date) 🔎 Finding statefulesets that use configmap $cm..."
    NOI_STS+=($(oc get statefulset --namespace ${NAMESPACE} -o go-template="
                  {{- range \$i, \$element := .items -}}
                    {{- range \$j, \$volume := \$element.spec.template.spec.volumes -}}
                      {{- if eq \$volume.configMap.name \"$cm\" -}}
                        {{- printf \"%s \" \$element.metadata.name -}}
                      {{- end -}}
                      {{- range \$k,\$source := \$volume.projected.sources -}}
                        {{- if eq \$source.configMap.name \"$cm\" -}}
                          {{- printf \"%s \" \$element.metadata.name -}}
                        {{- end -}}
                      {{- end -}}
                    {{- end -}}
                  {{- end -}}"))
done

# SORT AND DEDUPLICATE LIST OF STATEFULSETS 
IFS=$'\n' NOI_STS_SORTED=($(sort -u <<<"${NOI_STS[*]}"))
unset IFS
echo "$(date) Here are the statefulsets we need to restart:"
for sts in ${NOI_STS_SORTED[@]}; do echo "                             - $sts"; done 

# SORT AND DEDUPLICATE LIST OF DEPLOYMENTS 
IFS=$'\n' NOI_DEPLOYMENTS_SORTED=($(sort -u <<<"${NOI_DEPLOYMENTS[*]}"))
unset IFS
echo "$(date) Here are the deployments we need to restart:"
for dep in ${NOI_DEPLOYMENTS_SORTED[@]}; do echo "                             - $dep"; done

for svc in ${NOI_STS_SORTED[@]}; do 
  echo "$(date) Restarting ${svc}"
  oc rollout restart statefulset/${svc} --namespace ${NAMESPACE} ; 
done  
for svc in ${NOI_DEPLOYMENTS_SORTED[@]}; do 
  echo "$(date) Restarting ${svc}"
  oc rollout restart deployment/${svc} --namespace ${NAMESPACE} ; 
done 
for svc in ${NOI_STS_SORTED[@]}; do 
  echo "$(date) Waiting for ${svc}"
  oc rollout status statefulset/${svc} --namespace ${NAMESPACE} --timeout 15m; 
done  
for svc in ${NOI_DEPLOYMENTS_SORTED[@]}; do 
  echo "$(date) Waiting for ${svc}"
  oc rollout status deployment/${svc} --namespace ${NAMESPACE} --timeout 15m; 
done