How to collect Instana doc for Self-hosted Instana backend environments on Kubernetes/OpenShift

Answer

Use the attached script to get a complete set of doc for all information related to the Instana backend components running on Kubernetes/OpenShift in your cluster (Custom Edition backend).

This script can be used for collecting additional info for Standard Edition backends too, as long as kubectl plugin is installed on that backend's host.

Running this script will produce a file named instana-BE-mustgather-yyyy.mm.dd-hh.mm.ss.tgz which can be shared with Instana Support Team.

Content of script is:

#!/bin/bash
###############################################################################
#
# This script is used to collect data for
# the Instana operator-based on-prem running on Kubernetes / Openshift
#
# ./instana-k8s-BE-mustgather.sh
#

current_time=$(date "+%Y.%m.%d-%H.%M.%S")
export MGDIR=instana-BE-mustgather-$current_time
mkdir -p $MGDIR
if command -v oc > /dev/null
then
  CMD=oc
else
  CMD=kubectl
fi

$CMD version --output=yaml 2>/dev/null > $MGDIR/kubernetes-version.yaml
$CMD get crds > $MGDIR/crd_list.txt
$CMD get cores,units -A > $MGDIR/cores_and_units.txt
$CMD get namespaces > $MGDIR/namespaces.txt

if [ $CMD == "oc" ]
then
  $CMD get routes -n instana-core > $MGDIR/routes.txt
fi

DATASTORE_SUBSTRINGS="clickhouse|kafka|elastic|cass|postgres|strimzi|beeinst|datastore"
NS_SUBSTRINGS="instana|clickhouse|kafka|elastic|cass|postgres|strimzi|datastores|unit|core"

LIST_ALL_NS=$($CMD get namespaces | awk '{ print $1 }')

echo "============================"
echo "=== Gathering node data: ==="
echo "============================"
mkdir -p $MGDIR/_nodes
$CMD get nodes > $MGDIR/_nodes/nodes_list.txt && echo "listing all nodes"
$CMD describe nodes > $MGDIR/_nodes/nodes_describe.txt && echo "describing nodes"
$CMD get nodes -o yaml > $MGDIR/_nodes/nodes.yaml && echo "gathering nodes yamls"

echo "==================================================="
echo "=== List of Instana backend related namespaces: ==="
echo "==================================================="
INSTANA_NS=""
for NS in $LIST_ALL_NS; do
  if echo "$NS" | grep -qE "$NS_SUBSTRINGS"; then
    INSTANA_NS+="$NS "
    echo "$NS"
  fi
done

echo "============================================="
echo "=== Gathering list of all running images: ==="
echo "============================================="
$CMD get pods --all-namespaces -o jsonpath="{.items[*].spec['initContainers', 'containers'][*].image}" |\
tr -s '[[:space:]]' '\n' |\
sort |\
uniq -c > $MGDIR/all-image-list.txt && echo "all-image-list.txt"

echo "================================================"
echo "=== Gathering describe and yaml outputs for: ==="
echo "================================================"
CORE_NAMESPACE=$($CMD get cores -A | awk 'NR==2 {print $1}')
CORE_NAME=$($CMD get cores -A | awk 'NR==2 {print $2}')
echo "core name: " $CORE_NAME
echo "core namespace: " $CORE_NAMESPACE
mkdir $MGDIR/$CORE_NAMESPACE
$CMD describe core $CORE_NAME -n $CORE_NAMESPACE > $MGDIR/$CORE_NAMESPACE/"$CORE_NAME"_describe.txt
$CMD get core $CORE_NAME -n $CORE_NAMESPACE -o yaml > $MGDIR/$CORE_NAMESPACE/"$CORE_NAME".yaml
$CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' > $MGDIR/$CORE_NAMESPACE/"$CORE_NAME"_secret.yaml
$CMD get secret instana-registry -n $CORE_NAMESPACE --template='{{ index .data ".dockerconfigjson" | base64decode}}' > $MGDIR/$CORE_NAMESPACE/instana-registry_secret.json

echo "================================================"
echo "=== Gathering describe and yaml outputs for: ==="
echo "================================================"
$CMD get units -A | awk 'NR>1' | while read UNIT; do
  UNIT_NAMESPACE=$(echo "$UNIT" | awk '{print $1}')
  UNIT_NAME=$(echo "$UNIT" | awk '{print $2}')
  echo "unit name: " $UNIT_NAME
  echo "unit namespace: " $UNIT_NAMESPACE
  mkdir $MGDIR/$UNIT_NAMESPACE
  $CMD describe unit $UNIT_NAME -n $UNIT_NAMESPACE > $MGDIR/$UNIT_NAMESPACE/"$UNIT_NAME"_describe.txt
  $CMD get unit $UNIT_NAME -n $UNIT_NAMESPACE -o yaml > $MGDIR/$UNIT_NAMESPACE/"$UNIT_NAME".yaml
  $CMD get secret $UNIT_NAME -n $UNIT_NAMESPACE --template '{{ index .data "config.yaml" | base64decode}}' > $MGDIR/$UNIT_NAMESPACE/"$UNIT_NAME"_secret.yaml
  $CMD get secret instana-registry -n $UNIT_NAMESPACE --template='{{ index .data ".dockerconfigjson" | base64decode}}' > $MGDIR/$UNIT_NAMESPACE/instana-registry_secret.json
done

echo "========================================="
echo "=== Gathering Instana datastore CRDs: ==="
echo "========================================="
mkdir -p $MGDIR/_datastore_crds
ALL_CRDS=$($CMD get crds -A)
for CRD in $ALL_CRDS; do
  if echo "$CRD" | grep -qE "$DATASTORE_SUBSTRINGS"; then
    $CMD describe "$CRD" -A 2>/dev/null > "$MGDIR/_datastore_crds/$CRD-describe.txt"
    if [ -s "$MGDIR/_datastore_crds/$CRD-describe.txt" ]; then
      echo "describe output for $CRD"
      $CMD get "$CRD" -A -o yaml > "$MGDIR/_datastore_crds/$CRD.yaml" && echo "CRD yaml for $CRD"
    else
      rm -f "$MGDIR/_datastore_crds/$CRD-describe.txt"
    fi
  fi
done

echo "======================================================================"
echo "=== Gathering storage info and disk usage from Instana datastores: ==="
echo "======================================================================"
mkdir -p $MGDIR/_storage
$CMD get pvc -A > $MGDIR/_storage/PersistentVolumeClaims-list.txt
$CMD get pv > $MGDIR/_storage/PersistentVolumes-list.txt
$CMD get StorageClass > $MGDIR/_storage/StorageClasses-list.txt
$CMD get pvc -A -o yaml > $MGDIR/_storage/PersistentVolumeClaims.yaml
$CMD get pv -o yaml > $MGDIR/_storage/PersistentVolumes.yaml
$CMD get StorageClass -o yaml > $MGDIR/_storage/StorageClasses.yaml
$CMD describe pvc -A > $MGDIR/_storage/PersistentVolumeClaims-describe.txt
$CMD describe pv > $MGDIR/_storage/PersistentVolumes-describe.txt
$CMD describe StorageClass > $MGDIR/_storage/StorageClasses-describe.txt
for NS in $INSTANA_NS; do
  export NS=$NS
  if echo "$NS" | grep -qE "$DATASTORE_SUBSTRINGS"; then
    echo -e "\n### Namespace: $NS ###" >> $MGDIR/_storage/datastores_disk_usage.txt
    $CMD get pods -n $NS | awk 'NR>1{print $1}' | while read POD_NAME; do
      if [[ $POD_NAME == *"operator"* ]]; then
        echo "skipping $POD_NAME"
      else
        echo -e "\nDisk Usage for: $POD_NAME" >> $MGDIR/_storage/datastores_disk_usage.txt
        $CMD -n $NS exec $POD_NAME -- df -h 2>/dev/null >> "$MGDIR/_storage/datastores_disk_usage.txt" && echo "getting disk usage for $POD_NAME"
      fi
    done
  fi
  if [[ $NS == *"core"* ]]; then
    $CMD get pods -n $NS | awk 'NR>1{print $1}' | while read POD_NAME; do
      echo -e "\nDisk Usage for: $POD_NAME" >> $MGDIR/_storage/core_disk_usage.txt
      $CMD -n $NS exec $POD_NAME -- df -h 2>/dev/null >> "$MGDIR/_storage/core_disk_usage.txt" && echo "getting disk usage for $POD_NAME"
    done
  fi
done

echo "========================================================================================"
echo "=== Gathering logs, yamls and describe outputs for all pods from Instana namespaces: ==="
echo "========================================================================================"
for NS in $INSTANA_NS; do
  export NS=$NS
  [ -d $MGDIR/$NS ] || mkdir $MGDIR/$NS
  $CMD get all,events,secrets -n $NS -o wide &> $MGDIR/$NS/all-list.txt
  $CMD get pods -n $NS | awk 'NR>1{print $1}' | while read POD_NAME; do
      mkdir $MGDIR/$NS/$POD_NAME
      $CMD describe pod $POD_NAME -n $NS  > "$MGDIR/$NS/$POD_NAME/$POD_NAME-describe.txt" && echo "described $POD_NAME"
      $CMD get pod $POD_NAME -n $NS -o yaml > "$MGDIR/$NS/$POD_NAME/$POD_NAME.yaml" && echo "yaml output for $POD_NAME"
      CONTAINERS=$($CMD get pod $POD_NAME -n $NS -o jsonpath='{range .spec.containers[*]}{.name}{" "}{end}')
      for CNAME in $CONTAINERS; do 
        $CMD logs $POD_NAME -n $NS -c $CNAME > "$MGDIR/$NS/$POD_NAME/$CNAME.log" && echo "collected logs for $CNAME in $POD_NAME"
        $CMD logs "$POD_NAME" -n "$NS" -c "$CNAME" -p 2>/dev/null > "$MGDIR/$NS/$POD_NAME/$CNAME-previous.log"
        if [ -s "$MGDIR/$NS/$POD_NAME/$CNAME-previous.log" ]; then
          echo "collected previous logs for $CNAME"
        else
          rm -f "$MGDIR/$NS/$POD_NAME/$CNAME-previous.log"
        fi
      done
  done
done

echo "==========================================="
echo "=== Gathering component configurations: ==="
echo "==========================================="
IGNORED_SECRETS_SUBSTRINGS="sh.helm.release|core|internal|tls|registry"
mkdir $MGDIR/_component_configs
for NS in $INSTANA_NS; do
  export NS=$NS
  if [[ $NS == *"core"*  || $NS == *"unit"* ]]; then
    ALL_SECRETS=$($CMD get secrets -n $NS | awk 'NR>1{print $1}')
    for SECRET in $ALL_SECRETS; do
      if echo "$SECRET" | grep -qE "$IGNORED_SECRETS_SUBSTRINGS"; then
        echo "skipping $SECRET"
      else
	if [[ "$secret" = "contourcert" || "$secret" = "envoycert" ]]; then
		echo "skipping $secret"
	else
        $CMD -n $NS get secrets "$SECRET" --template='{{ index .data "config.yaml" | base64decode }}' > "$MGDIR/_component_configs/$SECRET.yaml" && echo "Config for $SECRET"
	fi
      fi
    done
  fi
done

echo "=============================================="
echo "=== Gathering components Internal metrics: ==="
echo "=============================================="
for NS in $INSTANA_NS; do
  export NS=$NS
  mkdir -p $MGDIR/_internal_metrics
  ADMIN_API_USER=$(kubectl get secret instana-internal -n $CORE_NAMESPACE --template='{{ index .data.adminAPIUser  | base64decode }}')
  ADMIN_API_PASS=$(kubectl get secret instana-internal -n $CORE_NAMESPACE --template='{{ index .data.adminAPIPassword  | base64decode }}')      
  if [[ $NS == *"core"*  || $NS == *"unit"* ]]; then
    $CMD get deployments -n $NS | awk 'NR>1{print $1}' | while read DEPLOYMENT_NAME; do
      $CMD exec deployment/$DEPLOYMENT_NAME -n $NS -- curl -s -X GET localhost:8601/metrics -u $ADMIN_API_USER:$ADMIN_API_PASS 2>/dev/null > "$MGDIR/_internal_metrics/$DEPLOYMENT_NAME.json" && echo "Metrics for $DEPLOYMENT_NAME"
    done
  fi
done

echo "========================================="
echo "=== Gathering data stores healthchecks ==="
echo "========================================="
echo "== Cassandra =="
CASSANDRA_POD_NAME=$($CMD get pods -A -l app.kubernetes.io/instance=cassandra-instana | awk 'NR==2 {print $2}')
CASSANDRA_NS=$($CMD get pods -A -l app.kubernetes.io/instance=cassandra-instana | awk 'NR==2 {print $1}')
CASSANDRA_PASS=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*cassandraConfigs:/ { in_block = 1 } in_block && /^[[:space:]]*adminPassword:/ { split($0, a, /:\s*/); print a[2]; exit }')
CASSANDRA_PASS=$(echo "$CASSANDRA_PASS" | xargs)
CASSANDRA_PASS=${CASSANDRA_PASS//\"/}
CASSANDRA_USER=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*cassandraConfigs:/ { in_block = 1 } in_block && /^[[:space:]]*adminUser:/ { split($0, a, /:\s*/); print a[2]; exit }')
CASSANDRA_USER=$(echo "$CASSANDRA_USER" | xargs)
CASSANDRA_USER=${CASSANDRA_USER//\"/}
mkdir -p $MGDIR/_datastore_healthchecks/cassandra
$CMD exec $CASSANDRA_POD_NAME -n $CASSANDRA_NS -c cassandra -- nodetool status 2>/dev/null > "$MGDIR/_datastore_healthchecks/cassandra/nodetool_status.txt" && echo "nodetool status"
$CMD exec $CASSANDRA_POD_NAME -n $CASSANDRA_NS -c cassandra -- nodetool describecluster 2>/dev/null > "$MGDIR/_datastore_healthchecks/cassandra/nodetool_describecluster.txt" && echo "nodetool describe cluster"
$CMD exec $CASSANDRA_POD_NAME -n $CASSANDRA_NS -c cassandra -- nodetool profileload 2>/dev/null > "$MGDIR/_datastore_healthchecks/cassandra/nodetool_profileload.txt" && echo "nodetool profileload"
$CMD exec $CASSANDRA_POD_NAME -n $CASSANDRA_NS -c cassandra -- nodetool tpstats 2>/dev/null > "$MGDIR/_datastore_healthchecks/cassandra/nodetool_tpstats.txt" && echo "nodetool tpstats"
$CMD exec $CASSANDRA_POD_NAME -n $CASSANDRA_NS -c cassandra -- cqlsh -u $CASSANDRA_USER -p $CASSANDRA_PASS -e "select * from system_schema.keyspaces;" 2>/dev/null > "$MGDIR/_datastore_healthchecks/cassandra/keyspaces.txt" && echo "keyspaces"
echo "== Elasticsearch =="
ELASTIC_POD_NAME=$($CMD get pods -A -l common.k8s.elastic.co/type=elasticsearch | awk 'NR==2 {print $2}')
ELASTIC_NS=$($CMD get pods -A -l common.k8s.elastic.co/type=elasticsearch | awk 'NR==2 {print $1}')
ELASTIC_PASS=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*elasticsearchConfig:/ { in_block = 1 } in_block && /^[[:space:]]*adminPassword:/ { split($0, a, /:\s*/); print a[2]; exit }')
ELASTIC_PASS=$(echo "$ELASTIC_PASS" | xargs)
ELASTIC_PASS=${ELASTIC_PASS//\"/}
ELASTIC_USER=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*elasticsearchConfig:/ { in_block = 1 } in_block && /^[[:space:]]*adminUser:/ { split($0, a, /:\s*/); print a[2]; exit }')
ELASTIC_USER=$(echo "$ELASTIC_USER" | xargs)
ELASTIC_USER=${ELASTIC_USER//\"/}
mkdir $MGDIR/_datastore_healthchecks/elasticsearch
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_cluster/health?filter_path=status,*_shards&pretty" > "$MGDIR/_datastore_healthchecks/elasticsearch/cluster_health.txt" && echo "cluster health"
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_cat/indices?pretty" > "$MGDIR/_datastore_healthchecks/elasticsearch/index_list.txt" && echo "index list"
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_cat/nodes?v=true&h=name,node*,heap*&pretty" > "$MGDIR/_datastore_healthchecks/elasticsearch/node_heap.txt" && echo "node heap"
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_cat/nodes?v=true&s=cpu:desc" > "$MGDIR/_datastore_healthchecks/elasticsearch/node_cpu.txt" && echo "node cpu"
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_cat/shards?h=index,shard,prirep,state,docs,store,unassigned.reason" > "$MGDIR/_datastore_healthchecks/elasticsearch/shards.txt" && echo "shards list"
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_nodes/stats/fs?level=node&pretty" > "$MGDIR/_datastore_healthchecks/elasticsearch/nodes_filesystem.json" && echo "nodes filesystem"
echo "== Postgres =="
POSTGRES_POD_NAME=$($CMD get pods -A -l cnpg.io/cluster=postgres | awk 'NR==2 {print $2}')
POSTGRES_NS=$($CMD get pods -A -l cnpg.io/cluster=postgres | awk 'NR==2 {print $1}')
mkdir $MGDIR/_datastore_healthchecks/postgres
$CMD exec $POSTGRES_POD_NAME -n $POSTGRES_NS -c postgres -- psql -c "\l+" > "$MGDIR/_datastore_healthchecks/postgres/postgres_size.txt" && echo "postgres size"
echo "== Kafka =="
KAFKA_POD_NAME=$($CMD get pods -A -l strimzi.io/pool-name=kafka | awk 'NR==2 {print $2}')
KAFKA_NS=$($CMD get pods -A -l strimzi.io/pool-name=kafka | awk 'NR==2 {print $1}')
KAFKA_PASS=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*kafkaConfig:/ { in_block = 1 } in_block && /^[[:space:]]*adminPassword:/ { split($0, a, /:\s*/); print a[2]; exit }')
KAFKA_PASS=$(echo "$KAFKA_PASS" | xargs)
KAFKA_PASS=${KAFKA_PASS//\"/}
KAFKA_USER=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*kafkaConfig:/ { in_block = 1 } in_block && /^[[:space:]]*adminUser:/ { split($0, a, /:\s*/); print a[2]; exit }')
KAFKA_USER=$(echo "$KAFKA_USER" | xargs)
KAFKA_USER=${KAFKA_USER//\"/}
mkdir $MGDIR/_datastore_healthchecks/kafka
$CMD exec $KAFKA_POD_NAME -n $KAFKA_NS -- bash -c "cat > /tmp/my1.conf <<-EOF
security.protocol=SASL_PLAINTEXT
sasl.mechanism=SCRAM-SHA-512
sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username='$KAFKA_USER' password='$KAFKA_PASS';
EOF
" 2>/dev/null
$CMD exec $KAFKA_POD_NAME -n $KAFKA_NS -c kafka -- /opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --command-config /tmp/my1.conf --list > "$MGDIR/_datastore_healthchecks/kafka/topic_list.txt" && echo "topic list"
$CMD exec $KAFKA_POD_NAME -n $KAFKA_NS -c kafka -- /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --command-config /tmp/my1.conf --list > "$MGDIR/_datastore_healthchecks/kafka/consumer_groups_list.txt" && echo "consumer groups list"
$CMD exec $KAFKA_POD_NAME -n $KAFKA_NS -c kafka -- /opt/kafka/bin/kafka-log-dirs.sh --describe --bootstrap-server localhost:9092 --command-config /tmp/my1.conf --topic-list "*" > "$MGDIR/_datastore_healthchecks/kafka/topics_size.json" && echo "topic size"

tar czf $MGDIR.tgz $MGDIR/