#!/bin/bash
###############################################################################
#
# This script is used to collect data for
# the Instana operator-based on-prem running on Kubernetes / Openshift
#
# ./instana-k8s-BE-mustgather.sh
#

current_time=$(date "+%Y.%m.%d-%H.%M.%S")
export MGDIR=instana-BE-mustgather-$current_time
mkdir -p $MGDIR
if command -v oc > /dev/null
then
  CMD=oc
else
  CMD=kubectl
fi

$CMD version --output=yaml 2>/dev/null > $MGDIR/kubernetes-version.yaml
$CMD get crds > $MGDIR/crd_list.txt
$CMD get cores,units -A > $MGDIR/cores_and_units.txt
$CMD get namespaces > $MGDIR/namespaces.txt

if [ $CMD == "oc" ]
then
  $CMD get routes -n instana-core > $MGDIR/routes.txt
fi

DATASTORE_SUBSTRINGS="clickhouse|kafka|elastic|cass|postgres|strimzi|beeinst|datastore"
NS_SUBSTRINGS="instana|clickhouse|kafka|elastic|cass|postgres|strimzi|datastores|unit|core"

LIST_ALL_NS=$($CMD get namespaces | awk '{ print $1 }')

echo "============================"
echo "=== Gathering node data: ==="
echo "============================"
mkdir -p $MGDIR/_nodes
$CMD get nodes > $MGDIR/_nodes/nodes_list.txt && echo "listing all nodes"
$CMD describe nodes > $MGDIR/_nodes/nodes_describe.txt && echo "describing nodes"
$CMD get nodes -o yaml > $MGDIR/_nodes/nodes.yaml && echo "gathering nodes yamls"

echo "==================================================="
echo "=== List of Instana backend related namespaces: ==="
echo "==================================================="
INSTANA_NS=""
for NS in $LIST_ALL_NS; do
  if echo "$NS" | grep -qE "$NS_SUBSTRINGS"; then
    INSTANA_NS+="$NS "
    echo "$NS"
  fi
done

echo "============================================="
echo "=== Gathering list of all running images: ==="
echo "============================================="
$CMD get pods --all-namespaces -o jsonpath="{.items[*].spec['initContainers', 'containers'][*].image}" |\
tr -s '[[:space:]]' '\n' |\
sort |\
uniq -c > $MGDIR/all-image-list.txt && echo "all-image-list.txt"

echo "================================================"
echo "=== Gathering describe and yaml outputs for: ==="
echo "================================================"
CORE_NAMESPACE=$($CMD get cores -A | awk 'NR==2 {print $1}')
CORE_NAME=$($CMD get cores -A | awk 'NR==2 {print $2}')
echo "core name: " $CORE_NAME
echo "core namespace: " $CORE_NAMESPACE
mkdir $MGDIR/$CORE_NAMESPACE
$CMD describe core $CORE_NAME -n $CORE_NAMESPACE > $MGDIR/$CORE_NAMESPACE/"$CORE_NAME"_describe.txt
$CMD get core $CORE_NAME -n $CORE_NAMESPACE -o yaml > $MGDIR/$CORE_NAMESPACE/"$CORE_NAME".yaml
$CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' > $MGDIR/$CORE_NAMESPACE/"$CORE_NAME"_secret.yaml
$CMD get secret instana-registry -n $CORE_NAMESPACE --template='{{ index .data ".dockerconfigjson" | base64decode}}' > $MGDIR/$CORE_NAMESPACE/instana-registry_secret.json

echo "================================================"
echo "=== Gathering describe and yaml outputs for: ==="
echo "================================================"
$CMD get units -A | awk 'NR>1' | while read UNIT; do
  UNIT_NAMESPACE=$(echo "$UNIT" | awk '{print $1}')
  UNIT_NAME=$(echo "$UNIT" | awk '{print $2}')
  echo "unit name: " $UNIT_NAME
  echo "unit namespace: " $UNIT_NAMESPACE
  mkdir $MGDIR/$UNIT_NAMESPACE
  $CMD describe unit $UNIT_NAME -n $UNIT_NAMESPACE > $MGDIR/$UNIT_NAMESPACE/"$UNIT_NAME"_describe.txt
  $CMD get unit $UNIT_NAME -n $UNIT_NAMESPACE -o yaml > $MGDIR/$UNIT_NAMESPACE/"$UNIT_NAME".yaml
  $CMD get secret $UNIT_NAME -n $UNIT_NAMESPACE --template '{{ index .data "config.yaml" | base64decode}}' > $MGDIR/$UNIT_NAMESPACE/"$UNIT_NAME"_secret.yaml
  $CMD get secret instana-registry -n $UNIT_NAMESPACE --template='{{ index .data ".dockerconfigjson" | base64decode}}' > $MGDIR/$UNIT_NAMESPACE/instana-registry_secret.json
done

echo "========================================="
echo "=== Gathering Instana datastore CRDs: ==="
echo "========================================="
mkdir -p $MGDIR/_datastore_crds
ALL_CRDS=$($CMD get crds -A)
for CRD in $ALL_CRDS; do
  if echo "$CRD" | grep -qE "$DATASTORE_SUBSTRINGS"; then
    $CMD describe "$CRD" -A 2>/dev/null > "$MGDIR/_datastore_crds/$CRD-describe.txt"
    if [ -s "$MGDIR/_datastore_crds/$CRD-describe.txt" ]; then
      echo "describe output for $CRD"
      $CMD get "$CRD" -A -o yaml > "$MGDIR/_datastore_crds/$CRD.yaml" && echo "CRD yaml for $CRD"
    else
      rm -f "$MGDIR/_datastore_crds/$CRD-describe.txt"
    fi
  fi
done

echo "======================================================================"
echo "=== Gathering storage info and disk usage from Instana datastores: ==="
echo "======================================================================"
mkdir -p $MGDIR/_storage
$CMD get pvc -A > $MGDIR/_storage/PersistentVolumeClaims-list.txt
$CMD get pv > $MGDIR/_storage/PersistentVolumes-list.txt
$CMD get StorageClass > $MGDIR/_storage/StorageClasses-list.txt
$CMD get pvc -A -o yaml > $MGDIR/_storage/PersistentVolumeClaims.yaml
$CMD get pv -o yaml > $MGDIR/_storage/PersistentVolumes.yaml
$CMD get StorageClass -o yaml > $MGDIR/_storage/StorageClasses.yaml
$CMD describe pvc -A > $MGDIR/_storage/PersistentVolumeClaims-describe.txt
$CMD describe pv > $MGDIR/_storage/PersistentVolumes-describe.txt
$CMD describe StorageClass > $MGDIR/_storage/StorageClasses-describe.txt
for NS in $INSTANA_NS; do
  export NS=$NS
  if echo "$NS" | grep -qE "$DATASTORE_SUBSTRINGS"; then
    echo -e "\n### Namespace: $NS ###" >> $MGDIR/_storage/datastores_disk_usage.txt
    $CMD get pods -n $NS | awk 'NR>1{print $1}' | while read POD_NAME; do
      if [[ $POD_NAME == *"operator"* ]]; then
        echo "skipping $POD_NAME"
      else
        echo -e "\nDisk Usage for: $POD_NAME" >> $MGDIR/_storage/datastores_disk_usage.txt
        $CMD -n $NS exec $POD_NAME -- df -h 2>/dev/null >> "$MGDIR/_storage/datastores_disk_usage.txt" && echo "getting disk usage for $POD_NAME"
      fi
    done
  fi
  if [[ $NS == *"core"* ]]; then
    $CMD get pods -n $NS | awk 'NR>1{print $1}' | while read POD_NAME; do
      echo -e "\nDisk Usage for: $POD_NAME" >> $MGDIR/_storage/core_disk_usage.txt
      $CMD -n $NS exec $POD_NAME -- df -h 2>/dev/null >> "$MGDIR/_storage/core_disk_usage.txt" && echo "getting disk usage for $POD_NAME"
    done
  fi
done

echo "========================================================================================"
echo "=== Gathering logs, yamls and describe outputs for all pods from Instana namespaces: ==="
echo "========================================================================================"
for NS in $INSTANA_NS; do
  export NS=$NS
  [ -d $MGDIR/$NS ] || mkdir $MGDIR/$NS
  $CMD get all,events,secrets -n $NS -o wide &> $MGDIR/$NS/all-list.txt
  $CMD get pods -n $NS | awk 'NR>1{print $1}' | while read POD_NAME; do
      mkdir $MGDIR/$NS/$POD_NAME
      $CMD describe pod $POD_NAME -n $NS  > "$MGDIR/$NS/$POD_NAME/$POD_NAME-describe.txt" && echo "described $POD_NAME"
      $CMD get pod $POD_NAME -n $NS -o yaml > "$MGDIR/$NS/$POD_NAME/$POD_NAME.yaml" && echo "yaml output for $POD_NAME"
      CONTAINERS=$($CMD get pod $POD_NAME -n $NS -o jsonpath='{range .spec.containers[*]}{.name}{" "}{end}')
      for CNAME in $CONTAINERS; do 
        $CMD logs $POD_NAME -n $NS -c $CNAME > "$MGDIR/$NS/$POD_NAME/$CNAME.log" && echo "collected logs for $CNAME in $POD_NAME"
        $CMD logs "$POD_NAME" -n "$NS" -c "$CNAME" -p 2>/dev/null > "$MGDIR/$NS/$POD_NAME/$CNAME-previous.log"
        if [ -s "$MGDIR/$NS/$POD_NAME/$CNAME-previous.log" ]; then
          echo "collected previous logs for $CNAME"
        else
          rm -f "$MGDIR/$NS/$POD_NAME/$CNAME-previous.log"
        fi
      done
  done
done

echo "==========================================="
echo "=== Gathering component configurations: ==="
echo "==========================================="
IGNORED_SECRETS_SUBSTRINGS="sh.helm.release|core|internal|tls|registry"
mkdir $MGDIR/_component_configs
for NS in $INSTANA_NS; do
  export NS=$NS
  if [[ $NS == *"core"*  || $NS == *"unit"* ]]; then
    ALL_SECRETS=$($CMD get secrets -n $NS | awk 'NR>1{print $1}')
    for SECRET in $ALL_SECRETS; do
      if echo "$SECRET" | grep -qE "$IGNORED_SECRETS_SUBSTRINGS"; then
        echo "skipping $SECRET"
      else
	if [[ "$secret" = "contourcert" || "$secret" = "envoycert" ]]; then
		echo "skipping $secret"
	else
        $CMD -n $NS get secrets "$SECRET" --template='{{ index .data "config.yaml" | base64decode }}' > "$MGDIR/_component_configs/$SECRET.yaml" && echo "Config for $SECRET"
	fi
      fi
    done
  fi
done

echo "=============================================="
echo "=== Gathering components Internal metrics: ==="
echo "=============================================="
for NS in $INSTANA_NS; do
  export NS=$NS
  mkdir -p $MGDIR/_internal_metrics
  ADMIN_API_USER=$(kubectl get secret instana-internal -n $CORE_NAMESPACE --template='{{ index .data.adminAPIUser  | base64decode }}')
  ADMIN_API_PASS=$(kubectl get secret instana-internal -n $CORE_NAMESPACE --template='{{ index .data.adminAPIPassword  | base64decode }}')      
  if [[ $NS == *"core"*  || $NS == *"unit"* ]]; then
    $CMD get deployments -n $NS | awk 'NR>1{print $1}' | while read DEPLOYMENT_NAME; do
      $CMD exec deployment/$DEPLOYMENT_NAME -n $NS -- curl -s -X GET localhost:8601/metrics -u $ADMIN_API_USER:$ADMIN_API_PASS 2>/dev/null > "$MGDIR/_internal_metrics/$DEPLOYMENT_NAME.json" && echo "Metrics for $DEPLOYMENT_NAME"
    done
  fi
done

echo "========================================="
echo "=== Gathering data stores healthchecks ==="
echo "========================================="
echo "== Cassandra =="
CASSANDRA_POD_NAME=$($CMD get pods -A -l app.kubernetes.io/instance=cassandra-instana | awk 'NR==2 {print $2}')
CASSANDRA_NS=$($CMD get pods -A -l app.kubernetes.io/instance=cassandra-instana | awk 'NR==2 {print $1}')
CASSANDRA_PASS=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*cassandraConfigs:/ { in_block = 1 } in_block && /^[[:space:]]*adminPassword:/ { split($0, a, /:\s*/); print a[2]; exit }')
CASSANDRA_PASS=$(echo "$CASSANDRA_PASS" | xargs)
CASSANDRA_PASS=${CASSANDRA_PASS//\"/}
CASSANDRA_USER=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*cassandraConfigs:/ { in_block = 1 } in_block && /^[[:space:]]*adminUser:/ { split($0, a, /:\s*/); print a[2]; exit }')
CASSANDRA_USER=$(echo "$CASSANDRA_USER" | xargs)
CASSANDRA_USER=${CASSANDRA_USER//\"/}
mkdir -p $MGDIR/_datastore_healthchecks/cassandra
$CMD exec $CASSANDRA_POD_NAME -n $CASSANDRA_NS -c cassandra -- nodetool status 2>/dev/null > "$MGDIR/_datastore_healthchecks/cassandra/nodetool_status.txt" && echo "nodetool status"
$CMD exec $CASSANDRA_POD_NAME -n $CASSANDRA_NS -c cassandra -- nodetool describecluster 2>/dev/null > "$MGDIR/_datastore_healthchecks/cassandra/nodetool_describecluster.txt" && echo "nodetool describe cluster"
$CMD exec $CASSANDRA_POD_NAME -n $CASSANDRA_NS -c cassandra -- nodetool profileload 2>/dev/null > "$MGDIR/_datastore_healthchecks/cassandra/nodetool_profileload.txt" && echo "nodetool profileload"
$CMD exec $CASSANDRA_POD_NAME -n $CASSANDRA_NS -c cassandra -- nodetool tpstats 2>/dev/null > "$MGDIR/_datastore_healthchecks/cassandra/nodetool_tpstats.txt" && echo "nodetool tpstats"
$CMD exec $CASSANDRA_POD_NAME -n $CASSANDRA_NS -c cassandra -- cqlsh -u $CASSANDRA_USER -p $CASSANDRA_PASS -e "select * from system_schema.keyspaces;" 2>/dev/null > "$MGDIR/_datastore_healthchecks/cassandra/keyspaces.txt" && echo "keyspaces"
echo "== Elasticsearch =="
ELASTIC_POD_NAME=$($CMD get pods -A -l common.k8s.elastic.co/type=elasticsearch | awk 'NR==2 {print $2}')
ELASTIC_NS=$($CMD get pods -A -l common.k8s.elastic.co/type=elasticsearch | awk 'NR==2 {print $1}')
ELASTIC_PASS=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*elasticsearchConfig:/ { in_block = 1 } in_block && /^[[:space:]]*adminPassword:/ { split($0, a, /:\s*/); print a[2]; exit }')
ELASTIC_PASS=$(echo "$ELASTIC_PASS" | xargs)
ELASTIC_PASS=${ELASTIC_PASS//\"/}
ELASTIC_USER=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*elasticsearchConfig:/ { in_block = 1 } in_block && /^[[:space:]]*adminUser:/ { split($0, a, /:\s*/); print a[2]; exit }')
ELASTIC_USER=$(echo "$ELASTIC_USER" | xargs)
ELASTIC_USER=${ELASTIC_USER//\"/}
mkdir $MGDIR/_datastore_healthchecks/elasticsearch
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_cluster/health?filter_path=status,*_shards&pretty" > "$MGDIR/_datastore_healthchecks/elasticsearch/cluster_health.txt" && echo "cluster health"
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_cat/indices?pretty" > "$MGDIR/_datastore_healthchecks/elasticsearch/index_list.txt" && echo "index list"
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_cat/nodes?v=true&h=name,node*,heap*&pretty" > "$MGDIR/_datastore_healthchecks/elasticsearch/node_heap.txt" && echo "node heap"
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_cat/nodes?v=true&s=cpu:desc" > "$MGDIR/_datastore_healthchecks/elasticsearch/node_cpu.txt" && echo "node cpu"
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_cat/shards?h=index,shard,prirep,state,docs,store,unassigned.reason" > "$MGDIR/_datastore_healthchecks/elasticsearch/shards.txt" && echo "shards list"
$CMD exec $ELASTIC_POD_NAME -c elasticsearch -n $ELASTIC_NS -- curl -u $ELASTIC_USER:$ELASTIC_PASS -s -X GET "http://localhost:9200/_nodes/stats/fs?level=node&pretty" > "$MGDIR/_datastore_healthchecks/elasticsearch/nodes_filesystem.json" && echo "nodes filesystem"
echo "== Postgres =="
POSTGRES_POD_NAME=$($CMD get pods -A -l cnpg.io/cluster=postgres | awk 'NR==2 {print $2}')
POSTGRES_NS=$($CMD get pods -A -l cnpg.io/cluster=postgres | awk 'NR==2 {print $1}')
mkdir $MGDIR/_datastore_healthchecks/postgres
$CMD exec $POSTGRES_POD_NAME -n $POSTGRES_NS -c postgres -- psql -c "\l+" > "$MGDIR/_datastore_healthchecks/postgres/postgres_size.txt" && echo "postgres size"
echo "== Kafka =="
KAFKA_POD_NAME=$($CMD get pods -A -l strimzi.io/pool-name=kafka | awk 'NR==2 {print $2}')
KAFKA_NS=$($CMD get pods -A -l strimzi.io/pool-name=kafka | awk 'NR==2 {print $1}')
KAFKA_PASS=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*kafkaConfig:/ { in_block = 1 } in_block && /^[[:space:]]*adminPassword:/ { split($0, a, /:\s*/); print a[2]; exit }')
KAFKA_PASS=$(echo "$KAFKA_PASS" | xargs)
KAFKA_PASS=${KAFKA_PASS//\"/}
KAFKA_USER=$($CMD get secret $CORE_NAME -n $CORE_NAMESPACE --template='{{ index .data "config.yaml" | base64decode}}' | awk '/^[[:space:]]*kafkaConfig:/ { in_block = 1 } in_block && /^[[:space:]]*adminUser:/ { split($0, a, /:\s*/); print a[2]; exit }')
KAFKA_USER=$(echo "$KAFKA_USER" | xargs)
KAFKA_USER=${KAFKA_USER//\"/}
mkdir $MGDIR/_datastore_healthchecks/kafka
$CMD exec $KAFKA_POD_NAME -n $KAFKA_NS -- bash -c "cat > /tmp/my1.conf <<-EOF
security.protocol=SASL_PLAINTEXT
sasl.mechanism=SCRAM-SHA-512
sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username='$KAFKA_USER' password='$KAFKA_PASS';
EOF
" 2>/dev/null
$CMD exec $KAFKA_POD_NAME -n $KAFKA_NS -c kafka -- /opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --command-config /tmp/my1.conf --list > "$MGDIR/_datastore_healthchecks/kafka/topic_list.txt" && echo "topic list"
$CMD exec $KAFKA_POD_NAME -n $KAFKA_NS -c kafka -- /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --command-config /tmp/my1.conf --list > "$MGDIR/_datastore_healthchecks/kafka/consumer_groups_list.txt" && echo "consumer groups list"
$CMD exec $KAFKA_POD_NAME -n $KAFKA_NS -c kafka -- /opt/kafka/bin/kafka-log-dirs.sh --describe --bootstrap-server localhost:9092 --command-config /tmp/my1.conf --topic-list "*" > "$MGDIR/_datastore_healthchecks/kafka/topics_size.json" && echo "topic size"

tar czf $MGDIR.tgz $MGDIR/