#!/bin/bash

######## User Configuration ########

# Option to include a logging stack, either by extending the example provided or your own log setup
ENABLE_LOG_STACK=false

# Option to disable JMX settings.  This is not necessary because the JMX
# interface can only be accessed from inside the Docker Swarm.  But for
# environments that must disable it anyway, set this to true.
DISABLE_JMX=false

# Value in minutes to wait for a stack to start before declaring failure
STACK_START_TIMEOUT=3

# Directory where the support packages will be generated
SUPPORT_DIR=./support

# Fully qualified directory where the backup packages will be stored
BACKUP_DIR=

# Option to specify an alternate location to find ISIQ YML files
# Leave empty to find them in the yml subdirectory of this Starter Kit
YML_DIR=

# Required settings for performing an automatic upgrade
# Location of prior version starter kit to read in customizations
ORIGINAL_STARTER_KIT_DIR=

# true|false - whether to save the merged files in original directory (true)
# or the new starter kit directory (false)
USE_ORIGINAL_DIR=false

###### End User Configuration ######

DOWN_NODES=
STACKS_UP="broker connect app logs"
STACKS_DOWN="logs app connect broker"

isiq_poc() {
	if [ "x$1" != "x" ]; then
		# Update YML files
		sed -i 's/ISIQ_SKIP_OIDC=false/ISIQ_SKIP_OIDC=true/i' "${YML_DIR}app-stack.yml"
		sed -i 's/ISIQ_SKIP_OIDC=false/ISIQ_SKIP_OIDC=true/i' "${YML_DIR}app-stack-nojmx.yml"
		sed -i 's/ISIQ_OIDC_ALLOW_SELF_SIGNED=false/ISIQ_OIDC_ALLOW_SELF_SIGNED=true/i' "${YML_DIR}app-stack.yml"
		sed -i 's/ISIQ_OIDC_ALLOW_SELF_SIGNED=false/ISIQ_OIDC_ALLOW_SELF_SIGNED=true/i' "${YML_DIR}app-stack-nojmx.yml"
		sed -i 's/ISIQ_AUTOMATICALLY_IMPORT_CERTIFICATE=false/ISIQ_AUTOMATICALLY_IMPORT_CERTIFICATE=true/i' "${YML_DIR}connect-stack.yml"
		sed -i 's/ISIQ_AUTOMATICALLY_IMPORT_CERTIFICATE=false/ISIQ_AUTOMATICALLY_IMPORT_CERTIFICATE=true/i' "${YML_DIR}connect-stack-nojmx.yml"
		isiq_start
	fi
} # isiq_poc

isiq_init() {
	NUM_NODES=1
	if [ "$1" != "skip_nodes" ]; then
		get_nodes $1
	fi
	if [ $NUM_NODES -gt 1 ]; then
		ISIQ_DEPLOYMENT=cluster
	else
		ISIQ_DEPLOYMENT=single_node
	fi
	# Make sure user supplied values are formatted correctly
	ENABLE_LOG_STACK=$(awk -vs1="$ENABLE_LOG_STACK" 'BEGIN { print tolower(s1) }')
	DISABLE_JMX=$(awk -vs1="$DISABLE_JMX" 'BEGIN { print tolower(s1) }')
	if [ $ENABLE_LOG_STACK == "false" ]; then
		STACKS_UP=$(echo "$STACKS_UP" | sed "s/logs//")
		STACKS_DOWN=$(echo "$STACKS_DOWN" | sed "s/logs//")
	fi
	STACK_TIMEOUT_CHECK=$(echo $STACK_START_TIMEOUT | tr '[0-9]' 'x' | sed 's/x*/x/')
	if [ $STACK_TIMEOUT_CHECK == "x" ]; then
		STACK_START_TIMEOUT=$(($STACK_START_TIMEOUT*12))
	else
		printf "Invalid stack timeout setting.  Using default of 3 minutes.\n" >&2
		STACK_START_TIMEOUT=$((3*12))
	fi
} #isiq_init

log_check() {
	MAX_MAP_COUNT=$(sysctl -n vm.max_map_count 2>/dev/null)
	if [ $(echo $?) -eq 0 ]; then
		if [ $MAX_MAP_COUNT -lt 262144 ]; then
			WHY_ES_BAD="insufficient"
			log_warning
		fi
	else
		WHY_ES_BAD="missing"
		log_warning
	fi
} # log_check

log_warning() {
	STACKS_UP=$(echo "$STACKS_UP" | sed "s/logs//")
	echo ""
	echo "**********************************************************************"
	echo "Unable to start logs stack due to $WHY_ES_BAD memory map setting."
	echo "Please review Step 4 of the ISIQ Deployment Guide's Summary of"
	echo "Installation & Configuration Steps for details on how to set"
	echo "vm.max_map_count for Elasticsearch."
	echo "**********************************************************************"
	echo ""

} #log_warning

get_down_nodes() {
	DOWN_NODES_ARRAY=$(docker node ls | grep Down | awk '{ print $2 }')
	PIFS=$IFS
	IFS=$'\n'
	DOWN_COUNTER=1
	for DOWN_NODE in $DOWN_NODES_ARRAY; do
		PIN_NODE=$(docker inspect $DOWN_NODE | grep isiq | awk '{ print $2 }' | cut -d '"' -f 2)
		if [ $DOWN_COUNTER -eq 1 ]; then
			DOWN_NODES=$PIN_NODE
			DOWN_COUNTER=2
		else
			DOWN_NODES="$DOWN_NODES $PIN_NODE"
		fi
	done
	IFS=$PIFS
} #get_down_nodes


set_yaml_dir() {
	if [ "x$YML_DIR" == "x" ]; then
		case ${ISIQ_DEPLOYMENT} in
			single_node)
				YML_DIR=yml/single_node/
				;;
			cluster)
				YML_DIR=yml/cluster/
				;;
			*)
				echo "Invalid configuration"
				echo "You must edit $0 to set ISIQ_DEPLOYMENT as single_node or cluster"
				exit 1
		esac
	fi
} #set_yaml_dir

get_services() {
	if [ "x$1" == "x" ]; then
		SERVICES=$(docker service ls 2> /dev/null | grep -v REPLICAS)
	else
		SERVICES=$(docker service ls | grep ${1}_)
	fi
} #get_services

show_docker_errors() {
	echo "Failed services:"
	PIFS=$IFS
	IFS=$'\n'
	get_services $1
	for SERVICE in $SERVICES; do
		RUNNING=$(echo "$SERVICE" | awk '{ print $4 }' | tr '/' ' ' | awk '{ print $2-$1 }')
		if [ $RUNNING -ne 0 ]; then
			printf "$SERVICE\n\n"
			SERVICE_NAME=$(echo "$SERVICE" | awk '{ print $2 }')
			echo "Corresponding container(s):"
			docker service ps --no-trunc $SERVICE_NAME | tail -n +2 
			printf "\n\nLogs:\n"
			docker service logs "$SERVICE_NAME"
			printf "\n\n"
		fi
	done
	IFS=$PIFS
} #show_docker_errors

show_help() {
	printf "\nIBM Security Verify Information Queue Utility\n\n"
	printf "Usage: $0 {setup|start|stop|restart|status|support|upgrade|backup|restore|images|logs|help}\n\n"
	printf "%-8s - %s\n" "setup" "Initial Configuration of ISIQ"
	printf "%-8s - %s\n" "start" "Start the ISIQ application"
	printf "%-8s - %s\n" "stop" "Stop the ISIQ application"
	printf "%-8s - %s\n" "restart" "Stop then Start the ISIQ application"
	printf "%-8s - %s\n" "status" "Show whether ISIQ is running or not"
	printf "%-8s - %s\n" "support" "Generate support file for IBM"
	printf "%-8s - %s\n" "upgrade" "Upgrade existing ISIQ run area to a new release"
	printf "%-8s - %s\n" "backup" "Perform offline backup of volumes used by ISIQ"
	printf "%-9s  %s\n" "" "*** see backup_readme.txt in docs dir BEFORE using ***"
	printf "%-8s - %s\n" "restore" "Perform offline restore of volumes from a backup"
	printf "%-8s - %s\n" "images" "On cluster nodes, retrieve containers needed by ISIQ"
	printf "%-8s - %s\n" "renew" "Renew nginx SSL certificate"
	printf "%-8s - %s\n" "logs" "Show console logs for a container"
	printf "%-9s  %s\n" "" "usage: $0 logs [-f (follow)] container_name"
	printf "%-9s  %s\n" "" "e.g. $0 logs -f connect_connect"
	printf "%-8s - %s\n\n" "help" "This message"
} #show_help

show_logs() {
	get_nodes skip_swarm
	if [ "x$1" == "x" ]; then
		printf "Must specify a container name.  e.g. app_rest\n"
		show_running_containers
		exit 1
	fi
	if [ "$1" == "-f" ]; then
		if [ "x$2" != "x" ]; then
			docker service logs -f "$2"
		else
			printf "Must specify a container name.  e.g. app_rest\n"
			show_running_containers
			exit 1
		fi
	else
		docker service logs "$1"
	fi
} #show_logs

show_running_containers() {
	SERVICES=$(docker service ls 2> /dev/null | grep -v REPLICAS | awk '{ print $2 }')
	NUM_SERVICES=$(echo "$SERVICES" | wc -w)
	if [ $NUM_SERVICES -gt 0 ]; then
		printf "List of currently running containers:\n\n"
		COL=1
		for SERVICE in $SERVICES; do
			if [ $COL -eq 1 ]; then
				printf "%-30s" "$SERVICE"
				COL=2
			else
				printf "$SERVICE\n"
				COL=1
			fi
		done
		printf "\n"
	else
		printf "No containers currently running.\n\n"
	fi
} #show_running_containers

generate_support_file() {
	docker version > /dev/null
	if [ $(echo $?) -ne 0 ]; then
		echo "Docker not installed. ISIQ cannot function."
		exit 1
	fi
	if [ ! -e "$SUPPORT_DIR" ]; then
		mkdir "$SUPPORT_DIR" 2> /dev/null
		if [ $(echo $?) -ne 0 ]; then
			echo "Support directory $SUPPORT_DIR does not exist and create attempt failed."
			exit 1
		fi
	fi
	if [ ! -d "$SUPPORT_DIR" ]; then
		echo "Support directory $SUPPORT_DIR is not a directory"
		exit 1
	fi
	if [ ! -w "$SUPPORT_DIR" ]; then
		chmod u+w "$SUPPORT_DIR" 2> /dev/null
		if [ $(echo $?) -ne 0 ]; then
			echo "Support directory $SUPPORT_DIR is not writable and change mod failed."
			exit 1
		fi
	fi
	isiq_init skip_swarm
	set_yaml_dir
	FILENAME=$(date +"%Y%m%d_%H%M%S")
	mkdir "${SUPPORT_DIR}/$FILENAME"
	mkdir "${SUPPORT_DIR}/$FILENAME/yml"
	mkdir "${SUPPORT_DIR}/$FILENAME/cfg"
	mkdir "${SUPPORT_DIR}/$FILENAME/logs"
	echo "Retrieving configuration information..."
	cp "${YML_DIR}"*.yml "${SUPPORT_DIR}/$FILENAME/yml"
	cp "cfg/connect/txdef.json" "${SUPPORT_DIR}/$FILENAME/cfg"
	cp "cfg/oidc/oidcSettings.json" "${SUPPORT_DIR}/$FILENAME/cfg"
	cp "cfg/rest/teamlist.json" "${SUPPORT_DIR}/$FILENAME/cfg"
	echo "Kernel version" > "${SUPPORT_DIR}/$FILENAME/os.txt"
	uname -a >> "${SUPPORT_DIR}/$FILENAME/os.txt"
	echo "" >> "${SUPPORT_DIR}/$FILENAME/os.txt"
	echo "Memory" >> "${SUPPORT_DIR}/$FILENAME/os.txt"
	free -th >> "${SUPPORT_DIR}/$FILENAME/os.txt"
	echo "Version" > "${SUPPORT_DIR}/$FILENAME/docker.txt"
	docker version >> "${SUPPORT_DIR}/$FILENAME/docker.txt" 2> /dev/null
	echo "" >> "${SUPPORT_DIR}/$FILENAME/docker.txt"
	echo "Nodes" >> "${SUPPORT_DIR}/$FILENAME/docker.txt"
	docker node ls 2>&1 >> "${SUPPORT_DIR}/$FILENAME/docker.txt"
	retrieve_logs "${SUPPORT_DIR}/$FILENAME/logs"
	cd ${SUPPORT_DIR}
	echo "Compressing support package"
	tar zcvf $FILENAME.tar.gz $FILENAME > /dev/null
	rm -rf $FILENAME
	echo "Finished creating ${SUPPORT_DIR}/$FILENAME.tar.gz"
} #generate_support_file

retrieve_logs() {
	if [ $ENABLE_LOG_STACK == "true" ]; then
		LOG_HOST=$(docker service ps app_nginx 2> /dev/null | grep Running | awk '{ print $4 }' | tail -n 1)
		if [ "x$LOG_HOST" != "x" ]; then
			curl -V > /dev/null
			if [ $(echo $?) -eq 0 ]; then
				CMD="curl -f -k https://${LOG_HOST}/api/logs/health"
				RESULT=$($CMD 2> /dev/null)
				if [ $(echo $?) -eq 0 ]; then
					echo "Retrieving logs from Elasticsearch"
					START_TIME=$(date -u -d "-2 days" +"%Y-%m-%dT%TZ")
					STOP_TIME=$(date -u +"%Y-%m-%dT%TZ")
					PAYLOAD=$(echo "\\\"version\\\":true,\\\"size\\\":5000,\\\"sort\\\":[{\\\"@timestamp\\\":{\\\"order\\\":\\\"desc\\\",\\\"unmapped_type\\\":\\\"boolean\\\"}}],\\\"_source\\\":{\\\"excludes\\\":[]},\\\"stored_fields\\\":[\\\"*\\\"],\\\"script_fields\\\":{},\\\"docvalue_fields\\\":[{\\\"field\\\":\\\"@timestamp\\\",\\\"format\\\":\\\"date_time\\\"}],\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[],\\\"filter\\\":[{\\\"match_all\\\":{}},{\\\"match_phrase\\\":{\\\"severity\\\":{\\\"query\\\":\\\"ERROR\\\"}}},{\\\"range\\\":{\\\"@timestamp\\\":{\\\"format\\\":\\\"strict_date_optional_time\\\",\\\"gte\\\":\\\"$START_TIME\\\",\\\"lte\\\":\\\"$STOP_TIME\\\"}}}],\\\"should\\\":[],\\\"must_not\\\":[]}}")
					CMD="curl -f -k https://${LOG_HOST}/api/logs/search -d payload=$PAYLOAD"
					RESULT=$($CMD > ${1}/connect.log 2> /dev/null)
					if [ $(echo $?) -eq 0 ]; then
						return
					else
						echo "Unable to retrieve logs from Elasticsearch, attempting to retrieve container logs"
					fi
				else
					echo "Unable to connect to Elasticsearch, attempting to retrieve container logs"
				fi
			else
				echo "curl command not found, attempting to retrieve container logs"
			fi
		else
			echo "Nginx service not found, attempting to retrieve container logs"
		fi
	fi
	docker service logs connect_connect 2>/dev/null | grep -i "Error\|Exception\|Warn" > ${1}/connect.log
	if [ $(echo $?) -ne 0 ]; then
		docker service ps connect_connect
		if [ $(echo $?) -ne 0 ]; then
			echo "Connect service not running, no logs available"
			echo "Connect service not running" >> ${1}/connect.log
		else
			echo "No error messages found" >> ${1}/connect.log
		fi
	fi
} #retrieve_logs

renew_cert() {
	if [ "x$1" == "x" ]; then
		cfg/nginx/gencert.sh renew
	else
		cfg/nginx/gencert.sh isiq_setup renew
	fi
} # renew_cert

check_cert_expiration() {
	# Only check renewal if we are the Certificate Authority
	if [ -f cfg/nginx/rootCA.crt ]; then
		NOW=$(date '+%s')
		EXP_DATE=$(cat cfg/nginx/site.crt | openssl x509 -noout -enddate | cut -d "=" -f 2)
		EXP_TIME=$(date -d "$EXP_DATE" '+%s')
		DAYS=$(( (EXP_TIME - NOW) / 86400 ))
		if [ $DAYS -lt 180 ]; then
			renew_cert isiq_setup
		fi
	fi
} #check_cert_expiration

check_stack() {
	TIMER=0
	WAIT=0
	CMD=$(docker service ls | grep ${1}_)
	while [ $(echo $?) -ne 0 ]; do
		if [ $TIMER -eq 0 ]; then
			printf "Waiting"
			WAIT=1
		else
			printf "."
		fi
		sleep 5
		TIMER=$(($TIMER+1))
		if [ $TIMER -gt 24 ]; then
			printf "\nFailed to start $1 stack in a timely manner\n"
			exit 2
		else
			CMD=$(docker service ls | grep ${1}_)
		fi
	done
	NOT_DONE=true
	TIMER=0
	PIFS=$IFS
	IFS=$'\n'
	while ($NOT_DONE == "true"); do
		NOT_DONE=false
		SERVICES=$(docker service ls | grep ${1}_)
		for SERVICE in $SERVICES; do
			RUNNING=$(echo "$SERVICE" | awk '{ print $4 }' | tr '/' ' ' | awk '{ print $2-$1 }')
			if [ $RUNNING -ne 0 ]; then
				SERVICE_NAME=$(echo "$SERVICE" | awk '{ print $2 }')
				SERVICE_PIN_NODE=$(docker inspect $SERVICE_NAME | grep node.labels.isiq)
				if [ $(echo $?) -eq 0 ]; then
					SERVICE_PIN_NODE=$(echo "$SERVICE_PIN_NODE" | awk '{ print $3 }' | cut -d '"' -f 1)
					CMD=$(grep -q $SERVICE_PIN_NODE <<< "$DOWN_NODES")
					if [ $(echo $?) -ne 0 ]; then
						NOT_DONE=true
					fi
				else
					NOT_DONE=true
				fi
			fi
		done
		if [ $NOT_DONE == "true" ]; then
			if [ $TIMER -eq 0 ] && [ $WAIT -eq 0 ]; then
				printf "Waiting"
			else
				printf "."
			fi
			sleep 5
			TIMER=$(($TIMER+1))
			if [ $TIMER -gt $STACK_START_TIMEOUT ]; then
				printf "\nFailed to start $1 stack in a timely manner\n\n\n"
				show_docker_errors $1
				exit 2
			fi
		fi
		# If services were already running, return immediately
		if [ $TIMER -eq 0 ]; then
			return
		fi
	done
	# Wait 30 seconds to verify services really started
	sleep 30
	SERVICES=$(docker service ls | grep ${1}_)
	for SERVICE in $SERVICES; do
		RUNNING=$(echo "$SERVICE" | awk '{ print $4 }' | tr '/' ' ' | awk '{ print $2-$1 }')
		if [ $RUNNING -ne 0 ]; then
			SERVICE_NAME=$(echo "$SERVICE" | awk '{ print $2 }')
			SERVICE_PIN_NODE=$(docker inspect $SERVICE_NAME | grep node.labels.isiq)
			if [ $(echo $?) -eq 0 ]; then
				SERVICE_PIN_NODE=$(echo "$SERVICE_PIN_NODE" | awk '{ print $3 }' | cut -d '"' -f 1)
				CMD=$(grep -q $SERVICE_PIN_NODE <<< "$DOWN_NODES")
				if [ $(echo $?) -ne 0 ]; then
					NOT_DONE=true
				fi
			else
				NOT_DONE=true
			fi
		fi
	done
	IFS=$PIFS
	if [ $NOT_DONE == "true" ]; then
		printf "\nFailed to start $1 stack in a timely manner\n\n\n"
		show_docker_errors $1
		exit 2
	fi
} # check_stack

get_nodes() {
	DOCKER_CHECK=$(docker version)
	if [ $(echo $?) -ne 0 ]; then
		echo "Docker not installed. ISIQ cannot continue."
		exit 1
	fi
	NODE_CHECK="$(docker node ls 2> /dev/null)"
	if [ $(echo $?) -ne 0 ]; then
		if [ "x$1" == "x" ]; then
			printf "No swarm detected. Creating swarm.\n\n"
			printf "        !!! Make a note of the information below !!!\n"
			printf "************************************************************\n"
			docker swarm init
			if [ $(echo $?) -ne 0 ]; then
				echo "Failed to initialize the swarm.  See errors above."
				exit 1
			fi
			printf "************************************************************\n\n"
			sleep 5
		else
			printf "No swarm detected.  Please run $0 setup to initialize ISIQ.\n"
			exit 1
		fi
	fi
	DOCKER_NODES="$(docker node ls | grep -v HOSTNAME | tr -d '*' | awk '{ print $2 }' | tr '\n' ' ' | sed 's/.$//')"
	NUM_NODES=$(echo "$DOCKER_NODES" | wc -w)
} #get_nodes

pull_images() {
	if [ "x$YML_DIR" == "x" ]; then
		if [ -d "yml/cluster/" ]; then
			YML_DIR="yml/cluster/"
		else
			if [ -d "yml/single_node" ]; then
				YML_DIR="yml/single_node/"
			else
				printf "YAML files not found, no image names could be processed.\n"
				exit 1
			fi
		fi
	fi
	printf "\n\nNow downloading the images needed for ISIQ.\n"
	IMAGES=$(grep image: ${YML_DIR}*.yml | awk '{ print $3 }' | sort | uniq)
	for IMAGE in $IMAGES; do
		echo "=== "
		echo "=== Pulling the image: $IMAGE"
		echo "=== "
		docker pull $IMAGE
	done
	if [ "x$NUM_NODES" == "x" ]; then
		NUM_NODES=1
	fi
	if [ $NUM_NODES -gt 1 ]; then
		printf "\n\nPlease copy the util and yml directories from the starter kit to\n"
		printf "all of the other nodes, and then execute isiq images on them.\n\n"
	fi
} # pull_images

isiq_setup() {
	# Can optionally pass in poc flag to avoid SSL and OIDC warning
	printf "\nGenerating Java token secret...\n"
	cfg/crypto/gentoken.sh isiq_setup
	if [ $(echo $?) -ne 0 ]; then
		printf "Failed to generate token. Please review the above errors.\n"
		exit 1
	fi
	printf "\nGenerating SSL truststore...\n"
	cfg/connect/ssl/genstore.sh isiq_setup
	if [ $(echo $?) -ne 0 ]; then
		printf "Failed to create truststore. Please review the above errors.\n"
		exit 1
	fi
	printf "\nGenerating SSL certificate...\n"
	read -p "ISIQ server hostname: " FQDN_HOST
	sed -i "s/ example.com/ $FQDN_HOST/" "${YML_DIR}../../cfg/nginx/cert.conf"
	cfg/nginx/gencert.sh isiq_setup
	if [ $(echo $?) -ne 0 ]; then
		printf "Failed to generate certificate. Please review the above errors.\n"
		exit 1
	fi
	printf "\nImport <StarterKit>/cfg/nginx/rootCA.crt into your browser for SSL to work.\n"
	pull_images
	if [ "x$1" == "x" ]; then
		printf "\n\n*******************************************************************************\n"
		printf "*                              !!! ATTENTION !!!                              *\n"
		printf "* You will still need to edit cfg/oidc/oidcSettings.json to specify your OIDC *\n"
		printf "* provider details. If you want a team of users to share product definitions, *\n"
		printf "* then use the Sharing Center in the ISIQ UI to create and manage groups.     *\n"
		printf "*******************************************************************************\n\n"
	fi
} #isiq_setup

isiq_status() {
	PIFS=$IFS
	IFS=$'\n'
	FOUND_BAD_SERVICE=0
	get_services
	NUM_SERVICES=$(echo "$SERVICES" | wc -w)
	if [ $NUM_SERVICES -eq 0 ]; then
		echo "ISIQ is stopped"
		exit 1
	fi
	IFS=$PIFS
	# Check if all stacks are up
	FOUND_MISSING_STACK=0
	RUNNING_STACKS="$(docker stack ls | awk '{ print $1 }')"
	for STACK in $STACKS_UP; do
		FOUND=0
		for RUN_STACK in $RUNNING_STACKS; do
			if [ "$RUN_STACK" == "$STACK" ]; then
				FOUND=1
			fi
		done
		if [ $FOUND -eq 0 ]; then
			if [ $FOUND_MISSING_STACK -eq 0 ]; then
				echo "Missing stack(s):"
				FOUND_MISSING_STACK=1
			fi
			echo "$STACK"
		fi
	done
	if [ $FOUND_MISSING_STACK -gt 0 ]; then
		printf "\nISIQ is not fully functional\n"
		exit 2
	fi
	# Check if any services are not fully replicated
	for SERVICE in $SERVICES; do
		RUNNING=$(echo "$SERVICE" | awk '{ print $4 }' | tr '/' ' ' | awk '{ print $2-$1 }')
		if [ $RUNNING -ne 0 ]; then
			if [ $FOUND_BAD_SERVICE -eq 0 ]; then
				printf "Failed service(s):\n\n"
				FOUND_BAD_SERVICE=1
			fi
			echo "$SERVICE" | awk '{ printf "%s: %s\n",$2,$4 }'
		fi
	done
	if [ $FOUND_BAD_SERVICE -gt 0 ]; then
		echo "ISIQ is not fully functional"
		exit 2
	fi
	echo "ISIQ is running"
} #isiq_status

isiq_start() {
	APP_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
	cd "$APP_DIR"
	if [ ! -f cfg/crypto/isiq.key ] && [ ! -f cfg/nginx/site.crt ] && [ ! -f cfg/crypto/isiq.jwt ]; then
		echo "ISIQ configuration is missing.  If this is a new install, please run $0 setup.  If this is an upgrade, please run $0 upgrade, or be sure to copy all of your cfg/* files into this new starter kit directory."
		exit 1
	fi
	if [ ! -f cfg/crypto/isiq.jwt ]; then
		echo "isiq.jwt is missing.  Please run ./cfg/crypto/gentoken.sh"
		exit 1
	fi
	if [ ! -f cfg/nginx/site.crt ]; then
		if [ -f cfg/nginx/site.req.pem ]; then
			echo "Nginx SSL certificate is missing!"
			echo "Please take ./cfg/nginx/site.req.pem to your Certificate Authority"
			echo "to generate a signed certificate.  That file needs to be placed"
			echo "in ./cfg/nginx/site.crt."
		else
			echo "site.crt is missing.  Please edit ./cfg/nginx/cert.conf to set"
			echo "DNS.1 in the alternate_names section to the hostname of your master"
			echo "swarm node.  Then run ./cfg/nginx/gencert.sh and select ca option"
		fi
		exit 1
	fi
	isiq_init
	if [ "x$1" != "x" ]; then
		STACKS_UP="$1"
		STACKS_DOWN="$1"
	fi
	if [ $ENABLE_LOG_STACK == "true" ]; then
		log_check
	fi
	if [ $DISABLE_JMX != "true" ]; then
		SUFFIX=stack.yml
	else
		SUFFIX=stack-nojmx.yml
	fi
	check_cert_expiration
	set_yaml_dir
	get_down_nodes
	cd ${YML_DIR}
	for STACK in $STACKS_UP; do
		printf "Starting $STACK stack..."
		CMD=$(docker service ls | grep ${STACK}_)
		if [ $(echo $?) -ne 0 ]; then
			docker stack deploy --detach=false -c ${STACK}-${SUFFIX} ${STACK} > /dev/null
			if [ $(echo $?) -ne 0 ]; then
				echo "Failed to start $STACK stack.  See errors above"
				exit 1
			fi
		fi
		check_stack $STACK
		printf "...Started\n"
	done
	if [ "x$1" == "x" ]; then
		echo "ISIQ Started"
	fi
} #isiq_start

isiq_stop() {
	if [ "x$1" != "x" ]; then
		STACKS_DOWN="$1"
	fi
	STACK_CHECK=$(docker stack ls 2> /dev/null)
	if [ $(echo $?) -ne 0 ]; then
		echo "ISIQ Stopped"
		return
	fi
	RUNNING_STACKS="$(docker stack ls | awk '{ print $1 }')"
	NUM_RUNNING_STACKS=$(echo "$RUNNING_STACKS" | wc -w)
	if [ $NUM_RUNNING_STACKS -eq 0 ]; then
		echo "ISIQ Stopped"
		return
	fi
	NUM_STACKS=$(echo "$STACKS_DOWN" | wc -w)
	for STACK in $STACKS_DOWN; do
		while read -r line; do
			if [ "x$line" == "x$STACK" ]; then
				echo "Stopping $line stack"
				docker stack rm $line > /dev/null
			fi
		done <<< "$RUNNING_STACKS"
	done
	get_nodes
	NOT_DONE=true
	# Give it up to 5 minutes to stop everything
	LOOP_TIMEOUT=$(( $(date +%s) + 300 ))
	while ($NOT_DONE == "true"); do
		sleep 5
		for STACK in $STACKS_DOWN; do
			NODE_COUNT=0
			for NODE in $DOCKER_NODES; do
				CMD="$(docker node ps $NODE | grep Remove)"
				if [ "$(echo $?)" -eq 1 ]; then
					CMD="$(docker node ps $NODE | grep ${STACK}_)"
					if [ "$(echo $?)" -eq 1 ]; then
						NODE_COUNT=$((NODE_COUNT+1))
					fi
				fi
				if [ $NODE_COUNT -eq $NUM_NODES ]; then
					STACKS_DOWN=$(echo "$STACKS_DOWN" | sed "s/$STACK//")
					NUM_STACKS=$((NUM_STACKS-1))
				fi
			done
		done
		if [ $NUM_STACKS -eq 0 ]; then
			NOT_DONE=false
		fi
		if [ $(date +%s) -gt $LOOP_TIMEOUT ]; then
			NOT_DONE=false
		fi
	done
	if [ "x$1" == "x" ]; then
		echo "ISIQ Stopped"
	fi
} #isiq_stop

isiq_backup() {
	if [ "x$BACKUP_DIR" == "x" ]; then
		echo "Please edit the isiq script to specify a BACKUP_DIR and then try again."
		exit 1
	fi
	isiq_stop
	DOCKER_VOLUME_PATH=/var/lib/docker/volumes/
	DATE_STR=$(/bin/date +"%Y%m%d%H%M")
	get_nodes skip_swarm	
	if [ ! -d "$BACKUP_DIR" ]; then
        echo "Creating directory $BACKUP_DIR"
        mkdir -p $BACKUP_DIR
	fi
	echo "Starting backup"
	for FILE in $(sudo /bin/ls "$DOCKER_VOLUME_PATH" | grep _isiq_); do
		echo "Adding ${FILE}"
		sudo tar rfP ${BACKUP_DIR}/isiq_backup_${DATE_STR}.tar ${DOCKER_VOLUME_PATH}${FILE}
	done
	echo "Compressing backup"
	gzip ${BACKUP_DIR}/isiq_backup_${DATE_STR}.tar
	if [ $NUM_NODES -gt 1 ]; then
		for NODE in $DOCKER_NODES; do
			if [ $NODE != $(hostname) ]; then
				echo "Backing up $NODE"
				    ssh -T ${NODE} <<EOF > /dev/null
if [ ! -d "$BACKUP_DIR" ]; then
	>&2 echo "Creating directory $BACKUP_DIR"
	mkdir -p "$BACKUP_DIR"
fi
DOCKER_FILES=\$(sudo /bin/ls $DOCKER_VOLUME_PATH | grep _isiq_)
for FILE in \$DOCKER_FILES; do
	>&2 echo "Adding \$FILE"
	sudo tar rfP ${BACKUP_DIR}/isiq_backup_${DATE_STR}.tar ${DOCKER_VOLUME_PATH}\${FILE}
done
>&2 echo "Compressing backup"
gzip ${BACKUP_DIR}/isiq_backup_${DATE_STR}.tar
exit
EOF
				if [ $? -eq 0 ]; then
					echo "Completed backup of $NODE"
				else
					echo "Failed to backup ${NODE}.  Backup will NOT be complete"
				fi
			fi
		done
	fi
	echo "ISIQ backup complete"
} #isiq_backup

isiq_restore() {
	if [ "x$BACKUP_DIR" == "x" ]; then
		echo "No backup directory found!"
		echo "Please edit the isiq script to specify a BACKUP_DIR and run a backup before attempting to restore."
		exit 1
	fi
	isiq_stop
	RECENT_BACKUP=$(ls -t $BACKUP_DIR | cut -d ' ' -f 1 | head -n 1)
	DOCKER_VOLUME_PATH=/var/lib/docker/volumes/
	get_nodes skip_swarm
	echo "Starting restore"
	for FILE in $(sudo /bin/ls "$DOCKER_VOLUME_PATH" | grep _isiq_); do
		echo "Removing current $FILE"
		sudo rm -rf ${DOCKER_VOLUME_PATH}${FILE}
	done
	echo "Unpacking restore file $RECENT_BACKUP"
	sudo tar zxfP ${BACKUP_DIR}/${RECENT_BACKUP}
	if [ $NUM_NODES -gt 1 ]; then
		for NODE in $DOCKER_NODES; do
			if [ $NODE != $(hostname) ]; then
				echo "Restoring $NODE"
					ssh -T ${NODE} <<EOF > /dev/null
for FILE in \$(sudo /bin/ls "$DOCKER_VOLUME_PATH" | grep _isiq_); do
	>&2 echo "Removing current \$FILE"
	sudo rm -rf ${DOCKER_VOLUME_PATH}\${FILE}
done
>&2 echo "Unpacking restore file $RECENT_BACKUP"
sudo tar zxfP ${BACKUP_DIR}/${RECENT_BACKUP}
if [ \$? -ne 0 ]; then
	>&2 echo "Error processing backup file!"
	exit 1
fi
exit
EOF
				if [ $? -eq 0 ]; then
					echo "Completed restore of $NODE"
				else
					echo "Failed to restore ${NODE}.  Restore will NOT be complete"
				fi
			fi
		done
	fi
	echo "ISIQ restore complete"
} #isiq_restore

isiq_upgrade() {
	PYTHON=python3
	PYTHON_CHECK=$(python3 -V 2>&1 > /dev/null)
	if [ $(echo $?) -ne 0 ]; then
		PYTHON_CHECK2=$(python -V 2>&1 | cut -d ' ' -f 2 | cut -d '.' -f 1)
		if [ "x$PYTHON_CHECK2" != "x3" ]; then
			echo "Python3 not found in path.  Unable to perform upgrade."
			exit 1
		else
			PYTHON=python
		fi
	fi
	if [ "x$ORIGINAL_STARTER_KIT_DIR" == "x" ]; then
		echo "Please edit the isiq script to specify an ORIGINAL_STARTER_KIT_DIR and then run the script again."
		exit 1
	fi
	ORIGINAL_DIR=$(awk -vs1="$USE_ORIGINAL_DIR" 'BEGIN { print tolower(s1) }')
	FILENAME=$(date +"%Y%m%d_%H%M%S")
	echo "Backing up existing YML and configuration files..."
	if [ "$ORIGINAL_DIR" == "true" ]; then
		(cd "$ORIGINAL_STARTER_KIT_DIR"; tar zcvf "$FILENAME.backup_cfg.tar.gz" yml cfg) > /dev/null
	else
		tar zcvf "$FILENAME.backup_cfg.tar.gz" "yml" "cfg" > /dev/null
	fi
	if [ $(echo $?) -ne 0 ]; then
		echo "Unable to backup YML files, aborting upgrade"
		exit 1
	fi
	$PYTHON util/isiq_upgrade.py "$ORIGINAL_STARTER_KIT_DIR" $ORIGINAL_DIR $ISIQ_DEPLOYMENT $DISABLE_JMX
	echo "Updating configuration files..."
	if [ "$ORIGINAL_DIR" == "true" ]; then
		rsync -aqrtI --exclude=yml --exclude=cfg . "$ORIGINAL_STARTER_KIT_DIR"
		rsync -aqrtI --ignore-existing cfg "$ORIGINAL_STARTER_KIT_DIR"
		rsync -aqrtI --include='*/' --include='*.sh' --exclude='*' cfg "$ORIGINAL_STARTER_KIT_DIR"
		if [ $(echo $?) -ne 0 ]; then
			echo "Error occurred while copying configuration files"
		fi
		isiq_upgrade_10.0.2 original
	else
		isiq_upgrade_10.0.2 new
		rsync -aqrtI --exclude='*.sh' "${ORIGINAL_STARTER_KIT_DIR}/cfg" .
		if [ $(echo $?) -ne 0 ]; then
			echo "Error occurred while copying configuration files"
		fi
		isiq_upgrade_10.0.2 post
	fi
} #isiq_upgrade

isiq_upgrade_10.0.2() {
	if [ "$1" == "original" ]; then
		isiq_upgrade_10.0.2_hash
		cp cfg/logstash/logstash.conf "${ORIGINAL_STARTER_KIT_DIR}/cfg/logstash/logstash.conf"
		cp cfg/grafana/grafana.ini "${ORIGINAL_STARTER_KIT_DIR}/cfg/grafana/grafana.ini"
	elif [ "$1" == "new" ]; then
		mv cfg/logstash/logstash.conf cfg/logstash/logstash.orig
		mv cfg/grafana/grafana.ini cfg/grafana/grafana.orig
	elif [ "$1" == "post" ]; then
		isiq_upgrade_10.0.2_hash
		mv cfg/logstash/logstash.orig cfg/logstash/logstash.conf
		mv cfg/grafana/grafana.orig cfg/grafana/grafana.ini
	fi
} #isiq_upgrade_10.0.2

isiq_upgrade_10.0.2_hash() {
	LOGSTASH_10_HASH=0da895ce13139585b2f0cf82a481fbbfe5eca296
	GRAFANA_10_HASH=3d7c5b768271fe0092a92c40169fccc39d4bb2c9
	LOGSTASH_HASH=$(${PYTHON} util/hash.py "${ORIGINAL_STARTER_KIT_DIR}/cfg/logstash/logstash.conf")
	GRAFANA_HASH=$(${PYTHON} util/hash.py "${ORIGINAL_STARTER_KIT_DIR}/cfg/grafana/grafana.ini")
	if [ $LOGSTASH_HASH != $LOGSTASH_10_HASH ]; then
		echo "Modified cfg/logstash/logstash.conf overwritten with 10.0.2.  Please merge your changes manually."
	fi
	if [ $GRAFANA_HASH != $GRAFANA_10_HASH ]; then
		echo "Modified cfg/grafana/grafana.ini overwritten with 10.0.2.  Please merge your changes manually."
	fi
} #isiq_upgrade_10.0.2_hash

case $(awk -vs1="$1" 'BEGIN { print tolower(s1) }') in
	start)
		isiq_start $2
		;;
	stop)
		isiq_stop $2
		;;
	restart)
		isiq_stop
		isiq_start
		;;
	upgrade)
		isiq_init
		set_yaml_dir
		isiq_upgrade
		;;
	status)
		isiq_init skip_nodes
		isiq_status
		;;
	setup)
		isiq_init
		set_yaml_dir
		isiq_setup $2
		isiq_poc $2
		;;
	support)
		generate_support_file
		;;
	backup)
		isiq_backup
		;;
	restore)
		isiq_restore
		;;
	logs)
		show_logs $2 $3
		;;
	log)
		show_logs $2 $3
		;;
	images)
		isiq_init skip_nodes
		set_yaml_dir
		pull_images
		;;
	renew)
		renew_cert
		;;
	help)
		show_help
		;;
	*)
	echo "Usage: $0 {setup|start|stop|restart|status|support|upgrade|backup|restore|images|renew|logs|help}"
	exit 1
esac

exit 0
