#!/bin/bash
# ---------------------------------------------------------------------------------------------------
# Identify all the service replicas/machines and permits you to toggle maintenance mode on or off via
# an SSM document execution on the host. If the json output from the SSM executions is not enough to
# debug just look in AWS and you'll see the whole execution history in SSM Command Execution History.
# ---------------------------------------------------------------------------------------------------
# Stop immediately if anything goes wrong, let's not create too much
# mess if John's shell is poor
set -eo pipefail
# Find & Import all the supporting functions from the supporting folder
# Get the directory of the current script to figure out where the
# Supporting funcs are
IMPORT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
for script in ${IMPORT_DIR}/supporting-funcs/*.sh; do
if [[ -f "$script" ]]; then
source "$script"
fi
done
# Usage for this script
usage() {
echo
title "toggle-maintenance"
section "----------------------------------"
echo "This script will open an SSM session to all available"
echo "service nodes in the region and will attempt to toggle on"
echo "or off the maintenance mode via a USR2 linux signal"
section "----------------------------------"
usage_text "Usage:" "toggle-maintenance [-p profile] [-r region] [-a automatic] [-s service] [-m maintenance]"
echo " $(option "-p profile") [pull-from-env/<profile-name>] AWS profile to use"
echo " $(option "-r region") AWS region to use"
echo " $(option "-s service") [sdf/rebaser/pinga/veritech/forklift/edda] SI Service to filter by, defaults to all"
echo " $(option "-a automatic") [Y/N] Run through automatically/no-interact"
echo " $(option "-m maintenance") [Y/N] Whether maintenance should be on"
section "----------------------------------"
example "e.g. ./awsi.sh toggle-maintenance -p pull-from-env -r us-east-1 -s sdf -a y -m y"
exit 0
}
# Add a check to see if the script is being sourced or executed
if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then
usage
fi
# Parse flags
while getopts ":p:r:a:s:m:" opt; do
case ${opt} in
p)
profile=$OPTARG
;;
r)
region=$OPTARG
;;
a)
automatic=$OPTARG
;;
s)
service=$OPTARG
;;
m)
maintenance=$OPTARG
;;
\?)
echo "Invalid option: -$OPTARG" >&2
usage
;;
:)
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
# ---------------------------------------------------------------------------------------------------
# Main script
# ---------------------------------------------------------------------------------------------------
echo "$0 being invoked"
# Use the profile if in the invocation
if [[ "$profile" != "pull-from-env" ]]; then
profile=$(get_param_or_env "$profile" "AWS_PROFILE" "Enter the AWS profile to use")
export AWS_PROFILE="$profile"
fi
region=$(get_param_or_env "$region" "AWS_REGION" "Enter the AWS region (e.g., us-west-2)")
export AWS_REGION="$region"
# List instances with fixed-width columns and filter for the upgradeable instances
# Based on the filter provided, always include SDF for the maintenance mode enable/disable
instances=$(list_instances "${service,,}" )
if [ -z "$instances" ]; then
echo "No running instances found."
exit 1
fi
echo "----------------------------------------"
echo "Running instances of service $service in the region $region:"
printf "%-5s %-20s %-20s %-20s %-20s\n" "Index" "Name" "InstanceId" "InstanceType" "PrivateIpAddress"
i=1
while read -r line; do
name=$(echo "$line" | awk '{print $1}')
instance_id=$(echo "$line" | awk '{print $2}')
instance_type=$(echo "$line" | awk '{print $3}')
private_ip=$(echo "$line" | awk '{print $4}')
printf "%-5s %-20s %-20s %-20s %-20s\n" "$i" "$name" "$instance_id" "$instance_type" "$private_ip"
((i++))
done <<< "$instances"
echo "----------------------------------------"
[[ "${automatic,,}" == "y" ]] || read -p "Would you like to toggle maintenance mode to $maintenance on these hosts? (Y/N) [takes ~10 seconds] " selection
[[ "${automatic,,}" == "y" ]] || sassy_selection_check $selection
# Setup somewhere unique to push the results of the check into if they chose to continue
# Reset this results_directory variable between each execution run.
results_directory="./results/$(date +"%Y-%m-%d_%H-%M-%S")"
check_results_file=check_results.json
start_results_file=start_results.json
stop_results_file=stop_results.json
upgrade_results_file=upgrade_results.json
mkdir -p "$results_directory/"
i=1
while read -r line; do
instance_id=$(echo "$line" | awk '{print $2}')
service_name=$(echo "$line" | awk '{print $1}' | awk -F- '{print $2}')
start_and_track_ssm_session "$instance_id" "$service_maintenance_script" "$results_directory" "InstanceId=$instance_id,Service=$service_name,Action=$maintenance" &
((i++))
done <<< "$instances"
await_file_results "$results_directory" $((i - 1))
concat_and_output_json "$results_directory" "$check_results_file"
if jq --arg maintenance="$maintenance" -e 'all(.[]; .status == "success") and all(.[]; .maintenance == "$maintenance")' "$results_directory/$check_results_file" > /dev/null; then
echo "All running service nodes of ${service} have had maintenance mode set to $maintenance"
echo "----------------------------------------"
exit 0
else
echo "Error: One or more of the checks failed to push a node into maintenance mode, try again later or look at the logs"
exit 2
fi