Skip to main content
Bounty Awarded with 100 reputation awarded by user2023
edited body
Source Link
Walter A
  • 361
  • 1
  • 5

I am not sure about introducinsintroducing a new function for all your ssh calls:

I am not sure about introducins a new function for all your ssh calls:

I am not sure about introducing a new function for all your ssh calls:

Source Link
Walter A
  • 361
  • 1
  • 5

Changes:

  • Quote variables unless you have a good reason not to quote them.
    How about a password * $(touch /tmp/sorry) * ?
  • Use ${parameter:-word} for turning an empty vaultenv into prod.
    If parameter is unset or null, the expansion of word is substituted.
    Otherwise, the value of parameter is substituted.
  • Translate uppercase to lowercase only once. Use ${parameter,,} for this.
  • Combine tests on vaultenv into one block.
    Your code will exit for ram and iqa, I accept these. Change when you want to exit.
  • Changed variable names to lowercase
    UPPERCASE is reserved for system vars like PATH.
  • Removed unused DATE.
  • I combined the two ansible-vault calls into one and assign to SHOW_AUTH without echo.
  • Removed assignment to RETURNCODE

The resulting script:

#!/bin/bash
clusters=(udcl101 udcl2011 udcl3011 udcl4011)
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin:$PATH

incident="$1" # Unused ???????
sysid=UNKNOWN

vaultenv="${vaultenv:-prod}"
vaultenvlc="${vaultenv,,}"
vaultenvuc="${vaultenv^^}"

case "${vaultenvlc}" in
  ram)
    # or do you want to exit here?
    snow_instance=udc2dev.service-now.com
    ;;
  iqa)
    # or do you want to exit here?
    snow_instance=udc2dev.service-now.com
    ;;
  dev)
    snow_instance=udc2dev.service-now.com
    ;;
  test)
    snow_instance=udc2trn.service-now.com
    ;;
  acc)
    snow_instance=udc2qa.service-now.com
    ;;
  prod)
    snow_instance=udc2.service-now.com
    ;;
  *)
    echo "unknown vault environment specified: ${vaultenv}"
    exit 1
esac

snow_auth=$(export EDITOR=cat;
             ansible-vault view --vault-password-file=~/.ssh/"sss.str.${vaultenvlc}" ~/iss/vaults/"vault-${vaultenvuc}.yml" |
             awk '
               /vault_servicenow_username/ {gsub(/"/, ""); usr=$2}
               /vault_servicenow_password/ {gsub(/"/, ""); pw=$2}
               END {printf("%s:%s", usr, pw)}
             ')

create_incident()
{
  url="https://${snow_instance}/api/now/table/incident"
  if curloutput=$(curl -sslv1 -u "${snow_auth}" -X POST -s -H "Accept:application/json" -H "Content-Type:application/json" "${url}" -d "${variables}" 2>/dev/null); then
    incident_number=$(echo "${curloutput}" |  python -c $'import sys, json\nprint json.load(sys.stdin)["result"]["number"]')
    sysid=$(echo "${curloutput}" |  python -c $'import sys, json\nprint json.load(sys.stdin)["result"]["sys_id"]')
    echo "OK: created incident ${incident_number} with sysid ${sysid}"
  else
    echo "ERROR creating incident:"
    echo "======================================="
    echo "${curloutput}"
    echo "======================================="
  fi
}


# sets the variables for the incident
set_variables()
{
read -r -d '' variables <<- EOM
{
  "short_description": "${snshort_description}",
  "assignment_group":  "RD-DI-Infra-Storage",
  "contact_type":      "interface",
  "state":             "New",
  "urgency":           "3 - Low",
  "impact":            "3 - Low",
  "cmdb_ci":           "${udcl}",
  "u_sec_env":         "Normal Secure",
  "description":       ${body}
}
EOM
}


# checks if file is empty *or* has only "", if not create incident
chk_body()
{
  if [[ "${body}" == "" || "${body}" == '""' ]]; then
    echo empty body
   else
    set_variables
    echo "${variables}"
    echo ""
    create_incident
  fi
}

# actual storage health checks

for udcl in "${clusters[@]}";
do

  body=$(ssh admin@"${udcl}" aggr show -root false |
     awk '/^udc/ && $4 >= 92 {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="aggr utilization above limit"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 0;snapshot show -create-time <21d" |
     awk '/^stv/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="snapshots older then 21 days"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 1;event log show -time <8h"|
     awk '/netinet.ethr.duplct.ipAdr|secd.ldap.noServers|object.store.unavailable/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0} 
          END {printf "\"%s\"\n", s}')
  snshort_description="eventlog netinet.ethr.duplct.ipAdr, secd.ldap.noServers, object.store.unavailable messages"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 1;event log show -severity EMERGENCY -time <8h"|
     awk '/udc/ && !/netinet.ethr.duplct.ipAdr|secd.ldap.noServers|object.store.unavailable/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="EMERGENCY messages"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 0;system healt alert show"|
     awk '/Node|Severity|Proba/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="system healt alert show messages"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 0;vol show -volume *esx* -percent-used >=85" |
     awk '/stv/ && !/dr/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="ESX volumes utilization more then 85"
  chk_body

  body=$(ssh admin@"${udcl}" "ro 0;vol show -state offline -fields aggregate" |
     awk '/stv/ {sub(/\r$/, ""); s = (s == "" ? "" : s "\\n") $0}
          END {printf "\"%s\"\n", s}')
  snshort_description="Validate offline volumes"
  chk_body

done

I am not sure about introducins a new function for all your ssh calls:

for udcl in "${clusters[@]}";
do

  body=$(check "${udcl}" aggr)
  chk_body

  body=$(check "${udcl}" snapshot)
  chk_body

  body=$(check "${udcl}" showtime)
  chk_body

  ...
}

check() {
   case "$2" in
   "aggr") 
      body=$(ssh admin@"${udcl}" aggr show -root false |
         awk '
           /^udc/ && $4 >= 92 {sub(/\r$/, "");
           s = (s == "" ? "" : s "\\n") $0}
           END    {printf "\"%s\"\n", s}
         ')
     snshort_description="aggr utilization above limit"
     ;;
    ...
   esac
}