Simple DevOps shell library for Linux/Mac/Windows

README.md

Simple DevOps shell library for Linux/Mac/Windows

Usage: devops [options] <command> [args]

Sourcing:
  Sourcing script defines functions and passes entrypoint function to init.

  Samples when devops is located in the same/relative/absolute path:
    . "`dirname "${BASH_SOURCE[0]}"`/devops" && init <entrypoint> "$@"
    . "`dirname "${BASH_SOURCE[0]}"`/<relpath>/devops" && init <entrypoint> "$@"
    . "<abspath>/devops" && init <entrypoint> "$@"

Options:
  -h, --help      Show this help and exit.
  --debug         Output debug messages.

Settings:
  Settings can be set in devops.env in the devops script path.
  Set defaults as NAME="${NAME:-DefaultValue}".

  DEVOPS_VARS:    Regexp to match env variables in list-settings. Defaults to /.*/.
  DEVOPS_DEBUG:   Output debug messages if set to anything except 0, n, no, off, false.
                  Alias DEBUG_DEVOPS works too.

Execution functions:

  @[+] <command> [options]
    Run the command and log the execution to stderr if debugging.
    Use @+ to log each argument on separate line.

  @@ <command> [options]
    Run the command if debugging.

  -|--|--- <command> [options]
    Run the command with stdout/stderr/both muted.

  all <command> [args]
    Run command for all args until first fails.

  any <command> [args]
    Run command for all args until first succeeds.

  within[-ss] <path> <command> [options]
    Run command within given path, optionally using a sub-shell.

  hascmd <commands>
    Test if command is available in path.

  require [commands]
    Fail if commands are not available in path.

Logging functions:

  I|info <message>
    Log info message to stderr.

  warn <message> [errorcode=1]
    Log warning message to stderr and return errorcode.

  fail [message] [exitcode=1]
    Log error message to stderr and exit with exitcode.

  error <message> [errorcode=1]
    Log error message to stderr and return errorcode.

  list-settings [regexp=$DEVOPS_VARS|.*] [prefix]
    Output env variables matching regexp.

Boolean functions:

  is-enabled <value>
    Test if value is not falsy (empty, 0, n, no, off, false).

  is-disabled <value>
    Test if value is falsy (empty, 0, n, no, off, false).

String functions:

  includes <list> <word>
    Test if list contains non-empty word.

  contains <string> <substring>
    Test if string contains non-empty substring.

  startswith <string> <substring>
    Test if string starts with non-empty substring.

  endswith <string> <substring>
    Test if string ends with non-empty substring.

  uppercase [string]
    Output string or input in uppercase.

  lowercase [string]
    Output string or input in lowercase.

Path functions:

  binpath [relative-to-bin=.] [script=self]
    Output absolute path to directory containing script, or relative to it

  abspath [path=.]
    Output absolute path

  relpath [path=.] [relative-to=.]
    Output relative path

Google Cloud functions:

  run-gcloud [-s|--shell] [args]
    Run gcloud command locally, or inside a gcloud docker container.
    Run `gcloud auth login` first, if default gcloud configuration
    is missing in ~/.config/gcloud/configurations/config_default,
    or in $APPDATA/gcloud/configurations/config_default on Windows.

Docker functions:

  is-docker
    Test if called inside docker container.

  run-in-docker <image> [options] <command> [args]
    Run command in a temporary docker image container
    in a /root/cd directory mapped to host current directory
    and with /root/home directory mapped to host home directory.
    Use the --rm option to remove the container on exit.

  run-in-container <name[~image]> <command> [args]
    Run command in named docker image container.
    Create missing container from image (defaults to name)
    with /root/cd directory mapped to host current directory
    and /root/home directory mapped to host home directory.


  has-container <container>
    Test if docker container exists.

  has-started-container <container>
    Test if docker container exists and is started.

  get-volume-path <path>
    Output the path in windows format as required by docker on windows.

Windows functions:

  is-windows
    Test if called on windows system.

  encode-abs-path <path>
    Output the path with the initial slash doubled to avoid MSYS/Git conversion.
    Pass absolute remote/container paths with the initial slash doubled
    to avoid MSYS/Git prefixing the absolute unix path argument on Windows.
    See http://www.mingw.org/wiki/Posix_path_conversion.

  decode-abs-path <path>
    Output the path without the initial slash doubled

Environment functions:

  hasopt <-o|--option> [args]
    Test if args include -o or --option.

  hasvar <variables>
    Test if environment variable is set.

  setvar <variable> [value]
    Set variable to value.

  require-vars [vars[:source-vars]]
    Fail if environment variables are not set.

  dotenv [--export] [paths]
    Load .env, .env.local, .env.$NODE_ENV, .env.$NODE_ENV.local files if present
    in given paths. Optionally export the variables. Env defaults to development. 
    Follows https://create-react-app.dev/docs/adding-custom-environment-variables.

  loadenv [--export] [files]
    Load env files if present. Optionally export the variables.

  loadfile <file> [label]
    Output file or report error.

Initialization functions:

  usage [script=executed]
    Output script usage.

  is-sourced [script=self]
    Test if script is sourced.

  from-sourced
    Test if called from a sourced script.

  from-self
    Test if called from the same script script.

  init [--pass-help|--debug] <command> [args]
    If caller is sourced or devops and help is not requested with -h or --help,
    then load devops.env from the devops script path.
    If caller is sourced, then return.   
    If help is requested with -h or --help, then output caller usage and return. 
    Set DEVOPS_DEBUG=1 on --debug.
    Run command with args.

devops

#!/bin/bash
#|
#|Usage: devops [options] <command> [args]
#|
#|Sourcing:
#|  Sourcing script defines functions and passes entrypoint function to init.
#|
#|  Samples when devops is located in the same/relative/absolute path:
#|    . "`dirname "${BASH_SOURCE[0]}"`/devops" && init <entrypoint> "$@"
#|    . "`dirname "${BASH_SOURCE[0]}"`/<relpath>/devops" && init <entrypoint> "$@"
#|    . "<abspath>/devops" && init <entrypoint> "$@"
#|
#|Options:
#|  -h, --help      Show this help and exit.
#|  --debug         Output debug messages.
#|
#|Settings:
#|  Settings can be set in devops.env in the devops script path.
#|  Set defaults as NAME="${NAME:-DefaultValue}".
#|
#|  DEVOPS_VARS:    Regexp to match env variables in list-settings. Defaults to /.*/.
#|  DEVOPS_DEBUG:   Output debug messages if set to anything except 0, n, no, off, false.
#|                  Alias DEBUG_DEVOPS works too.
export DEVOPS_DEBUG="${DEVOPS_DEBUG:-$DEBUG_DEVOPS}"
export DEBUG_DEVOPS="$DEVOPS_DEBUG"

#|
#|Execution functions:
#|
#|  @[+] <command> [options]
#|    Run the command and log the execution to stderr if debugging.
#|    Use @+ to log each argument on separate line.
@() { @@ echo ">>> $@" >&2; "$@"; }
@+() { local arg cmd="$1"; shift;
  @@ echo ">>> $cmd" >&2; for arg in "$@"; do @@ echo "... $arg" >&2; done; "$cmd" "$@"; }
#|
#|  @@ <command> [options]
#|    Run the command if debugging.
@@() { is-disabled "${DEVOPS_DEBUG:-$DEBUG_DEVOPS}" || "$@"; }
#|
#|  -|--|--- <command> [options]
#|    Run the command with stdout/stderr/both muted.
-() { "$@" >/dev/null; }
--() { "$@" 2>/dev/null; }
---() { "$@" >/dev/null 2>&1; }
#|
#|  all <command> [args]
#|    Run command for all args until first fails.
all() { local arg cmd="$1"; shift; for arg in "$@"; do "$cmd" "$arg" || return $?; done; }
#|
#|  any <command> [args]
#|    Run command for all args until first succeeds.
any() { local arg cmd="$1"; shift; for arg in "$@"; do "$cmd" "$arg" && return; done; return 1; }
#|
#|  within[-ss] <path> <command> [options]
#|    Run command within given path, optionally using a sub-shell.
within() { local d="$PWD" r=; cd "$1" && shift && "$@"; r=$?; cd "$d"; return "$r"; }
within-ss() { ( cd "$1" && shift && "$@"; ); }
#|
#|  hascmd <commands>
#|    Test if command is available in path.
hascmd() { local _a; for _a in "$@"; do --- which "$1" || return 1; done; }
#|
#|  require [commands]
#|    Fail if commands are not available in path.
require-command() { hascmd "$1" || fail "missing required command: $1"; }
require() { all require-command "$@"; }

#|
#|Logging functions:
#|
#|  I|info <message>
#|    Log info message to stderr.
I() { info "$@"; }
info() { echo "=== $@" >&2; }
#|
#|  warn <message> [errorcode=1]
#|    Log warning message to stderr and return errorcode.
warn() { echo "*** warning: $1" >&2; return "${2:-1}"; }
#|
#|  fail [message] [exitcode=1]
#|    Log error message to stderr and exit with exitcode.
fail() { [ -z "$1" ] || error "$1"; exit "${2:-1}"; } 
#|
#|  error <message> [errorcode=1]
#|    Log error message to stderr and return errorcode.
error() { echo "${__cmd:-${BASH_SOURCE[1]##*[\\/]}}: error: $1" >&2; return "${2:-1}"; } 
#|
#|  list-settings [regexp=$DEVOPS_VARS|.*] [prefix]
#|    Output env variables matching regexp.
list-settings() { local c re="${1:-${DEVOPS_VARS:-.*}}"; for c in '(' ')' '|'; do re="${re//$c/\\$c}"; done;
  env | sed -n "/$re/p;s/^/$2/" | sort | uniq; return 0; }

#|
#|Boolean functions:
#|
#|  is-enabled <value>
#|    Test if value is not falsy (empty, 0, n, no, off, false).
is-enabled() { case $1 in (''|'0'|'n'|'no'|'off'|'false') return 1;; esac; return 0; } 
#|
#|  is-disabled <value>
#|    Test if value is falsy (empty, 0, n, no, off, false).
is-disabled() { ! is-enabled $1; } 

#|
#|String functions:
#|
#|  includes <list> <word>
#|    Test if list contains non-empty word.
includes() { [ -n "$2" ] && contains " $1 " " $2 "; }
#|
#|  contains <string> <substring>
#|    Test if string contains non-empty substring.
contains() { [ -n "$2" -a "${1#*$2}" != "$1" ]; }
#|
#|  startswith <string> <substring>
#|    Test if string starts with non-empty substring.
startswith() { [ -n "$2" -a "${1#$2}" != "$1" ]; }
#|
#|  endswith <string> <substring>
#|    Test if string ends with non-empty substring.
endswith() { [ -n "$2" -a "${1%$2}" != "$1" ]; }
#|
#|  uppercase [string]
#|    Output string or input in uppercase.
uppercase() { if [ $# = 0 ]; then tr '[:lower:]' '[:upper:]'; else echo -n "$@" | uppercase; fi; }
#|
#|  lowercase [string]
#|    Output string or input in lowercase.
lowercase() { if [ $# = 0 ]; then tr '[:upper:]' '[:lower:]'; else echo -n "$@" | lowercase; fi; }


#|
#|Path functions:
#|
#|  binpath [relative-to-bin=.] [script=self]
#|    Output absolute path to directory containing script, or relative to it
binpath() { local s="${2:-${BASH_SOURCE[1]}}"; abspath "${s%[\\/]*}${1:+/$1}"; }       
#|
#|  abspath [path=.]
#|    Output absolute path
abspath() { local d="$PWD" r=; cd "${1:-.}" && pwd; r=$?; cd "$d"; return "$r"; }
#|
#|  relpath [path=.] [relative-to=.]
#|    Output relative path
relpath() { local p r up=; p="$(abspath "$1")/" && r="$(abspath "$2")" || return $?; while [ "$p" = "${p#$r[\\/]}" ];
  do r="${r%[\\/]*}"; up="../$up";done; p="${up}${p#$r/}"; p="${p%/}"; echo "${p:-.}"; }


#|
#|Google Cloud functions:
#|
#|  run-gcloud [-s|--shell] [args]
#|    Run gcloud command locally, or inside a gcloud docker container.
#|    Run `gcloud auth login` first, if default gcloud configuration
#|    is missing in ~/.config/gcloud/configurations/config_default,
#|    or in $APPDATA/gcloud/configurations/config_default on Windows.
run-gcloud() {
  local shell config config_loc=.config/gcloud/configurations/config_default
  is-windows && config="$APPDATA/${config_loc#.config/}" || config=~/"$config_loc"
  hasopt '-s|--shell' "$@" && shell=bash
  
  if [ -z "$shell" ] && hascmd gcloud; then
    [ -s "$config" -o "$1 $2" = 'auth login' ] || gcloud auth login || return $? 
    [ -s "$config" ] || error "missing gloud config: '$config'" || return $?
    @ gcloud "$@"
  else
    run-in-container gcloud@google/cloud-sdk ${shell:-bash -c \
      "[ -s ~/'$config_loc' -o '$1 $2' = 'auth login' ] || gcloud auth login && gcloud $*"
    }
  fi
}

#|
#|Docker functions:
#|
#|  is-docker
#|    Test if called inside docker container.
is-docker() { [ -f '/.dockerenv' ] || grep -q 'docker' '/proc/self/cgroup'; }
#|
#|  run-in-docker <image> [options] <command> [args]
#|    Run command in a temporary docker image container
#|    in a /root/cd directory mapped to host current directory
#|    and with /root/home directory mapped to host home directory.
#|    Use the --rm option to remove the container on exit.
run-in-docker() {
  require docker
  @ docker run -it \
    -v "`get-volume-path ~`:/root/home" \
    -v "`get-volume-path "$PWD"`:/root/cd" \
    -w "`encode-abs-path /root/cd`" \
    "$@"
}
#|
#|  run-in-container <name[~image]> <command> [args]
#|    Run command in named docker image container.
#|    Create missing container from image (defaults to name)
#|    with /root/cd directory mapped to host current directory
#|    and /root/home directory mapped to host home directory.
run-in-container() {
  local name="${1%%@*}" image="${1#*@}"; shift 
  require docker
#|
  if has-container "$name"; then
    has-started-container "$name" || @ docker start "$name" || return $?
    @ docker exec -it "$name" "$@"
  else
    run-in-docker --name "$name" "$image" "$@"  
  fi
}
#|
#|  has-container <container>
#|    Test if docker container exists.
has-container() { require docker; docker ps -a -q -f name="$1" | grep -q .; }
#|
#|  has-started-container <container>
#|    Test if docker container exists and is started.
has-started-container() { require docker; docker ps -q -f name="$1" | grep -q .; }
#|
#|  get-volume-path <path>
#|    Output the path in windows format as required by docker on windows.
get-volume-path() { if is-windows; then echo -n "$1" | sed 's|^/\([a-z]\)/|\1:/|'; else echo -n "$1"; fi; }

#|
#|Windows functions:
#|
#|  is-windows
#|    Test if called on windows system.
is-windows() { [ "$OS" = 'Windows_NT' ]; }
#|
#|  encode-abs-path <path>
#|    Output the path with the initial slash doubled to avoid MSYS/Git conversion.
#|    Pass absolute remote/container paths with the initial slash doubled
#|    to avoid MSYS/Git prefixing the absolute unix path argument on Windows.
#|    See http://www.mingw.org/wiki/Posix_path_conversion.
encode-abs-path() { is-windows && [ "${1#/}" != "$1" ] && echo "/$1" || echo "$1"; }
#|
#|  decode-abs-path <path>
#|    Output the path without the initial slash doubled
decode-abs-path() { [ "${1#//}" != "$1" ] && echo "${1#/}" || echo "$1"; }

#|
#|Environment functions:
#|
#|  hasopt <-o|--option> [args]
#|    Test if args include -o or --option.
hasopt() { local a o="$1"; shift;
  for a in "$@"; do case "$a" in ("${o#*|}"|"${o%|*}") return 0;; ('--') break;; esac; done; return 1; }
#|
#|  hasvar <variables>
#|    Test if environment variable is set.
hasvar() { local _a; for _a in "$@"; do [ -n "${!_a}" ] || return 1; done; }
#|
#|  setvar <variable> [value]
#|    Set variable to value.
setvar() { eval "$1=${2:+\"\$2\"}"; }
#|
#|  require-vars [vars[:source-vars]]
#|    Fail if environment variables are not set.
require-var() { local v="${1%%:*}" s="${1#*:}"; hasvar "$v" || fail "missing required settings: $s"; }
require-vars() { all require-var "$@"; }
#|
#|  dotenv [--export] [paths]
#|    Load .env, .env.local, .env.$NODE_ENV, .env.$NODE_ENV.local files if present
#|    in given paths. Optionally export the variables. Env defaults to development. 
#|    Follows https://create-react-app.dev/docs/adding-custom-environment-variables.
dotenv() { local p e ne="${NODE_ENV:-development}"; [ "$1" = '--export' ] && e="$1" && shift;
  for p in "$@"; do [ ! -d "$p" ] || loadenv $e "$d/.env" "$d/.env.local" "$d/.env.$ne" "$d/.env.$ne.local"; done; }
#|
#|  loadenv [--export] [files]
#|    Load env files if present. Optionally export the variables.
loadenv() { local f e; [ "$1" = '--export' ] && e="$1" && shift && set -o allexport;
  for f in "$@"; do [ ! -f "$f" ] || @ source "$f"; done; [ -z "$e" ] || set +o allexport; }
#|
#|  loadfile <file> [label]
#|    Output file or report error.
loadfile() { [ -f "$1" -a -r "$1" ] && -- cat "$1" || error "missing ${2:+$2 }file: '$1'"; }

#|
#|Initialization functions:
#|
#|  usage [script=executed]
#|    Output script usage.
usage() { sed -n '/^#|/s/#|//p;' "${1:-$0}"; return 0; }
#|
#|  is-sourced [script=self]
#|    Test if script is sourced.
is-sourced() { [ "${1:-${BASH_SOURCE[1]}}" != "$0" ]; }
#|
#|  from-sourced
#|    Test if called from a sourced script.
from-sourced() { is-sourced "${BASH_SOURCE[2]}"; }
#|
#|  from-self
#|    Test if called from the same script script.
from-self() { [ "${BASH_SOURCE[2]}" = "${BASH_SOURCE[1]}" ]; }
#|
#|  init [--pass-help|--debug] <command> [args]
#|    If caller is sourced or devops and help is not requested with -h or --help,
#|    then load devops.env from the devops script path.
#|    If caller is sourced, then return.   
#|    If help is requested with -h or --help, then output caller usage and return. 
#|    Set DEVOPS_DEBUG=1 on --debug.
#|    Run command with args.
init() {
  from-sourced || from-self && ! hasopt '-h|--help' "$@" && loadenv --export "${BASH_SOURCE[0]}.env"
  from-sourced && return
  [ "$1" = '--pass-help' ] && shift || { hasopt '-h|--help' "$@" && usage && return; }
  [ "$1" = '--debug' ] && export DEVOPS_DEBUG=1 DEBUG_DEVOPS=1 && shift
  [ -n "$1" ] || error "init: missing command, see ${BASH_SOURCE[0]##*[\\/]} -h, --help" || return 2
  local cmd="$1"; shift
  __cmd="${BASH_SOURCE[1]##*[\\/]}" __bin="${BASH_SOURCE[1]%[\\/]*}" "$cmd" "$@"
}

init "$@"

devops.cmd

@call bash "%~dpn0" %*
@exit /b %errorlevel%

devops.env

DEBUG_DEVOPS=1

sample-extract

#!/bin/bash

extract() {
  base=$(relpath $(binpath ..))
  docs=$(relpath $base/samples)

  [ $# != 0 ] || { extract "$docs/*.docx"; return $?; }

  @ ts-node "$base/extract.ts" "$@" || exit $?
}

. "`dirname "${BASH_SOURCE[0]}"`/devops" && init --pass-help extract "$@"

sample-extract.cmd

@call bash "%~dpn0" %*
@exit /b %errorlevel%

sample-gql

#!/usr/bin/env bash
#|
#|Usage: gql <environment> [options] [graphcurl-options]
#|
#|Options:
#|  -h, --help      Show this help and exit.
#|  --debug         Output debug messages.
#|
#|Environments:
#|  development|dev|d|local|l|.
#|  staging|s
#|  demo
#|  production|prod
#|
#|Settings:
#|  API_<ENV>_URL:  GraphQL API URL for given environment. See defaults below.
#|  API_<ENV>_KEY:  GraphQL API JWT (JSON Web Token) string/@file for given environment.
#|                  Defaults to '@<bin>/../.secrets/<ENV>-API_KEY'.
#|
#|  DEVOPS_DEBUG:   Output debug messages if set to anything except 0, n, no, off, false.
#|                  Alias DEBUG_DEVOPS works too.
#|
#|  Loads '<bin>/../.secrets/(init.sh|init-app-gql.sh|.env|gql.env)' if missing any vars.
#|
#|Defaults:
#|  API_DEVELOPMENT_URL=http://localhost:8681/graphql
#|  API_STAGING_URL=https://api.staging.cravedelivery.com/graphql
#|  API_DEMO_URL=https://api.demo.cravedelivery.com/graphql
#|  API_PRODUCTION_URL=https://api.cravedelivery.com/graphql
#|
#|ToDo (graphcurl):
#|- Support automatic/whitelisted persisted queries
#|- Select operation from a file that contains multiple queries/mutations
#|- Send multiplied operation in single request for array json/yaml data
#|- Load array data from csv with header
#|- Select default/environment endpoints from GraphQL Config and Apollo config
#|- Use default paths for graphql files from extended GraphQL Config
#|
#|Examples:
#|  gql . -q "{ authenticatedParty { id name addresses { id } } }" -v
#|
#|  gql . -q "mutation upsertOrderPromo($input: OrderPromoInput!) {
#|    upsertOrderPromo(input: $input) { id code definition validSince validTill } }" \
#|    -d 'input:{"id":"test","code":"TEST","type":"DISCOUNT_AMOUNT","definition":"5"}'
#|
#|  gql . @upsertOrderPromo.graphql \
#|    -d "input:{id:test, code:TEST, type:DISCOUNT_AMOUNT, definition:'5'}" -v
#|
#|  gql . -q @upsertOrderPromo.graphql -d @promo-mycode.json -v
#|

declare -A API_ENV_URLS=(
  [DEVELOPMENT]='http://localhost:8681/graphql'
  [STAGING]='https://api.staging.cravedelivery.com/graphql'
  [DEMO]='https://api.demo.cravedelivery.com/graphql'
  [PRODUCTION]='https://api.cravedelivery.com/graphql'
)

declare -A API_ENVS=(
  [development]='DEVELOPMENT'
  [dev]='DEVELOPMENT'
  [d]='DEVELOPMENT'
  [local]='DEVELOPMENT'
  [l]='DEVELOPMENT'
  [.]='DEVELOPMENT'
  [staging]='STAGING'
  [s]='STAGING'
  [demo]='DEMO'
  [production]='PRODUCTION'
  [prod]='PRODUCTION'
)

gql() {
  local url key env="${API_ENVS[$1]}" base="${__bin%[\\/]*}"; shift
  local etc="$base/.secrets" npmbin="$base/node_modules/.bin"
  [ -n "$1" ] || fail 'missing environment argument'
  [ -n "$env" ] || fail "unknown environment: '$1'"

  url="API_${env}_URL"
  key="API_${env}_KEY"

  hasvar "$url" "$key" || loadenv "$etc/init.sh" "$etc/init-app-gql.sh" "$etc/.env" "$etc/gql.env"
  hasvar "$url" && url="${!url}" || url="${API_ENV_URLS[$env]}"
  hasvar "$key" && key="${!key}" || key="@$etc/$env-API_KEY"
  ! startswith "$key" '@' || key="`loadfile "${key#@}" 'API key'`" && [ -n "$key" ] || fail

  # TODO: Switch to npx after fixed argument quoting is merged and released
  #       See https://github.com/npm/npx/issues/43
  @ "$npmbin/graphcurl" -e "$url" -H "Authorization:Bearer $key" "$@"
}

. "`dirname "${BASH_SOURCE[0]}"`/devops" && init gql "$@"

sample-gql.cmd

@call bash "%~dpn0" %*
@exit /b %errorlevel%