Dump Prometheus metrics from an OCP cluster

prometheus-metrics_dump.sh

#!/usr/bin/env bash
# run script like "bash prometheus-metrics_dump.sh"
# Based on Red Hat solution https://access.redhat.com/solutions/5482971

function queue() {
local TARGET="${1}"
shift
local LIVE
LIVE="$(jobs | wc -l)"
while [[ "${LIVE}" -ge 45 ]]; do
  sleep 1
  LIVE="$(jobs | wc -l)"
done
echo "${@}"
if [[ -n "${FILTER:-}" ]]; then
  "${@}" | "${FILTER}" >"${TARGET}" &
else
  "${@}" >"${TARGET}" &
fi
}

ARTIFACT_DIR=$PWD
mkdir -p $ARTIFACT_DIR/metrics
echo "Snapshotting prometheus (may take 15s) ..."
queue ${ARTIFACT_DIR}/metrics/prometheus.tar.gz oc --insecure-skip-tls-verify exec -n openshift-monitoring prometheus-k8s-0 -- tar cvzf - -C /prometheus .
FILTER=gzip queue ${ARTIFACT_DIR}/metrics/prometheus-target-metadata.json.gz oc --insecure-skip-tls-verify exec -n openshift-monitoring prometheus-k8s-0 -- /bin/bash -c "curl -G http://localhost:9090/api/v1/targets/metadata --data-urlencode 'match_target={instance!=\"\"}'"
wait