automate dashboards importing in PMM2

cleanup-dash.py

#!/usr/bin/env python

import sys
import json

""" Prometheus has been replaced by VictoriaMetrics since PMM 2.13.0 """
datasources = [['${DS_METRICS}','Metrics'], 
               ['${DS_PTSUMMARY}','PTSummary'], 
               ['${DS_CLICKHOUSE}','ClickHouse'],
               ['${DS_PROMETHEUS_ALERTMANAGER}','Prometheus AlertManager']]


def drop_some_internal_elements(dashboard):
    knownDatsource = False
    for element in enumerate(dashboard.copy()):
        if '__inputs' in element:
            for inputs_index, inputs in enumerate(dashboard['__inputs']):
                for datasource in datasources:
                    if datasource[0] == dashboard['__inputs'][inputs_index]['name']:
                        knownDatsource = True
                        break;
                if not (knownDatsource):
                    datasources.append(['${' + dashboard['__inputs'][inputs_index]['name'] + '}','Metrics']) 
            del dashboard['__inputs']
        if '__requires' in element:
            del dashboard['__requires']
#    for datasource in datasources:
#        print '%s:%s' % (datasource[0], datasource[1])
    return dashboard


def fix_datasource(dashboard):
    for element in enumerate(dashboard.copy()):
        if 'panels' in element:
            for panel_index, panel in enumerate(dashboard['panels']):
                if 'datasource' in panel:
                    for datasource in datasources:
                        if panel['datasource'] == datasource[0]:
                            dashboard['panels'][panel_index]['datasource'] = datasource[1] 

                if 'panels' in panel:
                        if (len(dashboard['panels'][panel_index]['panels']) > 0):
                            for panelIn_index, panelIn in enumerate(dashboard['panels'][panel_index]['panels']):
                                if 'datasource' in panelIn:
                                    for datasource in datasources:
                                        if dashboard['panels'][panel_index]['panels'][panelIn_index]['datasource'] == datasource[0]:
                                            dashboard['panels'][panel_index]['panels'][panelIn_index]['datasource'] = datasource[1] 

                if 'mappingTypes' in panel:
                        for mappingTypes_index, mappingTypes in enumerate(dashboard['panels'][panel_index]['mappingTypes']):
                            if 'datasource' in mappingTypes:
                                    for datasource in datasources:
                                        if dashboard['panels'][panel_index]['mappingTypes'][mappingTypes_index]['datasource'] == datasource[0]:
                                            dashboard['panels'][panel_index]['mappingTypes'][mappingTypes_index]['datasource'] = datasource[1]

        if 'templating' in element:
            for panel_index, panel in enumerate(dashboard['templating']['list']):
                    if 'datasource' in panel.keys():
                        for datasource in datasources:
                            if panel['datasource'] == datasource[0]:
                                dashboard['templating']['list'][panel_index]['datasource'] = datasource[1]

    return dashboard


def main():
    """Execute cleanups."""
    with open(sys.argv[1], 'r') as dashboard_file:
        dashboard = json.loads(dashboard_file.read())

    # registered cleanupers.
    CLEANUPERS = [drop_some_internal_elements, fix_datasource]

    for func in CLEANUPERS:
        dashboard = func(dashboard)

    dashboard_json = json.dumps(dashboard, sort_keys=True, indent=4,
                                separators=(',', ': '))

    with open(sys.argv[1], 'w') as dashboard_file:
        dashboard_file.write(dashboard_json)
        dashboard_file.write('\n')

if __name__ == '__main__':
    main()

import_dash.sh

#!/bin/bash

dashboards=(13266 12630 12470)
pmm_server="172.17.0.2"
user_pass="admin:admin"
folderName="General"

if [[ $# -eq 0 ]]; then
    echo " * No arguments are supplied. Default values will be used."
    echo " * Usage: import-dashboard-grafana-cloud  [FolderName] [<dashboardId> <dashboardId> ... ]"
    echo " * Dashboard(s) (${dashboards[@]}) will be uploaded into ${folderName} folder"
else
    if [[ $1 =~ ^[0-9]+$ ]]; then
        dashboards=( $@ )
        echo " * First argument isn't a folder name. $# dashbaord(s) ($@) will be uploaded into General folder."
        echo " * Usage: import-dashboard-grafana-cloud  [FolderName] [<dashboardId> <dashboardId> ... ]"
    else
        folderName="$1"
        if [[ $# > 1 ]]; then
            dashboards=( ${@/$folderName} )
        fi
        echo " * Dashbaord(s) (${dashboards[@]}) will be uploaded into $1 folder."
    fi
fi

if ! [[ ${folderName} == "General" ]]; then 
    folderId=$(curl -s -k -u ${user_pass} https://${pmm_server}/graph/api/folders | python -m json.tool | grep -B1 ${folderName} | head -1 | cut -d":" -f2 | cut -d"," -f1 | sed 's/ //g')
    if ! [[ ${folderId} =~ ^[0-9]+$ ]]; then
        echo " * Folder ${folderName} is NOT existed."
        folderId=$(curl -s -k -X POST -H "Content-Type: application/json" -d $(echo "{\"title\":\"${folderName}\"}") -u ${user_pass} https://${pmm_server}/graph/api/folders | cut -d":" -f2 | cut -d"," -f1)
        echo " * Folder ${folderName} has been created with id ${folderId}"    
    else 
        echo " * Found folderId ${folderId} for folder ${folderName}"
    fi;
else
    folderId=0
fi

for dashboard in "${dashboards[@]}";
    do 
        echo -e "\n * Dashboard: ${dashboard}"
        if ! [[ ${dashboard} =~ ^[0-9]+$ ]]
        then
            echo $(./cleanup-dash.py ${dashboard})
            response=$(echo "{\"dashboard\":$(cat ${dashboard}),\"overwrite\":true,\"folderId\":${folderId})}" | curl -s -k -X POST -H "Content-Type: application/json" -d @- -u ${user_pass} https://${pmm_server}/graph/api/dashboards/import)
            echo "   * Result of uploading file ${dashboard}: ${response}"
        fi;

        revision=1;
        while true; do
            response=$(curl -s https://grafana.com/api/dashboards/${dashboard}/revisions/${revision}/download | wc -l);
            if [ $response != 3 ]; then
                echo "   * Revision ${revision} exists"
            else
                echo "   * Revision ${revision} doesn't exist"
                        ((revision=revision-1));
                break;
            fi;

            ((revision=revision+1));
        done;

        if [ $revision != 0 ]; then
                response=$(echo "{\"dashboard\":$(curl -s https://grafana.com/api/dashboards/${dashboard}/revisions/${revision}/download),\"overwrite\":true,\"folderId\":${folderId}}" | curl -s -k -X POST -H "Content-Type: application/json" -d @- -u ${user_pass} https://${pmm_server}/graph/api/dashboards/import) 
                echo "   * Result of uploading ${revision} revision: ${response}"
                if [[ ${response} =~ "{\"message\":\"Failed to save dashboard\"}" ]]
                then
                    echo "   * Going to download dashbaord and modify datasources"
                    echo $(curl https://grafana.com/api/dashboards/${dashboard}/revisions/${revision}/download --output ${dashboard}-rev${revision}.json) 
                    echo $(./cleanup-dash.py ${dashboard}-rev${revision}.json)
                    response=$(echo "{\"dashboard\":$(cat ${dashboard}-rev${revision}.json),\"overwrite\":true,\"folderId\":${folderId}}" | curl -s -k -X POST -H "Content-Type: application/json" -d @- -u ${user_pass} https://${pmm_server}/graph/api/dashboards/import)
                    echo "   * Result of uploading file ${dashboard}-rev${revision}.json: ${response}"
                fi;

        else
                echo "   * Dashboard ${dashboard} doesn't exist"
        fi;

   done;