dataskew_ranges.py

#!/usr/env python

import json
import sys

#biggestRange = 0
#biggestRaftLog = 0

def queriesPerSecond(rangeInfo):
    for node in rangeInfo["nodes"]:
        nodeID = node["nodeId"]
        if nodeID == node["range"]["state"]["state"]["lease"]["replica"]["nodeId"]:
            return node["range"]["stats"]["queriesPerSecond"]
    return 0

def statsLiveCount(rangeInfo):
    for node in rangeInfo["nodes"]:
        nodeID = node["nodeId"]
        if nodeID == node["range"]["state"]["state"]["lease"]["replica"]["nodeId"]:
            return int(node["range"]["state"]["state"]["stats"]["liveCount"])
    return 0

def nodes(rangeInfo):
    return [node["nodeId"] for node in rangeInfo["nodes"]]


def leaseholder(rangeInfo):
    return rangeInfo["nodes"][0]["range"]["state"]["state"]["lease"]["replica"]["nodeId"]

def startKey(rangeInfo):
    return rangeInfo["nodes"][0]["range"]["span"]["startKey"]

def endKey(rangeInfo):
    return rangeInfo["nodes"][0]["range"]["span"]["endKey"]

if len(sys.argv) != 2:
    print "Usage: python %s <ranges-file>" % sys.argv[0]
    sys.exit(1)

with open(sys.argv[1]) as ranges_file:    
    ranges = json.load(ranges_file)["ranges"]

sorted_ranges = sorted(ranges.values(), key=lambda x: statsLiveCount(x), reverse=True)

print "order: rangeId\tliveCount\tnodes\tleaseholder\tstartkey\tendkey"
for i in range(min(len(sorted_ranges), 100)):
    print "%3d: %5s\t%d\t%17s\t%s\t%s\t%s" % (i+1, sorted_ranges[i]["rangeId"], statsLiveCount(sorted_ranges[i]), nodes(sorted_ranges[i]), leaseholder(sorted_ranges[i]),startKey(sorted_ranges[i]),endKey(sorted_ranges[i]))

hottest_nodes.py

#!/usr/env python

import json
import operator
import sys

from collections import defaultdict

def statsPerSecond(rangeInfo):
    for node in rangeInfo["nodes"]:
        nodeID = node["nodeId"]
        if nodeID == node["range"]["state"]["state"]["lease"]["replica"]["nodeId"]:
            return node["range"]["stats"]["queriesPerSecond"], node["range"]["stats"]["writesPerSecond"], nodeID
    return 0, 0, 0


if len(sys.argv) != 2:
    print "Usage: python %s <ranges-file>" % sys.argv[0]
    sys.exit(1)

with open(sys.argv[1]) as ranges_file:    
    ranges = json.load(ranges_file)["ranges"]

qpsPerNode = defaultdict(float)
wpsPerNode = defaultdict(float)
for rangeInfo in ranges.values():
    qps, wps, leaseholder = statsPerSecond(rangeInfo)
    for node in rangeInfo["nodes"]:
        nodeID = node["nodeId"]
        if nodeID == leaseholder:
            qpsPerNode[nodeID] += qps
        wpsPerNode[nodeID] += wps

sortedQPS = sorted(qpsPerNode.items(), key=operator.itemgetter(1), reverse=True)

for (nodeID, qps) in sortedQPS:
    print "%2d: %10f\t%10f" % (nodeID, qps, wpsPerNode[nodeID])

hottest_ranges.py

#!/usr/env python

import json
import sys

#biggestRange = 0
#biggestRaftLog = 0

def queriesPerSecond(rangeInfo):
    for node in rangeInfo["nodes"]:
        nodeID = node["nodeId"]
        if nodeID == node["range"]["state"]["state"]["lease"]["replica"]["nodeId"]:
            return node["range"]["stats"]["queriesPerSecond"]
    return 0


def nodes(rangeInfo):
    return [node["nodeId"] for node in rangeInfo["nodes"]]


def leaseholder(rangeInfo):
    return rangeInfo["nodes"][0]["range"]["state"]["state"]["lease"]["replica"]["nodeId"]



if len(sys.argv) != 2:
    print "Usage: python %s <ranges-file>" % sys.argv[0]
    sys.exit(1)

with open(sys.argv[1]) as ranges_file:    
    ranges = json.load(ranges_file)["ranges"]
sorted_ranges = sorted(ranges.values(), key=lambda x: queriesPerSecond(x), reverse=True)

for i in range(min(len(sorted_ranges), 50)):
    print "%3d: %5s\t%10f\t%10s\t%s" % (i+1, sorted_ranges[i]["rangeId"], queriesPerSecond(sorted_ranges[i]), nodes(sorted_ranges[i]), leaseholder(sorted_ranges[i]))

table_qps.py

#!/usr/env python

import json
import operator
import sys

from collections import defaultdict

def statsPerSecond(rangeInfo):
    for node in rangeInfo["nodes"]:
        nodeID = node["nodeId"]
        if nodeID == node["range"]["state"]["state"]["lease"]["replica"]["nodeId"]:
            return node["range"]["stats"]["queriesPerSecond"], node["range"]["stats"]["writesPerSecond"], nodeID
    return 0, 0, 0


if len(sys.argv) != 3:
    print "Usage: python %s <ranges-file> <table-id>" % sys.argv[0]
    sys.exit(1)

tableID = sys.argv[2]

with open(sys.argv[1]) as ranges_file:    
    ranges = json.load(ranges_file)["ranges"]

stats = dict()
for rangeID, rangeInfo in ranges.items():
    if rangeInfo["nodes"][0]["range"]["span"]["startKey"].startswith("/Table/%s/" % tableID):
        stats[rangeID] = statsPerSecond(rangeInfo)

sortedStats = sorted(stats.items(), key=lambda (rangeID, stats): stats[0], reverse=True)

for (rangeID, stats) in sortedStats:
    print "%5s:\t%10f\t%10f\tn%d" % (rangeID, stats[0], stats[1], stats[2])

tables_per_node.py

#!/usr/env python

import json
import operator
import sys
import re

from collections import defaultdict

def statsPerSecond(rangeInfo):
    for node in rangeInfo["nodes"]:
        nodeID = node["nodeId"]
        if nodeID == node["range"]["state"]["state"]["lease"]["replica"]["nodeId"]:
            return node["range"]["stats"]["queriesPerSecond"], node["range"]["stats"]["writesPerSecond"], nodeID
    return 0, 0, 0


if len(sys.argv) != 2:
    print "Usage: python %s <ranges-file>" % sys.argv[0]
    sys.exit(1)

with open(sys.argv[1]) as ranges_file:    
    ranges = json.load(ranges_file)["ranges"]


tableIDRegex = re.compile('/Table/([0-9]+)/')
qpsPerNode = defaultdict(float)
wpsPerNode = defaultdict(float)
replicasPerNodeAndTable = defaultdict(lambda: defaultdict(int))
leasesPerNodeAndTable = defaultdict(lambda: defaultdict(int))
qpsPerNodeAndTable = defaultdict(lambda: defaultdict(float))
wpsPerNodeAndTable = defaultdict(lambda: defaultdict(float))
for rangeInfo in ranges.values():
    match = tableIDRegex.match(rangeInfo["nodes"][0]["range"]["span"]["startKey"])
    tableID = match.group(1) if match else '0'
    qps, wps, leaseholder = statsPerSecond(rangeInfo)
    for node in rangeInfo["nodes"]:
        nodeID = node["nodeId"]
        replicasPerNodeAndTable[nodeID][tableID] += 1
        if nodeID == leaseholder:
            leasesPerNodeAndTable[nodeID][tableID] += 1
            qpsPerNode[nodeID] += qps
            qpsPerNodeAndTable[nodeID][tableID] += qps
        wpsPerNode[nodeID] += wps
        wpsPerNodeAndTable[nodeID][tableID] += wps

sortedNodeQPS = sorted(qpsPerNode.items(), key=operator.itemgetter(1), reverse=True)

for (nodeID, qps) in sortedNodeQPS:
    print "%3s: %10f\t%10f" % ('n'+str(nodeID), qps, wpsPerNode[nodeID])
    sortedTableReplicas = sorted(replicasPerNodeAndTable[nodeID].items(), key=operator.itemgetter(1), reverse=True)
    for (tableID, tableReplicas) in sortedTableReplicas:
        print "\t%4s: %5d\t%5d\t%10f\t%10f" % (tableID, tableReplicas, leasesPerNodeAndTable[nodeID][tableID], qpsPerNodeAndTable[nodeID][tableID], wpsPerNodeAndTable[nodeID][tableID])