Skip to content

Instantly share code, notes, and snippets.

@slenky

slenky/stagepage.js

Created January 13, 2021 11:24
Show Gist options
  • Save slenky/f89ee5de18a2f075a481e3d4452a427c to your computer and use it in GitHub Desktop.
Save slenky/f89ee5de18a2f075a481e3d4452a427c to your computer and use it in GitHub Desktop.
Download ZIP
Fixing the SparkUI + Jupyter Server Proxy
Raw
stagepage.js
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var shouldBlockUI = true;
$(document).ajaxStop(function () {
if (shouldBlockUI) {
$.unblockUI();
shouldBlockUI = false;
}
});
$(document).ajaxStart(function () {
if (shouldBlockUI) {
$.blockUI({message: '<h3>Loading Stage Page...</h3>'});
}
});
$.extend( $.fn.dataTable.ext.type.order, {
"duration-pre": ConvertDurationString,
"duration-asc": function ( a, b ) {
a = ConvertDurationString( a );
b = ConvertDurationString( b );
return ((a < b) ? -1 : ((a > b) ? 1 : 0));
},
"duration-desc": function ( a, b ) {
a = ConvertDurationString( a );
b = ConvertDurationString( b );
return ((a < b) ? 1 : ((a > b) ? -1 : 0));
}
} );
// This function will only parse the URL under certain format
// e.g. (history) https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4&attempt=1
// e.g. (proxy) https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4&attempt=1
function stageEndPoint(appId) {
var queryString = document.baseURI.split('?');
var words = document.baseURI.split('/');
var indexOfProxy = words.indexOf("proxy");
var indexOfJupyterProxy = words.indexOf("4040");
var stageId = queryString[1].split("&").filter(word => word.includes("id="))[0].split("=")[1];
if (indexOfJupyterProxy > 0) {
var newBaseURI = words.slice(0, indexOfJupyterProxy + 1).join('/');
return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
}
if (indexOfProxy > 0) {
var appId = words[indexOfProxy + 1];
var newBaseURI = words.slice(0, words.indexOf("proxy") + 2).join('/');
return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
}
var indexOfHistory = words.indexOf("history");
if (indexOfHistory > 0) {
var appId = words[indexOfHistory + 1];
var appAttemptId = words[indexOfHistory + 2];
var newBaseURI = words.slice(0, words.indexOf("history")).join('/');
if (isNaN(appAttemptId) || appAttemptId == "0") {
return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
} else {
return newBaseURI + "/api/v1/applications/" + appId + "/" + appAttemptId + "/stages/" + stageId;
}
}
return location.origin + "/api/v1/applications/" + appId + "/stages/" + stageId;
}
function getColumnNameForTaskMetricSummary(columnKey) {
switch(columnKey) {
case "executorRunTime":
return "Duration";
case "jvmGcTime":
return "GC Time";
case "gettingResultTime":
return "Getting Result Time";
case "inputMetrics":
return "Input Size / Records";
case "outputMetrics":
return "Output Size / Records";
case "peakExecutionMemory":
return "Peak Execution Memory";
case "resultSerializationTime":
return "Result Serialization Time";
case "schedulerDelay":
return "Scheduler Delay";
case "diskBytesSpilled":
return "Spill (disk)";
case "memoryBytesSpilled":
return "Spill (memory)";
case "shuffleReadMetrics":
return "Shuffle Read Size / Records";
case "shuffleWriteMetrics":
return "Shuffle Write Size / Records";
case "executorDeserializeTime":
return "Task Deserialization Time";
case "shuffleReadBlockedTime":
return "Shuffle Read Blocked Time";
case "shuffleRemoteReads":
return "Shuffle Remote Reads";
default:
return "NA";
}
}
function displayRowsForSummaryMetricsTable(row, type, columnIndex) {
switch(row.columnKey) {
case 'inputMetrics':
var str = formatBytes(row.data.bytesRead[columnIndex], type) + " / " +
row.data.recordsRead[columnIndex];
return str;
break;
case 'outputMetrics':
var str = formatBytes(row.data.bytesWritten[columnIndex], type) + " / " +
row.data.recordsWritten[columnIndex];
return str;
break;
case 'shuffleReadMetrics':
var str = formatBytes(row.data.readBytes[columnIndex], type) + " / " +
row.data.readRecords[columnIndex];
return str;
break;
case 'shuffleReadBlockedTime':
var str = formatDuration(row.data.fetchWaitTime[columnIndex]);
return str;
break;
case 'shuffleRemoteReads':
var str = formatBytes(row.data.remoteBytesRead[columnIndex], type);
return str;
break;
case 'shuffleWriteMetrics':
var str = formatBytes(row.data.writeBytes[columnIndex], type) + " / " +
row.data.writeRecords[columnIndex];
return str;
break;
default:
return (row.columnKey == 'peakExecutionMemory' || row.columnKey == 'memoryBytesSpilled'
|| row.columnKey == 'diskBytesSpilled') ? formatBytes(
row.data[columnIndex], type) : (formatDuration(row.data[columnIndex]));
}
}
function createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTable) {
var taskMetricsTable = "#summary-metrics-table";
if ($.fn.dataTable.isDataTable(taskMetricsTable)) {
taskSummaryMetricsDataTable.clear().draw();
taskSummaryMetricsDataTable.rows.add(taskSummaryMetricsTable).draw();
} else {
var taskConf = {
"data": taskSummaryMetricsTable,
"columns": [
{data : 'metric'},
// Min
{
data: function (row, type) {
return displayRowsForSummaryMetricsTable(row, type, 0);
}
},
// 25th percentile
{
data: function (row, type) {
return displayRowsForSummaryMetricsTable(row, type, 1);
}
},
// Median
{
data: function (row, type) {
return displayRowsForSummaryMetricsTable(row, type, 2);
}
},
// 75th percentile
{
data: function (row, type) {
return displayRowsForSummaryMetricsTable(row, type, 3);
}
},
// Max
{
data: function (row, type) {
return displayRowsForSummaryMetricsTable(row, type, 4);
}
}
],
"columnDefs": [
{ "type": "duration", "targets": 1 },
{ "type": "duration", "targets": 2 },
{ "type": "duration", "targets": 3 },
{ "type": "duration", "targets": 4 },
{ "type": "duration", "targets": 5 }
],
"paging": false,
"searching": false,
"order": [[0, "asc"]],
"bSort": false,
"bAutoWidth": false,
"oLanguage": {
"sEmptyTable": "No tasks have reported metrics yet"
}
};
taskSummaryMetricsDataTable = $(taskMetricsTable).DataTable(taskConf);
}
taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTable.slice();
}
function createRowMetadataForColumn(colKey, data, checkboxId) {
var row = {
"metric": getColumnNameForTaskMetricSummary(colKey),
"data": data,
"checkboxId": checkboxId,
"columnKey": colKey
};
return row;
}
function reselectCheckboxesBasedOnTaskTableState() {
var allChecked = true;
var taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
if (typeof taskTableSelector !== 'undefined' && taskSummaryMetricsTableCurrentStateArray.length > 0) {
for (var k = 0; k < optionalColumns.length; k++) {
if (taskTableSelector.column(optionalColumns[k]).visible()) {
$("#box-"+optionalColumns[k]).prop('checked', true);
taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == optionalColumns[k])[0]);
taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
} else {
allChecked = false;
}
}
if (allChecked) {
$("#box-0").prop('checked', true);
}
createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableCurrentFilteredArray);
}
}
function getStageAttemptId() {
var words = document.baseURI.split('?');
var digitsRegex = /[0-9]+/;
// We are using regex here to extract the stage attempt id as there might be certain url's with format
// like /proxy/application_1539986433979_27115/stages/stage/?id=0&attempt=0#tasksTitle
var stgAttemptId = words[1].split("&").filter(
word => word.includes("attempt="))[0].split("=")[1].match(digitsRegex);
return stgAttemptId;
}
var taskSummaryMetricsTableArray = [];
var taskSummaryMetricsTableCurrentStateArray = [];
var taskSummaryMetricsDataTable;
var optionalColumns = [11, 12, 13, 14, 15, 16, 17];
var taskTableSelector;
$(document).ready(function () {
setDataTableDefaults();
$("#showAdditionalMetrics").append(
"<div><a id='additionalMetrics'>" +
"<span class='expand-input-rate-arrow arrow-closed' id='arrowtoggle1'></span>" +
" Show Additional Metrics" +
"</a></div>" +
"<div class='container-fluid container-fluid-div' id='toggle-metrics' hidden>" +
"<div id='select_all' class='select-all-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-0' data-column='0'> Select All</div>" +
"<div id='scheduler_delay' class='scheduler-delay-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-11' data-column='11'> Scheduler Delay</div>" +
"<div id='task_deserialization_time' class='task-deserialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-12' data-column='12'> Task Deserialization Time</div>" +
"<div id='shuffle_read_blocked_time' class='shuffle-read-blocked-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-13' data-column='13'> Shuffle Read Blocked Time</div>" +
"<div id='shuffle_remote_reads' class='shuffle-remote-reads-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-14' data-column='14'> Shuffle Remote Reads</div>" +
"<div id='result_serialization_time' class='result-serialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-15' data-column='15'> Result Serialization Time</div>" +
"<div id='getting_result_time' class='getting-result-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-16' data-column='16'> Getting Result Time</div>" +
"<div id='peak_execution_memory' class='peak-execution-memory-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-17' data-column='17'> Peak Execution Memory</div>" +
"</div>");
$('#scheduler_delay').attr("data-toggle", "tooltip")
.attr("data-placement", "top")
.attr("title", "Scheduler delay includes time to ship the task from the scheduler to the executor, and time to send " +
"the task result from the executor to the scheduler. If scheduler delay is large, consider decreasing the size of tasks or decreasing the size of task results.");
$('#task_deserialization_time').attr("data-toggle", "tooltip")
.attr("data-placement", "top")
.attr("title", "Time spent deserializing the task closure on the executor, including the time to read the broadcasted task.");
$('#shuffle_read_blocked_time').attr("data-toggle", "tooltip")
.attr("data-placement", "top")
.attr("title", "Time that the task spent blocked waiting for shuffle data to be read from remote machines.");
$('#shuffle_remote_reads').attr("data-toggle", "tooltip")
.attr("data-placement", "top")
.attr("title", "Total shuffle bytes read from remote executors. This is a subset of the shuffle read bytes; the remaining shuffle data is read locally. ");
$('#result_serialization_time').attr("data-toggle", "tooltip")
.attr("data-placement", "top")
.attr("title", "Time spent serializing the task result on the executor before sending it back to the driver.");
$('#getting_result_time').attr("data-toggle", "tooltip")
.attr("data-placement", "top")
.attr("title", "Time that the driver spends fetching task results from workers. If this is large, consider decreasing the amount of data returned from each task.");
$('#peak_execution_memory').attr("data-toggle", "tooltip")
.attr("data-placement", "top")
.attr("title", "Execution memory refers to the memory used by internal data structures created during " +
"shuffles, aggregations and joins when Tungsten is enabled. The value of this accumulator " +
"should be approximately the sum of the peak sizes across all such data structures created " +
"in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and " +
"external sort.");
$('[data-toggle="tooltip"]').tooltip();
var tasksSummary = $("#parent-container");
getStandAloneAppId(function (appId) {
// rendering the UI page
$.get(createTemplateURI(appId, "stagespage"), function(template) {
tasksSummary.append(Mustache.render($(template).filter("#stages-summary-template").html()));
$("#additionalMetrics").click(function(){
$("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
$("#toggle-metrics").toggle();
if (window.localStorage) {
window.localStorage.setItem("arrowtoggle1class", $("#arrowtoggle1").attr('class'));
}
});
$("#aggregatedMetrics").click(function(){
$("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
$("#toggle-aggregatedMetrics").toggle();
if (window.localStorage) {
window.localStorage.setItem("arrowtoggle2class", $("#arrowtoggle2").attr('class'));
}
});
var endPoint = stageEndPoint(appId);
var stageAttemptId = getStageAttemptId();
$.getJSON(endPoint + "/" + stageAttemptId, function(response, status, jqXHR) {
var responseBody = response;
var dataToShow = {};
dataToShow.showInputData = responseBody.inputBytes > 0;
dataToShow.showOutputData = responseBody.outputBytes > 0;
dataToShow.showShuffleReadData = responseBody.shuffleReadBytes > 0;
dataToShow.showShuffleWriteData = responseBody.shuffleWriteBytes > 0;
dataToShow.showBytesSpilledData =
(responseBody.diskBytesSpilled > 0 || responseBody.memoryBytesSpilled > 0);
if (!dataToShow.showShuffleReadData) {
$('#shuffle_read_blocked_time').remove();
$('#shuffle_remote_reads').remove();
optionalColumns.splice(2, 2);
}
// prepare data for executor summary table
var stageExecutorSummaryInfoKeys = Object.keys(responseBody.executorSummary);
$.getJSON(createRESTEndPointForExecutorsPage(appId),
function(executorSummaryResponse, status, jqXHR) {
var executorDetailsMap = {};
executorSummaryResponse.forEach(function (executorDetail) {
executorDetailsMap[executorDetail.id] = executorDetail;
});
var executorSummaryTable = [];
stageExecutorSummaryInfoKeys.forEach(function (columnKeyIndex) {
var executorSummary = responseBody.executorSummary[columnKeyIndex];
var executorDetail = executorDetailsMap[columnKeyIndex.toString()];
executorSummary.id = columnKeyIndex;
executorSummary.executorLogs = {};
executorSummary.hostPort = "CANNOT FIND ADDRESS";
if (executorDetail) {
if (executorDetail["executorLogs"]) {
responseBody.executorSummary[columnKeyIndex].executorLogs =
executorDetail["executorLogs"];
}
if (executorDetail["hostPort"]) {
responseBody.executorSummary[columnKeyIndex].hostPort =
executorDetail["hostPort"];
}
}
executorSummaryTable.push(responseBody.executorSummary[columnKeyIndex]);
});
// building task aggregated metrics by executor table
var executorSummaryConf = {
"data": executorSummaryTable,
"columns": [
{data : "id"},
{data : "executorLogs", render: formatLogsCells},
{data : "hostPort"},
{
data : function (row, type) {
return type === 'display' ? formatDuration(row.taskTime) : row.taskTime;
}
},
{
data : function (row, type) {
var totaltasks = row.succeededTasks + row.failedTasks + row.killedTasks;
return type === 'display' ? totaltasks : totaltasks.toString();
}
},
{data : "failedTasks"},
{data : "killedTasks"},
{data : "succeededTasks"},
{data : "isBlacklistedForStage"},
{
data : function (row, type) {
return row.inputRecords != 0 ? formatBytes(row.inputBytes, type) + " / " + row.inputRecords : "";
}
},
{
data : function (row, type) {
return row.outputRecords != 0 ? formatBytes(row.outputBytes, type) + " / " + row.outputRecords : "";
}
},
{
data : function (row, type) {
return row.shuffleReadRecords != 0 ? formatBytes(row.shuffleRead, type) + " / " + row.shuffleReadRecords : "";
}
},
{
data : function (row, type) {
return row.shuffleWriteRecords != 0 ? formatBytes(row.shuffleWrite, type) + " / " + row.shuffleWriteRecords : "";
}
},
{
data : function (row, type) {
return typeof row.memoryBytesSpilled != 'undefined' ? formatBytes(row.memoryBytesSpilled, type) : "";
}
},
{
data : function (row, type) {
return typeof row.diskBytesSpilled != 'undefined' ? formatBytes(row.diskBytesSpilled, type) : "";
}
}
],
"order": [[0, "asc"]],
"bAutoWidth": false,
"oLanguage": {
"sEmptyTable": "No data to show yet"
}
};
var executorSummaryTableSelector =
$("#summary-executor-table").DataTable(executorSummaryConf);
$('#parent-container [data-toggle="tooltip"]').tooltip();
executorSummaryTableSelector.column(9).visible(dataToShow.showInputData);
if (dataToShow.showInputData) {
$('#executor-summary-input').attr("data-toggle", "tooltip")
.attr("data-placement", "top")
.attr("title", "Bytes and records read from Hadoop or from Spark storage.");
$('#executor-summary-input').tooltip(true);
}
executorSummaryTableSelector.column(10).visible(dataToShow.showOutputData);
if (dataToShow.showOutputData) {
$('#executor-summary-output').attr("data-toggle", "tooltip")
.attr("data-placement", "top")
.attr("title", "Bytes and records written to Hadoop.");
$('#executor-summary-output').tooltip(true);
}
executorSummaryTableSelector.column(11).visible(dataToShow.showShuffleReadData);
if (dataToShow.showShuffleReadData) {
$('#executor-summary-shuffle-read').attr("data-toggle", "tooltip")
.attr("data-placement", "top")
.attr("title", "Total shuffle bytes and records read (includes both data read locally and data read from remote executors).");
$('#executor-summary-shuffle-read').tooltip(true);
}
executorSummaryTableSelector.column(12).visible(dataToShow.showShuffleWriteData);
if (dataToShow.showShuffleWriteData) {
$('#executor-summary-shuffle-write').attr("data-toggle", "tooltip")
.attr("data-placement", "top")
.attr("title", "Bytes and records written to disk in order to be read by a shuffle in a future stage.");
$('#executor-summary-shuffle-write').tooltip(true);
}
executorSummaryTableSelector.column(13).visible(dataToShow.showBytesSpilledData);
executorSummaryTableSelector.column(14).visible(dataToShow.showBytesSpilledData);
});
// prepare data for accumulatorUpdates
var accumulatorTable = responseBody.accumulatorUpdates.filter(accumUpdate =>
!(accumUpdate.name).toString().includes("internal."));
var quantiles = "0,0.25,0.5,0.75,1.0";
$.getJSON(endPoint + "/" + stageAttemptId + "/taskSummary?quantiles=" + quantiles,
function(taskMetricsResponse, status, jqXHR) {
var taskMetricKeys = Object.keys(taskMetricsResponse);
taskMetricKeys.forEach(function (columnKey) {
switch(columnKey) {
case "shuffleReadMetrics":
var row1 = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 3);
var row2 = createRowMetadataForColumn(
"shuffleReadBlockedTime", taskMetricsResponse[columnKey], 13);
var row3 = createRowMetadataForColumn(
"shuffleRemoteReads", taskMetricsResponse[columnKey], 14);
if (dataToShow.showShuffleReadData) {
taskSummaryMetricsTableArray.push(row1);
taskSummaryMetricsTableArray.push(row2);
taskSummaryMetricsTableArray.push(row3);
}
break;
case "schedulerDelay":
var row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 11);
taskSummaryMetricsTableArray.push(row);
break;
case "executorDeserializeTime":
var row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 12);
taskSummaryMetricsTableArray.push(row);
break;
case "resultSerializationTime":
var row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 15);
taskSummaryMetricsTableArray.push(row);
break;
case "gettingResultTime":
var row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 16);
taskSummaryMetricsTableArray.push(row);
break;
case "peakExecutionMemory":
var row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 17);
taskSummaryMetricsTableArray.push(row);
break;
case "inputMetrics":
var row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 1);
if (dataToShow.showInputData) {
taskSummaryMetricsTableArray.push(row);
}
break;
case "outputMetrics":
var row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 2);
if (dataToShow.showOutputData) {
taskSummaryMetricsTableArray.push(row);
}
break;
case "shuffleWriteMetrics":
var row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 4);
if (dataToShow.showShuffleWriteData) {
taskSummaryMetricsTableArray.push(row);
}
break;
case "diskBytesSpilled":
var row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 5);
if (dataToShow.showBytesSpilledData) {
taskSummaryMetricsTableArray.push(row);
}
break;
case "memoryBytesSpilled":
var row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 6);
if (dataToShow.showBytesSpilledData) {
taskSummaryMetricsTableArray.push(row);
}
break;
default:
if (getColumnNameForTaskMetricSummary(columnKey) != "NA") {
var row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 0);
taskSummaryMetricsTableArray.push(row);
}
break;
}
});
var taskSummaryMetricsTableFilteredArray =
taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTableFilteredArray.slice();
reselectCheckboxesBasedOnTaskTableState();
});
// building accumulator update table
var accumulatorConf = {
"data": accumulatorTable,
"columns": [
{data : "id"},
{data : "name"},
{data : "value"}
],
"paging": false,
"searching": false,
"order": [[0, "asc"]],
"bAutoWidth": false
};
$("#accumulator-table").DataTable(accumulatorConf);
// building tasks table that uses server side functionality
var totalTasksToShow = responseBody.numCompleteTasks + responseBody.numActiveTasks +
responseBody.numKilledTasks + responseBody.numFailedTasks;
var taskTable = "#active-tasks-table";
var taskConf = {
"serverSide": true,
"paging": true,
"info": true,
"processing": true,
"lengthMenu": [[20, 40, 60, 100, totalTasksToShow], [20, 40, 60, 100, "All"]],
"orderMulti": false,
"bAutoWidth": false,
"ajax": {
"url": endPoint + "/" + stageAttemptId + "/taskTable",
"data": function (data) {
var columnIndexToSort = 0;
var columnNameToSort = "Index";
if (data.order[0].column && data.order[0].column != "") {
columnIndexToSort = parseInt(data.order[0].column);
columnNameToSort = data.columns[columnIndexToSort].name;
}
delete data.columns;
data.numTasks = totalTasksToShow;
data.columnIndexToSort = columnIndexToSort;
data.columnNameToSort = columnNameToSort;
},
"dataSrc": function (jsons) {
var jsonStr = JSON.stringify(jsons);
var tasksToShow = JSON.parse(jsonStr);
return tasksToShow.aaData;
},
"error": function (jqXHR, textStatus, errorThrown) {
alert("Unable to connect to the server. Looks like the Spark " +
"application must have ended. Please Switch to the history UI.");
$("#active-tasks-table_processing").css("display","none");
}
},
"columns": [
{data: function (row, type) {
return type !== 'display' ? (isNaN(row.index) ? 0 : row.index ) : row.index;
},
name: "Index"
},
{data : "taskId", name: "ID"},
{data : "attempt", name: "Attempt"},
{data : "status", name: "Status"},
{data : "taskLocality", name: "Locality Level"},
{data : "executorId", name: "Executor ID"},
{data : "host", name: "Host"},
{data : "executorLogs", name: "Logs", render: formatLogsCells},
{data : "launchTime", name: "Launch Time", render: formatDate},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.executorRunTime) {
return type === 'display' ? formatDuration(row.taskMetrics.executorRunTime) : row.taskMetrics.executorRunTime;
} else {
return "";
}
},
name: "Duration"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.jvmGcTime) {
return type === 'display' ? formatDuration(row.taskMetrics.jvmGcTime) : row.taskMetrics.jvmGcTime;
} else {
return "";
}
},
name: "GC Time"
},
{
data : function (row, type) {
if (row.schedulerDelay) {
return type === 'display' ? formatDuration(row.schedulerDelay) : row.schedulerDelay;
} else {
return "";
}
},
name: "Scheduler Delay"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.executorDeserializeTime) {
return type === 'display' ? formatDuration(row.taskMetrics.executorDeserializeTime) : row.taskMetrics.executorDeserializeTime;
} else {
return "";
}
},
name: "Task Deserialization Time"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
return type === 'display' ? formatDuration(row.taskMetrics.shuffleReadMetrics.fetchWaitTime) : row.taskMetrics.shuffleReadMetrics.fetchWaitTime;
} else {
return "";
}
},
name: "Shuffle Read Blocked Time"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
return type === 'display' ? formatBytes(row.taskMetrics.shuffleReadMetrics.remoteBytesRead, type) : row.taskMetrics.shuffleReadMetrics.remoteBytesRead;
} else {
return "";
}
},
name: "Shuffle Remote Reads"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.resultSerializationTime) {
return type === 'display' ? formatDuration(row.taskMetrics.resultSerializationTime) : row.taskMetrics.resultSerializationTime;
} else {
return "";
}
},
name: "Result Serialization Time"
},
{
data : function (row, type) {
if (row.gettingResultTime) {
return type === 'display' ? formatDuration(row.gettingResultTime) : row.gettingResultTime;
} else {
return "";
}
},
name: "Getting Result Time"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.peakExecutionMemory) {
return type === 'display' ? formatBytes(row.taskMetrics.peakExecutionMemory, type) : row.taskMetrics.peakExecutionMemory;
} else {
return "";
}
},
name: "Peak Execution Memory"
},
{
data : function (row, type) {
if (accumulatorTable.length > 0 && row.accumulatorUpdates.length > 0) {
var allAccums = "";
row.accumulatorUpdates.forEach(function(accumulator) {
allAccums += accumulator.name + ': ' + accumulator.update + "<BR>";
})
return allAccums;
} else {
return "";
}
},
name: "Accumulators"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.inputMetrics && row.taskMetrics.inputMetrics.bytesRead > 0) {
if (type === 'display') {
return formatBytes(row.taskMetrics.inputMetrics.bytesRead, type) + " / " + row.taskMetrics.inputMetrics.recordsRead;
} else {
return row.taskMetrics.inputMetrics.bytesRead + " / " + row.taskMetrics.inputMetrics.recordsRead;
}
} else {
return "";
}
},
name: "Input Size / Records"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.outputMetrics && row.taskMetrics.outputMetrics.bytesWritten > 0) {
if (type === 'display') {
return formatBytes(row.taskMetrics.outputMetrics.bytesWritten, type) + " / " + row.taskMetrics.outputMetrics.recordsWritten;
} else {
return row.taskMetrics.outputMetrics.bytesWritten + " / " + row.taskMetrics.outputMetrics.recordsWritten;
}
} else {
return "";
}
},
name: "Output Size / Records"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.writeTime > 0) {
return type === 'display' ? formatDuration(parseInt(row.taskMetrics.shuffleWriteMetrics.writeTime) / 1000000) : row.taskMetrics.shuffleWriteMetrics.writeTime;
} else {
return "";
}
},
name: "Write Time"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.bytesWritten > 0) {
if (type === 'display') {
return formatBytes(row.taskMetrics.shuffleWriteMetrics.bytesWritten, type) + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
} else {
return row.taskMetrics.shuffleWriteMetrics.bytesWritten + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
}
} else {
return "";
}
},
name: "Shuffle Write Size / Records"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics && row.taskMetrics.shuffleReadMetrics.localBytesRead > 0) {
var totalBytesRead = parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) + parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead);
if (type === 'display') {
return formatBytes(totalBytesRead, type) + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
} else {
return totalBytesRead + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
}
} else {
return "";
}
},
name: "Shuffle Read Size / Records"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.memoryBytesSpilled && row.taskMetrics.memoryBytesSpilled > 0) {
return type === 'display' ? formatBytes(row.taskMetrics.memoryBytesSpilled, type) : row.taskMetrics.memoryBytesSpilled;
} else {
return "";
}
},
name: "Spill (Memory)"
},
{
data : function (row, type) {
if (row.taskMetrics && row.taskMetrics.diskBytesSpilled && row.taskMetrics.diskBytesSpilled > 0) {
return type === 'display' ? formatBytes(row.taskMetrics.diskBytesSpilled, type) : row.taskMetrics.diskBytesSpilled;
} else {
return "";
}
},
name: "Spill (Disk)"
},
{
data : function (row, type) {
var msg = row.errorMessage;
if (typeof msg === 'undefined') {
return "";
} else {
var formHead = msg.substring(0, msg.indexOf("at"));
var form = "<span onclick=\"this.parentNode.querySelector('.stacktrace-details').classList.toggle('collapsed')\" class=\"expand-details\">+details</span>";
var formMsg = "<div class=\"stacktrace-details collapsed\"><pre>" + row.errorMessage + "</pre></div>";
return formHead + form + formMsg;
}
},
name: "Errors"
}
],
"columnDefs": [
{ "visible": false, "targets": 11 },
{ "visible": false, "targets": 12 },
{ "visible": false, "targets": 13 },
{ "visible": false, "targets": 14 },
{ "visible": false, "targets": 15 },
{ "visible": false, "targets": 16 },
{ "visible": false, "targets": 17 },
{ "visible": false, "targets": 18 }
],
"deferRender": true
};
taskTableSelector = $(taskTable).DataTable(taskConf);
$('#active-tasks-table_filter input').unbind();
var searchEvent;
$('#active-tasks-table_filter input').bind('keyup', function(e) {
if (typeof searchEvent !== 'undefined') {
window.clearTimeout(searchEvent);
}
var value = this.value;
searchEvent = window.setTimeout(function(){
taskTableSelector.search( value ).draw();}, 500);
});
reselectCheckboxesBasedOnTaskTableState();
// hide or show columns dynamically event
$('input.toggle-vis').on('click', function(e){
// Get the column
var para = $(this).attr('data-column');
if (para == "0") {
var allColumns = taskTableSelector.columns(optionalColumns);
if ($(this).is(":checked")) {
$(".toggle-vis").prop('checked', true);
allColumns.visible(true);
createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableArray);
} else {
$(".toggle-vis").prop('checked', false);
allColumns.visible(false);
var taskSummaryMetricsTableFilteredArray =
taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
}
} else {
var column = taskTableSelector.column(para);
// Toggle the visibility
column.visible(!column.visible());
var taskSummaryMetricsTableFilteredArray = [];
if ($(this).is(":checked")) {
taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == para)[0]);
taskSummaryMetricsTableFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
} else {
taskSummaryMetricsTableFilteredArray =
taskSummaryMetricsTableCurrentStateArray.filter(row => (row.checkboxId).toString() != para);
}
createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
}
});
// title number and toggle list
$("#summaryMetricsTitle").html("Summary Metrics for " + "<a href='#tasksTitle'>" + responseBody.numCompleteTasks + " Completed Tasks" + "</a>");
$("#tasksTitle").html("Tasks (" + totalTasksToShow + ")");
// hide or show the accumulate update table
if (accumulatorTable.length == 0) {
$("#accumulator-update-table").hide();
} else {
taskTableSelector.column(18).visible(true);
$("#accumulator-update-table").show();
}
// Showing relevant stage data depending on stage type for task table and executor
// summary table
taskTableSelector.column(19).visible(dataToShow.showInputData);
taskTableSelector.column(20).visible(dataToShow.showOutputData);
taskTableSelector.column(21).visible(dataToShow.showShuffleWriteData);
taskTableSelector.column(22).visible(dataToShow.showShuffleWriteData);
taskTableSelector.column(23).visible(dataToShow.showShuffleReadData);
taskTableSelector.column(24).visible(dataToShow.showBytesSpilledData);
taskTableSelector.column(25).visible(dataToShow.showBytesSpilledData);
if (window.localStorage) {
if (window.localStorage.getItem("arrowtoggle1class") !== null &&
window.localStorage.getItem("arrowtoggle1class").includes("arrow-open")) {
$("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
$("#toggle-metrics").toggle();
}
if (window.localStorage.getItem("arrowtoggle2class") !== null &&
window.localStorage.getItem("arrowtoggle2class").includes("arrow-open")) {
$("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
$("#toggle-aggregatedMetrics").toggle();
}
}
});
});
});
});
Raw
utils.js
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// this function works exactly the same as UIUtils.formatDuration
function formatDuration(milliseconds) {
if (milliseconds < 100) {
return parseInt(milliseconds).toFixed(1) + " ms";
}
var seconds = milliseconds * 1.0 / 1000;
if (seconds < 1) {
return seconds.toFixed(1) + " s";
}
if (seconds < 60) {
return seconds.toFixed(0) + " s";
}
var minutes = seconds / 60;
if (minutes < 10) {
return minutes.toFixed(1) + " min";
} else if (minutes < 60) {
return minutes.toFixed(0) + " min";
}
var hours = minutes / 60;
return hours.toFixed(1) + " h";
}
function formatBytes(bytes, type) {
if (type !== 'display') return bytes;
if (bytes == 0) return '0.0 B';
var k = 1024;
var dm = 1;
var sizes = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
var i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
}
function padZeroes(num) {
return ("0" + num).slice(-2);
}
function formatTimeMillis(timeMillis) {
if (timeMillis <= 0) {
return "-";
} else {
var dt = new Date(timeMillis);
return formatDateString(dt);
}
}
function formatDateString(dt) {
return dt.getFullYear() + "-" +
padZeroes(dt.getMonth() + 1) + "-" +
padZeroes(dt.getDate()) + " " +
padZeroes(dt.getHours()) + ":" +
padZeroes(dt.getMinutes()) + ":" +
padZeroes(dt.getSeconds());
}
function getTimeZone() {
try {
return Intl.DateTimeFormat().resolvedOptions().timeZone;
} catch(ex) {
// Get time zone from a string representing the date,
// eg. "Thu Nov 16 2017 01:13:32 GMT+0800 (CST)" -> "CST"
return new Date().toString().match(/\((.*)\)/)[1];
}
}
function formatLogsCells(execLogs, type) {
if (type !== 'display') return Object.keys(execLogs);
if (!execLogs) return;
var result = '';
$.each(execLogs, function (logName, logUrl) {
result += '<div><a href=' + logUrl + '>' + logName + '</a></div>'
});
return result;
}
function getStandAloneAppId(cb) {
var words = document.baseURI.split('/');
//Custom jupyterhub workaround that parses port number in URI
var ind = words.indexOf("4040");
if (ind > 0) {
$.getJSON(location.origin + "/notebook/" + words[4] + "/" + words[5] + "/proxy/4040/api/v1/applications", function(response, status, jqXHR) {
if (response && response.length > 0) {
var appId = response[0].id
cb(appId);
return;
}
});
}
var ind = words.indexOf("proxy");
var indp = words.indexOf("4040");
if ((ind > 0) && (indp < 1)) {
var appId = words[ind + 1];
cb(appId);
return;
}
ind = words.indexOf("history");
if (ind > 0) {
var appId = words[ind + 1];
cb(appId);
return;
}
// Looks like Web UI is running in standalone mode
// Let's get application-id using REST End Point
$.getJSON(location.origin + "/notebook/" + words[4] + "/" + words[5] + "/proxy/4040/api/v1/applications", function(response, status, jqXHR) {
if (response && response.length > 0) {
var appId = response[0].id;
cb(appId);
return;
}
});
}
// This function is a helper function for sorting in datatable.
// When the data is in duration (e.g. 12ms 2s 2min 2h )
// It will convert the string into integer for correct ordering
function ConvertDurationString(data) {
data = data.toString();
var units = data.replace(/[\d\.]/g, '' )
.replace(' ', '')
.toLowerCase();
var multiplier = 1;
switch(units) {
case 's':
multiplier = 1000;
break;
case 'min':
multiplier = 600000;
break;
case 'h':
multiplier = 3600000;
break;
default:
break;
}
return parseFloat(data) * multiplier;
}
function createTemplateURI(appId, templateName) {
var words = document.baseURI.split('/');
var ind = words.indexOf("4040");
if (ind > 0) {
var baseURI = words.slice(0, ind + 1).join('/') + '/static/' + templateName + '-template.html';
return baseURI;
}
var ind = words.indexOf("proxy");
if (ind > 0) {
var baseURI = words.slice(0, ind + 1).join('/') + '/' + appId + '/static/' + templateName + '-template.html';
return baseURI;
}
ind = words.indexOf("history");
if(ind > 0) {
var baseURI = words.slice(0, ind).join('/') + '/static/' + templateName + '-template.html';
return baseURI;
}
return location.origin + "/static/" + templateName + "-template.html";
}
function setDataTableDefaults() {
$.extend($.fn.dataTable.defaults, {
stateSave: true,
lengthMenu: [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]],
pageLength: 20
});
}
function formatDate(date) {
if (date <= 0) return "-";
else {
var dt = new Date(date.replace("GMT", "Z"))
return formatDateString(dt);
}
}
function createRESTEndPointForExecutorsPage(appId) {
var words = document.baseURI.split('/');
var ind = words.indexOf("4040");
if (ind > 0) {
var newBaseURI = words.slice(0, ind + 1).join('/');
return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors";
}
var ind = words.indexOf("proxy");
if (ind > 0) {
var appId = words[ind + 1];
var newBaseURI = words.slice(0, ind + 2).join('/');
return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors";
}
ind = words.indexOf("history");
if (ind > 0) {
var appId = words[ind + 1];
var attemptId = words[ind + 2];
var newBaseURI = words.slice(0, ind).join('/');
if (isNaN(attemptId)) {
return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors";
} else {
return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors";
}
}
return location.origin + "/api/v1/applications/" + appId + "/allexecutors";
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment